diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dd7c872c5a..92b46025d6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -92,6 +92,22 @@ updates: - dependency-name: "mcr.microsoft.com/vscode/devcontainers/typescript-node" update-types: ["version-update:semver-major"] + - package-ecosystem: "docker" + directory: "/lambdas/services/scale-set" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "chore(scale-set)" + # Keep the service runtime aligned with the supported Node.js major. + ignore: + - dependency-name: "node" + update-types: ["version-update:semver-major"] + - package-ecosystem: "pip" directory: "/.github/workflows/mkdocs" schedule: diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index 8eba8f46aa..1a270a4461 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -50,3 +50,29 @@ jobs: name: coverage-reports path: ./**/coverage retention-days: 5 + + scale-set-container: + name: Build scale-set service container + runs-on: ubuntu-latest + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + - name: Build scale-set service image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64,linux/arm64 + push: false + cache-from: type=gha,scope=scale-set-service + cache-to: type=gha,mode=max,scope=scale-set-service diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8fcb90d72e..188cd35d20 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,7 @@ name: Release build +env: + SCALE_SET_IMAGE: ghcr.io/${{ github.repository_owner }}/terraform-aws-github-runner-scale-set-service + on: push: branches: @@ -21,6 +24,8 @@ jobs: actions: write # for release-please-action to trigger other workflows id-token: write # for actions/attest-build-provenance to generate attestations attestations: write # for actions/attest-build-provenance to write attestations + artifact-metadata: write # for publishing linked container attestations + packages: write # for publishing the scale-set service image to GHCR environment: release steps: - name: Harden the runner (Audit all outbound calls) @@ -55,6 +60,47 @@ jobs: target-branch: ${{ steps.branch.outputs.name }} release-type: terraform-module token: ${{ steps.token.outputs.token }} + - name: Set up QEMU + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + - name: Set up Docker Buildx + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + - name: Log in to the GitHub Container Registry + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish scale-set service image + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: scale-set-image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.SCALE_SET_IMAGE }}:${{ steps.release.outputs.tag_name }} + ${{ env.SCALE_SET_IMAGE }}:latest + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ steps.release.outputs.tag_name }} + sbom: true + provenance: mode=max + cache-from: type=gha,scope=scale-set-service + cache-to: type=gha,mode=max,scope=scale-set-service + - name: Attest scale-set service image + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: scale-set-image-attest + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-name: ${{ env.SCALE_SET_IMAGE }} + subject-digest: ${{ steps.scale-set-image.outputs.digest }} + push-to-registry: true - name: Attest if: ${{ steps.release.outputs.releases_created == 'true' }} id: attest @@ -65,20 +111,26 @@ jobs: if: ${{ steps.release.outputs.releases_created == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ github.event.inputs.version }} TAG_NAME: ${{ steps.release.outputs.tag_name }} ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }} + CONTAINER_ATTESTATION_URL: ${{ steps.scale-set-image-attest.outputs.attestation-url }} + CONTAINER_IMAGE: ${{ env.SCALE_SET_IMAGE }} + CONTAINER_DIGEST: ${{ steps.scale-set-image.outputs.digest }} REPOSITORY: ${{ github.repository }} run: | - version="${VERSION}" tag_name="${TAG_NAME}" attestation_url="${ATTESTATION_URL}" + container_attestation_url="${CONTAINER_ATTESTATION_URL}" + container_image="${CONTAINER_IMAGE}" + container_digest="${CONTAINER_DIGEST}" repository="${REPOSITORY}" - gh release view $version --json body -q '.body' > new-release-notes.md + gh release view "$tag_name" --json body -q '.body' > new-release-notes.md echo "## Attestation" >> new-release-notes.md echo "Attestation url: $attestation_url" >> new-release-notes.md echo "Verify the artifacts by running \`gh attestation verify --repo ${repository}\`" >> new-release-notes.md - gh release edit $tag_name -F new-release-notes.md -t $tag_name + echo "Scale-set service image: \`${container_image}@${container_digest}\`" >> new-release-notes.md + echo "Container attestation url: $container_attestation_url" >> new-release-notes.md + gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name" - name: Upload release assets if: ${{ steps.release.outputs.releases_created == 'true' }} env: diff --git a/docs/security.md b/docs/security.md index a94688b234..4ef4d17b94 100644 --- a/docs/security.md +++ b/docs/security.md @@ -14,6 +14,14 @@ The examples are using standard AMI's for different operating systems. Instances ## Attestation -The module is released using GitHub actions and the lambda artifacts are attached to the release as attachment. During the release attestations are created. The attestations are created by the release pipeline. You find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release. The attestations are not a security guarantee. We recommend you to verify the attestation after downloading the lambda artifacts. +The module is released using GitHub Actions and the Lambda artifacts are attached to the release. The release pipeline creates provenance attestations for those artifacts. You can find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release; it is not a security guarantee. We recommend verifying the attestation after downloading the Lambda artifacts. + +Releases also publish the multi-architecture scale-set service image to the GitHub Container Registry with an SBOM, build provenance, and a registry attestation. The convenience image default follows the latest module release. Production deployments should override it with the immutable image digest printed in the release notes, then verify that image with: + +```bash +gh attestation verify \ + oci://ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256: \ + --repo github-aws-runners/terraform-aws-github-runner +``` --8<-- "SECURITY.md:mkdocsrunners" diff --git a/lambdas/libs/compute-providers/aws/ec2/scale-set.ts b/lambdas/libs/compute-providers/aws/ec2/scale-set.ts new file mode 100644 index 0000000000..671ddc69f7 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/scale-set.ts @@ -0,0 +1,32 @@ +import type { ScaleSetComputeProviderModule, ScaleSetComputeProviderPlugin } from '../../scale-set'; + +import { createEc2ScaleSetProvider, type Ec2ScaleSetProviderDependencies } from './src/scale-set/provider'; + +export type { Ec2ScaleSetProviderConfig, Ec2ScaleSetProviderDependencies } from './src/scale-set/provider'; +export { createEc2ScaleSetProvider, parseEc2ScaleSetProviderConfig } from './src/scale-set/provider'; + +export function createEc2ScaleSetPlugin( + dependencies: Ec2ScaleSetProviderDependencies = {}, +): ScaleSetComputeProviderPlugin<'ec2'> { + return { + type: 'ec2', + capabilities: { + environmentVariables: {}, + create: ({ runnerConfigName, scaleSetId, githubScope, configuration }) => + createEc2ScaleSetProvider( + { + runnerConfigName, + scaleSetId, + githubScope, + configuration: configuration as Parameters[0]['configuration'], + }, + dependencies, + ), + }, + }; +} + +export const provider = { + type: 'ec2', + createPlugin: createEc2ScaleSetPlugin, +} satisfies ScaleSetComputeProviderModule<'ec2'>; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts new file mode 100644 index 0000000000..d6483f3204 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; + +import { parseEc2ScaleSetProviderConfig } from './configuration'; +import { config } from './test/fixtures'; + +describe('EC2 scale-set provider configuration', () => { + it('strictly parses the supported provider-owned configuration', () => { + expect(parseEc2ScaleSetProviderConfig(config)).toMatchObject(config); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: '' })).toMatchObject({ + runnerNamePrefix: '', + }); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: 'r'.repeat(45) })).toMatchObject({ + runnerNamePrefix: 'r'.repeat(45), + }); + }); + + it.each([ + [{ ...config, region: 'eu-west-one' }], + [{ ...config, subnets: ['subnet-12345678', 'subnet-12345678'] }], + [{ ...config, ec2instanceCriteria: { ...config.ec2instanceCriteria, instanceAllocationStrategy: 'diversified' } }], + [{ ...config, ec2OverrideConfig: { UserData: 'untrusted' } }], + [{ ...config, scaleErrors: ['ThrottlingException'] }], + [{ ...config, ssmParameterTags: [{ Key: 'aws:owner', Value: 'untrusted' }] }], + [{ ...config, runnerNamePrefix: 'r'.repeat(46) }], + [{ ...config, bootTimeoutMinutes: 10 }], + ])('rejects invalid or unsupported values instead of forwarding them to AWS', (invalid) => { + expect(() => parseEc2ScaleSetProviderConfig(invalid)).toThrow(); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts new file mode 100644 index 0000000000..1d03728cb8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts @@ -0,0 +1,346 @@ +import type { Tag as SsmTag } from '@aws-sdk/client-ssm'; + +import type { Ec2OverrideConfig, RunnerInputParameters } from '../runners.d'; +import { isRecord, Ec2ScaleSetValidationError } from './reconcile'; + +const SPOT_ALLOCATION_STRATEGIES = new Set([ + 'lowest-price', + 'diversified', + 'capacity-optimized', + 'capacity-optimized-prioritized', + 'price-capacity-optimized', +]); +const ON_DEMAND_ALLOCATION_STRATEGIES = new Set(['lowest-price', 'prioritized']); + +export interface Ec2ScaleSetProviderConfig { + region: string; + environment: string; + runnerNamePrefix: string; + jitConfigParameterPath: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; + ec2OverrideConfig?: Ec2OverrideConfig; + amiIdSsmParameterName?: string; + tracingEnabled?: boolean; + onDemandFailoverOnError?: string[]; + useDedicatedHost?: boolean; + ssmKmsKeyId?: string; + ssmParameterTags?: SsmTag[]; +} + +export interface CreateEc2ScaleSetProviderInput { + runnerConfigName: string; + scaleSetId: number; + githubScope: string; + configuration: Ec2ScaleSetProviderConfig; +} + +function rejectUnknownKeys(value: Record, allowedKeys: ReadonlySet, name: string): void { + const unknownKey = Object.keys(value).find((key) => !allowedKeys.has(key)); + if (unknownKey !== undefined) { + throw new Ec2ScaleSetValidationError(`Unsupported EC2 scale-set configuration field '${name}.${unknownKey}'`); + } +} + +function requireString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length === 0 || value.length > maximumLength || !pattern.test(value)) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requirePossiblyEmptyString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length > maximumLength || !pattern.test(value)) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function optionalString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string | undefined { + if (value === undefined) return undefined; + return requireString(value, name, pattern, maximumLength); +} + +function optionalBoolean(value: unknown, name: string): boolean | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'boolean') { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requireStringArray( + value: unknown, + name: string, + pattern: RegExp, + maximumItemLength: number, + allowEmpty = false, +): string[] { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.length > 100) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + const parsed = value.map((item, index) => requireString(item, `${name}[${index}]`, pattern, maximumItemLength)); + if (new Set(parsed).size !== parsed.length) { + throw new Ec2ScaleSetValidationError(`EC2 scale-set configuration field '${name}' contains duplicate values`); + } + return parsed; +} + +function parseInstanceTypePriorities(value: unknown): Record | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'instanceTypePriorities'"); + } + + const result = Object.create(null) as Record; + for (const [instanceType, priority] of Object.entries(value)) { + requireString(instanceType, 'instanceTypePriorities key', /^[a-z0-9][a-z0-9.-]*$/, 64); + if (typeof priority !== 'number' || !Number.isSafeInteger(priority) || priority < 0 || priority > 1000) { + throw new Ec2ScaleSetValidationError( + `Invalid EC2 scale-set configuration priority for instance type '${instanceType}'`, + ); + } + result[instanceType] = priority; + } + return result; +} + +function requireSsmTagValue(value: unknown): string { + if (typeof value !== 'string' || value.length > 256) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + for (const character of value) { + const codePoint = character.codePointAt(0)!; + if (codePoint < 32 || codePoint === 127) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + } + return value; +} + +function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ec2OverrideConfig'"); + } + + const supportedKeys = new Set([ + 'InstanceType', + 'MaxPrice', + 'SubnetId', + 'AvailabilityZone', + 'AvailabilityZoneId', + 'WeightedCapacity', + 'Priority', + 'ImageId', + ]); + if (Object.keys(value).some((key) => !supportedKeys.has(key))) { + throw new Ec2ScaleSetValidationError('EC2 scale-set configuration contains an unsupported launch override'); + } + + const weightedCapacity = value.WeightedCapacity; + const priority = value.Priority; + for (const [name, number] of [ + ['WeightedCapacity', weightedCapacity], + ['Priority', priority], + ] as const) { + if (number !== undefined && (typeof number !== 'number' || !Number.isFinite(number) || number < 0)) { + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); + } + } + + return { + InstanceType: optionalString( + value.InstanceType, + 'ec2OverrideConfig.InstanceType', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ) as Ec2OverrideConfig['InstanceType'], + MaxPrice: optionalString(value.MaxPrice, 'ec2OverrideConfig.MaxPrice', /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, 32), + SubnetId: optionalString(value.SubnetId, 'ec2OverrideConfig.SubnetId', /^subnet-[0-9a-f]+$/, 32), + AvailabilityZone: optionalString( + value.AvailabilityZone, + 'ec2OverrideConfig.AvailabilityZone', + /^[a-z]{2}(?:-[a-z0-9]+)+-\d[a-z]$/, + 64, + ), + AvailabilityZoneId: optionalString( + value.AvailabilityZoneId, + 'ec2OverrideConfig.AvailabilityZoneId', + /^[a-z0-9-]+$/, + 64, + ), + WeightedCapacity: weightedCapacity as number | undefined, + Priority: priority as number | undefined, + ImageId: optionalString(value.ImageId, 'ec2OverrideConfig.ImageId', /^ami-[0-9a-f]+$/, 32), + }; +} + +function parseSsmTags(value: unknown): SsmTag[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.length > 45) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + + const tags: SsmTag[] = []; + const keys = new Set(); + for (const item of value) { + if (!isRecord(item)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + const key = requireString(item.Key, 'ssmParameterTags.Key', /^[A-Za-z0-9_.:/=+@-]+$/, 128); + const tagValue = requireSsmTagValue(item.Value); + if (key.toLowerCase().startsWith('aws:') || keys.has(key)) { + throw new Ec2ScaleSetValidationError(`Invalid or duplicate SSM tag key '${key}'`); + } + keys.add(key); + tags.push({ Key: key, Value: tagValue }); + } + return tags; +} + +export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProviderConfig { + if (!isRecord(value)) { + throw new Ec2ScaleSetValidationError('EC2 scale-set provider configuration must be an object'); + } + rejectUnknownKeys( + value, + new Set([ + 'region', + 'environment', + 'runnerNamePrefix', + 'jitConfigParameterPath', + 'subnets', + 'launchTemplateName', + 'ec2instanceCriteria', + 'ec2OverrideConfig', + 'amiIdSsmParameterName', + 'tracingEnabled', + 'onDemandFailoverOnError', + 'useDedicatedHost', + 'ssmKmsKeyId', + 'ssmParameterTags', + ]), + 'configuration', + ); + if (!isRecord(value.ec2instanceCriteria)) { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ec2instanceCriteria'"); + } + rejectUnknownKeys( + value.ec2instanceCriteria, + new Set([ + 'instanceTypes', + 'instanceTypePriorities', + 'targetCapacityType', + 'maxSpotPrice', + 'instanceAllocationStrategy', + ]), + 'ec2instanceCriteria', + ); + + const targetCapacityType = value.ec2instanceCriteria.targetCapacityType; + if (targetCapacityType !== 'on-demand' && targetCapacityType !== 'spot') { + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'targetCapacityType'"); + } + const instanceAllocationStrategy = requireString( + value.ec2instanceCriteria.instanceAllocationStrategy, + 'instanceAllocationStrategy', + /^[a-z-]+$/, + 64, + ) as RunnerInputParameters['ec2instanceCriteria']['instanceAllocationStrategy']; + const allowedAllocationStrategies = + targetCapacityType === 'spot' ? SPOT_ALLOCATION_STRATEGIES : ON_DEMAND_ALLOCATION_STRATEGIES; + if (!allowedAllocationStrategies.has(instanceAllocationStrategy)) { + throw new Ec2ScaleSetValidationError( + `Invalid allocation strategy '${instanceAllocationStrategy}' for '${targetCapacityType}' capacity`, + ); + } + + const jitConfigParameterPath = requireString( + value.jitConfigParameterPath, + 'jitConfigParameterPath', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ).replace(/\/$/, ''); + + return { + region: requireString(value.region, 'region', /^[a-z]{2}(?:-[a-z0-9]+)+-\d$/, 32), + environment: requireString(value.environment, 'environment', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128), + runnerNamePrefix: requirePossiblyEmptyString(value.runnerNamePrefix, 'runnerNamePrefix', /^[A-Za-z0-9._-]*$/, 45), + jitConfigParameterPath, + subnets: requireStringArray(value.subnets, 'subnets', /^subnet-[0-9a-f]+$/, 32), + launchTemplateName: requireString(value.launchTemplateName, 'launchTemplateName', /^[A-Za-z0-9()./_-]+$/, 128), + ec2instanceCriteria: { + instanceTypes: requireStringArray( + value.ec2instanceCriteria.instanceTypes, + 'instanceTypes', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ), + instanceTypePriorities: parseInstanceTypePriorities(value.ec2instanceCriteria.instanceTypePriorities), + targetCapacityType, + maxSpotPrice: optionalString( + value.ec2instanceCriteria.maxSpotPrice, + 'maxSpotPrice', + /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, + 32, + ), + instanceAllocationStrategy, + }, + ec2OverrideConfig: parseEc2OverrideConfig(value.ec2OverrideConfig), + amiIdSsmParameterName: optionalString( + value.amiIdSsmParameterName, + 'amiIdSsmParameterName', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ), + tracingEnabled: optionalBoolean(value.tracingEnabled, 'tracingEnabled'), + onDemandFailoverOnError: requireStringArray( + value.onDemandFailoverOnError ?? [], + 'onDemandFailoverOnError', + /^[A-Za-z0-9._-]+$/, + 128, + true, + ), + useDedicatedHost: optionalBoolean(value.useDedicatedHost, 'useDedicatedHost'), + ssmKmsKeyId: optionalString(value.ssmKmsKeyId, 'ssmKmsKeyId', /^[A-Za-z0-9_:/+=,.@-]+$/, 2048), + ssmParameterTags: parseSsmTags(value.ssmParameterTags), + }; +} + +export function validateFactoryInput(input: CreateEc2ScaleSetProviderInput): void { + requireString(input.runnerConfigName, 'runnerConfigName', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128); + if (!Number.isSafeInteger(input.scaleSetId) || input.scaleSetId <= 0) { + throw new Ec2ScaleSetValidationError('scaleSetId must be a positive safe integer'); + } + validateCanonicalGitHubScope(input.githubScope); +} + +function validateCanonicalGitHubScope(value: unknown): string { + if (typeof value !== 'string' || value.length === 0 || value.length > 2048) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + let url: URL; + try { + url = new URL(value); + } catch { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + const parts = url.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean); + if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + url.pathname = `/${parts.join('/')}`; + const canonical = url.toString().replace(/\/$/, ''); + if (canonical !== value) { + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + return value; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts new file mode 100644 index 0000000000..c4b0347376 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts @@ -0,0 +1,123 @@ +import { createHash } from 'node:crypto'; + +import { CreateFleetCommand, DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { EC2_GITHUB_SCOPE_HASH_TAG, EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG } from './inventory'; +import { createRequest, githubScopeHash, githubState, ownedInstance } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set inventory', () => { + it('lists only the exact runner-config and scale-set ownership boundary', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ + ownedInstance('i-owned', { runnerId: 101, runnerName: 'runner-i-owned' }), + ownedInstance('i-other', undefined, { runnerConfigName: 'other' }), + ownedInstance( + 'i-other-scope', + { runnerId: 102, runnerName: 'runner-i-other-scope' }, + { + githubScopeHash: createHash('sha256').update('https://github.com/another', 'utf8').digest('hex'), + }, + ), + ], + }, + ], + }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ status: 'converged', desiredRunners: 1, currentRunners: 1 }); + expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { + Filters: expect.arrayContaining([ + { Name: 'tag:ghr:Application', Values: ['github-action-runner'] }, + { Name: 'tag:ghr:created_by', Values: ['scale-set-service'] }, + { Name: 'tag:ghr:environment', Values: ['unit-test'] }, + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: ['linux'] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: ['42'] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash] }, + ]), + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts a young handed-off instance as serving during its bounded boot window', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-booting', { runnerId: 101, runnerName: 'runner-i-booting' })], + }, + ], + }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:09:59Z').getTime() }).reconcile( + createRequest(), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('uses the orchestration request boot window instead of provider configuration', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-at-timeout', { runnerId: 101, runnerName: 'runner-i-at-timeout' })], + }, + ], + }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:05:00Z').getTime() }).reconcile( + createRequest({ bootTimeoutMinutes: 5 }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts tagged capacity after the boot window without public runner inventory', async () => { + const instance = ownedInstance('i-old', { runnerId: 101, runnerName: 'runner-i-old' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:10:00Z').getTime() }).reconcile( + createRequest({ busyRunners: 1 }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts an exact JobStarted identity as serving without waiting for public inventory', async () => { + const instance = ownedInstance('i-started', { runnerId: 101, runnerName: 'runner-i-started' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T12:00:00Z').getTime() }).reconcile( + createRequest({ + runnerStates: [ + githubState(101, 'runner-i-started', { status: 'unknown', busy: undefined, lifecycle: 'started' }), + ], + }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts new file mode 100644 index 0000000000..91941b0139 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts @@ -0,0 +1,249 @@ +import { createHash } from 'node:crypto'; + +import { DescribeInstancesCommand, type EC2Client, type Instance, type Tag } from '@aws-sdk/client-ec2'; + +import type { ScaleSetReconcileRequest, ScaleSetRunnerState } from '../../../../scale-set'; +import type { CreateEc2ScaleSetProviderInput } from './configuration'; +import { retainUnknown, type MutableReconcileState } from './reconcile'; + +export const EC2_RUNNER_CONFIG_TAG = 'ghr:runner_config'; +export const EC2_SCALE_SET_ID_TAG = 'ghr:scale_set_id'; +export const EC2_GITHUB_SCOPE_HASH_TAG = 'ghr:github_scope_hash'; +export const EC2_SCALE_SET_STATE_TAG = 'ghr:scale_set_state'; +export const EC2_RUNNER_NAME_TAG = 'ghr:runner_name'; +export const EC2_GITHUB_RUNNER_ID_TAG = 'ghr:github_runner_id'; + +const APPLICATION_TAG = 'ghr:Application'; +const APPLICATION_VALUE = 'github-action-runner'; +const CREATED_BY_TAG = 'ghr:created_by'; +export const SCALE_SET_RUNNER_SOURCE = 'scale-set-service'; +const ENVIRONMENT_TAG = 'ghr:environment'; +export const GITHUB_RUNNER_NAME_MAX_LENGTH = 64; + +type Ec2ScaleSetState = 'provisioning' | 'publishing' | 'config-published' | 'retiring'; + +export interface OwnedEc2Runner { + instanceId: string; + launchTime?: Date; + githubRunnerId?: number; + runnerName?: string; + scaleSetState?: Ec2ScaleSetState; +} + +export function githubScopeHash(githubScope: string): string { + return createHash('sha256').update(githubScope, 'utf8').digest('hex'); +} + +export function runnerIdentityFromGitHubScope(githubScope: string): { + runnerOwner: string; + runnerType: 'Org' | 'Repo'; +} { + const pathParts = new URL(githubScope).pathname.replace(/^\/+|\/+$/g, '').split('/'); + if (pathParts.length === 2 && pathParts[0].toLowerCase() !== 'enterprises') { + return { runnerOwner: pathParts.join('/'), runnerType: 'Repo' }; + } + + // The legacy EC2 tags do not have an enterprise discriminator. They remain + // informational here; exact ownership is fenced by runner config, scale-set + // ID, and the canonical GitHub-scope hash. + return { + runnerOwner: pathParts[0].toLowerCase() === 'enterprises' ? pathParts[1] : pathParts[0], + runnerType: 'Org', + }; +} + +export function ownershipTags(input: CreateEc2ScaleSetProviderInput): Tag[] { + return [ + { Key: ENVIRONMENT_TAG, Value: input.configuration.environment }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + ]; +} + +export async function listOwnedRunners( + input: CreateEc2ScaleSetProviderInput, + ec2Client: EC2Client, + signal: AbortSignal, +): Promise { + const runners: OwnedEc2Runner[] = []; + let nextToken: string | undefined; + do { + const response = await ec2Client.send( + new DescribeInstancesCommand({ + Filters: [ + { Name: 'instance-state-name', Values: ['pending', 'running'] }, + { Name: `tag:${APPLICATION_TAG}`, Values: [APPLICATION_VALUE] }, + { Name: `tag:${CREATED_BY_TAG}`, Values: [SCALE_SET_RUNNER_SOURCE] }, + { Name: `tag:${ENVIRONMENT_TAG}`, Values: [input.configuration.environment] }, + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: [input.runnerConfigName] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: [String(input.scaleSetId)] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash(input.githubScope)] }, + ], + NextToken: nextToken, + }), + { abortSignal: signal }, + ); + nextToken = response.NextToken; + + for (const instance of response.Reservations?.flatMap((reservation) => reservation.Instances ?? []) ?? []) { + const runner = parseOwnedRunner(instance, input); + if (runner) runners.push(runner); + } + } while (nextToken); + + return runners; +} + +function parseOwnedRunner(instance: Instance, input: CreateEc2ScaleSetProviderInput): OwnedEc2Runner | undefined { + if (!instance.InstanceId) return undefined; + const tags = new Map((instance.Tags ?? []).flatMap((tag) => (tag.Key ? [[tag.Key, tag.Value]] : []))); + + if ( + tags.get(APPLICATION_TAG) !== APPLICATION_VALUE || + tags.get(CREATED_BY_TAG) !== SCALE_SET_RUNNER_SOURCE || + tags.get(ENVIRONMENT_TAG) !== input.configuration.environment || + tags.get(EC2_RUNNER_CONFIG_TAG) !== input.runnerConfigName || + tags.get(EC2_SCALE_SET_ID_TAG) !== String(input.scaleSetId) || + tags.get(EC2_GITHUB_SCOPE_HASH_TAG) !== githubScopeHash(input.githubScope) + ) { + return undefined; + } + + const taggedRunnerId = tags.get(EC2_GITHUB_RUNNER_ID_TAG); + const githubRunnerId = taggedRunnerId === undefined ? undefined : Number(taggedRunnerId); + const rawScaleSetState = tags.get(EC2_SCALE_SET_STATE_TAG); + const scaleSetState = ['provisioning', 'publishing', 'config-published', 'retiring'].includes(rawScaleSetState ?? '') + ? (rawScaleSetState as Ec2ScaleSetState) + : undefined; + + return { + instanceId: instance.InstanceId, + launchTime: instance.LaunchTime, + githubRunnerId: Number.isSafeInteger(githubRunnerId) && githubRunnerId! > 0 ? githubRunnerId : undefined, + runnerName: tags.get(EC2_RUNNER_NAME_TAG), + scaleSetState, + }; +} + +function validRunnerState(value: ScaleSetRunnerState): boolean { + return ( + Number.isSafeInteger(value.runnerId) && + value.runnerId > 0 && + Number.isSafeInteger(value.scaleSetId) && + value.scaleSetId > 0 && + typeof value.runnerName === 'string' && + value.runnerName.length > 0 && + value.runnerName.length <= GITHUB_RUNNER_NAME_MAX_LENGTH && + ['online', 'offline', 'unknown'].includes(value.status) && + (typeof value.busy === 'boolean' || value.busy === undefined) && + ['started', 'completed', 'unknown'].includes(value.lifecycle) + ); +} + +export function indexRunnerStates( + runnerStates: readonly ScaleSetRunnerState[], + scaleSetId: number, +): { byName: Map; ambiguousNames: Set; ambiguousIds: Set } { + const byName = new Map(); + const byId = new Map(); + const ambiguousNames = new Set(); + const ambiguousIds = new Set(); + + for (const state of runnerStates) { + if (!validRunnerState(state) || state.scaleSetId !== scaleSetId) continue; + if (byName.has(state.runnerName)) ambiguousNames.add(state.runnerName); + const existingName = byId.get(state.runnerId); + if (existingName !== undefined && existingName !== state.runnerName) { + ambiguousIds.add(state.runnerId); + ambiguousNames.add(existingName); + ambiguousNames.add(state.runnerName); + } + byName.set(state.runnerName, state); + byId.set(state.runnerId, state.runnerName); + } + return { byName, ambiguousNames, ambiguousIds }; +} + +export function matchingRunnerState( + runner: OwnedEc2Runner, + index: ReturnType, + scaleSetId: number, +): ScaleSetRunnerState | undefined { + if (!runner.runnerName || !runner.githubRunnerId) return undefined; + if (index.ambiguousNames.has(runner.runnerName) || index.ambiguousIds.has(runner.githubRunnerId)) return undefined; + const state = index.byName.get(runner.runnerName); + if ( + !state || + state.runnerId !== runner.githubRunnerId || + state.runnerName !== runner.runnerName || + state.scaleSetId !== scaleSetId + ) { + return undefined; + } + return state; +} + +function isWithinBootTimeout(runner: OwnedEc2Runner, bootTimeoutMinutes: number, now: number): boolean { + const launchTime = runner.launchTime?.getTime(); + if (launchTime === undefined || !Number.isFinite(launchTime)) return false; + const ageMilliseconds = now - launchTime; + return ageMilliseconds >= 0 && ageMilliseconds < bootTimeoutMinutes * 60_000; +} + +function isConfirmedServingState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.status === 'online'; +} + +export function servingCapacity( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + now: number, +): OwnedEc2Runner[] { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const serving: OwnedEc2Runner[] = []; + + for (const runner of runners) { + if (runner.scaleSetState !== 'config-published') { + // An interrupted publication may already have been consumed. Preserve it, + // but do not let it suppress replacement capacity indefinitely. + retainUnknown(state, runner.instanceId); + continue; + } + + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + if (githubState !== undefined && isConfirmedServingState(githubState)) { + serving.push(runner); + continue; + } + if (isWithinBootTimeout(runner, request.bootTimeoutMinutes, now)) { + serving.push(runner); + continue; + } + + // A config-published EC2 instance is provider-owned capacity. The exact + // Actions-service identity is used only when removing it; no public + // GitHub runner inventory is needed to count capacity. + if (runner.githubRunnerId !== undefined && runner.runnerName !== undefined) { + serving.push(runner); + continue; + } + + retainUnknown(state, runner.instanceId); + // Unknown lifecycle state is not allowed to suppress replacement capacity. + // Aggregate busy state still protects scale-down, while tagged EC2 + // inventory remains the source of current provider-owned capacity. + } + + return serving; +} + +export function isSafeScaleDownState(state: ScaleSetRunnerState): boolean { + return state.busy === false || (state.lifecycle === 'completed' && state.busy !== true); +} + +export function isBusyState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.busy === true; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts new file mode 100644 index 0000000000..704283ac64 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts @@ -0,0 +1,132 @@ +import { CreateFleetCommand, DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { PutParameterCommand } from '@aws-sdk/client-ssm'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { config, createRequest, githubState, ownedInstance } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks, ssmMock } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set provider orchestration', () => { + it('rejects non-canonical GitHub ownership scopes before creating clients', () => { + expect(() => createTestProvider({ githubScope: 'https://GITHUB.com/example/' })).toThrow( + 'githubScope must be a canonical HTTPS GitHub configuration URL', + ); + }); + + it('counts an old tagged handoff as provider capacity without public inventory', async () => { + const old = ownedInstance('i-old-offline', { runnerId: 100, runnerName: 'runner-i-old-offline' }); + const replacementId = 'i-1234567890abcdef0'; + const replacement = ownedInstance( + replacementId, + { runnerId: 101, runnerName: `runner-${replacementId}` }, + { + launchTime: new Date('2026-08-24T10:10:30Z'), + }, + ); + ec2Mock + .on(DescribeInstancesCommand) + .resolvesOnce({ Reservations: [{ Instances: [old] }] }) + .resolves({ Reservations: [{ Instances: [old, replacement] }] }); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacementId] }] }); + const computeProvider = createTestProvider({ now: () => new Date('2026-08-24T10:11:00Z').getTime() }); + const completeInventory = createRequest({ + runnerStates: [ + githubState(100, 'runner-i-old-offline', { + status: 'offline', + busy: false, + lifecycle: 'completed', + }), + ], + }); + + const result = await computeProvider.reconcile(completeInventory); + const nextResult = await computeProvider.reconcile(completeInventory); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(nextResult).toMatchObject({ + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it.each(['provisioning', 'publishing'])( + 'retains interrupted %s capacity but provisions a replacement', + async (scaleSetState) => { + const stuck = ownedInstance( + 'i-stuck', + scaleSetState === 'publishing' ? { runnerId: 100, runnerName: 'runner-i-stuck' } : undefined, + { scaleSetState }, + ); + const replacement = 'i-1234567890abcdef0'; + ec2Mock + .on(DescribeInstancesCommand) + .resolvesOnce({ Reservations: [{ Instances: [stuck] }] }) + .resolves({ + Reservations: [ + { + Instances: [stuck, ownedInstance(replacement, { runnerId: 101, runnerName: `runner-${replacement}` })], + }, + ], + }); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacement] }] }); + const computeProvider = createTestProvider(); + + const result = await computeProvider.reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 1, terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-stuck'] }); + expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { + Name: `${config.jitConfigParameterPath}/${replacement}`, + }); + + const nextResult = await computeProvider.reconcile(createRequest()); + expect(nextResult).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).toHaveReceivedCommandTimes(CreateFleetCommand, 1); + }, + ); + + it('caps retained-capacity replacement surge when every replacement remains ambiguous', async () => { + const ambiguous = [ + ownedInstance('i-stuck-1', undefined, { scaleSetState: 'provisioning' }), + ownedInstance('i-stuck-2', { runnerId: 102, runnerName: 'runner-i-stuck-2' }, { scaleSetState: 'publishing' }), + ]; + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: ambiguous }] }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedUnknown: 2 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('propagates cancellation instead of converting shutdown into a retry result', async () => { + const abort = new AbortController(); + abort.abort(new Error('service stopping')); + + await expect(createTestProvider().reconcile(createRequest({ signal: abort.signal }))).rejects.toThrow( + 'service stopping', + ); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts new file mode 100644 index 0000000000..3c9df100ed --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts @@ -0,0 +1,121 @@ +import { EC2Client } from '@aws-sdk/client-ec2'; +import { SSMClient } from '@aws-sdk/client-ssm'; + +import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '../../../../scale-set'; +import { createEc2RunnerClient } from '../runners'; +import { + parseEc2ScaleSetProviderConfig, + validateFactoryInput, + type CreateEc2ScaleSetProviderInput, + type Ec2ScaleSetProviderConfig, +} from './configuration'; +import { listOwnedRunners, servingCapacity, type OwnedEc2Runner } from './inventory'; +import { + emptyState, + finish, + safeError, + throwIfAborted, + validateBootTimeout, + validateBusyRunners, + validateDesiredRunners, +} from './reconcile'; +import { scaleDown } from './scale-down'; +import { scaleUp } from './scale-up'; + +export type { CreateEc2ScaleSetProviderInput, Ec2ScaleSetProviderConfig } from './configuration'; +export { parseEc2ScaleSetProviderConfig } from './configuration'; +export { + EC2_GITHUB_RUNNER_ID_TAG, + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from './inventory'; + +const RETAINED_CAPACITY_REPLACEMENT_SURGE = 1; + +export interface Ec2ScaleSetProviderDependencies { + ec2Client?: EC2Client; + ssmClient?: SSMClient; + now?: () => number; +} + +function createClients(config: Ec2ScaleSetProviderConfig, dependencies: Ec2ScaleSetProviderDependencies) { + return { + ec2Client: dependencies.ec2Client ?? new EC2Client({ region: config.region }), + ssmClient: + dependencies.ssmClient ?? + new SSMClient({ + region: config.region, + maxAttempts: 10, + retryMode: 'adaptive', + }), + }; +} + +export function createEc2ScaleSetProvider( + input: CreateEc2ScaleSetProviderInput, + dependencies: Ec2ScaleSetProviderDependencies = {}, +): ScaleSetComputeProvider { + const normalizedInput = { + ...input, + configuration: parseEc2ScaleSetProviderConfig(input.configuration), + }; + validateFactoryInput(normalizedInput); + const clients = createClients(normalizedInput.configuration, dependencies); + const runnerClient = createEc2RunnerClient(clients.ec2Client); + const now = dependencies.now ?? Date.now; + + return { + async reconcile(request): Promise { + request.signal.throwIfAborted(); + const validationError = + validateDesiredRunners(request.desiredRunners) ?? + validateBusyRunners(request.busyRunners) ?? + validateBootTimeout(request.bootTimeoutMinutes); + if (validationError) { + const state = emptyState(0); + state.errors.push(validationError); + return finish(state, request.desiredRunners); + } + + const runnerOperations = runnerClient.forRequest({ signal: request.signal }); + let ownedRunners: OwnedEc2Runner[]; + try { + ownedRunners = await listOwnedRunners(normalizedInput, clients.ec2Client, request.signal); + } catch (error) { + throwIfAborted(request.signal, error); + const state = emptyState(0); + state.errors.push(safeError('list', error)); + return finish(state, request.desiredRunners); + } + + const state = emptyState(ownedRunners.length); + const servingRunners = servingCapacity(normalizedInput, ownedRunners, request, state, now()); + + if (servingRunners.length < request.desiredRunners) { + const capacityDeficit = request.desiredRunners - servingRunners.length; + const availableReplacementSlots = Math.max( + 0, + request.desiredRunners + RETAINED_CAPACITY_REPLACEMENT_SURGE - ownedRunners.length, + ); + const launchCount = Math.min(capacityDeficit, availableReplacementSlots); + if (launchCount > 0) { + await scaleUp(normalizedInput, launchCount, request, state, runnerOperations, clients.ssmClient); + } + } else if (servingRunners.length > request.desiredRunners) { + await scaleDown( + normalizedInput, + servingRunners, + servingRunners.length - request.desiredRunners, + request, + state, + runnerOperations, + ); + } + + return finish(state, request.desiredRunners); + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts new file mode 100644 index 0000000000..a462b935b8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts @@ -0,0 +1,32 @@ +import { DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { createRequest } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set reconciliation validation', () => { + it('rejects an invalid desired count without touching AWS', async () => { + const result = await createTestProvider().reconcile(createRequest({ desiredRunners: -1 })); + + expect(result).toMatchObject({ + status: 'error', + desiredRunners: -1, + currentRunners: 0, + }); + expect(result.errors).toEqual([{ operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT' }]); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); + + it.each([0, 121, 1.5])('rejects invalid orchestration boot timeout %s without touching AWS', async (value) => { + const result = await createTestProvider().reconcile(createRequest({ bootTimeoutMinutes: value })); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 0, + }); + expect(result.errors).toEqual([{ operation: 'validate', code: 'INVALID_BOOT_TIMEOUT' }]); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts new file mode 100644 index 0000000000..a49e8c0293 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts @@ -0,0 +1,131 @@ +import type { + ScaleSetReconcileActions, + ScaleSetReconcileError, + ScaleSetReconcileOperation, + ScaleSetReconcileResult, +} from '../../../../scale-set'; + +const MAX_BOOT_TIMEOUT_MINUTES = 120; + +export interface MutableReconcileState { + currentRunners: number; + retainedUnknownResourceIds: Set; + actions: ScaleSetReconcileActions; + errors: ScaleSetReconcileError[]; +} + +export class Ec2ScaleSetValidationError extends Error { + constructor(message: string) { + super(message); + this.name = 'Ec2ScaleSetValidationError'; + } +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function safeError( + operation: ScaleSetReconcileOperation, + error: unknown, + details: Pick = {}, +): ScaleSetReconcileError { + return { + operation, + code: safeErrorCode(error), + ...details, + }; +} + +function safeErrorCode(error: unknown): string { + if (error instanceof Ec2ScaleSetValidationError) return 'INVALID_CONFIGURATION'; + if (!isRecord(error)) return 'UNEXPECTED_ERROR'; + for (const candidate of [error.name, error.code]) { + if (typeof candidate === 'string' && /^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(candidate)) { + return candidate; + } + } + return 'UNEXPECTED_ERROR'; +} + +export function throwIfAborted(signal: AbortSignal, error?: unknown): void { + if (signal.aborted || (isRecord(error) && error.name === 'AbortError')) { + signal.throwIfAborted(); + throw error; + } +} + +function resultStatus(errors: readonly ScaleSetReconcileError[], current: number, desired: number) { + if (errors.length > 0 || current < desired) return 'error' as const; + if (current > desired) return 'retained' as const; + return 'converged' as const; +} + +export function finish(state: MutableReconcileState, desiredRunners: number): ScaleSetReconcileResult { + if (desiredRunners >= 0 && state.currentRunners < desiredRunners && state.errors.length === 0) { + state.errors.push({ + operation: 'reconcile', + code: 'CAPACITY_NOT_PROVISIONED', + }); + } + return { + status: resultStatus(state.errors, state.currentRunners, desiredRunners), + desiredRunners, + currentRunners: state.currentRunners, + actions: state.actions, + errors: state.errors, + }; +} + +export function emptyState(currentRunners: number): MutableReconcileState { + return { + currentRunners, + retainedUnknownResourceIds: new Set(), + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }; +} + +export function retainUnknown(state: MutableReconcileState, resourceId?: string): void { + if (resourceId === undefined) { + state.actions.retainedUnknown++; + return; + } + if (state.retainedUnknownResourceIds.has(resourceId)) return; + state.retainedUnknownResourceIds.add(resourceId); + state.actions.retainedUnknown++; +} + +export function validateDesiredRunners(desiredRunners: number): ScaleSetReconcileError | undefined { + if (!Number.isSafeInteger(desiredRunners) || desiredRunners < 0 || desiredRunners > 10000) { + return { + operation: 'validate', + code: 'INVALID_DESIRED_RUNNER_COUNT', + }; + } + return undefined; +} + +export function validateBootTimeout(bootTimeoutMinutes: number): ScaleSetReconcileError | undefined { + if ( + !Number.isSafeInteger(bootTimeoutMinutes) || + bootTimeoutMinutes < 1 || + bootTimeoutMinutes > MAX_BOOT_TIMEOUT_MINUTES + ) { + return { + operation: 'validate', + code: 'INVALID_BOOT_TIMEOUT', + }; + } + return undefined; +} + +export function validateBusyRunners(busyRunners: number): ScaleSetReconcileError | undefined { + if (!Number.isSafeInteger(busyRunners) || busyRunners < 0 || busyRunners > 2_147_483_647) { + return { + operation: 'validate', + code: 'INVALID_BUSY_RUNNER_COUNT', + }; + } + return undefined; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts new file mode 100644 index 0000000000..40b64e1c2b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts @@ -0,0 +1,195 @@ +import { DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createRequest, githubState, ownedInstance, signal } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set scale down', () => { + it('terminates only exact known-idle or completed runners and retains busy or unknown runners', async () => { + const completed = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + const busy = ownedInstance('i-busy', { runnerId: 102, runnerName: 'runner-busy' }); + const unknown = ownedInstance('i-unknown', { runnerId: 103, runnerName: 'runner-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [completed, busy, unknown] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 2, + runnerStates: [ + githubState(101, 'runner-completed', { status: 'offline', busy: undefined, lifecycle: 'completed' }), + githubState(102, 'runner-busy', { busy: true, lifecycle: 'started' }), + ], + removeRunner, + }), + ); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 2, + currentRunners: 2, + actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 0 }, + errors: [], + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + expect(removeRunner).toHaveBeenCalledWith({ + runnerId: 101, + runnerName: 'runner-completed', + scaleSetId: 42, + signal, + }); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-completed'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-busy'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-unknown'] }); + }); + + it('uses aggregate idle state to remove a tagged runner after a restart', async () => { + const instance = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + const result = await createTestProvider().reconcile( + createRequest({ desiredRunners: 0, busyRunners: 0, runnerStates: [], removeRunner }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 0, + actions: { terminated: 1 }, + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + }); + + it('never lets a completed lifecycle marker override a current busy signal', async () => { + const instance = ownedInstance('i-completed-busy', { runnerId: 101, runnerName: 'runner-completed-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 1, + runnerStates: [ + githubState(101, 'runner-completed-busy', { + lifecycle: 'completed', + status: 'online', + busy: true, + }), + ], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedBusy: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner without an error when the exact removal check observes that it became busy', async () => { + const instance = ownedInstance('i-raced-busy', { runnerId: 101, runnerName: 'runner-raced-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_busy' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-raced-busy')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner and requests inventory when exact removal observes identity drift', async () => { + const instance = ownedInstance('i-raced-unknown', { runnerId: 101, runnerName: 'runner-raced-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_unknown' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-raced-unknown')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not trust a mutable EC2 GitHub-runner-id tag when controller identity disagrees', async () => { + const instance = ownedInstance('i-mismatch', { runnerId: 999, runnerName: 'runner-exact' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-exact')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not terminate compute when exact GitHub removal fails', async () => { + const instance = ownedInstance('i-idle', { runnerId: 101, runnerName: 'runner-idle' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi + .fn() + .mockRejectedValue(Object.assign(new Error('must not leak'), { name: 'ServiceUnavailable' })); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + busyRunners: 0, + runnerStates: [githubState(101, 'runner-idle')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'remove_runner', + code: 'ServiceUnavailable', + runnerName: 'runner-idle', + resourceId: 'i-idle', + }, + ]); + expect(JSON.stringify(result)).not.toContain('must not leak'); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts new file mode 100644 index 0000000000..78fddce9cd --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts @@ -0,0 +1,118 @@ +import type { ScaleSetReconcileRequest, ScaleSetRunnerState } from '../../../../scale-set'; +import type { Ec2RunnerResourceOperations } from '../runners'; +import type { CreateEc2ScaleSetProviderInput } from './configuration'; +import { + indexRunnerStates, + isBusyState, + isSafeScaleDownState, + matchingRunnerState, + type OwnedEc2Runner, +} from './inventory'; +import { retainUnknown, safeError, throwIfAborted, type MutableReconcileState } from './reconcile'; + +async function terminateKnownIdleRunner( + runner: OwnedEc2Runner, + githubState: ScaleSetRunnerState, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runnerOperations: Ec2RunnerResourceOperations, +): Promise { + let removalResult; + try { + removalResult = await request.removeRunner({ + runnerId: githubState.runnerId, + runnerName: githubState.runnerName, + scaleSetId: githubState.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('remove_runner', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } + + if (removalResult.status === 'retained_busy') { + state.actions.retainedBusy++; + return false; + } + if (removalResult.status !== 'removed') { + retainUnknown(state, runner.instanceId); + return false; + } + + try { + await runnerOperations.terminate(runner.instanceId); + state.currentRunners--; + state.actions.terminated++; + return true; + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('terminate', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } +} + +export async function scaleDown( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runnerOperations: Ec2RunnerResourceOperations, +): Promise { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const candidates: { runner: OwnedEc2Runner; githubState: ScaleSetRunnerState }[] = []; + + for (const runner of runners) { + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + const hasContradictoryState = runner.runnerName !== undefined && runnerStateIndex.byName.has(runner.runnerName); + if (!githubState) { + if ( + !hasContradictoryState && + request.busyRunners === 0 && + runner.githubRunnerId !== undefined && + runner.runnerName !== undefined + ) { + candidates.push({ + runner, + githubState: { + runnerId: runner.githubRunnerId, + runnerName: runner.runnerName, + scaleSetId: input.scaleSetId, + status: 'unknown', + busy: false, + lifecycle: 'unknown', + }, + }); + } else { + retainUnknown(state, runner.instanceId); + } + } else if (isBusyState(githubState)) { + state.actions.retainedBusy++; + } else if (isSafeScaleDownState(githubState)) { + candidates.push({ runner, githubState }); + } else { + retainUnknown(state, runner.instanceId); + } + } + + candidates.sort((left, right) => { + const launchOrder = (right.runner.launchTime?.getTime() ?? 0) - (left.runner.launchTime?.getTime() ?? 0); + return launchOrder || left.runner.instanceId.localeCompare(right.runner.instanceId); + }); + + let remaining = count; + for (const candidate of candidates) { + if (remaining === 0) break; + request.signal.throwIfAborted(); + if (await terminateKnownIdleRunner(candidate.runner, candidate.githubState, request, state, runnerOperations)) { + remaining--; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts new file mode 100644 index 0000000000..d99d19ebbc --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts @@ -0,0 +1,200 @@ +import { + CreateFleetCommand, + CreateTagsCommand, + DescribeInstancesCommand, + TerminateInstancesCommand, +} from '@aws-sdk/client-ec2'; +import { DeleteParameterCommand, GetParameterCommand, PutParameterCommand } from '@aws-sdk/client-ssm'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from './inventory'; +import { config, createRequest, githubScopeHash, jitResult, signal } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks, ssmMock } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set scale up', () => { + it('launches owned compute, verifies JIT identity, and publishes only a SecureString', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const generateJitConfiguration = vi.fn().mockResolvedValue(jitResult(instanceId)); + + const result = await createTestProvider().reconcile(createRequest({ generateJitConfiguration })); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 1, + currentRunners: 1, + actions: { launched: 1, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }); + expect(generateJitConfiguration).toHaveBeenCalledWith({ runnerName: `runner-${instanceId}`, signal }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateFleetCommand, { + TagSpecifications: expect.arrayContaining([ + expect.objectContaining({ + ResourceType: 'instance', + Tags: expect.arrayContaining([ + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:Owner', Value: 'example' }, + { Key: 'ghr:Type', Value: 'Org' }, + ]), + }), + ]), + }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateTagsCommand, { + Resources: [instanceId], + Tags: expect.arrayContaining([ + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, + ]), + }); + expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + Value: 'sensitive-encoded-jit-configuration', + Type: 'SecureString', + Overwrite: false, + Tags: expect.arrayContaining([ + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + ]), + }); + }); + + it('resolves an AMI parameter through the provider-owned SSM client', async () => { + const instanceId = 'i-1234567890abcdef0'; + const amiIdSsmParameterName = '/github-action-runners/unit-test/ami'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(GetParameterCommand).resolves({ Parameter: { Value: 'ami-0123456789abcdef0' } }); + + const result = await createTestProvider({ + configuration: { ...config, amiIdSsmParameterName }, + }).reconcile(createRequest()); + + expect(result.actions.launched).toBe(1); + expect(ssmMock).toHaveReceivedCommandWith(GetParameterCommand, { + Name: amiIdSsmParameterName, + WithDecryption: true, + }); + }); + + it('does not remove an unrelated GitHub runner when JIT identity validation fails', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + generateJitConfiguration: vi.fn().mockResolvedValue({ + ...jitResult(instanceId), + runnerName: 'runner-owned-by-another-config', + }), + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 0, + actions: { launched: 0, terminated: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'generate_jit_configuration', + code: 'INVALID_CONFIGURATION', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ssmMock).not.toHaveReceivedCommand(PutParameterCommand); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: [instanceId] }); + }); + + it('retains compute when failed JIT publication cannot be safely cancelled', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('redacted secret'), { name: 'TimeoutError' })); + ssmMock.on(DeleteParameterCommand).rejects(Object.assign(new Error('missing'), { name: 'ParameterNotFound' })); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'publish_jit_configuration', + code: 'TimeoutError', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); + expect(JSON.stringify(result)).not.toContain('redacted secret'); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not treat a successful DeleteParameter as proof that bootstrap did not read JIT first', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('throttled'), { name: 'ThrottlingException' })); + ssmMock.on(DeleteParameterCommand).resolves({}); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await createTestProvider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + }); + expect(result.errors).toEqual([ + { + operation: 'publish_jit_configuration', + code: 'ThrottlingException', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); + expect(ssmMock).toHaveReceivedCommandWith(DeleteParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it.each(['ThrottlingException', 'InvalidParameterValue'])( + 'collapses EC2 launch failure %s into one scale-set error', + async (errorCode) => { + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Errors: [{ ErrorCode: errorCode }] }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 0, + actions: { launched: 0, terminated: 0 }, + }); + expect(result.errors).toEqual([{ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }]); + }, + ); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts new file mode 100644 index 0000000000..a0fc3b984b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts @@ -0,0 +1,250 @@ +import { DeleteParameterCommand, PutParameterCommand, type SSMClient, type Tag as SsmTag } from '@aws-sdk/client-ssm'; + +import type { GenerateScaleSetJitConfigurationResult, ScaleSetReconcileRequest } from '../../../../scale-set'; +import type { Ec2RunnerResourceOperations } from '../runners'; +import type { CreateEc2ScaleSetProviderInput, Ec2ScaleSetProviderConfig } from './configuration'; +import { + EC2_GITHUB_RUNNER_ID_TAG, + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, + GITHUB_RUNNER_NAME_MAX_LENGTH, + githubScopeHash, + ownershipTags, + runnerIdentityFromGitHubScope, + SCALE_SET_RUNNER_SOURCE, +} from './inventory'; +import { + Ec2ScaleSetValidationError, + retainUnknown, + safeError, + throwIfAborted, + type MutableReconcileState, +} from './reconcile'; + +const SSM_STANDARD_TIER_THRESHOLD = 4000; +const SSM_ADVANCED_TIER_MAX_BYTES = 8192; + +function jitParameterName(config: Ec2ScaleSetProviderConfig, instanceId: string): string { + return `${config.jitConfigParameterPath}/${instanceId}`; +} + +function jitParameterTags(input: CreateEc2ScaleSetProviderInput, instanceId: string): SsmTag[] { + const reserved = new Set(['InstanceId', EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG, EC2_GITHUB_SCOPE_HASH_TAG]); + return [ + ...(input.configuration.ssmParameterTags ?? []).filter((tag) => tag.Key && !reserved.has(tag.Key)), + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + ]; +} + +async function publishJitConfiguration( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + encodedJitConfiguration: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + const valueSize = Buffer.byteLength(encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new Ec2ScaleSetValidationError('JIT configuration must be between 1 and 8192 bytes'); + } + + await ssmClient.send( + new PutParameterCommand({ + Name: jitParameterName(input.configuration, instanceId), + Value: encodedJitConfiguration, + Type: 'SecureString', + KeyId: input.configuration.ssmKmsKeyId, + Overwrite: false, + Tier: valueSize >= SSM_STANDARD_TIER_THRESHOLD ? 'Advanced' : 'Standard', + Tags: jitParameterTags(input, instanceId), + }), + { abortSignal: signal }, + ); +} + +async function bestEffortCancelJitPublication( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + try { + await ssmClient.send(new DeleteParameterCommand({ Name: jitParameterName(input.configuration, instanceId) }), { + abortSignal: signal, + }); + } catch (error) { + throwIfAborted(signal, error); + } +} + +function validateJitResult( + result: GenerateScaleSetJitConfigurationResult, + expectedRunnerName: string, + scaleSetId: number, +): void { + if ( + !Number.isSafeInteger(result.runnerId) || + result.runnerId <= 0 || + result.runnerName !== expectedRunnerName || + result.scaleSetId !== scaleSetId + ) { + throw new Ec2ScaleSetValidationError('JIT configuration returned an unexpected runner identity'); + } + const valueSize = Buffer.byteLength(result.encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new Ec2ScaleSetValidationError('JIT configuration has an invalid size'); + } +} + +async function terminateUnpublishedRunner( + instanceId: string, + state: MutableReconcileState, + runners: Ec2RunnerResourceOperations, + signal: AbortSignal, +): Promise { + try { + await runners.terminate(instanceId); + state.currentRunners--; + state.actions.terminated++; + } catch (error) { + throwIfAborted(signal, error); + retainUnknown(state, instanceId); + state.errors.push(safeError('terminate', error, { resourceId: instanceId })); + } +} + +async function cleanGitHubRunner( + jit: GenerateScaleSetJitConfigurationResult, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, +): Promise { + try { + await request.removeRunner({ + runnerId: jit.runnerId, + runnerName: jit.runnerName, + scaleSetId: jit.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('remove_runner', error, { runnerName: jit.runnerName })); + } +} + +async function configureLaunchedRunner( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runners: Ec2RunnerResourceOperations, + ssmClient: SSMClient, +): Promise { + const runnerName = `${input.configuration.runnerNamePrefix}${instanceId}`; + if (runnerName.length > GITHUB_RUNNER_NAME_MAX_LENGTH) { + state.errors.push({ + operation: 'generate_jit_configuration', + code: 'RUNNER_NAME_TOO_LONG', + resourceId: instanceId, + }); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + let jit: GenerateScaleSetJitConfigurationResult; + try { + jit = await request.generateJitConfiguration({ runnerName, signal: request.signal }); + validateJitResult(jit, runnerName, input.scaleSetId); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('generate_jit_configuration', error, { runnerName, resourceId: instanceId })); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + try { + await runners.tag(instanceId, [ + ...ownershipTags(input), + { Key: EC2_RUNNER_NAME_TAG, Value: jit.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(jit.runnerId) }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, + ]); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + await cleanGitHubRunner(jit, request, state); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + try { + await publishJitConfiguration(input, instanceId, jit.encodedJitConfiguration, ssmClient, request.signal); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('publish_jit_configuration', error, { runnerName, resourceId: instanceId })); + await bestEffortCancelJitPublication(input, instanceId, ssmClient, request.signal); + // Main's bootstrap reads before deleting. Even a successful controller-side + // DeleteParameter can race after that read and cannot prove non-consumption. + // Preserve both GitHub and compute state until an exact lifecycle signal is observed. + retainUnknown(state, instanceId); + return; + } + + state.actions.launched++; + try { + await runners.tag(instanceId, [{ Key: EC2_SCALE_SET_STATE_TAG, Value: 'config-published' }]); + } catch (error) { + throwIfAborted(request.signal, error); + // Publication may already have been consumed. Preserve the instance and exact GitHub identity. + retainUnknown(state, instanceId); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + } +} + +export async function scaleUp( + input: CreateEc2ScaleSetProviderInput, + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runners: Ec2RunnerResourceOperations, + ssmClient: SSMClient, +): Promise { + const runnerIdentity = runnerIdentityFromGitHubScope(input.githubScope); + let createResult; + try { + createResult = await runners.create({ + environment: input.configuration.environment, + runnerOwner: runnerIdentity.runnerOwner, + runnerType: runnerIdentity.runnerType, + subnets: input.configuration.subnets, + launchTemplateName: input.configuration.launchTemplateName, + ec2instanceCriteria: input.configuration.ec2instanceCriteria, + ec2OverrideConfig: input.configuration.ec2OverrideConfig, + numberOfRunners: count, + source: SCALE_SET_RUNNER_SOURCE, + amiIdSsmParameterName: input.configuration.amiIdSsmParameterName, + tracingEnabled: input.configuration.tracingEnabled, + onDemandFailoverOnError: input.configuration.onDemandFailoverOnError, + useDedicatedHost: input.configuration.useDedicatedHost, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error)); + return; + } + + state.currentRunners += createResult.instances.length; + if (createResult.failedInstanceCount > 0) { + state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }); + } + + for (const instanceId of createResult.instances) { + request.signal.throwIfAborted(); + await configureLaunchedRunner(input, instanceId, request, state, runners, ssmClient); + } +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts new file mode 100644 index 0000000000..abbd63a16c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts @@ -0,0 +1,111 @@ +import { createHash } from 'node:crypto'; + +import type { Instance } from '@aws-sdk/client-ec2'; +import { vi } from 'vitest'; + +import type { + GenerateScaleSetJitConfigurationResult, + ScaleSetReconcileRequest, + ScaleSetRunnerState, +} from '../../../../../scale-set'; +import type { Ec2ScaleSetProviderConfig } from '../configuration'; +import { + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_GITHUB_RUNNER_ID_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from '../inventory'; + +export const signal = new AbortController().signal; +export const githubScope = 'https://github.com/example'; +export const githubScopeHash = createHash('sha256').update(githubScope, 'utf8').digest('hex'); + +export const config: Ec2ScaleSetProviderConfig = { + region: 'eu-west-1', + environment: 'unit-test', + runnerNamePrefix: 'runner-', + jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', + subnets: ['subnet-12345678'], + launchTemplateName: 'unit-test-runners', + ec2instanceCriteria: { + instanceTypes: ['m7i.large'], + targetCapacityType: 'on-demand', + instanceAllocationStrategy: 'lowest-price', + }, + ssmParameterTags: [{ Key: 'Project', Value: 'runner-tests' }], +}; + +export function ownedInstance( + instanceId: string, + identity?: { runnerId: number; runnerName: string }, + overrides: { + runnerConfigName?: string; + scaleSetId?: number; + scaleSetState?: string; + githubScopeHash?: string; + launchTime?: Date; + } = {}, +): Instance { + return { + InstanceId: instanceId, + LaunchTime: overrides.launchTime ?? new Date('2026-08-24T10:00:00Z'), + Tags: [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: overrides.runnerConfigName ?? 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(overrides.scaleSetId ?? 42) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: overrides.githubScopeHash ?? githubScopeHash }, + { + Key: EC2_SCALE_SET_STATE_TAG, + Value: overrides.scaleSetState ?? (identity ? 'config-published' : 'provisioning'), + }, + ...(identity + ? [ + { Key: EC2_RUNNER_NAME_TAG, Value: identity.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(identity.runnerId) }, + ] + : []), + ], + }; +} + +export function githubState( + runnerId: number, + runnerName: string, + overrides: Partial = {}, +): ScaleSetRunnerState { + return { + runnerId, + runnerName, + scaleSetId: 42, + status: 'online', + busy: false, + lifecycle: 'unknown', + ...overrides, + }; +} + +export function jitResult(instanceId = 'i-1234567890abcdef0'): GenerateScaleSetJitConfigurationResult { + return { + encodedJitConfiguration: 'sensitive-encoded-jit-configuration', + runnerId: 101, + runnerName: `runner-${instanceId}`, + scaleSetId: 42, + }; +} + +export function createRequest(overrides: Partial = {}): ScaleSetReconcileRequest { + return { + desiredRunners: 1, + busyRunners: 0, + bootTimeoutMinutes: 10, + runnerStates: [], + signal, + generateJitConfiguration: vi.fn().mockResolvedValue(jitResult()), + removeRunner: vi.fn().mockResolvedValue({ status: 'removed' }), + ...overrides, + }; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts new file mode 100644 index 0000000000..510009001a --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts @@ -0,0 +1,40 @@ +import { CreateTagsCommand, EC2Client, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { DeleteParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; + +import type { Ec2ScaleSetProviderConfig } from '../configuration'; +import { createEc2ScaleSetProvider } from '../provider'; +import { config, githubScope } from './fixtures'; + +export const ec2Mock = mockClient(EC2Client); +export const ssmMock = mockClient(SSMClient); +const ec2Client = new EC2Client({ region: 'eu-west-1' }); +const ssmClient = new SSMClient({ region: 'eu-west-1' }); + +export function createTestProvider( + options: { githubScope?: string; now?: () => number; configuration?: Ec2ScaleSetProviderConfig } = {}, +) { + return createEc2ScaleSetProvider( + { + runnerConfigName: 'linux', + scaleSetId: 42, + githubScope: options.githubScope ?? githubScope, + configuration: options.configuration ?? config, + }, + { + ec2Client, + ssmClient, + now: options.now ?? (() => new Date('2026-08-24T10:05:00Z').getTime()), + }, + ); +} + +export function resetAwsMocks(): void { + ec2Mock.reset(); + ssmMock.reset(); + ec2Mock.on(CreateTagsCommand).resolves({}); + ec2Mock.on(TerminateInstancesCommand).resolves({}); + ssmMock.on(PutParameterCommand).resolves({}); + ssmMock.on(DeleteParameterCommand).resolves({}); +} diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index c5560942fa..ece1bc603e 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -6,7 +6,7 @@ export interface ComputeProvider { type: ComputeProviderType; } -export type RunnerSource = 'scale-up-lambda' | 'pool-lambda'; +export type RunnerSource = 'scale-up-lambda' | 'pool-lambda' | 'scale-set-service'; export type RunnerType = 'Org' | 'Repo'; export interface CreateGitHubRunnerConfig { diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index c1806818cc..0ab5bb25de 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -8,8 +8,10 @@ "./provider-types": "./provider-types.ts", "./webhook": "./webhook.ts", "./control-plane": "./control-plane.ts", + "./scale-set": "./scale-set.ts", "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", + "./aws/ec2/scale-set": "./aws/ec2/scale-set.ts", "./aws/ec2/runners": "./aws/ec2/src/runners.ts", "./aws/ec2/control-plane/runner-creation": "./aws/ec2/src/control-plane/runner-creation.ts" }, @@ -27,6 +29,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-ssm": "^3.1009.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/providers.config.scale-set.ts b/lambdas/libs/compute-providers/providers.config.scale-set.ts new file mode 100644 index 0000000000..7f8f0ecc64 --- /dev/null +++ b/lambdas/libs/compute-providers/providers.config.scale-set.ts @@ -0,0 +1,5 @@ +import { provider as ec2 } from './aws/ec2/scale-set'; +import type { ScaleSetComputeProviderModule } from './scale-set'; + +/** Provider plugins included in the scale-set service bundle. */ +export const enabledScaleSetProviders = [ec2] as const satisfies readonly ScaleSetComputeProviderModule[]; diff --git a/lambdas/libs/compute-providers/scale-set.test.ts b/lambdas/libs/compute-providers/scale-set.test.ts new file mode 100644 index 0000000000..f56e044d86 --- /dev/null +++ b/lambdas/libs/compute-providers/scale-set.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createEc2ScaleSetPlugin } from './aws/ec2/scale-set'; +import { + createScaleSetComputeProviderRegistry, + type ScaleSetComputeProviderPlugin, + validateScaleSetProviderEnvironmentVariables, +} from './scale-set'; + +const configuration = { + region: 'eu-west-1', + environment: 'unit-test', + runnerNamePrefix: 'runner-', + jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', + subnets: ['subnet-12345678'], + launchTemplateName: 'unit-test-runners', + ec2instanceCriteria: { + instanceTypes: ['m7i.large'], + targetCapacityType: 'on-demand', + instanceAllocationStrategy: 'lowest-price', + }, +}; + +describe('scale-set compute-provider registry', () => { + it('creates a separate provider instance for every runner config', () => { + const registry = createScaleSetComputeProviderRegistry([createEc2ScaleSetPlugin()]); + const first = registry.create('ec2', { + runnerConfigName: 'shared', + scaleSetId: 1, + githubScope: 'https://github.com/first', + configuration, + }); + const second = registry.create('ec2', { + runnerConfigName: 'shared', + scaleSetId: 1, + githubScope: 'https://github.com/second', + configuration, + }); + + expect(first).not.toBe(second); + expect(first.reconcile).toEqual(expect.any(Function)); + expect(second.reconcile).toEqual(expect.any(Function)); + expect(registry.environmentVariables('ec2')).toEqual({}); + expect(Object.isFrozen(registry.environmentVariables('ec2'))).toBe(true); + }); + + it('rejects duplicate and missing plugins explicitly', () => { + const plugin: ScaleSetComputeProviderPlugin = { + type: 'test', + capabilities: { + environmentVariables: {}, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }; + + expect(() => createScaleSetComputeProviderRegistry([plugin, plugin])).toThrow( + "Duplicate scale-set compute provider plugin 'test'", + ); + expect(() => + createScaleSetComputeProviderRegistry([]).create('missing', { + runnerConfigName: 'runner', + scaleSetId: 1, + githubScope: 'https://github.com/example', + configuration: {}, + }), + ).toThrow("No scale-set compute provider plugin registered for 'missing'"); + expect(() => createScaleSetComputeProviderRegistry([]).environmentVariables('missing')).toThrow( + "No scale-set compute provider plugin registered for 'missing'", + ); + }); + + it('returns a validated immutable provider environment', () => { + const source = { EC2_ENDPOINT_MODE: 'regional' }; + const registry = createScaleSetComputeProviderRegistry([ + { + type: 'test', + capabilities: { + environmentVariables: source, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }, + ]); + + const environment = registry.environmentVariables('test'); + source.EC2_ENDPOINT_MODE = 'changed-after-registration'; + expect(environment).toEqual({ EC2_ENDPOINT_MODE: 'regional' }); + expect(Object.isFrozen(environment)).toBe(true); + }); + + it.each>>([ + { AWS_REGION: 'eu-west-1' }, + { SCALE_SET_OVERRIDE: 'unsafe' }, + { NODE_OPTIONS: '--import=untrusted' }, + { PATH: '/untrusted' }, + { lower_case: 'value' }, + { VALID_NAME: 'line\nbreak' }, + { VALID_NAME: 'x'.repeat(4097) }, + ])('rejects reserved or unsafe provider environment variables: %o', (environmentVariables) => { + expect(() => + createScaleSetComputeProviderRegistry([ + { + type: 'test', + capabilities: { + environmentVariables, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }, + ]), + ).toThrow(/reserved or invalid|invalid value/); + }); + + it('rejects malformed or oversized provider environments', () => { + expect(() => validateScaleSetProviderEnvironmentVariables(null as never)).toThrow('must be an object'); + expect(() => validateScaleSetProviderEnvironmentVariables([] as never)).toThrow('must be an object'); + expect(() => validateScaleSetProviderEnvironmentVariables({ VALID_NAME: 42 } as never)).toThrow( + 'has an invalid value', + ); + expect(() => + validateScaleSetProviderEnvironmentVariables( + Object.fromEntries(Array.from({ length: 65 }, (_, index) => [`PROVIDER_${index}`, 'value'])), + ), + ).toThrow('must contain at most 64 entries'); + }); +}); diff --git a/lambdas/libs/compute-providers/scale-set.ts b/lambdas/libs/compute-providers/scale-set.ts new file mode 100644 index 0000000000..860138bbe9 --- /dev/null +++ b/lambdas/libs/compute-providers/scale-set.ts @@ -0,0 +1,205 @@ +import { enabledScaleSetProviders } from './providers.config.scale-set'; + +export type ScaleSetRunnerStatus = 'online' | 'offline' | 'unknown'; +export type ScaleSetRunnerLifecycle = 'started' | 'completed' | 'unknown'; + +/** + * Controller-observed GitHub state for one runner. + * + * The compute provider treats missing, duplicate, or unrecognized state as + * unknown. Callers must not infer `busy: false` when GitHub did not provide a + * busy state. + */ +export interface ScaleSetRunnerState { + runnerId: number; + runnerName: string; + scaleSetId: number; + status: ScaleSetRunnerStatus; + busy: boolean | undefined; + lifecycle: ScaleSetRunnerLifecycle; +} + +export interface GenerateScaleSetJitConfigurationInput { + runnerName: string; + signal?: AbortSignal; +} + +export interface GenerateScaleSetJitConfigurationResult { + encodedJitConfiguration: string; + runnerId: number; + runnerName: string; + scaleSetId: number; +} + +export type GenerateScaleSetJitConfiguration = ( + input: GenerateScaleSetJitConfigurationInput, +) => Promise; + +export interface RemoveScaleSetRunnerInput { + runnerId: number; + runnerName: string; + scaleSetId: number; + signal?: AbortSignal; +} + +export type ScaleSetRemoveRunnerStatus = 'removed' | 'retained_busy' | 'retained_unknown'; + +export interface ScaleSetRemoveRunnerResult { + status: ScaleSetRemoveRunnerStatus; +} + +export type RemoveScaleSetRunner = (input: RemoveScaleSetRunnerInput) => Promise; + +export interface ScaleSetReconcileRequest { + desiredRunners: number; + /** Aggregate busy-runner count reported by the GitHub Actions scale-set session. */ + busyRunners: number; + bootTimeoutMinutes: number; + runnerStates: readonly ScaleSetRunnerState[]; + signal: AbortSignal; + generateJitConfiguration: GenerateScaleSetJitConfiguration; + removeRunner: RemoveScaleSetRunner; +} + +export type ScaleSetReconcileStatus = 'converged' | 'retained' | 'error'; + +export type ScaleSetReconcileOperation = + | 'validate' + | 'reconcile' + | 'list' + | 'launch' + | 'generate_jit_configuration' + | 'publish_jit_configuration' + | 'remove_runner' + | 'terminate'; + +/** Error metadata is deliberately bounded and never contains a JIT configuration or raw upstream error message. */ +export interface ScaleSetReconcileError { + operation: ScaleSetReconcileOperation; + code: string; + runnerName?: string; + resourceId?: string; +} + +export interface ScaleSetReconcileActions { + launched: number; + terminated: number; + retainedBusy: number; + retainedUnknown: number; +} + +export interface ScaleSetReconcileResult { + status: ScaleSetReconcileStatus; + desiredRunners: number; + /** Best-known owned capacity after actions completed; the next reconciliation re-observes AWS. */ + currentRunners: number; + actions: ScaleSetReconcileActions; + errors: readonly ScaleSetReconcileError[]; +} + +export interface ScaleSetComputeProvider { + reconcile(request: ScaleSetReconcileRequest): Promise; +} + +export interface ScaleSetComputeProviderFactoryInput { + runnerConfigName: string; + scaleSetId: number; + /** Canonical GitHub configuration URL used as an immutable provider ownership scope. */ + githubScope: string; + /** Provider-owned configuration. The selected provider validates it before use. */ + configuration: unknown; +} + +export interface ScaleSetComputeProviderCapabilities { + /** Provider-owned, non-secret task environment. Values are validated and immutable after registration. */ + environmentVariables: Readonly>; + create(input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider; +} + +export interface ScaleSetComputeProviderPlugin { + type: TType; + capabilities: ScaleSetComputeProviderCapabilities; +} + +export interface ScaleSetComputeProviderModule { + type: TType; + createPlugin(): ScaleSetComputeProviderPlugin; +} + +const SCALE_SET_ENVIRONMENT_KEY = /^[A-Z][A-Z0-9_]{0,127}$/; +const RESERVED_SCALE_SET_ENVIRONMENT_PREFIXES = ['AWS_', 'ECS_', 'GITHUB_', 'SCALE_SET_', 'NODE_']; +const RESERVED_SCALE_SET_ENVIRONMENT_KEYS = new Set(['PATH', 'HOME', 'HOSTNAME', 'PWD', 'SHLVL']); +const MAX_SCALE_SET_ENVIRONMENT_VARIABLES = 64; +const MAX_SCALE_SET_ENVIRONMENT_VALUE_BYTES = 4096; + +export function validateScaleSetProviderEnvironmentVariables( + value: Readonly>, +): Readonly> { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Scale-set provider environmentVariables must be an object'); + } + const entries = Object.entries(value); + if (entries.length > MAX_SCALE_SET_ENVIRONMENT_VARIABLES) { + throw new Error( + `Scale-set provider environmentVariables must contain at most ${MAX_SCALE_SET_ENVIRONMENT_VARIABLES} entries`, + ); + } + + const normalized = Object.create(null) as Record; + for (const [key, environmentValue] of entries) { + if ( + !SCALE_SET_ENVIRONMENT_KEY.test(key) || + RESERVED_SCALE_SET_ENVIRONMENT_KEYS.has(key) || + RESERVED_SCALE_SET_ENVIRONMENT_PREFIXES.some((prefix) => key.startsWith(prefix)) + ) { + throw new Error(`Scale-set provider environment variable '${key}' is reserved or invalid`); + } + if ( + typeof environmentValue !== 'string' || + Buffer.byteLength(environmentValue, 'utf8') > MAX_SCALE_SET_ENVIRONMENT_VALUE_BYTES || + [...environmentValue].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }) + ) { + throw new Error(`Scale-set provider environment variable '${key}' has an invalid value`); + } + normalized[key] = environmentValue; + } + return Object.freeze(normalized); +} + +export function createScaleSetComputeProviderRegistry( + plugins: readonly ScaleSetComputeProviderPlugin[] = enabledScaleSetProviders.map((provider) => + provider.createPlugin(), + ), +) { + const pluginsByType = new Map(); + const environmentVariablesByType = new Map>>(); + for (const plugin of plugins) { + if (pluginsByType.has(plugin.type)) { + throw new Error(`Duplicate scale-set compute provider plugin '${plugin.type}'`); + } + pluginsByType.set(plugin.type, plugin); + environmentVariablesByType.set( + plugin.type, + validateScaleSetProviderEnvironmentVariables(plugin.capabilities.environmentVariables), + ); + } + + function get(type: string): ScaleSetComputeProviderPlugin { + const plugin = pluginsByType.get(type); + if (!plugin) throw new Error(`No scale-set compute provider plugin registered for '${type}'`); + return plugin; + } + + return { + create(type: string, input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider { + return get(type).capabilities.create(input); + }, + environmentVariables(type: string): Readonly> { + get(type); + return environmentVariablesByType.get(type)!; + }, + }; +} diff --git a/lambdas/libs/compute-providers/vitest.config.ts b/lambdas/libs/compute-providers/vitest.config.ts index fd62b358c0..24c4554995 100644 --- a/lambdas/libs/compute-providers/vitest.config.ts +++ b/lambdas/libs/compute-providers/vitest.config.ts @@ -16,7 +16,7 @@ export default mergeConfig(defaultConfig, { 'core/**/*.ts', 'aws/**/*.ts', ], - exclude: ['**/*.test.ts', '**/*.d.ts', 'templates/**/*'], + exclude: ['**/*.test.ts', '**/test/**/*.ts', '**/*.d.ts', 'templates/**/*'], thresholds: { statements: 96.16, branches: 95.32, diff --git a/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset b/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset new file mode 100644 index 0000000000..28a50fa226 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset @@ -0,0 +1,21 @@ +MIT License + +Copyright GitHub, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lambdas/libs/github-actions-scale-set/README.md b/lambdas/libs/github-actions-scale-set/README.md new file mode 100644 index 0000000000..6696627a11 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/README.md @@ -0,0 +1,69 @@ +# GitHub Actions runner scale-set client for TypeScript + +This workspace package implements the GitHub Actions runner scale-set protocol with native `fetch`. It is intended for the `terraform-aws-github-runner` control plane and can also be reused by other Node.js scale-set listeners. + +The protocol is currently a GitHub Public Preview. This implementation follows the public [`actions/scaleset`](https://github.com/actions/scaleset) Go client at commit [`cb0405b`](https://github.com/actions/scaleset/tree/cb0405b2d874500e75ae34eff8d582ab75956b45). + +The upstream copyright and MIT permission notice are retained in [`LICENSE.actions-scaleset`](./LICENSE.actions-scaleset). + +## What it provides + +- HTTPS organization, repository, enterprise, GitHub.com, and GHES registration URLs; +- PAT authentication or an asynchronous access-token provider for existing GitHub App authentication; +- runner scale-set CRUD and runner-group lookup; +- just-in-time runner configuration generation; +- runner lookup and removal; +- message-session creation, refresh, long polling, acknowledgement, and job acquisition; +- bounded retries for idempotent requests that encounter network failures, HTTP 429, or HTTP 5xx responses. + +There is no scale-up or scale-down REST operation. A listener polls the message queue and reports its maximum capacity. GitHub returns `statistics.totalAssignedJobs`; the caller reconciles its compute capacity to that value and terminates ephemeral compute after `JobCompleted` messages. + +## Basic usage + +```ts +import { GitHubActionsScaleSetClient } from '@aws-github-runner/github-actions-scale-set'; + +const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + accessTokenProvider: async () => installationAccessToken, + systemInfo: { + system: 'terraform-aws-github-runner', + subsystem: 'scale-set-listener', + }, + retry: { + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 5 * 60_000, + }, +}); + +const scaleSet = await client.createRunnerScaleSet({ + name: 'linux-x64', + runnerGroupId: 1, + runnerSetting: { disableUpdate: true }, +}); + +const session = await client.createMessageSessionClient(scaleSet.id!, 'listener-01'); + +try { + const message = await session.getMessage(0, 20); + if (message) { + await session.deleteMessage(message.messageId); + + const availableIds = message.jobAvailableMessages.map((job) => job.runnerRequestId); + await session.acquireJobs(availableIds); + + // Reconcile compute from message.statistics.totalAssignedJobs and use + // client.generateJitRunnerConfig(...) for every runner being created. + } +} finally { + await session.close(); +} +``` + +Treat encoded JIT configurations and all access tokens as secrets. The upstream Go listener acknowledges a message before job acquisition and scaling callbacks; a later callback failure is returned from the listener and does not cause message redelivery. + +The retry values shown above are the defaults. Automatic transport retries apply only to idempotent methods (`GET`, `HEAD`, `OPTIONS`, `PUT`, and `DELETE`). Non-idempotent `POST` and `PATCH` operations, including JIT generation, session creation, and job acquisition, are attempted once so a lost response cannot cause the operation to be replayed. `Retry-After` is honored for eligible 429/5xx responses and capped by `maxBackoffMs`. Caller cancellation interrupts both an active request and retry backoff. + +The `/actions/runner-registration` admin bootstrap is the sole narrowly scoped exception: its `POST` retries transient transport failures and 429/5xx responses, plus 401/403 while RemoteAuth propagates. Queue 401 responses are not transport-retried; they trigger the message-session token refresh flow once. diff --git a/lambdas/libs/github-actions-scale-set/package.json b/lambdas/libs/github-actions-scale-set/package.json new file mode 100644 index 0000000000..8e0d87576a --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/package.json @@ -0,0 +1,35 @@ +{ + "name": "@aws-github-runner/github-actions-scale-set", + "version": "1.0.0", + "main": "src/index.ts", + "type": "module", + "license": "MIT", + "exports": { + ".": "./src/index.ts", + "./config": "./src/config.ts", + "./errors": "./src/errors.ts", + "./types": "./src/types.ts" + }, + "scripts": { + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "typecheck": "tsc --noEmit", + "all": "yarn format && yarn lint && yarn typecheck && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "typescript": "^5.9.3" + }, + "nx": { + "includedScripts": [ + "format", + "format-check", + "lint", + "typecheck", + "all" + ] + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/client.test.ts b/lambdas/libs/github-actions-scale-set/src/client.test.ts new file mode 100644 index 0000000000..246ee8cbb8 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/client.test.ts @@ -0,0 +1,434 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { actionsServiceUrl, GitHubActionsScaleSetClient } from './client'; +import { SCALE_SET_ERROR_CODES, ScaleSetHttpError, ScaleSetProtocolError } from './errors'; +import { ScaleSetFetch } from './types'; + +type RequestInput = Parameters[0]; +type ServiceHandler = (url: URL, init: RequestInit) => Response | Promise; + +function requestUrl(input: RequestInput): URL { + if (input instanceof Request) { + return new URL(input.url); + } + return new URL(input.toString()); +} + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function actionsAdminToken(expiresAt = Math.floor(Date.now() / 1000) + 60 * 60): string { + const header = Buffer.from(JSON.stringify({ alg: 'RS256', typ: 'JWT' })).toString('base64url'); + const payload = Buffer.from(JSON.stringify({ exp: expiresAt })).toString('base64url'); + return `${header}.${payload}.signature`; +} + +function clientFixture(serviceHandler: ServiceHandler, options: { adminToken?: () => string; now?: () => Date } = {}) { + const accessTokenProvider = vi.fn(async () => 'github-access-token'); + const registrationRequests: Array<{ url: URL; init: RequestInit }> = []; + const serviceRequests: Array<{ url: URL; init: RequestInit }> = []; + const fetchImplementation = vi.fn(async (input, init = {}) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + registrationRequests.push({ url, init }); + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + registrationRequests.push({ url, init }); + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: options.adminToken?.() ?? actionsAdminToken(), + }); + } + + serviceRequests.push({ url, init }); + return serviceHandler(url, init); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + accessTokenProvider, + fetch: fetchImplementation, + systemInfo: { system: 'unit-test', subsystem: 'sdk' }, + now: options.now, + }); + + return { + accessTokenProvider, + client, + fetchImplementation, + registrationRequests, + serviceRequests, + }; +} + +describe('actionsServiceUrl', () => { + it('normalizes a long trailing slash sequence before joining the request path', () => { + const base = `https://actions.example/tenant/123${'/'.repeat(10_000)}`; + + expect(actionsServiceUrl(base, '').pathname).toBe('/tenant/123'); + expect(actionsServiceUrl(base, '_apis/runtime/runnerscalesets').pathname).toBe( + '/tenant/123/_apis/runtime/runnerscalesets', + ); + const url = actionsServiceUrl(base, '/_apis/runtime/runnerscalesets'); + expect(url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets'); + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + }); +}); + +describe('GitHubActionsScaleSetClient', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('bootstraps Actions authentication once and reuses the unexpired admin token', async () => { + const fixture = clientFixture(() => jsonResponse({ count: 0, value: [] })); + + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + + expect(fixture.accessTokenProvider).toHaveBeenCalledOnce(); + expect(fixture.registrationRequests).toHaveLength(2); + expect(fixture.serviceRequests).toHaveLength(2); + + const registrationTokenRequest = fixture.registrationRequests[0]; + expect(registrationTokenRequest.url.toString()).toBe( + 'https://api.github.com/orgs/example/actions/runners/registration-token', + ); + expect(new Headers(registrationTokenRequest.init.headers).get('Authorization')).toBe('Bearer github-access-token'); + expect(new Headers(registrationTokenRequest.init.headers).get('Content-Type')).toBe( + 'application/vnd.github.v3+json', + ); + expect(JSON.parse(new Headers(registrationTokenRequest.init.headers).get('User-Agent') as string)).toMatchObject({ + build_commit_sha: '', + kind: 'scaleset', + system: 'unit-test', + }); + + const adminConnectionRequest = fixture.registrationRequests[1]; + expect(adminConnectionRequest.url.toString()).toBe('https://api.github.com/actions/runner-registration'); + expect(new Headers(adminConnectionRequest.init.headers).get('Authorization')).toBe( + 'RemoteAuth runner-registration-token', + ); + expect(JSON.parse(adminConnectionRequest.init.body as string)).toEqual({ + url: 'https://github.com/example', + runner_event: 'register', + }); + + for (const { url, init } of fixture.serviceRequests) { + expect(url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets'); + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + expect(url.searchParams.get('runnerGroupId')).toBe('4'); + expect(url.searchParams.get('name')).toBe('linux'); + expect(new Headers(init.headers).get('Authorization')).toMatch(/^Bearer /); + } + }); + + it.each([ + 'http://actions.example/tenant/123', + 'https://user:password@actions.example/tenant/123', + 'https://actions.example/tenant/123?signature=secret', + 'https://actions.example/tenant/123#fragment', + ])('rejects an unsafe Actions service admin URL: %s', async (unsafeUrl) => { + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ url: unsafeUrl, token: actionsAdminToken() }); + } + return new Response(null, { status: 500 }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + }); + + await expect(client.getRunnerScaleSetById(42)).rejects.toBeInstanceOf(ScaleSetProtocolError); + }); + + it('refreshes the Actions admin token when it enters the 60-second expiry window', async () => { + let nowMs = Date.UTC(2026, 7, 14, 12, 0, 0); + let tokenIssue = 0; + const fixture = clientFixture(() => jsonResponse({ count: 0, value: [] }), { + now: () => new Date(nowMs), + adminToken: () => { + tokenIssue += 1; + const lifetimeSeconds = tokenIssue === 1 ? 120 : 3_600; + return actionsAdminToken(Math.floor(nowMs / 1000) + lifetimeSeconds); + }, + }); + + await fixture.client.getRunnerScaleSet(4, 'linux'); + nowMs += 70_000; + await fixture.client.getRunnerScaleSet(4, 'linux'); + + expect(fixture.accessTokenProvider).toHaveBeenCalledTimes(2); + expect(fixture.registrationRequests).toHaveLength(4); + expect(tokenIssue).toBe(2); + }); + + it('retries transient and propagation failures only while bootstrapping the admin connection', async () => { + let registrationTokenRequests = 0; + let adminConnectionRequests = 0; + let actionsRequests = 0; + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + registrationTokenRequests += 1; + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + adminConnectionRequests += 1; + if (adminConnectionRequests === 1) { + return jsonResponse({ message: 'temporarily unavailable' }, 503); + } + if (adminConnectionRequests === 2) { + return jsonResponse({ message: 'not propagated' }, 401); + } + if (adminConnectionRequests === 3) { + return jsonResponse({ message: 'not propagated' }, 403); + } + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + + actionsRequests += 1; + return jsonResponse({ count: 0, value: [] }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + retry: { + maxRetries: 3, + initialBackoffMs: 0, + maxBackoffMs: 0, + requestTimeoutMs: 1_000, + }, + }); + + await expect(client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + + expect(registrationTokenRequests).toBe(1); + expect(adminConnectionRequests).toBe(4); + expect(actionsRequests).toBe(1); + }); + + it('does not replay JIT generation when its POST receives a transient failure', async () => { + let jitRequests = 0; + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + if (url.pathname.endsWith('/generatejitconfig')) { + jitRequests += 1; + return jsonResponse({ message: 'temporarily unavailable' }, 503); + } + return new Response(null, { status: 500 }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + retry: { + maxRetries: 4, + initialBackoffMs: 0, + maxBackoffMs: 0, + requestTimeoutMs: 1_000, + }, + }); + + await expect(client.generateJitRunnerConfig({ name: 'runner-71', workFolder: '_work' }, 42)).rejects.toMatchObject({ + status: 503, + }); + expect(jitRequests).toBe(1); + }); + + it('raises a typed HTTP error with Actions exception and request metadata', async () => { + const fixture = clientFixture( + () => + new Response( + JSON.stringify({ + typeName: 'Microsoft.TeamFoundation.DistributedTask.WebApi.AgentExistsException', + message: 'runner already exists', + }), + { + status: 409, + headers: { + ActivityId: 'activity-123', + 'Content-Type': 'application/json', + 'X-GitHub-Request-Id': 'github-456', + }, + }, + ), + ); + + const request = fixture.client.getRunner(71); + await expect(request).rejects.toBeInstanceOf(ScaleSetHttpError); + await expect(request).rejects.toMatchObject({ + code: SCALE_SET_ERROR_CODES.runnerExists, + status: 409, + activityId: 'activity-123', + githubRequestId: 'github-456', + exceptionName: 'Microsoft.TeamFoundation.DistributedTask.WebApi.AgentExistsException', + }); + }); + + it('uses the exact scale-set CRUD endpoints and legacy request body casing', async () => { + const fixture = clientFixture((url, init) => { + const method = init.method; + if (method === 'GET' && url.pathname.endsWith('/runnerscalesets')) { + return jsonResponse({ + count: url.searchParams.has('name') ? 1 : 2, + value: [ + { id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }, + { id: 12, name: 'windows', RunnerSetting: {} }, + ], + }); + } + if (method === 'GET' && url.pathname.endsWith('/runnerscalesets/11')) { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }); + } + if (method === 'POST') { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }); + } + if (method === 'PATCH') { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: false } }); + } + if (method === 'DELETE') { + return new Response(null, { status: 204 }); + } + return new Response(null, { status: 500 }); + }); + const createInput = { + name: 'linux', + runnerGroupId: 4, + runnerSetting: { disableUpdate: true }, + }; + const updateInput = { + labels: [{ name: 'arm64' }], + runnerSetting: { disableUpdate: false }, + }; + + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toMatchObject({ + id: 11, + runnerSetting: { disableUpdate: true }, + }); + await expect(fixture.client.listRunnerScaleSets(4)).resolves.toHaveLength(2); + await expect(fixture.client.getRunnerScaleSetById(11)).resolves.toMatchObject({ id: 11 }); + await expect(fixture.client.createRunnerScaleSet(createInput)).resolves.toMatchObject({ id: 11 }); + await expect(fixture.client.updateRunnerScaleSet(11, updateInput)).resolves.toMatchObject({ + id: 11, + }); + await expect(fixture.client.deleteRunnerScaleSet(11)).resolves.toBeUndefined(); + + expect(createInput).toMatchObject({ labels: [{ name: 'linux', type: 'System' }] }); + expect(updateInput).toMatchObject({ labels: [{ name: 'arm64', type: 'System' }] }); + + const createRequest = fixture.serviceRequests.find(({ init }) => init.method === 'POST'); + const updateRequest = fixture.serviceRequests.find(({ init }) => init.method === 'PATCH'); + expect(createRequest).toBeDefined(); + expect(updateRequest).toBeDefined(); + + const createBody = JSON.parse(createRequest?.init.body as string) as Record; + expect(createBody).toMatchObject({ + name: 'linux', + runnerGroupId: 4, + labels: [{ name: 'linux', type: 'System' }], + RunnerSetting: { disableUpdate: true }, + }); + expect(createBody).not.toHaveProperty('runnerSetting'); + + const updateBody = JSON.parse(updateRequest?.init.body as string) as Record; + expect(updateBody).toMatchObject({ + labels: [{ name: 'arm64', type: 'System' }], + RunnerSetting: { disableUpdate: false }, + }); + expect(updateBody).not.toHaveProperty('runnerSetting'); + + expect(fixture.serviceRequests.map(({ url, init }) => [init.method, url.pathname])).toEqual([ + ['GET', '/tenant/123/_apis/runtime/runnerscalesets'], + ['GET', '/tenant/123/_apis/runtime/runnerscalesets'], + ['GET', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ['POST', '/tenant/123/_apis/runtime/runnerscalesets'], + ['PATCH', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ['DELETE', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ]); + for (const { url } of fixture.serviceRequests) { + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + } + }); + + it('uses the exact runner-group, JIT, and agent endpoints', async () => { + const fixture = clientFixture((url, init) => { + if (url.pathname.endsWith('/runnergroups/')) { + return jsonResponse({ + count: 1, + value: [{ id: 8, name: 'default', size: 0, isDefaultGroup: true }], + }); + } + if (url.pathname.endsWith('/generatejitconfig')) { + return jsonResponse({ + runner: { id: 71, name: 'runner-71', runnerScaleSetId: 42 }, + encodedJITConfig: 'encoded-jit', + }); + } + if (init.method === 'GET' && url.pathname.endsWith('/agents/71')) { + return jsonResponse({ id: 71, name: 'runner-71', runnerScaleSetId: 42 }); + } + if (init.method === 'GET' && url.pathname.endsWith('/agents')) { + return jsonResponse({ + count: 1, + value: [{ id: 71, name: 'runner-71', runnerScaleSetId: 42 }], + }); + } + if (init.method === 'DELETE' && url.pathname.endsWith('/agents/71')) { + return new Response(null, { status: 204 }); + } + return new Response(null, { status: 500 }); + }); + + await expect(fixture.client.getRunnerGroupByName('default')).resolves.toMatchObject({ id: 8 }); + await expect( + fixture.client.generateJitRunnerConfig({ name: 'runner-71', workFolder: '_work' }, 42), + ).resolves.toEqual({ + runner: { id: 71, name: 'runner-71', runnerScaleSetId: 42 }, + encodedJITConfig: 'encoded-jit', + }); + await expect(fixture.client.getRunner(71)).resolves.toMatchObject({ id: 71 }); + await expect(fixture.client.getRunnerByName('runner-71')).resolves.toMatchObject({ id: 71 }); + await expect(fixture.client.removeRunner(71)).resolves.toBeUndefined(); + + expect(fixture.serviceRequests.map(({ url, init }) => [init.method, url.pathname])).toEqual([ + ['GET', '/tenant/123/_apis/runtime/runnergroups/'], + ['POST', '/tenant/123/_apis/runtime/runnerscalesets/42/generatejitconfig'], + ['GET', '/tenant/123/_apis/distributedtask/pools/0/agents/71'], + ['GET', '/tenant/123/_apis/distributedtask/pools/0/agents'], + ['DELETE', '/tenant/123/_apis/distributedtask/pools/0/agents/71'], + ]); + expect(fixture.serviceRequests[0].url.searchParams.get('groupName')).toBe('default'); + expect(fixture.serviceRequests[3].url.searchParams.get('agentName')).toBe('runner-71'); + expect(JSON.parse(fixture.serviceRequests[1].init.body as string)).toEqual({ + name: 'runner-71', + workFolder: '_work', + }); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/client.ts b/lambdas/libs/github-actions-scale-set/src/client.ts new file mode 100644 index 0000000000..36e89cd586 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/client.ts @@ -0,0 +1,554 @@ +import { githubApiUrl, ParsedGitHubConfig, parseGitHubConfigUrl, runnerRegistrationTokenPath } from './config'; +import { ACTIONS_API_VERSION, RUNNER_ENDPOINT, RUNNER_GROUP_ENDPOINT, SCALE_SET_ENDPOINT } from './endpoints'; +import { ScaleSetProtocolError } from './errors'; +import { createRetryingFetch, executeRequest, HttpResult, parseJsonResponse } from './http'; +import { MessageSessionClient } from './message-session-client'; +import { + AccessTokenProvider, + GitHubActionsScaleSetClientOptions, + RunnerGroup, + RunnerReference, + RunnerScaleSet, + RunnerScaleSetJitRunnerConfig, + RunnerScaleSetJitRunnerSetting, + ScaleSetFetch, + ScaleSetRequestOptions, + SystemInfo, +} from './types'; +import { trimTrailingSlashes } from './url'; + +const ADMIN_TOKEN_REFRESH_SKEW_MS = 60_000; +const SUCCESS_STATUSES = Array.from({ length: 100 }, (_, index) => index + 200); + +interface RegistrationTokenResponse { + token?: string; + expires_at?: string; +} + +interface ActionsServiceAdminConnectionResponse { + url?: string; + token?: string; +} + +interface ActionsServiceAdminToken { + token: string; + expiresAt: Date; + url: string; +} + +interface RunnerScaleSetListResponse { + count: number; + value: RunnerScaleSet[]; +} + +interface RunnerGroupListResponse { + count: number; + value: RunnerGroup[]; +} + +interface RunnerReferenceListResponse { + count: number; + value: RunnerReference[]; +} + +interface ActionsRequestOptions extends ScaleSetRequestOptions { + query?: Record; + body?: unknown; + expectedStatuses: readonly number[]; + authorization?: string; +} + +function joinUrlPath(base: string, path: string): string { + const normalizedBase = trimTrailingSlashes(base); + if (normalizedBase === '') { + if (path === '') { + return ''; + } + return path.startsWith('/') ? path : `/${path}`; + } + if (path === '') { + return normalizedBase; + } + return `${normalizedBase}${path.startsWith('/') ? '' : '/'}${path}`; +} + +export function actionsServiceUrl( + base: string, + path: string, + query: Record = {}, +): URL { + const [pathOnly, pathQuery = ''] = path.split('?', 2); + const result = new URL(joinUrlPath(base, pathOnly)); + const mergedQuery = new URLSearchParams(pathQuery); + + for (const [name, value] of Object.entries(query)) { + if (value !== undefined) { + mergedQuery.set(name, String(value)); + } + } + if (!mergedQuery.get('api-version')) { + mergedQuery.set('api-version', ACTIONS_API_VERSION); + } + result.search = mergedQuery.toString(); + return result; +} + +function encodeSystemUserAgent(systemInfo: SystemInfo): string { + return JSON.stringify({ + system: systemInfo.system ?? '', + version: systemInfo.version ?? '', + commit_sha: systemInfo.commitSha ?? '', + scale_set_id: systemInfo.scaleSetId ?? 0, + subsystem: systemInfo.subsystem ?? '', + build_version: '1.0.0', + build_commit_sha: '', + kind: 'scaleset', + }); +} + +function parseJwtExpiration(token: string): Date { + const parts = token.split('.'); + if (parts.length < 2) { + throw new ScaleSetProtocolError('Actions service admin token is not a JWT'); + } + + let claims: unknown; + try { + claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) as unknown; + } catch (error) { + throw new ScaleSetProtocolError('failed to decode Actions service admin token claims', { + cause: error, + }); + } + + if ( + typeof claims !== 'object' || + claims === null || + !('exp' in claims) || + typeof claims.exp !== 'number' || + !Number.isFinite(claims.exp) + ) { + throw new ScaleSetProtocolError('Actions service admin token is missing a numeric exp claim'); + } + + return new Date(claims.exp * 1000); +} + +function applyDefaultLabelTypes(scaleSet: RunnerScaleSet): void { + for (const label of scaleSet.labels ?? []) { + label.type ||= 'System'; + } +} + +function ensureLabels(scaleSet: RunnerScaleSet): void { + if ((scaleSet.labels?.length ?? 0) > 0) { + return; + } + if (!scaleSet.name) { + throw new ScaleSetProtocolError('runner scale set must have a name or at least one label'); + } + scaleSet.labels = [{ name: scaleSet.name, type: 'System' }]; +} + +function runnerScaleSetRequestBody(scaleSet: RunnerScaleSet): Record { + const wire = { ...scaleSet } as Record; + delete wire.runnerSetting; + // The capital R is intentional and matches the Actions scale set wire contract. + wire.RunnerSetting = scaleSet.runnerSetting ?? {}; + return wire; +} + +function normalizeRunnerScaleSet(scaleSet: RunnerScaleSet | null): RunnerScaleSet | null { + if (scaleSet === null) { + return null; + } + + const wire = scaleSet as RunnerScaleSet & { RunnerSetting?: RunnerScaleSet['runnerSetting'] }; + if (wire.runnerSetting === undefined && wire.RunnerSetting !== undefined) { + wire.runnerSetting = wire.RunnerSetting; + } + delete wire.RunnerSetting; + return wire; +} + +/** A native-fetch client for the GitHub Actions runner scale set APIs. */ +export class GitHubActionsScaleSetClient { + private readonly config: ParsedGitHubConfig; + private readonly fetchImplementation: ScaleSetFetch; + private readonly adminConnectionFetchImplementation: ScaleSetFetch; + private readonly accessTokenProvider: AccessTokenProvider; + private readonly now: () => Date; + private readonly customUserAgent?: string; + private currentSystemInfo: SystemInfo; + private currentUserAgent: string; + private adminToken?: ActionsServiceAdminToken; + private adminTokenRefresh?: Promise; + + constructor(options: GitHubActionsScaleSetClientOptions) { + this.config = parseGitHubConfigUrl(options.gitHubConfigUrl, options.forceGhes); + const fetchImplementation = options.fetch ?? globalThis.fetch; + if (typeof fetchImplementation !== 'function') { + throw new TypeError('a fetch implementation is required'); + } + this.fetchImplementation = createRetryingFetch(fetchImplementation, options.retry); + // Upstream retries transient RemoteAuth propagation failures only for this + // bootstrap POST. Queue 401s must remain owned by session-token refresh, + // and other non-idempotent SDK requests must never use this opt-in wrapper. + this.adminConnectionFetchImplementation = createRetryingFetch(fetchImplementation, options.retry, { + additionalRetryStatuses: [401, 403], + additionalRetryMethods: ['POST'], + }); + + const hasPersonalAccessToken = + typeof options.personalAccessToken === 'string' && options.personalAccessToken !== ''; + const hasAccessTokenProvider = typeof options.accessTokenProvider === 'function'; + if (hasPersonalAccessToken === hasAccessTokenProvider) { + throw new TypeError('provide exactly one of personalAccessToken or accessTokenProvider'); + } + + this.accessTokenProvider = hasPersonalAccessToken + ? async () => options.personalAccessToken as string + : (options.accessTokenProvider as AccessTokenProvider); + this.now = options.now ?? (() => new Date()); + this.currentSystemInfo = { ...options.systemInfo }; + this.customUserAgent = options.userAgent; + this.currentUserAgent = options.userAgent ?? encodeSystemUserAgent(this.currentSystemInfo); + } + + get gitHubConfig(): ParsedGitHubConfig { + return { + ...this.config, + configUrl: new URL(this.config.configUrl), + }; + } + + get systemInfo(): SystemInfo { + return { ...this.currentSystemInfo }; + } + + setSystemInfo(systemInfo: SystemInfo): void { + this.currentSystemInfo = { ...systemInfo }; + if (this.customUserAgent === undefined) { + this.currentUserAgent = encodeSystemUserAgent(systemInfo); + } + } + + debugInfo(): string { + return JSON.stringify({ + system_info: this.currentUserAgent, + }); + } + + async getRunnerScaleSet( + runnerGroupId: number, + runnerScaleSetName: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + const { result, url } = await this.actionsRequest('GET', SCALE_SET_ENDPOINT, { + expectedStatuses: [200], + query: { runnerGroupId, name: runnerScaleSetName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + return null; + } + if (list.count !== 1) { + throw new ScaleSetProtocolError( + `multiple runner scale sets found with name ${JSON.stringify(runnerScaleSetName)}`, + ); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner scale set response count was 1 but value was empty'); + } + return normalizeRunnerScaleSet(list.value[0]); + } + + async listRunnerScaleSets(runnerGroupId: number, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', SCALE_SET_ENDPOINT, { + expectedStatuses: [200], + query: { runnerGroupId }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + return list.value.map((scaleSet) => normalizeRunnerScaleSet(scaleSet) as RunnerScaleSet); + } + + async getRunnerScaleSetById( + runnerScaleSetId: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`; + const { result, url } = await this.actionsRequest('GET', path, { + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'GET', url)); + } + + async getRunnerGroupByName(runnerGroupName: string, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', `/${RUNNER_GROUP_ENDPOINT}`, { + expectedStatuses: [200], + query: { groupName: runnerGroupName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + throw new ScaleSetProtocolError(`no runner group found with name ${JSON.stringify(runnerGroupName)}`); + } + if (list.count !== 1) { + throw new ScaleSetProtocolError(`multiple runner groups found with name ${JSON.stringify(runnerGroupName)}`); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner group response count was 1 but value was empty'); + } + return list.value[0]; + } + + async createRunnerScaleSet(scaleSet: RunnerScaleSet, options: ScaleSetRequestOptions = {}): Promise { + ensureLabels(scaleSet); + applyDefaultLabelTypes(scaleSet); + const { result, url } = await this.actionsRequest('POST', SCALE_SET_ENDPOINT, { + body: runnerScaleSetRequestBody(scaleSet), + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'POST', url)) as RunnerScaleSet; + } + + async updateRunnerScaleSet( + runnerScaleSetId: number, + scaleSet: RunnerScaleSet, + options: ScaleSetRequestOptions = {}, + ): Promise { + applyDefaultLabelTypes(scaleSet); + const path = `${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`; + const { result, url } = await this.actionsRequest('PATCH', path, { + body: runnerScaleSetRequestBody(scaleSet), + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'PATCH', url)) as RunnerScaleSet; + } + + async deleteRunnerScaleSet(runnerScaleSetId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.actionsRequest('DELETE', `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + async generateJitRunnerConfig( + setting: RunnerScaleSetJitRunnerSetting, + runnerScaleSetId: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}/generatejitconfig`; + const { result, url } = await this.actionsRequest('POST', path, { + body: setting, + expectedStatuses: [200], + signal: options.signal, + }); + return parseJsonResponse(result, 'POST', url); + } + + async getRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { + const path = `/${RUNNER_ENDPOINT}/${runnerId}`; + const { result, url } = await this.actionsRequest('GET', path, { + expectedStatuses: [200], + signal: options.signal, + }); + return parseJsonResponse(result, 'GET', url); + } + + async getRunnerByName(runnerName: string, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', RUNNER_ENDPOINT, { + expectedStatuses: [200], + query: { agentName: runnerName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + return null; + } + if (list.count !== 1) { + throw new ScaleSetProtocolError(`multiple runners found with name ${JSON.stringify(runnerName)}`); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner response count was 1 but value was empty'); + } + return list.value[0]; + } + + async removeRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.actionsRequest('DELETE', `/${RUNNER_ENDPOINT}/${runnerId}`, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + async createMessageSessionClient( + runnerScaleSetId: number, + owner: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + return MessageSessionClient.create({ + runnerScaleSetId, + owner, + fetchImplementation: this.fetchImplementation, + userAgent: () => this.currentUserAgent, + actionsRequest: (method, path, requestOptions) => this.actionsRequest(method, path, requestOptions), + signal: options.signal, + }); + } + + /** Alias matching the upstream Go client's factory name. */ + async messageSessionClient( + runnerScaleSetId: number, + owner: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.createMessageSessionClient(runnerScaleSetId, owner, options); + } + + private async actionsRequest( + method: string, + path: string, + options: ActionsRequestOptions, + ): Promise<{ result: HttpResult; url: URL }> { + const adminToken = await this.getAdminToken(options.signal); + const url = actionsServiceUrl(adminToken.url, path, options.query); + const headers: Record = { + 'Content-Type': 'application/json', + Authorization: options.authorization ?? `Bearer ${adminToken.token}`, + 'User-Agent': this.currentUserAgent, + }; + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method, + headers, + body, + signal: options.signal, + }, + options.expectedStatuses, + ); + return { result, url }; + } + + private async getAdminToken(signal?: AbortSignal): Promise { + if (this.adminTokenIsUsable(this.adminToken)) { + return this.adminToken; + } + + if (this.adminTokenRefresh === undefined) { + this.adminTokenRefresh = this.refreshAdminToken(signal).finally(() => { + this.adminTokenRefresh = undefined; + }); + } + return this.adminTokenRefresh; + } + + private adminTokenIsUsable(token?: ActionsServiceAdminToken): token is ActionsServiceAdminToken { + return token !== undefined && this.now().getTime() + ADMIN_TOKEN_REFRESH_SKEW_MS < token.expiresAt.getTime(); + } + + private async refreshAdminToken(signal?: AbortSignal): Promise { + const registrationToken = await this.getRunnerRegistrationToken(signal); + const adminConnection = await this.getActionsServiceAdminConnection(registrationToken, signal); + const refreshedToken: ActionsServiceAdminToken = { + token: adminConnection.token, + expiresAt: parseJwtExpiration(adminConnection.token), + url: adminConnection.url, + }; + this.adminToken = refreshedToken; + return refreshedToken; + } + + private async getRunnerRegistrationToken(signal?: AbortSignal): Promise { + const accessToken = await this.getAccessToken(); + const url = githubApiUrl(this.config, runnerRegistrationTokenPath(this.config)); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.github.v3+json', + Authorization: `Bearer ${accessToken}`, + 'User-Agent': this.currentUserAgent, + }, + body: '', + signal, + }, + [201], + ); + const response = parseJsonResponse(result, 'POST', url); + if (!response.token) { + throw new ScaleSetProtocolError('runner registration token response is missing token'); + } + return response.token; + } + + private async getAccessToken(): Promise { + const providedToken = await this.accessTokenProvider(); + const accessToken = typeof providedToken === 'string' ? providedToken : providedToken.token; + if (accessToken === '') throw new ScaleSetProtocolError('access token provider returned an empty token'); + return accessToken; + } + + private async getActionsServiceAdminConnection( + registrationToken: string, + signal?: AbortSignal, + ): Promise<{ url: string; token: string }> { + const url = githubApiUrl(this.config, '/actions/runner-registration'); + const result = await executeRequest( + this.adminConnectionFetchImplementation, + url, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `RemoteAuth ${registrationToken}`, + 'User-Agent': this.currentUserAgent, + }, + body: JSON.stringify({ + url: this.config.configUrl.toString(), + runner_event: 'register', + }), + signal, + }, + SUCCESS_STATUSES, + ); + const response = parseJsonResponse(result, 'POST', url); + if (!response.url) { + throw new ScaleSetProtocolError('Actions service admin connection is missing url'); + } + if (!response.token) { + throw new ScaleSetProtocolError('Actions service admin connection is missing token'); + } + let actionsServiceUrl: URL; + try { + actionsServiceUrl = new URL(response.url); + } catch (error) { + throw new ScaleSetProtocolError('Actions service admin connection contains an invalid url', { cause: error }); + } + if ( + actionsServiceUrl.protocol !== 'https:' || + actionsServiceUrl.username || + actionsServiceUrl.password || + actionsServiceUrl.search || + actionsServiceUrl.hash + ) { + throw new ScaleSetProtocolError( + 'Actions service admin connection url must be HTTPS and contain no credentials, query, or fragment', + ); + } + return { url: trimTrailingSlashes(actionsServiceUrl.toString()), token: response.token }; + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/config.test.ts b/lambdas/libs/github-actions-scale-set/src/config.test.ts new file mode 100644 index 0000000000..316fbab015 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/config.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; + +import { InvalidGitHubConfigUrlError, parseGitHubConfigUrl } from './config'; + +describe('GitHub configuration URL parsing', () => { + it('requires HTTPS for GitHub.com and GHES configuration URLs', () => { + expect(() => parseGitHubConfigUrl('http://github.com/example')).toThrow(InvalidGitHubConfigUrlError); + expect(() => parseGitHubConfigUrl('http://github.example.com/example', true)).toThrow(/should be HTTPS/); + }); + + it('continues to accept an HTTPS GitHub configuration URL', () => { + expect(parseGitHubConfigUrl('https://github.com/example')).toMatchObject({ + scope: 'organization', + organization: 'example', + isHosted: true, + }); + }); + + it('normalizes long slash sequences', () => { + const slashSequence = '/'.repeat(10_000); + + expect(parseGitHubConfigUrl(`https://github.com${slashSequence}example${slashSequence}`)).toMatchObject({ + configUrl: new URL('https://github.com/example'), + scope: 'organization', + organization: 'example', + isHosted: true, + }); + }); + + it.each(['https://github.com////', 'https://github.com/org//repository', 'https://github.com/org/repository/extra'])( + 'continues to reject an invalid path after slash normalization: %s', + (configUrl) => { + expect(() => parseGitHubConfigUrl(configUrl)).toThrow(InvalidGitHubConfigUrlError); + }, + ); + + it.each([ + ['https://github.com/org/repository////', 'repository'], + ['https://github.com/enterprises/example////', 'enterprise'], + ])('retains the scope when normalizing %s', (configUrl, scope) => { + expect(parseGitHubConfigUrl(configUrl)).toMatchObject({ scope }); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/config.ts b/lambdas/libs/github-actions-scale-set/src/config.ts new file mode 100644 index 0000000000..e9b5e1a4a3 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/config.ts @@ -0,0 +1,119 @@ +import { trimSurroundingSlashes, trimTrailingSlashes } from './url'; + +export const GITHUB_SCOPES = { + enterprise: 'enterprise', + organization: 'organization', + repository: 'repository', +} as const; + +export type GitHubScope = (typeof GITHUB_SCOPES)[keyof typeof GITHUB_SCOPES]; + +export interface ParsedGitHubConfig { + configUrl: URL; + scope: GitHubScope; + enterprise?: string; + organization?: string; + repository?: string; + isHosted: boolean; +} + +export class InvalidGitHubConfigUrlError extends Error { + constructor(configUrl: string, options?: ErrorOptions) { + super( + `${JSON.stringify(configUrl)}: invalid config URL, should be HTTPS and point to an enterprise, org, or repository`, + options, + ); + this.name = 'InvalidGitHubConfigUrlError'; + } +} + +function environmentForcesGhes(): boolean { + return ( + typeof process !== 'undefined' && Object.prototype.hasOwnProperty.call(process.env, 'GITHUB_ACTIONS_FORCE_GHES') + ); +} + +function isHostedGitHubUrl(configUrl: URL, forceGhes?: boolean): boolean { + if (forceGhes ?? environmentForcesGhes()) { + return false; + } + + const host = configUrl.host.toLowerCase(); + return host === 'github.com' || host === 'www.github.com' || host === 'github.localhost' || host.endsWith('.ghe.com'); +} + +/** Parse a repository, organization, or enterprise registration URL. */ +export function parseGitHubConfigUrl(configUrl: string, forceGhes?: boolean): ParsedGitHubConfig { + let parsedUrl: URL; + try { + parsedUrl = new URL(trimTrailingSlashes(configUrl.trim())); + } catch (error) { + throw new InvalidGitHubConfigUrlError(configUrl, { cause: error }); + } + + if (parsedUrl.protocol !== 'https:') { + throw new InvalidGitHubConfigUrlError(configUrl); + } + + const pathParts = trimSurroundingSlashes(parsedUrl.pathname).split('/'); + const isHosted = isHostedGitHubUrl(parsedUrl, forceGhes); + + if (pathParts.length === 1 && pathParts[0] !== '') { + parsedUrl.pathname = `/${pathParts[0]}`; + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.organization, + organization: pathParts[0], + isHosted, + }; + } + + if (pathParts.length === 2 && pathParts.every((part) => part !== '')) { + parsedUrl.pathname = `/${pathParts.join('/')}`; + if (pathParts[0].toLowerCase() === 'enterprises') { + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.enterprise, + enterprise: pathParts[1], + isHosted, + }; + } + + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.repository, + organization: pathParts[0], + repository: pathParts[1], + isHosted, + }; + } + + throw new InvalidGitHubConfigUrlError(configUrl); +} + +/** Build a GitHub REST API URL for GitHub.com, ghe.com, or GHES. */ +export function githubApiUrl(config: ParsedGitHubConfig, path: string): URL { + const result = new URL(config.configUrl.origin); + const normalizedPath = path.startsWith('/') ? path : `/${path}`; + + if (config.isHosted) { + result.host = + config.configUrl.host.toLowerCase() === 'www.github.com' ? 'api.github.com' : `api.${config.configUrl.host}`; + result.pathname = normalizedPath; + return result; + } + + result.pathname = `/api/v3${normalizedPath}`; + return result; +} + +export function runnerRegistrationTokenPath(config: ParsedGitHubConfig): string { + switch (config.scope) { + case GITHUB_SCOPES.organization: + return `/orgs/${config.organization}/actions/runners/registration-token`; + case GITHUB_SCOPES.enterprise: + return `/enterprises/${config.enterprise}/actions/runners/registration-token`; + case GITHUB_SCOPES.repository: + return `/repos/${config.organization}/${config.repository}/actions/runners/registration-token`; + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/endpoints.ts b/lambdas/libs/github-actions-scale-set/src/endpoints.ts new file mode 100644 index 0000000000..d409f73704 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/endpoints.ts @@ -0,0 +1,4 @@ +export const RUNNER_ENDPOINT = '_apis/distributedtask/pools/0/agents'; +export const SCALE_SET_ENDPOINT = '_apis/runtime/runnerscalesets'; +export const RUNNER_GROUP_ENDPOINT = '_apis/runtime/runnergroups/'; +export const ACTIONS_API_VERSION = '6.0-preview'; diff --git a/lambdas/libs/github-actions-scale-set/src/errors.ts b/lambdas/libs/github-actions-scale-set/src/errors.ts new file mode 100644 index 0000000000..860e4aaa7d --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/errors.ts @@ -0,0 +1,171 @@ +export const SCALE_SET_ERROR_CODES = { + badRequest: 'BAD_REQUEST', + conflict: 'CONFLICT', + jobStillRunning: 'JOB_STILL_RUNNING', + messageQueueTokenExpired: 'MESSAGE_QUEUE_TOKEN_EXPIRED', + notFound: 'NOT_FOUND', + runnerExists: 'RUNNER_EXISTS', + runnerNotFound: 'RUNNER_NOT_FOUND', + unauthorized: 'UNAUTHORIZED', + unexpectedStatus: 'UNEXPECTED_STATUS', +} as const; + +export type ScaleSetErrorCode = (typeof SCALE_SET_ERROR_CODES)[keyof typeof SCALE_SET_ERROR_CODES]; + +export interface ScaleSetHttpErrorDetails { + method: string; + url: string; + status: number; + statusText: string; + headers: Headers; + responseBody: string; + code?: ScaleSetErrorCode; + cause?: unknown; +} + +interface ActionsException { + typeName?: unknown; + message?: unknown; +} + +export function redactUrlForError(value: string): string { + try { + const url = new URL(value); + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return url.toString(); + } catch { + return ''; + } +} + +function statusErrorCode(status: number): ScaleSetErrorCode { + switch (status) { + case 400: + return SCALE_SET_ERROR_CODES.badRequest; + case 401: + return SCALE_SET_ERROR_CODES.unauthorized; + case 404: + return SCALE_SET_ERROR_CODES.notFound; + case 409: + return SCALE_SET_ERROR_CODES.conflict; + default: + return SCALE_SET_ERROR_CODES.unexpectedStatus; + } +} + +function exceptionErrorCode(typeName?: string): ScaleSetErrorCode | undefined { + if (typeName?.includes('AgentExistsException')) { + return SCALE_SET_ERROR_CODES.runnerExists; + } + if (typeName?.includes('AgentNotFoundException')) { + return SCALE_SET_ERROR_CODES.runnerNotFound; + } + if (typeName?.includes('JobStillRunningException')) { + return SCALE_SET_ERROR_CODES.jobStillRunning; + } + return undefined; +} + +function parseActionsException(responseBody: string): { typeName?: string; message?: string } { + if (responseBody === '') { + return {}; + } + + try { + const parsed = JSON.parse(responseBody) as ActionsException; + return { + typeName: typeof parsed.typeName === 'string' ? parsed.typeName : undefined, + message: typeof parsed.message === 'string' ? parsed.message : undefined, + }; + } catch { + return {}; + } +} + +/** An unsuccessful HTTP response from either GitHub or the Actions service. */ +export class ScaleSetHttpError extends Error { + readonly code: ScaleSetErrorCode; + readonly status: number; + readonly statusText: string; + readonly method: string; + readonly url: string; + readonly activityId?: string; + readonly githubRequestId?: string; + readonly exceptionName?: string; + readonly responseBody: string; + + constructor(details: ScaleSetHttpErrorDetails) { + const safeUrl = redactUrlForError(details.url); + const exception = parseActionsException(details.responseBody); + const activityId = details.headers.get('ActivityId') ?? undefined; + const githubRequestId = details.headers.get('X-GitHub-Request-Id') ?? undefined; + const responseDescription = [details.status, details.statusText].filter(Boolean).join(' '); + const metadata = [ + `status=${JSON.stringify(responseDescription)}`, + activityId ? `activity_id=${JSON.stringify(activityId)}` : undefined, + githubRequestId ? `github_request_id=${JSON.stringify(githubRequestId)}` : undefined, + ] + .filter((part): part is string => part !== undefined) + .join(', '); + const responseMessage = exception.message ?? (details.responseBody || 'unknown error'); + const exceptionPrefix = exception.typeName ? `${exception.typeName}: ` : ''; + + super(`request ${details.method} ${safeUrl} failed (${metadata}): ${exceptionPrefix}${responseMessage}`, { + cause: details.cause, + }); + this.name = 'ScaleSetHttpError'; + this.code = details.code ?? exceptionErrorCode(exception.typeName) ?? statusErrorCode(details.status); + this.status = details.status; + this.statusText = details.statusText; + this.method = details.method; + this.url = safeUrl; + this.activityId = activityId; + this.githubRequestId = githubRequestId; + this.exceptionName = exception.typeName; + this.responseBody = details.responseBody; + } +} + +export class ScaleSetRequestError extends Error { + readonly method: string; + readonly url: string; + readonly attempts: number; + + constructor(method: string, url: string, cause: unknown, attempts = 1) { + const safeUrl = redactUrlForError(url); + super(`request ${method} ${safeUrl} failed before receiving a response after ${attempts} attempt(s)`, { cause }); + this.name = 'ScaleSetRequestError'; + this.method = method; + this.url = safeUrl; + this.attempts = attempts; + } +} + +export class ScaleSetRequestTimeoutError extends ScaleSetRequestError { + readonly timeoutMs: number; + + constructor(method: string, url: string, timeoutMs: number, attempts: number) { + super(method, url, new Error(`request attempt exceeded ${timeoutMs}ms`), attempts); + this.name = 'ScaleSetRequestTimeoutError'; + this.timeoutMs = timeoutMs; + } +} + +export class ScaleSetProtocolError extends Error { + readonly method?: string; + readonly url?: string; + + constructor(message: string, options: { method?: string; url?: string; cause?: unknown } = {}) { + super(message, { cause: options.cause }); + this.name = 'ScaleSetProtocolError'; + this.method = options.method; + this.url = options.url; + } +} + +export function isScaleSetHttpError(error: unknown): error is ScaleSetHttpError { + return error instanceof ScaleSetHttpError; +} diff --git a/lambdas/libs/github-actions-scale-set/src/http.test.ts b/lambdas/libs/github-actions-scale-set/src/http.test.ts new file mode 100644 index 0000000000..76f506d13f --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/http.test.ts @@ -0,0 +1,238 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { ScaleSetRequestError, ScaleSetRequestTimeoutError } from './errors'; +import { + createRetryingFetch, + DEFAULT_SCALE_SET_RETRY_OPTIONS, + executeRequest, + resolveScaleSetRetryOptions, +} from './http'; +import { ScaleSetFetch } from './types'; + +function okResponse(): Response { + return new Response('{"ok":true}', { status: 200 }); +} + +describe('retrying fetch', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('uses bounded defaults close to the upstream client and validates overrides', () => { + expect(resolveScaleSetRetryOptions()).toEqual({ + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 300_000, + }); + expect(DEFAULT_SCALE_SET_RETRY_OPTIONS).toEqual(resolveScaleSetRetryOptions()); + expect(() => resolveScaleSetRetryOptions({ maxRetries: -1 })).toThrow(/retry\.maxRetries/); + expect(() => resolveScaleSetRetryOptions({ maxRetries: 1.5 })).toThrow(/integer/); + expect(() => resolveScaleSetRetryOptions({ requestTimeoutMs: 0 })).toThrow(/requestTimeoutMs/); + }); + + it('retries a network error after deterministic exponential backoff', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi + .fn() + .mockRejectedValueOnce(new TypeError('socket closed')) + .mockResolvedValueOnce(okResponse()); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 25, + maxBackoffMs: 100, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.advanceTimersByTimeAsync(24); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + + await expect(request).resolves.toMatchObject({ status: 200 }); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + }); + + it('honors Retry-After for 429 responses and caps the wait at maxBackoffMs', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi + .fn() + .mockResolvedValueOnce( + new Response('{"message":"slow down"}', { + status: 429, + headers: { 'Retry-After': '120' }, + }), + ) + .mockResolvedValueOnce(okResponse()); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 10, + maxBackoffMs: 30_000, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.advanceTimersByTimeAsync(29_999); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + + await expect(request).resolves.toMatchObject({ status: 200 }); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + }); + + it('retries 5xx responses only up to maxRetries and returns the final response', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn( + async () => + new Response('{"message":"unavailable"}', { + status: 503, + headers: { 'Retry-After': 'invalid' }, + }), + ); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 2, + initialBackoffMs: 10, + maxBackoffMs: 15, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.runAllTimersAsync(); + + await expect(request).resolves.toMatchObject({ status: 503 }); + expect(underlyingFetch).toHaveBeenCalledTimes(3); + }); + + it('does not retry a queue 401 so message-session refresh remains the owner', async () => { + const underlyingFetch = vi.fn(async () => new Response(null, { status: 401 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect(fetchWithRetry('https://queue.example/messages')).resolves.toMatchObject({ status: 401 }); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('does not replay a POST after a retryable HTTP response', async () => { + const underlyingFetch = vi.fn(async () => new Response(null, { status: 503 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect( + fetchWithRetry('https://actions.example/generatejitconfig', { method: 'POST' }), + ).resolves.toMatchObject({ status: 503 }); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('does not replay a POST carried by a Request after a network failure', async () => { + const underlyingFetch = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect( + fetchWithRetry(new Request('https://actions.example/sessions', { method: 'POST' })), + ).rejects.toMatchObject({ + name: 'ScaleSetRequestError', + method: 'POST', + attempts: 1, + } satisfies Partial); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('times out each attempt and stops after the configured retry bound', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(() => new Promise(() => undefined)); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 10, + maxBackoffMs: 10, + requestTimeoutMs: 50, + }); + + const request = fetchWithRetry('https://actions.example/hangs'); + const rejection = expect(request).rejects.toMatchObject({ + name: 'ScaleSetRequestTimeoutError', + attempts: 2, + timeoutMs: 50, + } satisfies Partial); + await vi.advanceTimersByTimeAsync(50); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(10); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(50); + + await rejection; + }); + + it('interrupts retry backoff immediately when the caller aborts', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(async () => new Response(null, { status: 503 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 30_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 60_000, + }); + const controller = new AbortController(); + const request = fetchWithRetry('https://actions.example/test', { signal: controller.signal }); + await vi.advanceTimersByTimeAsync(0); + + controller.abort(new Error('caller cancelled')); + + await expect(request).rejects.toThrow('caller cancelled'); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('wraps an exhausted idempotent network failure with the final attempt count', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + const request = fetchWithRetry('https://actions.example/test', { method: 'GET' }); + const rejection = expect(request).rejects.toMatchObject({ + name: 'ScaleSetRequestError', + method: 'GET', + attempts: 2, + } satisfies Partial); + await vi.runAllTimersAsync(); + + await rejection; + }); + + it('redacts signed query strings from request errors', async () => { + const fetchImplementation = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const request = executeRequest( + fetchImplementation, + 'https://queue.example/messages?signature=do-not-log&token=also-secret', + { method: 'GET' }, + [200], + ); + + await expect(request).rejects.toMatchObject({ + url: 'https://queue.example/messages', + message: expect.not.stringContaining('do-not-log'), + } satisfies Partial); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/http.ts b/lambdas/libs/github-actions-scale-set/src/http.ts new file mode 100644 index 0000000000..ef5f47bab9 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/http.ts @@ -0,0 +1,327 @@ +import { + ScaleSetErrorCode, + ScaleSetHttpError, + ScaleSetProtocolError, + ScaleSetRequestError, + ScaleSetRequestTimeoutError, + redactUrlForError, +} from './errors'; +import { ScaleSetFetch, ScaleSetRetryOptions } from './types'; + +export interface ResolvedScaleSetRetryOptions { + maxRetries: number; + initialBackoffMs: number; + maxBackoffMs: number; + requestTimeoutMs: number; +} + +export const DEFAULT_SCALE_SET_RETRY_OPTIONS: Readonly = Object.freeze({ + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 5 * 60_000, +}); + +export interface HttpResult { + response: Response; + body: string; +} + +interface RetryingFetchPolicy { + additionalRetryStatuses?: readonly number[]; + /** + * Escape hatch for a known-safe, operation-scoped wrapper. Non-idempotent + * methods are never retried by the default transport policy. + */ + additionalRetryMethods?: readonly string[]; +} + +const MAX_RESPONSE_BODY_BYTES = 1024 * 1024; +const IDEMPOTENT_RETRY_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE']); + +function trimByteOrderMark(body: string): string { + return body.startsWith('\uFEFF') ? body.slice(1) : body; +} + +function boundedNumber(name: string, value: number, minimum: number, integer: boolean): number { + if (!Number.isFinite(value) || value < minimum || (integer && !Number.isInteger(value))) { + throw new TypeError(`${name} must be ${integer ? 'an integer' : 'a number'} greater than or equal to ${minimum}`); + } + return value; +} + +export function resolveScaleSetRetryOptions(options: ScaleSetRetryOptions = {}): ResolvedScaleSetRetryOptions { + return { + maxRetries: boundedNumber( + 'retry.maxRetries', + options.maxRetries ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.maxRetries, + 0, + true, + ), + initialBackoffMs: boundedNumber( + 'retry.initialBackoffMs', + options.initialBackoffMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.initialBackoffMs, + 0, + false, + ), + maxBackoffMs: boundedNumber( + 'retry.maxBackoffMs', + options.maxBackoffMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.maxBackoffMs, + 0, + false, + ), + requestTimeoutMs: boundedNumber( + 'retry.requestTimeoutMs', + options.requestTimeoutMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.requestTimeoutMs, + 1, + false, + ), + }; +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException('The operation was aborted', 'AbortError'); +} + +function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(abortReason(signal)); + } + if (delayMs === 0) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timeout); + reject(abortReason(signal as AbortSignal)); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +function retryableStatus(status: number, additionalRetryStatuses: ReadonlySet): boolean { + return status === 429 || (status >= 500 && status <= 599) || additionalRetryStatuses.has(status); +} + +function requestMethod(input: RequestInput, init: RequestInit): string { + return (init.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase(); +} + +function exponentialBackoffMs(retryIndex: number, options: ResolvedScaleSetRetryOptions): number { + return Math.min(options.initialBackoffMs * 2 ** retryIndex, options.maxBackoffMs); +} + +function retryAfterMs(response: Response, options: ResolvedScaleSetRetryOptions): number | undefined { + const value = response.headers.get('Retry-After')?.trim(); + if (!value) { + return undefined; + } + + let delayMs: number; + if (/^\d+$/.test(value)) { + delayMs = Number(value) * 1_000; + } else { + const retryAt = Date.parse(value); + if (!Number.isFinite(retryAt)) { + return undefined; + } + delayMs = Math.max(0, retryAt - Date.now()); + } + return Math.min(delayMs, options.maxBackoffMs); +} + +async function fetchAttempt( + fetchImplementation: ScaleSetFetch, + input: RequestInput, + init: RequestInit, + options: ResolvedScaleSetRetryOptions, + attempt: number, +): Promise { + const method = requestMethod(input, init); + const url = input instanceof Request ? input.url : input.toString(); + const callerSignal = init.signal ?? undefined; + if (callerSignal?.aborted) { + throw abortReason(callerSignal); + } + + const attemptController = new AbortController(); + const forwardAbort = () => attemptController.abort(abortReason(callerSignal as AbortSignal)); + callerSignal?.addEventListener('abort', forwardAbort, { once: true }); + const timeoutError = new ScaleSetRequestTimeoutError(method, url, options.requestTimeoutMs, attempt); + const timeout = setTimeout(() => attemptController.abort(timeoutError), options.requestTimeoutMs); + + let onAttemptAbort: (() => void) | undefined; + const aborted = new Promise((_, reject) => { + onAttemptAbort = () => reject(abortReason(attemptController.signal)); + attemptController.signal.addEventListener('abort', onAttemptAbort, { once: true }); + }); + + try { + return await Promise.race([fetchImplementation(input, { ...init, signal: attemptController.signal }), aborted]); + } finally { + clearTimeout(timeout); + callerSignal?.removeEventListener('abort', forwardAbort); + if (onAttemptAbort !== undefined) { + attemptController.signal.removeEventListener('abort', onAttemptAbort); + } + } +} + +type RequestInput = Parameters[0]; + +/** Wrap native fetch with the bounded retry and timeout policy used by every SDK request. */ +export function createRetryingFetch( + fetchImplementation: ScaleSetFetch, + retryOptions: ScaleSetRetryOptions = {}, + policy: RetryingFetchPolicy = {}, +): ScaleSetFetch { + const options = resolveScaleSetRetryOptions(retryOptions); + const additionalRetryStatusSet = new Set(policy.additionalRetryStatuses ?? []); + const additionalRetryMethodSet = new Set((policy.additionalRetryMethods ?? []).map((method) => method.toUpperCase())); + + return async (input, init = {}) => { + const method = requestMethod(input, init); + const url = input instanceof Request ? input.url : input.toString(); + const maxRetries = + IDEMPOTENT_RETRY_METHODS.has(method) || additionalRetryMethodSet.has(method) ? options.maxRetries : 0; + + for (let retryIndex = 0; retryIndex <= maxRetries; retryIndex += 1) { + const attempt = retryIndex + 1; + try { + const response = await fetchAttempt(fetchImplementation, input, init, options, attempt); + if (!retryableStatus(response.status, additionalRetryStatusSet) || retryIndex === maxRetries) { + return response; + } + + const delayMs = retryAfterMs(response, options) ?? exponentialBackoffMs(retryIndex, options); + await response.body?.cancel().catch(() => undefined); + await waitForRetry(delayMs, init.signal ?? undefined); + } catch (error) { + if (init.signal?.aborted) { + throw abortReason(init.signal); + } + if (retryIndex === maxRetries) { + if (error instanceof ScaleSetRequestTimeoutError) { + throw error; + } + throw new ScaleSetRequestError(method, url, error, attempt); + } + await waitForRetry(exponentialBackoffMs(retryIndex, options), init.signal ?? undefined); + } + } + + throw new ScaleSetRequestError(method, url, new Error('retry loop exhausted'), maxRetries + 1); + }; +} + +export async function executeRequest( + fetchImplementation: ScaleSetFetch, + url: string | URL, + init: RequestInit, + expectedStatuses: readonly number[], + errorCode?: ScaleSetErrorCode | ((response: Response) => ScaleSetErrorCode | undefined), +): Promise { + const method = init.method ?? 'GET'; + const urlString = url.toString(); + const displayUrl = redactUrlForError(urlString); + let response: Response; + + try { + response = await fetchImplementation(url, { ...init, redirect: 'error' }); + } catch (error) { + if (init.signal?.aborted || (error instanceof Error && error.name === 'AbortError')) { + throw error; + } + if (error instanceof ScaleSetRequestError) { + throw error; + } + throw new ScaleSetRequestError(method, urlString, error); + } + + let body: string; + try { + const contentLength = response.headers.get('content-length'); + if (contentLength !== null && /^\d+$/.test(contentLength) && Number(contentLength) > MAX_RESPONSE_BODY_BYTES) { + throw new ScaleSetProtocolError( + `response body from ${method} ${displayUrl} exceeds ${MAX_RESPONSE_BODY_BYTES} bytes`, + { + method, + url: displayUrl, + }, + ); + } + if (response.body === null) { + body = ''; + } else { + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MAX_RESPONSE_BODY_BYTES) { + await reader.cancel().catch(() => undefined); + throw new ScaleSetProtocolError( + `response body from ${method} ${displayUrl} exceeds ${MAX_RESPONSE_BODY_BYTES} bytes`, + { method, url: displayUrl }, + ); + } + chunks.push(value); + } + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + body = trimByteOrderMark(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + } + } catch (error) { + if (error instanceof ScaleSetProtocolError) throw error; + throw new ScaleSetProtocolError(`failed to read the response body from ${method} ${displayUrl}`, { + method, + url: displayUrl, + cause: error, + }); + } + + if (!expectedStatuses.includes(response.status)) { + throw new ScaleSetHttpError({ + method, + url: displayUrl, + status: response.status, + statusText: response.statusText, + headers: response.headers, + responseBody: body, + code: typeof errorCode === 'function' ? errorCode(response) : errorCode, + }); + } + + return { response, body }; +} + +export function parseJsonResponse(result: HttpResult, method: string, url: string | URL): T { + const urlString = redactUrlForError(url.toString()); + if (result.body === '') { + throw new ScaleSetProtocolError(`empty JSON response from ${method} ${urlString}`, { + method, + url: urlString, + }); + } + + try { + return JSON.parse(result.body) as T; + } catch (error) { + throw new ScaleSetProtocolError(`invalid JSON response from ${method} ${urlString}`, { + method, + url: urlString, + cause: error, + }); + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/index.ts b/lambdas/libs/github-actions-scale-set/src/index.ts new file mode 100644 index 0000000000..ab0aad76db --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/index.ts @@ -0,0 +1,29 @@ +export { + actionsServiceUrl, + GitHubActionsScaleSetClient as Client, + GitHubActionsScaleSetClient, + GitHubActionsScaleSetClient as ScaleSetClient, +} from './client'; +export { ACTIONS_API_VERSION, RUNNER_ENDPOINT, RUNNER_GROUP_ENDPOINT, SCALE_SET_ENDPOINT } from './endpoints'; +export { + GITHUB_SCOPES, + githubApiUrl, + InvalidGitHubConfigUrlError, + parseGitHubConfigUrl, + runnerRegistrationTokenPath, +} from './config'; +export type { GitHubScope, ParsedGitHubConfig } from './config'; +export { + isScaleSetHttpError, + redactUrlForError, + SCALE_SET_ERROR_CODES, + ScaleSetHttpError, + ScaleSetProtocolError, + ScaleSetRequestError, + ScaleSetRequestTimeoutError, +} from './errors'; +export type { ScaleSetErrorCode, ScaleSetHttpErrorDetails } from './errors'; +export { DEFAULT_SCALE_SET_RETRY_OPTIONS } from './http'; +export type { ResolvedScaleSetRetryOptions } from './http'; +export { HEADER_SCALE_SET_MAX_CAPACITY, MessageSessionClient } from './message-session-client'; +export * from './types'; diff --git a/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts b/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts new file mode 100644 index 0000000000..ed4e7d41ae --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts @@ -0,0 +1,281 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { GitHubActionsScaleSetClient } from './client'; +import { ScaleSetProtocolError } from './errors'; +import { HEADER_SCALE_SET_MAX_CAPACITY } from './message-session-client'; +import { RunnerScaleSetStatistic, ScaleSetFetch } from './types'; + +type RequestInput = Parameters[0]; +type RequestHandler = (url: URL, init: RequestInit) => Response | Promise; + +const statistics: RunnerScaleSetStatistic = { + totalAvailableJobs: 2, + totalAcquiredJobs: 1, + totalAssignedJobs: 1, + totalRunningJobs: 1, + totalRegisteredRunners: 2, + totalBusyRunners: 1, + totalIdleRunners: 1, +}; + +function requestUrl(input: RequestInput): URL { + if (input instanceof Request) { + return new URL(input.url); + } + return new URL(input.toString()); +} + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function actionsAdminToken(): string { + const payload = Buffer.from(JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 60 * 60 })).toString('base64url'); + return `header.${payload}.signature`; +} + +function sessionFixture(handler: RequestHandler) { + const requests: Array<{ url: URL; init: RequestInit }> = []; + const fetchImplementation = vi.fn(async (input, init = {}) => { + const url = requestUrl(input); + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + + requests.push({ url, init }); + return handler(url, init); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-token', + fetch: fetchImplementation, + systemInfo: { system: 'unit-test', subsystem: 'listener' }, + }); + + return { client, fetchImplementation, requests }; +} + +function sessionResponse(token = 'queue-token') { + return { + sessionId: '11111111-1111-1111-1111-111111111111', + ownerName: 'listener-1', + runnerScaleSet: { id: 42, name: 'linux' }, + messageQueueUrl: 'https://queue.example/messages?existing=1', + messageQueueAccessToken: token, + statistics, + }; +} + +describe('MessageSessionClient', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('normalizes the capitalized RunnerSetting returned when creating a session', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse({ + ...sessionResponse(), + runnerScaleSet: { id: 42, name: 'linux', RunnerSetting: { disableUpdate: true } }, + }); + } + return new Response(null, { status: 500 }); + }); + + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + expect(session.session.runnerScaleSet?.runnerSetting).toEqual({ disableUpdate: true }); + expect(session.session.runnerScaleSet).not.toHaveProperty('RunnerSetting'); + }); + + it('maps a 202 poll to null and sends the queue capacity contract', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + return new Response(null, { status: 202 }); + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 17)).resolves.toBeNull(); + + expect(session.session.statistics).toEqual(statistics); + const queueRequest = fixture.requests.find(({ url }) => url.hostname === 'queue.example'); + expect(queueRequest).toBeDefined(); + expect(queueRequest?.url.toString()).toBe('https://queue.example/messages?existing=1'); + const headers = new Headers(queueRequest?.init.headers); + expect(headers.get('Accept')).toBe('application/json; api-version=6.0-preview'); + expect(headers.get('Authorization')).toBe('Bearer queue-token'); + expect(headers.get(HEADER_SCALE_SET_MAX_CAPACITY)).toBe('17'); + expect(headers.get('User-Agent')).toContain('"kind":"scaleset"'); + }); + + it('decodes known batched messages, ignores unknown types, acknowledges, and acquires jobs', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + return jsonResponse({ + messageId: 19, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([ + { + messageType: 'JobAvailable', + runnerRequestId: 501, + acquireJobUrl: 'https://actions.example/acquire/501', + }, + { + messageType: 'FutureMessageType', + runnerRequestId: 999, + }, + { + messageType: 'JobCompleted', + runnerRequestId: 500, + runnerId: 71, + runnerName: 'runner-71', + result: 'Succeeded', + }, + ]), + }); + } + if (url.hostname === 'queue.example' && init.method === 'DELETE') { + return new Response(null, { status: 204 }); + } + if (url.pathname.endsWith('/runnerscalesets/42/acquirejobs') && init.method === 'POST') { + return jsonResponse({ count: 1, value: [501] }); + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + const message = await session.getMessage(18, 4); + expect(message).toMatchObject({ messageId: 19, statistics }); + expect(message?.jobAvailableMessages).toEqual([ + expect.objectContaining({ messageType: 'JobAvailable', runnerRequestId: 501 }), + ]); + expect(message?.jobCompletedMessages).toEqual([ + expect.objectContaining({ messageType: 'JobCompleted', runnerName: 'runner-71' }), + ]); + expect(message?.jobAssignedMessages).toEqual([]); + expect(message?.jobStartedMessages).toEqual([]); + + await expect(session.deleteMessage(19)).resolves.toBeUndefined(); + await expect(session.acquireJobs([501, 999])).resolves.toEqual([501]); + + const pollRequest = fixture.requests.find( + ({ url, init }) => url.hostname === 'queue.example' && init.method === 'GET', + ); + expect(pollRequest?.url.searchParams.get('lastMessageId')).toBe('18'); + + const ackRequest = fixture.requests.find( + ({ url, init }) => url.hostname === 'queue.example' && init.method === 'DELETE', + ); + expect(ackRequest?.url.pathname).toBe('/messages/19'); + expect(ackRequest?.url.searchParams.get('existing')).toBe('1'); + expect(new Headers(ackRequest?.init.headers).get('Authorization')).toBe('Bearer queue-token'); + + const acquireRequest = fixture.requests.find(({ url }) => url.pathname.endsWith('/acquirejobs')); + expect(acquireRequest?.url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets/42/acquirejobs'); + expect(acquireRequest?.url.searchParams.get('api-version')).toBe('6.0-preview'); + expect(new Headers(acquireRequest?.init.headers).get('Authorization')).toBe('Bearer queue-token'); + expect(JSON.parse(acquireRequest?.init.body as string)).toEqual([501, 999]); + }); + + it.each([ + ['message id', { messageId: 0, messageType: 'RunnerScaleSetJobMessages', statistics, body: '[]' }], + [ + 'statistics', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics: { ...statistics, totalAssignedJobs: -1 }, + body: '[]', + }, + ], + [ + 'runner request id', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([{ messageType: 'JobAvailable', runnerRequestId: 0 }]), + }, + ], + [ + 'runner identity', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([ + { messageType: 'JobCompleted', runnerRequestId: 1, runnerId: 2, runnerName: 'bad\nname' }, + ]), + }, + ], + ])('rejects malformed known message %s fields', async (_name, payload) => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') return jsonResponse(payload); + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 10)).rejects.toBeInstanceOf(ScaleSetProtocolError); + }); + + it('refreshes the message session once on a queue 401 and retries with the new token', async () => { + let refreshCount = 0; + let oldTokenPolls = 0; + let newTokenPolls = 0; + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse('old-queue-token')); + } + if (url.pathname.includes('/runnerscalesets/42/sessions/') && init.method === 'PATCH') { + refreshCount += 1; + return jsonResponse({ + ...sessionResponse('new-queue-token'), + runnerScaleSet: { id: 42, name: 'linux', RunnerSetting: { disableUpdate: false } }, + }); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + const authorization = new Headers(init.headers).get('Authorization'); + if (authorization === 'Bearer old-queue-token') { + oldTokenPolls += 1; + return jsonResponse({ message: 'expired' }, 401); + } + if (authorization === 'Bearer new-queue-token') { + newTokenPolls += 1; + return new Response(null, { status: 202 }); + } + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 10)).resolves.toBeNull(); + + expect(oldTokenPolls).toBe(1); + expect(newTokenPolls).toBe(1); + expect(refreshCount).toBe(1); + expect(session.session.messageQueueAccessToken).toBe('new-queue-token'); + expect(session.session.runnerScaleSet?.runnerSetting).toEqual({ disableUpdate: false }); + expect(session.session.runnerScaleSet).not.toHaveProperty('RunnerSetting'); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/message-session-client.ts b/lambdas/libs/github-actions-scale-set/src/message-session-client.ts new file mode 100644 index 0000000000..130e68482e --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/message-session-client.ts @@ -0,0 +1,467 @@ +import { SCALE_SET_ERROR_CODES, ScaleSetHttpError, ScaleSetProtocolError } from './errors'; +import { executeRequest, HttpResult, parseJsonResponse } from './http'; +import { SCALE_SET_ENDPOINT } from './endpoints'; +import { + JobAssigned, + JobAvailable, + JobCompleted, + JobStarted, + MESSAGE_TYPES, + RunnerScaleSetMessage, + RunnerScaleSet, + RunnerScaleSetSession, + RunnerScaleSetStatistic, + ScaleSetFetch, + ScaleSetRequestOptions, +} from './types'; + +export const HEADER_SCALE_SET_MAX_CAPACITY = 'X-ScaleSetMaxCapacity'; + +interface ActionsRequestOptions extends ScaleSetRequestOptions { + body?: unknown; + expectedStatuses: readonly number[]; + authorization?: string; +} + +interface MessageSessionClientCreateOptions extends ScaleSetRequestOptions { + runnerScaleSetId: number; + owner: string; + fetchImplementation: ScaleSetFetch; + userAgent: () => string; + actionsRequest: ( + method: string, + path: string, + options: ActionsRequestOptions, + ) => Promise<{ result: HttpResult; url: URL }>; +} + +interface RunnerScaleSetMessageResponse { + messageId: number; + messageType: string; + body?: string; + statistics?: RunnerScaleSetStatistic | null; +} + +interface AcquireJobsResponse { + count: number; + value: number[]; +} + +interface JobMessageType { + messageType?: unknown; +} + +const STATISTIC_FIELDS = [ + 'totalAvailableJobs', + 'totalAcquiredJobs', + 'totalAssignedJobs', + 'totalRunningJobs', + 'totalRegisteredRunners', + 'totalBusyRunners', + 'totalIdleRunners', +] as const satisfies readonly (keyof RunnerScaleSetStatistic)[]; + +function positiveInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || (value as number) <= 0) { + throw new ScaleSetProtocolError(`${field} must be a positive integer`); + } + return value as number; +} + +function validateStatistics(value: unknown): RunnerScaleSetStatistic | null { + if (value === null || value === undefined) return null; + if (typeof value !== 'object' || Array.isArray(value)) { + throw new ScaleSetProtocolError('runner scale set statistics must be an object'); + } + for (const field of STATISTIC_FIELDS) { + const statistic = (value as Record)[field]; + if (!Number.isSafeInteger(statistic) || (statistic as number) < 0) { + throw new ScaleSetProtocolError(`statistics.${field} must be a non-negative integer`); + } + } + return value as RunnerScaleSetStatistic; +} + +function validateKnownJobMessage(rawMessage: Record, messageType: string): void { + positiveInteger(rawMessage.runnerRequestId, `${messageType}.runnerRequestId`); + if (messageType === MESSAGE_TYPES.jobStarted || messageType === MESSAGE_TYPES.jobCompleted) { + positiveInteger(rawMessage.runnerId, `${messageType}.runnerId`); + if ( + typeof rawMessage.runnerName !== 'string' || + rawMessage.runnerName.length === 0 || + rawMessage.runnerName.length > 256 || + hasAsciiControlCharacter(rawMessage.runnerName) + ) { + throw new ScaleSetProtocolError(`${messageType}.runnerName is invalid`); + } + } +} + +function hasAsciiControlCharacter(value: string): boolean { + return [...value].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }); +} + +function normalizeRunnerScaleSet(scaleSet: RunnerScaleSet | null | undefined): RunnerScaleSet | null | undefined { + if (scaleSet === null || scaleSet === undefined) { + return scaleSet; + } + + const wire = scaleSet as RunnerScaleSet & { RunnerSetting?: RunnerScaleSet['runnerSetting'] }; + if (wire.runnerSetting === undefined && wire.RunnerSetting !== undefined) { + wire.runnerSetting = wire.RunnerSetting; + } + delete wire.RunnerSetting; + return wire; +} + +function normalizeSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + session.runnerScaleSet = normalizeRunnerScaleSet(session.runnerScaleSet); + return session; +} + +function cloneSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + return { + ...session, + runnerScaleSet: + session.runnerScaleSet === undefined || session.runnerScaleSet === null + ? session.runnerScaleSet + : { + ...session.runnerScaleSet, + labels: session.runnerScaleSet.labels?.map((label) => ({ ...label })), + runnerSetting: + session.runnerScaleSet.runnerSetting === undefined + ? undefined + : { ...session.runnerScaleSet.runnerSetting }, + statistics: + session.runnerScaleSet.statistics === undefined || session.runnerScaleSet.statistics === null + ? session.runnerScaleSet.statistics + : { ...session.runnerScaleSet.statistics }, + }, + statistics: + session.statistics === undefined || session.statistics === null ? session.statistics : { ...session.statistics }, + }; +} + +function validateSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + if (!session.sessionId) { + throw new ScaleSetProtocolError('message session response is missing sessionId'); + } + if (!session.messageQueueUrl) { + throw new ScaleSetProtocolError('message session response is missing messageQueueUrl'); + } + if (!session.messageQueueAccessToken) { + throw new ScaleSetProtocolError('message session response is missing messageQueueAccessToken'); + } + let queueUrl: URL; + try { + queueUrl = new URL(session.messageQueueUrl); + } catch (error) { + throw new ScaleSetProtocolError('message session response contains an invalid messageQueueUrl', { cause: error }); + } + if (queueUrl.protocol !== 'https:' || queueUrl.username || queueUrl.password || queueUrl.hash) { + throw new ScaleSetProtocolError( + 'message session messageQueueUrl must be HTTPS and contain no credentials or fragment', + ); + } + return session; +} + +function parseRunnerScaleSetMessage(result: HttpResult, url: URL): RunnerScaleSetMessage { + const response = parseJsonResponse(result, 'GET', url); + positiveInteger(response.messageId, 'messageId'); + if (response.messageType !== 'RunnerScaleSetJobMessages') { + throw new ScaleSetProtocolError(`unsupported message type: ${response.messageType}`); + } + + let batchedMessages: unknown[] = []; + if (response.body) { + try { + const parsed = JSON.parse(response.body) as unknown; + if (!Array.isArray(parsed)) { + throw new TypeError('message body is not an array'); + } + batchedMessages = parsed; + if (batchedMessages.length > 50) { + throw new TypeError('message body contains more than 50 entries'); + } + } catch (error) { + throw new ScaleSetProtocolError('failed to unmarshal batched runner scale set messages', { + cause: error, + }); + } + } + + const message: RunnerScaleSetMessage = { + messageId: response.messageId, + statistics: validateStatistics(response.statistics), + jobAvailableMessages: [], + jobAssignedMessages: [], + jobStartedMessages: [], + jobCompletedMessages: [], + }; + + for (const rawMessage of batchedMessages) { + if (typeof rawMessage !== 'object' || rawMessage === null) { + throw new ScaleSetProtocolError('runner scale set job message is not an object'); + } + const messageType = (rawMessage as JobMessageType).messageType; + if (typeof messageType !== 'string') { + throw new ScaleSetProtocolError('runner scale set job message is missing messageType'); + } + switch (messageType) { + case MESSAGE_TYPES.jobAvailable: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobAvailableMessages.push(rawMessage as JobAvailable); + break; + case MESSAGE_TYPES.jobAssigned: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobAssignedMessages.push(rawMessage as JobAssigned); + break; + case MESSAGE_TYPES.jobStarted: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobStartedMessages.push(rawMessage as JobStarted); + break; + case MESSAGE_TYPES.jobCompleted: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobCompletedMessages.push(rawMessage as JobCompleted); + break; + default: + // The upstream client ignores unknown job message types for forward compatibility. + break; + } + } + + return message; +} + +/** A message queue session scoped to one runner scale set. */ +export class MessageSessionClient { + private readonly runnerScaleSetId: number; + private readonly fetchImplementation: ScaleSetFetch; + private readonly userAgent: () => string; + private readonly actionsRequest: MessageSessionClientCreateOptions['actionsRequest']; + private currentSession: RunnerScaleSetSession; + private sessionRefresh?: Promise; + + private constructor(options: MessageSessionClientCreateOptions, session: RunnerScaleSetSession) { + this.runnerScaleSetId = options.runnerScaleSetId; + this.fetchImplementation = options.fetchImplementation; + this.userAgent = options.userAgent; + this.actionsRequest = options.actionsRequest; + this.currentSession = session; + } + + static async create(options: MessageSessionClientCreateOptions): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${options.runnerScaleSetId}/sessions`; + const { result, url } = await options.actionsRequest('POST', path, { + body: { ownerName: options.owner }, + expectedStatuses: [200], + signal: options.signal, + }); + const session = validateSession(normalizeSession(parseJsonResponse(result, 'POST', url))); + return new MessageSessionClient(options, session); + } + + /** A defensive snapshot of the current session and its latest statistics. */ + get session(): RunnerScaleSetSession { + return cloneSession(this.currentSession); + } + + getSession(): RunnerScaleSetSession { + return this.session; + } + + async close(options: ScaleSetRequestOptions = {}): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/sessions/${this.currentSession.sessionId}`; + await this.actionsRequest('DELETE', path, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + /** + * Long-poll for a batched scale set message. A 202 response means no message + * is currently available and is represented as `null`. + */ + async getMessage( + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.withMessageTokenRefresh( + (session) => this.getMessageWithSession(session, lastMessageId, maxCapacity, options), + options, + ); + } + + async pollMessage( + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.getMessage(lastMessageId, maxCapacity, options); + } + + /** Delete a queue message after processing it, which acknowledges the batch. */ + async deleteMessage(messageId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.withMessageTokenRefresh( + (session) => this.deleteMessageWithSession(session, messageId, options), + options, + ); + } + + async acknowledgeMessage(messageId: number, options: ScaleSetRequestOptions = {}): Promise { + return this.deleteMessage(messageId, options); + } + + /** Return the authoritative subset of runner request IDs acquired by the service. */ + async acquireJobs(requestIds: number[], options: ScaleSetRequestOptions = {}): Promise { + return this.withMessageTokenRefresh( + (session) => this.acquireJobsWithSession(session, requestIds, options), + options, + ); + } + + private async getMessageWithSession( + session: RunnerScaleSetSession, + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions, + ): Promise { + const url = new URL(session.messageQueueUrl); + if (lastMessageId > 0) { + url.searchParams.set('lastMessageId', String(lastMessageId)); + } + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'GET', + headers: { + Accept: 'application/json; api-version=6.0-preview', + Authorization: `Bearer ${session.messageQueueAccessToken}`, + 'User-Agent': this.userAgent(), + [HEADER_SCALE_SET_MAX_CAPACITY]: String(maxCapacity), + }, + signal: options.signal, + }, + [200, 202], + (response) => (response.status === 401 ? SCALE_SET_ERROR_CODES.messageQueueTokenExpired : undefined), + ); + + if (result.response.status === 202) { + return null; + } + return parseRunnerScaleSetMessage(result, url); + } + + private async deleteMessageWithSession( + session: RunnerScaleSetSession, + messageId: number, + options: ScaleSetRequestOptions, + ): Promise { + const url = new URL(session.messageQueueUrl); + const valueAfterOrigin = session.messageQueueUrl.slice(url.origin.length); + const originalPath = valueAfterOrigin.startsWith('/') ? url.pathname : ''; + url.pathname = `${originalPath}/${messageId}`; + await executeRequest( + this.fetchImplementation, + url, + { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${session.messageQueueAccessToken}`, + 'User-Agent': this.userAgent(), + }, + signal: options.signal, + }, + [204], + (response) => (response.status === 401 ? SCALE_SET_ERROR_CODES.messageQueueTokenExpired : undefined), + ); + } + + private async acquireJobsWithSession( + session: RunnerScaleSetSession, + requestIds: number[], + options: ScaleSetRequestOptions, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/acquirejobs`; + try { + const { result, url } = await this.actionsRequest('POST', path, { + body: requestIds, + expectedStatuses: [200], + authorization: `Bearer ${session.messageQueueAccessToken}`, + signal: options.signal, + }); + return parseJsonResponse(result, 'POST', url).value; + } catch (error) { + if (error instanceof ScaleSetHttpError && error.status === 401) { + throw new ScaleSetHttpError({ + method: error.method, + url: error.url, + status: error.status, + statusText: error.statusText, + headers: new Headers({ + ...(error.activityId ? { ActivityId: error.activityId } : {}), + ...(error.githubRequestId ? { 'X-GitHub-Request-Id': error.githubRequestId } : {}), + }), + responseBody: error.responseBody, + code: SCALE_SET_ERROR_CODES.messageQueueTokenExpired, + cause: error, + }); + } + throw error; + } + } + + private async withMessageTokenRefresh( + operation: (session: RunnerScaleSetSession) => Promise, + options: ScaleSetRequestOptions, + ): Promise { + const expiredSession = this.currentSession; + try { + return await operation(expiredSession); + } catch (error) { + if (!(error instanceof ScaleSetHttpError) || error.code !== SCALE_SET_ERROR_CODES.messageQueueTokenExpired) { + throw error; + } + } + + await this.refreshMessageSession(expiredSession, options); + return operation(this.currentSession); + } + + private async refreshMessageSession( + expiredSession: RunnerScaleSetSession, + options: ScaleSetRequestOptions, + ): Promise { + if ( + this.currentSession.sessionId !== expiredSession.sessionId || + this.currentSession.messageQueueAccessToken !== expiredSession.messageQueueAccessToken + ) { + return; + } + + if (this.sessionRefresh === undefined) { + this.sessionRefresh = this.doRefreshMessageSession(options).finally(() => { + this.sessionRefresh = undefined; + }); + } + await this.sessionRefresh; + } + + private async doRefreshMessageSession(options: ScaleSetRequestOptions): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/sessions/${this.currentSession.sessionId}`; + const { result, url } = await this.actionsRequest('PATCH', path, { + expectedStatuses: [200], + signal: options.signal, + }); + this.currentSession = validateSession( + normalizeSession(parseJsonResponse(result, 'PATCH', url)), + ); + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/types.ts b/lambdas/libs/github-actions-scale-set/src/types.ts new file mode 100644 index 0000000000..11d387bf4c --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/types.ts @@ -0,0 +1,183 @@ +export const DEFAULT_RUNNER_GROUP = 'default'; + +export const MESSAGE_TYPES = { + jobAvailable: 'JobAvailable', + jobAssigned: 'JobAssigned', + jobStarted: 'JobStarted', + jobCompleted: 'JobCompleted', +} as const; + +export type MessageType = (typeof MESSAGE_TYPES)[keyof typeof MESSAGE_TYPES]; + +export interface JobMessageBase { + messageType: MessageType; + runnerRequestId: number; + repositoryName: string; + ownerName: string; + jobId: string; + jobWorkflowRef: string; + jobDisplayName: string; + workflowRunId: number; + eventName: string; + requestLabels: string[]; + queueTime: string; + scaleSetAssignTime: string; + runnerAssignTime: string; + finishTime: string; +} + +export interface JobAvailable extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobAvailable; + acquireJobUrl: string; +} + +export interface JobAssigned extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobAssigned; +} + +export interface JobStarted extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobStarted; + runnerId: number; + runnerName: string; +} + +export interface JobCompleted extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobCompleted; + result: string; + runnerId: number; + runnerName: string; +} + +export interface Label { + type?: string; + name: string; +} + +export interface RunnerGroup { + id: number; + name: string; + size: number; + isDefaultGroup: boolean; +} + +export interface RunnerSetting { + disableUpdate?: boolean; +} + +/** + * Runner scale set representation used by the Actions service. + * + * The same shape is accepted for create and update operations, so server-owned + * fields are optional. The client translates `runnerSetting` to the upstream + * wire key `RunnerSetting` when it sends a request. + */ +export interface RunnerScaleSet { + id?: number; + name?: string; + runnerGroupId?: number; + runnerGroupName?: string; + labels?: Label[]; + runnerSetting?: RunnerSetting; + createdOn?: string; + runnerJitConfigUrl?: string; + statistics?: RunnerScaleSetStatistic | null; +} + +export interface RunnerScaleSetJitRunnerSetting { + name: string; + workFolder?: string; +} + +export interface RunnerReference { + id: number; + name: string; + runnerScaleSetId: number; +} + +export interface RunnerScaleSetJitRunnerConfig { + runner: RunnerReference | null; + encodedJITConfig: string; +} + +export interface RunnerScaleSetStatistic { + totalAvailableJobs: number; + totalAcquiredJobs: number; + totalAssignedJobs: number; + totalRunningJobs: number; + totalRegisteredRunners: number; + totalBusyRunners: number; + totalIdleRunners: number; +} + +export interface RunnerScaleSetSession { + sessionId: string; + ownerName: string; + runnerScaleSet?: RunnerScaleSet | null; + messageQueueUrl: string; + messageQueueAccessToken: string; + statistics?: RunnerScaleSetStatistic | null; +} + +export interface RunnerScaleSetMessage { + messageId: number; + statistics: RunnerScaleSetStatistic | null; + jobAvailableMessages: JobAvailable[]; + jobAssignedMessages: JobAssigned[]; + jobStartedMessages: JobStarted[]; + jobCompletedMessages: JobCompleted[]; +} + +export interface SystemInfo { + system?: string; + version?: string; + commitSha?: string; + scaleSetId?: number; + subsystem?: string; +} + +export interface AccessToken { + token: string; + expiresAt?: string | Date; +} + +export type AccessTokenProvider = () => Promise; + +export type ScaleSetFetch = typeof globalThis.fetch; + +export interface ScaleSetRequestOptions { + signal?: AbortSignal; +} + +export interface ScaleSetRetryOptions { + /** Number of retries after the initial request for retry-eligible operations. */ + maxRetries?: number; + /** Initial exponential-backoff delay. */ + initialBackoffMs?: number; + /** Upper bound for exponential backoff and Retry-After delays. */ + maxBackoffMs?: number; + /** Timeout applied independently to each fetch attempt. */ + requestTimeoutMs?: number; +} + +interface ScaleSetClientBaseOptions { + gitHubConfigUrl: string; + systemInfo?: SystemInfo; + fetch?: ScaleSetFetch; + forceGhes?: boolean; + userAgent?: string; + /** Intended for deterministic tests. Defaults to `new Date()`. */ + now?: () => Date; + retry?: ScaleSetRetryOptions; +} + +export type GitHubActionsScaleSetClientOptions = ScaleSetClientBaseOptions & + ( + | { + personalAccessToken: string; + accessTokenProvider?: never; + } + | { + personalAccessToken?: never; + accessTokenProvider: AccessTokenProvider; + } + ); diff --git a/lambdas/libs/github-actions-scale-set/src/url.ts b/lambdas/libs/github-actions-scale-set/src/url.ts new file mode 100644 index 0000000000..55d9ecb468 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/url.ts @@ -0,0 +1,24 @@ +const FORWARD_SLASH = '/'.charCodeAt(0); + +export function trimTrailingSlashes(value: string): string { + let end = value.length; + while (end > 0 && value.charCodeAt(end - 1) === FORWARD_SLASH) { + end -= 1; + } + + return end === value.length ? value : value.slice(0, end); +} + +export function trimSurroundingSlashes(value: string): string { + let start = 0; + while (start < value.length && value.charCodeAt(start) === FORWARD_SLASH) { + start += 1; + } + + let end = value.length; + while (end > start && value.charCodeAt(end - 1) === FORWARD_SLASH) { + end -= 1; + } + + return start === 0 && end === value.length ? value : value.slice(start, end); +} diff --git a/lambdas/libs/github-actions-scale-set/tsconfig.json b/lambdas/libs/github-actions-scale-set/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/libs/github-actions-scale-set/vitest.config.ts b/lambdas/libs/github-actions-scale-set/vitest.config.ts new file mode 100644 index 0000000000..c52b8d7522 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/vitest.config.ts @@ -0,0 +1,22 @@ +import { mergeConfig } from 'vitest/config'; + +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + root: __dirname, + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/**/*.d.ts', 'src/index.ts'], + thresholds: { + // Measured by the package-scoped wire-contract suite. These floors keep + // meaningful regression protection without pretending every defensive + // parser/error branch is exercised by production-path tests. + statements: 75, + branches: 60, + functions: 80, + lines: 75, + }, + }, + }, +}); diff --git a/lambdas/package.json b/lambdas/package.json index c6fa5d72c3..0a0b0b088a 100644 --- a/lambdas/package.json +++ b/lambdas/package.json @@ -3,7 +3,8 @@ "private": true, "workspaces": [ "functions/*", - "libs/*" + "libs/*", + "services/*" ], "scripts": { "build": "nx run-many --target=build --all", diff --git a/lambdas/services/scale-set/Dockerfile b/lambdas/services/scale-set/Dockerfile new file mode 100644 index 0000000000..d1bfbe429f --- /dev/null +++ b/lambdas/services/scale-set/Dockerfile @@ -0,0 +1,25 @@ +FROM node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 AS build +WORKDIR /workspace/lambdas + +COPY lambdas/package.json lambdas/yarn.lock lambdas/.yarnrc.yml ./ +COPY lambdas/.yarn ./.yarn +COPY lambdas/tsconfig.json lambdas/vitest.base.config.ts ./ +COPY lambdas/functions ./functions +COPY lambdas/libs ./libs +COPY lambdas/services ./services + +RUN corepack enable && yarn install --immutable +RUN yarn workspace @aws-github-runner/scale-set-service build + +FROM node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 AS runtime +ENV NODE_ENV=production \ + SCALE_SET_HEALTH_PORT=8080 +WORKDIR /app + +COPY --from=build --chown=node:node /workspace/lambdas/services/scale-set/dist/ ./ +COPY --from=build --chown=node:node /workspace/lambdas/services/scale-set/healthcheck.cjs ./healthcheck.cjs + +USER node +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=90s --retries=3 CMD ["node", "/app/healthcheck.cjs"] +ENTRYPOINT ["node", "--disable-proto=delete", "/app/index.js"] diff --git a/lambdas/services/scale-set/Dockerfile.dockerignore b/lambdas/services/scale-set/Dockerfile.dockerignore new file mode 100644 index 0000000000..7c72e0e7ab --- /dev/null +++ b/lambdas/services/scale-set/Dockerfile.dockerignore @@ -0,0 +1,7 @@ +**/coverage +**/dist +**/node_modules +.git +.nx +lambdas/.yarn/install-state.gz +*.log diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md new file mode 100644 index 0000000000..e42ada323d --- /dev/null +++ b/lambdas/services/scale-set/README.md @@ -0,0 +1,107 @@ +# Scale-set controller service + +This workspace builds the long-running, topology-neutral controller used by the scale-set orchestration provider. + +Each controller group maps to one ECS service, one task definition, and normally one running task. The task contains one application container and one `ScaleSetController`, which supervises one independent reconciler per runner config: + +```text +ECS service (controller group) +└── one ECS task + └── one scale-set container + └── ScaleSetController + ├── reconciler: runner config A → scale set A → session A + └── reconciler: runner config B → scale set B → session B +``` + +A group is only a packing and deployment boundary. Every reconciler retains its own GitHub message session, lifecycle state, retry loop, health, and compute-provider instance. One reconciler failure does not exit the others. + +## Production configuration + +The ECS task receives only these group selectors: + +- `SCALE_SET_CONTROLLER_GROUP_NAME` +- `SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH` +- `SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION` + +The service reads every direct child under the SSM path with paginated `GetParametersByPath`. Each child name must equal its `runnerConfigName`, and each value uses this flat, versioned schema: + +```json +{ + "schemaVersion": 1, + "runnerConfigName": "linux-x64", + "githubConfigUrl": "https://github.com/example", + "scaleSetName": "linux-x64", + "runnerGroupName": "self-hosted-linux", + "runnerGroupIdParameterName": "/runners/github-app/runner-group/self-hosted-linux", + "minRunners": 0, + "maxRunners": 20, + "bootTimeoutMinutes": 10, + "sslVerify": true, + "githubApp": { + "appIdParameterName": "/runners/github-app/id", + "privateKeyParameterName": "/runners/github-app/key" + }, + "computeProvider": { + "type": "ec2", + "configuration": { + "region": "eu-west-1", + "environment": "example-linux-x64", + "runnerNamePrefix": "", + "jitConfigParameterPath": "/runners/example-linux-x64/tokens", + "subnets": ["subnet-0123456789abcdef0"], + "launchTemplateName": "example-linux-x64-action-runner", + "ec2instanceCriteria": { + "instanceTypes": ["m7i.large"], + "targetCapacityType": "spot", + "instanceAllocationStrategy": "price-capacity-optimized" + }, + "onDemandFailoverOnError": [], + "useDedicatedHost": false, + "ssmParameterTags": [] + } + } +} +``` + +`scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. `runnerGroupIdParameterName` is an optional SSM cache path. When present, the service reads the runner-group ID from that parameter; if it is missing, the service resolves the name through the configured GitHub Actions service endpoint and writes the ID back as a non-secret `String` parameter with overwrite enabled. The service resolves the scale-set ID from the group and scale-set names; if the named scale set does not exist, it registers it in the resolved runner group and uses the ID returned by GitHub. `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. Optional fields are `scaleSetId`, `runnerGroupIdParameterName`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. + +GitHub App ID and private-key values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. `installationIdParameterName` is optional; when it is absent or its parameter is not present, the service creates a short-lived App JWT and discovers the installation by matching the configured organization or enterprise account through `GET /app/installations`. This works with GitHub.com, GHES, and GitHub Enterprise Cloud data-residency API hosts derived from `githubConfigUrl`. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, App JWTs, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. + +`SCALE_SET_CONTROLLER_MANIFEST` is supported only as a bounded local/test convenience. It contains `{ "version": 1, "groupName": "...", "reconcilers": [...] }` and uses the same reconciler objects. + +Runtime settings: + +| Environment variable | Default | +| --------------------------------------------- | ------- | +| `SCALE_SET_HEALTH_PORT` | `8080` | +| `SCALE_SET_HEALTH_STALE_AFTER_SECONDS` | `180` | +| `SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS` | `110` | +| `SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS` | `10` | +| `SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS` | `1` | +| `SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS` | `30` | +| `LOG_LEVEL` | `info` | + +## Reconciliation and health + +Demand is calculated as `max(totalAssignedJobs, min(maxRunners, minRunners + totalAssignedJobs))`. The maximum therefore bounds requested idle capacity without ever requesting scale-down below work GitHub has already assigned. Job-started and job-completed messages maintain a bounded in-memory lifecycle cache. After a restart, lifecycle state is unknown, but provider-owned EC2 tags still identify current capacity. The aggregate scale-set busy count protects unknown runners from scale-down until it reaches zero. Runner deletion executes inside the serialized reconcile loop and re-checks the exact Actions-service runner identity by name before removal. + +The public GitHub runner inventory is not fetched by the scale-set service. The selected compute provider reconciles its own capacity inventory, while the scale-set session remains the source of demand and aggregate busy-runner statistics. This keeps the service aligned with the upstream scale-set API and supports GitHub.com, GHES, and data-residency endpoints without relying on a separate public REST runner endpoint. + +Messages follow the upstream scale-set listener order: acknowledge first, then acquire available jobs, update lifecycle state, and reconcile compute. Provider failures therefore stop that reconciler after the message has been acknowledged; provider results expose one error outcome rather than the control-plane scaling retry classification. A typed busy/unknown retention remains a successful reconciliation. Session and transport failures are handled separately by bounded client retries or session recreation. + +The EC2 provider uses the tagged, `config-published` EC2 instances as its capacity inventory. The GitHub Actions scale-set session supplies `desiredRunners` through `totalAssignedJobs` and the aggregate `totalBusyRunners` count. When capacity is above desired and the aggregate busy count is zero, tagged runners may be removed through the Actions service and their matching EC2 instances terminated; busy, contradictory, or unknown identities are retained. The provider uses the boot window (`bootTimeoutMinutes`, default `10`) for newly launched instances and keeps interrupted publication states from being counted as serving. EC2 ownership includes a SHA-256 hash of the canonical GitHub configuration scope, preventing the same runner-config name and numeric scale-set ID in another GitHub scope from colliding. No public GitHub REST runner inventory call is required. + +- `GET /healthz` reports controller liveness and is used by Docker/ECS. External GitHub outages remain live but degraded to avoid restart loops. +- `GET /readyz` reports readiness and returns 503 unless every reconciler is ready. + +## Container + +Build from the repository root: + +```shell +docker build --target runtime -f lambdas/services/scale-set/Dockerfile -t scale-set-controller . +``` + +The image supports `linux/amd64` and `linux/arm64`, uses a digest-pinned multi-stage Node image, runs as the unprivileged `node` user, includes a Node-based health check, and does not require filesystem writes. Deploy with a read-only root filesystem, all Linux capabilities dropped, no Docker socket, and only the task-role permissions required by the selected group. + +The module's official GHCR package must allow anonymous pulls so the default image works without registry credentials. Production deployments should select a released image by digest and verify its provenance/attestation. A private ECR override requires `container.ecr_repository.arn`; private non-ECR registry credentials are not currently exposed by the Terraform orchestration module. diff --git a/lambdas/services/scale-set/healthcheck.cjs b/lambdas/services/scale-set/healthcheck.cjs new file mode 100644 index 0000000000..66c833829c --- /dev/null +++ b/lambdas/services/scale-set/healthcheck.cjs @@ -0,0 +1,13 @@ +'use strict'; + +const http = require('node:http'); +const port = Number(process.env.SCALE_SET_HEALTH_PORT || '8080'); +const request = http.get( + { host: '127.0.0.1', port, path: '/healthz', timeout: 4000, headers: { Connection: 'close' } }, + (response) => { + response.resume(); + process.exit(response.statusCode === 200 ? 0 : 1); + }, +); +request.on('timeout', () => request.destroy(new Error('health check timed out'))); +request.on('error', () => process.exit(1)); diff --git a/lambdas/services/scale-set/package.json b/lambdas/services/scale-set/package.json new file mode 100644 index 0000000000..9a912211bd --- /dev/null +++ b/lambdas/services/scale-set/package.json @@ -0,0 +1,42 @@ +{ + "name": "@aws-github-runner/scale-set-service", + "version": "0.1.0", + "private": true, + "main": "dist/index.js", + "type": "module", + "license": "MIT", + "scripts": { + "build": "ncc build src/main.ts -o dist --minify", + "typecheck": "tsc --noEmit", + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "all": "yarn typecheck && yarn build && yarn format-check && yarn lint && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "@vercel/ncc": "0.38.4", + "typescript": "^5.9.3" + }, + "dependencies": { + "@aws-github-runner/aws-ssm-util": "*", + "@aws-github-runner/compute-providers": "*", + "@aws-github-runner/github-actions-scale-set": "*", + "@aws-sdk/client-ssm": "^3.1009.0", + "@octokit/auth-app": "8.2.0", + "@octokit/request": "^9.2.2", + "undici": "^6.19.2" + }, + "nx": { + "includedScripts": [ + "build", + "typecheck", + "format", + "format-check", + "lint", + "all" + ] + } +} diff --git a/lambdas/services/scale-set/src/config.test.ts b/lambdas/services/scale-set/src/config.test.ts new file mode 100644 index 0000000000..6fe4d30805 --- /dev/null +++ b/lambdas/services/scale-set/src/config.test.ts @@ -0,0 +1,205 @@ +import { + MAX_MANIFEST_BYTES, + parseScaleSetControllerManifest, + parseScaleSetReconcilerConfig, + parseScaleSetServiceConfig, +} from './config'; + +function runnerConfig(overrides: Record = {}) { + return { + schemaVersion: 1, + runnerConfigName: 'linux-x64', + githubConfigUrl: 'https://github.com/example', + scaleSetId: 123, + expectedScaleSetName: 'linux-x64', + expectedRunnerGroupId: null, + minRunners: 0, + maxRunners: 20, + githubApp: { + appIdParameterName: '/runner/app/id', + privateKeyParameterName: '/runner/app/key', + installationIdParameterName: '/runner/app/installation-id', + }, + computeProvider: { type: 'ec2', configuration: { subnetIds: ['subnet-1'] } }, + ...overrides, + }; +} + +describe('scale-set service configuration', () => { + it('parses the production SSM group source and runtime defaults', () => { + expect( + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'ec2-default', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/runner/groups/ec2-default', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '42', + }), + ).toEqual({ + groupName: 'ec2-default', + groupConfigPath: '/runner/groups/ec2-default', + groupRevision: '42', + healthPort: 8080, + healthStaleAfterMs: 180000, + shutdownTimeoutMs: 110000, + sessionCloseTimeoutMs: 10000, + reconnectInitialBackoffMs: 1000, + reconnectMaxBackoffMs: 30000, + }); + }); + + it('supports bounded inline manifests for local use', () => { + const manifest = JSON.stringify({ version: 1, groupName: 'local', reconcilers: [runnerConfig()] }); + expect(parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: manifest }).manifest).toBe(manifest); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_MANIFEST: manifest, + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/both', + }), + ).toThrow('provide exactly one'); + expect(() => + parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: 'x'.repeat(MAX_MANIFEST_BYTES + 1) }), + ).toThrow('must not exceed'); + }); + + it('validates production selectors and numeric runtime settings', () => { + expect(() => parseScaleSetServiceConfig({})).toThrow('provide exactly one'); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'bad name', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '1', + }), + ).toThrow('group name'); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '1', + SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS: '31', + SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS: '30', + }), + ).toThrow('must not exceed'); + }); +}); + +describe('parseScaleSetControllerManifest', () => { + it('parses the frozen flat runner-config schema and defaults', () => { + expect(parseScaleSetReconcilerConfig(runnerConfig(), 0, 'group')).toMatchObject({ + schemaVersion: 1, + runnerConfigName: 'linux-x64', + scaleSetName: 'linux-x64', + bootTimeoutMinutes: 10, + sessionOwner: 'group.linux-x64', + workFolder: '_work', + forceGhes: false, + sslVerify: true, + computeProvider: { type: 'ec2', configuration: { subnetIds: ['subnet-1'] } }, + }); + }); + + it('normalizes explicit optional settings', () => { + expect( + parseScaleSetReconcilerConfig( + runnerConfig({ + expectedRunnerGroupId: 7, + sessionOwner: 'owner/group', + workFolder: 'runner/_work', + forceGhes: true, + sslVerify: false, + userAgent: 'github-aws-runners/test', + bootTimeoutMinutes: 30, + }), + 0, + 'group', + ), + ).toMatchObject({ + expectedRunnerGroupId: 7, + sessionOwner: 'owner/group', + workFolder: 'runner/_work', + forceGhes: true, + sslVerify: false, + bootTimeoutMinutes: 30, + }); + }); + + it('accepts a runner-group name for runtime ID resolution', () => { + expect( + parseScaleSetReconcilerConfig(runnerConfig({ runnerGroupName: 'self-hosted-linux' }), 0, 'group'), + ).toMatchObject({ runnerGroupName: 'self-hosted-linux' }); + }); + + it('accepts scaleSetName without a GitHub-generated scale-set ID', () => { + const parsed = parseScaleSetReconcilerConfig( + runnerConfig({ + scaleSetName: 'linux-x64', + expectedScaleSetName: undefined, + runnerGroupName: 'self-hosted-linux', + scaleSetId: undefined, + }), + 0, + 'group', + ); + expect(parsed).toMatchObject({ scaleSetName: 'linux-x64', runnerGroupName: 'self-hosted-linux' }); + expect(parsed).not.toHaveProperty('scaleSetId'); + }); + + it('bounds the derived session owner for maximum-length names', () => { + const parsed = parseScaleSetReconcilerConfig( + runnerConfig({ runnerConfigName: 'r'.repeat(128) }), + 0, + 'g'.repeat(128), + ); + expect(parsed.sessionOwner).toHaveLength(256); + expect(parsed.sessionOwner).toMatch(/\.[a-f0-9]{16}$/); + }); + + it('rejects unsafe URLs, unknown fields, prototype keys, and schema drift', () => { + expect(() => + parseScaleSetReconcilerConfig(runnerConfig({ githubConfigUrl: 'http://github.com/example' }), 0, 'g'), + ).toThrow('must use HTTPS'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ extra: true }), 0, 'g')).toThrow('unknown field'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ schemaVersion: 2 }), 0, 'g')).toThrow('schemaVersion'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ bootTimeoutMinutes: 0 }), 0, 'g')).toThrow( + 'bootTimeoutMinutes', + ); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ bootTimeoutMinutes: 121 }), 0, 'g')).toThrow( + 'bootTimeoutMinutes', + ); + const polluted = JSON.parse('{"__proto__":{"admin":true}}') as unknown; + expect(() => + parseScaleSetReconcilerConfig( + runnerConfig({ computeProvider: { type: 'ec2', configuration: polluted } }), + 0, + 'g', + ), + ).toThrow('forbidden field'); + }); + + it('rejects duplicate runner names and scale-set IDs within an equivalent GitHub scope', () => { + expect(() => + parseScaleSetControllerManifest({ version: 1, groupName: 'g', reconcilers: [runnerConfig(), runnerConfig()] }), + ).toThrow('duplicated'); + expect(() => + parseScaleSetControllerManifest({ + version: 1, + groupName: 'g', + reconcilers: [ + runnerConfig(), + runnerConfig({ runnerConfigName: 'other', githubConfigUrl: 'https://GITHUB.com/example/' }), + ], + }), + ).toThrow('duplicated'); + }); + + it('allows the same numeric scale-set ID in different GitHub scopes', () => { + expect( + parseScaleSetControllerManifest({ + version: 1, + groupName: 'g', + reconcilers: [ + runnerConfig(), + runnerConfig({ runnerConfigName: 'other', githubConfigUrl: 'https://github.com/another' }), + ], + }).reconcilers, + ).toHaveLength(2); + }); +}); diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts new file mode 100644 index 0000000000..0f048dbe6c --- /dev/null +++ b/lambdas/services/scale-set/src/config.ts @@ -0,0 +1,515 @@ +import { createHash } from 'node:crypto'; + +export const SCALE_SET_CONTROLLER_MANIFEST_VERSION = 1; +export const MAX_MANIFEST_BYTES = 256 * 1024; + +export type JsonPrimitive = boolean | number | string | null; +export type JsonValue = JsonPrimitive | JsonValue[] | { readonly [key: string]: JsonValue }; + +export interface GitHubAppParameterReferences { + appIdParameterName: string; + installationIdParameterName?: string; + privateKeyParameterName: string; +} + +export interface ScaleSetReconcilerConfig { + schemaVersion: 1; + runnerConfigName: string; + scaleSetId?: number; + scaleSetName: string; + runnerGroupName?: string; + runnerGroupIdParameterName?: string; + expectedRunnerGroupId?: number; + githubConfigUrl: string; + githubApp: GitHubAppParameterReferences; + computeProvider: { + type: string; + configuration: Readonly>; + }; + minRunners: number; + maxRunners: number; + bootTimeoutMinutes: number; + sessionOwner: string; + workFolder: string; + forceGhes: boolean; + sslVerify: boolean; + userAgent?: string; +} + +export interface ScaleSetControllerManifest { + version: typeof SCALE_SET_CONTROLLER_MANIFEST_VERSION; + groupName: string; + revision?: string; + reconcilers: readonly ScaleSetReconcilerConfig[]; +} + +export interface ScaleSetServiceConfig { + manifest?: string; + groupConfigPath?: string; + groupName?: string; + groupRevision?: string; + healthPort: number; + healthStaleAfterMs: number; + shutdownTimeoutMs: number; + sessionCloseTimeoutMs: number; + reconnectInitialBackoffMs: number; + reconnectMaxBackoffMs: number; +} + +export type ScaleSetServiceEnvironment = Readonly>; + +const MAX_SCALE_SET_CAPACITY = 2_147_483_647; +const DEFAULT_BOOT_TIMEOUT_MINUTES = 10; +const MAX_BOOT_TIMEOUT_MINUTES = 120; +const DEFAULT_HEALTH_PORT = 8080; +const DEFAULT_HEALTH_STALE_AFTER_SECONDS = 180; +const DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 110; +const DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS = 10; +const DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS = 1; +const DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS = 30; +const MAX_RECONCILERS = 1000; +const MAX_PROVIDER_CONFIG_NODES = 10_000; +const MAX_PROVIDER_CONFIG_DEPTH = 32; +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const SAFE_PROVIDER_TYPE = /^[a-z][a-z0-9_-]{0,63}$/; +const SAFE_SSM_PARAMETER = /^\/[A-Za-z0-9_.\-/]{1,2047}$/; +const PROTOTYPE_KEYS = new Set(['__proto__', 'constructor', 'prototype']); + +export class ScaleSetConfigurationError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'ScaleSetConfigurationError'; + } +} + +function parseInteger( + environment: ScaleSetServiceEnvironment, + name: string, + options: { defaultValue: number; minimum: number; maximum: number }, +): number { + const raw = environment[name]?.trim(); + if (!raw) return options.defaultValue; + if (!/^\d+$/.test(raw)) throw new ScaleSetConfigurationError(`${name} must be an integer`); + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < options.minimum || value > options.maximum) { + throw new ScaleSetConfigurationError(`${name} must be between ${options.minimum} and ${options.maximum}`); + } + return value; +} + +export function parseScaleSetServiceConfig(environment: ScaleSetServiceEnvironment): ScaleSetServiceConfig { + const manifest = environment.SCALE_SET_CONTROLLER_MANIFEST?.trim(); + const groupConfigPath = environment.SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH?.trim(); + if ((manifest === undefined || manifest === '') === (groupConfigPath === undefined || groupConfigPath === '')) { + throw new ScaleSetConfigurationError( + 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + ); + } + if (manifest !== undefined && Buffer.byteLength(manifest, 'utf8') > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError(`SCALE_SET_CONTROLLER_MANIFEST must not exceed ${MAX_MANIFEST_BYTES} bytes`); + } + let groupName: string | undefined; + let groupRevision: string | undefined; + if (groupConfigPath !== undefined) { + validateSsmParameterName(groupConfigPath, 'group config path'); + groupName = validateSafeName(environment.SCALE_SET_CONTROLLER_GROUP_NAME?.trim() ?? '', 'group name'); + groupRevision = environment.SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION?.trim(); + if (!groupRevision || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(groupRevision)) { + throw new ScaleSetConfigurationError('SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION is invalid'); + } + } + + const reconnectInitialBackoffMs = + parseInteger(environment, 'SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS', { + defaultValue: DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS, + minimum: 1, + maximum: 300, + }) * 1000; + const reconnectMaxBackoffMs = + parseInteger(environment, 'SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS', { + defaultValue: DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS, + minimum: 1, + maximum: 3600, + }) * 1000; + if (reconnectInitialBackoffMs > reconnectMaxBackoffMs) { + throw new ScaleSetConfigurationError( + 'SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS must not exceed SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS', + ); + } + + return { + ...(manifest ? { manifest } : {}), + ...(groupConfigPath ? { groupConfigPath, groupName, groupRevision } : {}), + healthPort: parseInteger(environment, 'SCALE_SET_HEALTH_PORT', { + defaultValue: DEFAULT_HEALTH_PORT, + minimum: 1, + maximum: 65535, + }), + healthStaleAfterMs: + parseInteger(environment, 'SCALE_SET_HEALTH_STALE_AFTER_SECONDS', { + defaultValue: DEFAULT_HEALTH_STALE_AFTER_SECONDS, + minimum: 30, + maximum: 3600, + }) * 1000, + shutdownTimeoutMs: + parseInteger(environment, 'SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS', { + defaultValue: DEFAULT_SHUTDOWN_TIMEOUT_SECONDS, + minimum: 1, + maximum: 300, + }) * 1000, + sessionCloseTimeoutMs: + parseInteger(environment, 'SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS', { + defaultValue: DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS, + minimum: 1, + maximum: 60, + }) * 1000, + reconnectInitialBackoffMs, + reconnectMaxBackoffMs, + }; +} + +function objectValue(value: unknown, path: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ScaleSetConfigurationError(`${path} must be an object`); + } + return value as Record; +} + +function exactKeys(value: Record, allowed: readonly string[], path: string): void { + const allowedSet = new Set(allowed); + const unknown = Object.keys(value).filter((key) => !allowedSet.has(key)); + if (unknown.length > 0) + throw new ScaleSetConfigurationError(`${path} contains unknown field ${JSON.stringify(unknown[0])}`); +} + +function requiredString(value: Record, key: string, path: string): string { + const result = value[key]; + if (typeof result !== 'string' || result.trim() === '') { + throw new ScaleSetConfigurationError(`${path}.${key} must be a non-empty string`); + } + return result.trim(); +} + +function optionalString(value: Record, key: string, path: string): string | undefined { + const result = value[key]; + if (result === undefined) return undefined; + if (typeof result !== 'string' || result.trim() === '') { + throw new ScaleSetConfigurationError(`${path}.${key} must be a non-empty string when set`); + } + return result.trim(); +} + +function integer(value: Record, key: string, path: string, minimum: number, maximum: number): number { + const result = value[key]; + if (!Number.isSafeInteger(result) || (result as number) < minimum || (result as number) > maximum) { + throw new ScaleSetConfigurationError(`${path}.${key} must be an integer between ${minimum} and ${maximum}`); + } + return result as number; +} + +function optionalBoolean(value: Record, key: string, path: string, fallback: boolean): boolean { + const result = value[key]; + if (result === undefined) return fallback; + if (typeof result !== 'boolean') throw new ScaleSetConfigurationError(`${path}.${key} must be a boolean`); + return result; +} + +function validateSafeName(value: string, path: string): string { + if (!SAFE_NAME.test(value)) { + throw new ScaleSetConfigurationError( + `${path} must start with an ASCII letter or digit and contain only letters, digits, dots, underscores, or hyphens`, + ); + } + return value; +} + +function validateSsmParameterName(value: string, path: string): string { + if (!SAFE_SSM_PARAMETER.test(value) || value.includes('//') || value.endsWith('/')) { + throw new ScaleSetConfigurationError(`${path} must be an absolute SSM parameter name`); + } + return value; +} + +function validateGitHubConfigUrl(raw: string, path: string): string { + let url: URL; + try { + url = new URL(raw); + } catch (error) { + throw new ScaleSetConfigurationError(`${path} must be a valid URL`, { cause: error }); + } + if (url.protocol !== 'https:') throw new ScaleSetConfigurationError(`${path} must use HTTPS`); + if (url.username || url.password || url.search || url.hash) { + throw new ScaleSetConfigurationError(`${path} must not contain credentials, a query, or a fragment`); + } + const parts = url.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean); + if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { + throw new ScaleSetConfigurationError(`${path} must identify a GitHub organization, repository, or enterprise`); + } + url.pathname = `/${parts.join('/')}`; + return url.toString().replace(/\/$/, ''); +} + +function validateWorkFolder(value: string, path: string): string { + if ( + value.length > 128 || + value.startsWith('/') || + value.includes('\\') || + value.split('/').some((part) => part === '' || part === '.' || part === '..') || + !/^[A-Za-z0-9._/-]+$/.test(value) + ) { + throw new ScaleSetConfigurationError(`${path} must be a safe relative path`); + } + return value; +} + +function validateUserAgent(value: string | undefined, path: string): string | undefined { + if (value === undefined) return undefined; + if (value.length > 256 || !/^[\x20-\x7E]+$/.test(value)) { + throw new ScaleSetConfigurationError(`${path} must contain at most 256 visible ASCII characters`); + } + return value; +} + +function validateJsonValue(value: unknown, path: string, depth = 0, counter = { value: 0 }): JsonValue { + counter.value += 1; + if (counter.value > MAX_PROVIDER_CONFIG_NODES || depth > MAX_PROVIDER_CONFIG_DEPTH) { + throw new ScaleSetConfigurationError(`${path} exceeds the provider configuration complexity limit`); + } + if (value === null || typeof value === 'boolean' || typeof value === 'string') return value; + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new ScaleSetConfigurationError(`${path} contains a non-finite number`); + return value; + } + if (Array.isArray(value)) + return value.map((item, index) => validateJsonValue(item, `${path}[${index}]`, depth + 1, counter)); + const record = objectValue(value, path); + const result: Record = Object.create(null) as Record; + for (const [key, child] of Object.entries(record)) { + if (PROTOTYPE_KEYS.has(key)) + throw new ScaleSetConfigurationError(`${path} contains forbidden field ${JSON.stringify(key)}`); + if (key.length === 0 || key.length > 128) + throw new ScaleSetConfigurationError(`${path} contains an invalid field name`); + result[key] = validateJsonValue(child, `${path}.${key}`, depth + 1, counter); + } + return result; +} + +function parseGitHubApp(value: unknown, path: string): GitHubAppParameterReferences { + const record = objectValue(value, path); + exactKeys(record, ['appIdParameterName', 'installationIdParameterName', 'privateKeyParameterName'], path); + const installationIdParameterName = optionalString(record, 'installationIdParameterName', path); + return { + appIdParameterName: validateSsmParameterName( + requiredString(record, 'appIdParameterName', path), + `${path}.appIdParameterName`, + ), + ...(installationIdParameterName === undefined + ? {} + : { + installationIdParameterName: validateSsmParameterName( + installationIdParameterName, + `${path}.installationIdParameterName`, + ), + }), + privateKeyParameterName: validateSsmParameterName( + requiredString(record, 'privateKeyParameterName', path), + `${path}.privateKeyParameterName`, + ), + }; +} + +function parseComputeProvider(value: unknown, path: string): ScaleSetReconcilerConfig['computeProvider'] { + const record = objectValue(value, path); + exactKeys(record, ['type', 'configuration'], path); + const type = requiredString(record, 'type', path); + if (!SAFE_PROVIDER_TYPE.test(type)) throw new ScaleSetConfigurationError(`${path}.type is invalid`); + const configuration = validateJsonValue(record.configuration, `${path}.configuration`); + if (typeof configuration !== 'object' || configuration === null || Array.isArray(configuration)) { + throw new ScaleSetConfigurationError(`${path}.configuration must be an object`); + } + return { type, configuration }; +} + +export function parseScaleSetReconcilerConfig( + value: unknown, + index: number, + groupName: string, + basePath = 'manifest.reconcilers', +): ScaleSetReconcilerConfig { + const path = `${basePath}[${index}]`; + const record = objectValue(value, path); + exactKeys( + record, + [ + 'schemaVersion', + 'runnerConfigName', + 'scaleSetId', + 'scaleSetName', + 'expectedScaleSetName', + 'runnerGroupName', + 'runnerGroupIdParameterName', + 'expectedRunnerGroupId', + 'githubConfigUrl', + 'githubApp', + 'computeProvider', + 'minRunners', + 'maxRunners', + 'bootTimeoutMinutes', + 'sessionOwner', + 'workFolder', + 'forceGhes', + 'sslVerify', + 'userAgent', + ], + path, + ); + const runnerConfigName = validateSafeName( + requiredString(record, 'runnerConfigName', path), + `${path}.runnerConfigName`, + ); + const minRunners = integer(record, 'minRunners', path, 0, MAX_SCALE_SET_CAPACITY); + const maxRunners = integer(record, 'maxRunners', path, 0, MAX_SCALE_SET_CAPACITY); + const bootTimeoutMinutes = + record.bootTimeoutMinutes === undefined + ? DEFAULT_BOOT_TIMEOUT_MINUTES + : integer(record, 'bootTimeoutMinutes', path, 1, MAX_BOOT_TIMEOUT_MINUTES); + if (minRunners > maxRunners) throw new ScaleSetConfigurationError(`${path}.minRunners must not exceed maxRunners`); + const sessionOwner = optionalString(record, 'sessionOwner', path) ?? defaultSessionOwner(groupName, runnerConfigName); + if (!/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/.test(sessionOwner)) { + throw new ScaleSetConfigurationError(`${path}.sessionOwner is invalid`); + } + const userAgent = validateUserAgent(optionalString(record, 'userAgent', path), `${path}.userAgent`); + if (record.schemaVersion !== 1) throw new ScaleSetConfigurationError(`${path}.schemaVersion must be 1`); + const expectedRunnerGroupId = + record.expectedRunnerGroupId === undefined || record.expectedRunnerGroupId === null + ? undefined + : integer(record, 'expectedRunnerGroupId', path, 1, MAX_SCALE_SET_CAPACITY); + const runnerGroupName = + record.runnerGroupName === undefined + ? undefined + : validateScaleSetName(requiredString(record, 'runnerGroupName', path), `${path}.runnerGroupName`); + const runnerGroupIdParameterName = + record.runnerGroupIdParameterName === undefined + ? undefined + : validateSsmParameterName( + requiredString(record, 'runnerGroupIdParameterName', path), + `${path}.runnerGroupIdParameterName`, + ); + if (record.scaleSetName !== undefined && record.expectedScaleSetName !== undefined) { + throw new ScaleSetConfigurationError(`${path} must configure only one of scaleSetName or expectedScaleSetName`); + } + const scaleSetName = validateScaleSetName( + requiredString(record, record.scaleSetName === undefined ? 'expectedScaleSetName' : 'scaleSetName', path), + `${path}.scaleSetName`, + ); + const scaleSetId = + record.scaleSetId === undefined ? undefined : integer(record, 'scaleSetId', path, 1, MAX_SCALE_SET_CAPACITY); + if (scaleSetId === undefined && runnerGroupName === undefined) { + throw new ScaleSetConfigurationError(`${path}.runnerGroupName is required when scaleSetId is omitted`); + } + return { + schemaVersion: 1, + runnerConfigName, + ...(scaleSetId === undefined ? {} : { scaleSetId }), + scaleSetName, + ...(runnerGroupName === undefined ? {} : { runnerGroupName }), + ...(runnerGroupIdParameterName === undefined ? {} : { runnerGroupIdParameterName }), + ...(expectedRunnerGroupId === undefined ? {} : { expectedRunnerGroupId }), + githubConfigUrl: validateGitHubConfigUrl( + requiredString(record, 'githubConfigUrl', path), + `${path}.githubConfigUrl`, + ), + githubApp: parseGitHubApp(record.githubApp, `${path}.githubApp`), + computeProvider: parseComputeProvider(record.computeProvider, `${path}.computeProvider`), + minRunners, + maxRunners, + bootTimeoutMinutes, + sessionOwner, + workFolder: validateWorkFolder(optionalString(record, 'workFolder', path) ?? '_work', `${path}.workFolder`), + forceGhes: optionalBoolean(record, 'forceGhes', path, false), + sslVerify: optionalBoolean(record, 'sslVerify', path, true), + ...(userAgent === undefined ? {} : { userAgent }), + }; +} + +function defaultSessionOwner(groupName: string, runnerConfigName: string): string { + const candidate = `${groupName}.${runnerConfigName}`; + if (candidate.length <= 256) return candidate; + const suffix = createHash('sha256').update(candidate).digest('hex').slice(0, 16); + return `${candidate.slice(0, 239)}.${suffix}`; +} + +function validateScaleSetName(value: string, path: string): string { + if (value.length > 128 || !/^[\x20-\x7E]+$/.test(value)) { + throw new ScaleSetConfigurationError(`${path} must contain at most 128 visible ASCII characters`); + } + return value; +} + +export function parseScaleSetControllerManifest(input: string | unknown): ScaleSetControllerManifest { + let parsed = input; + if (typeof input === 'string') { + if (Buffer.byteLength(input, 'utf8') > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError(`controller manifest must not exceed ${MAX_MANIFEST_BYTES} bytes`); + } + try { + parsed = JSON.parse(input) as unknown; + } catch (error) { + throw new ScaleSetConfigurationError('controller manifest must contain valid JSON', { cause: error }); + } + } + const manifest = objectValue(parsed, 'manifest'); + exactKeys(manifest, ['version', 'groupName', 'revision', 'reconcilers'], 'manifest'); + if (manifest.version !== SCALE_SET_CONTROLLER_MANIFEST_VERSION) { + throw new ScaleSetConfigurationError(`manifest.version must be ${SCALE_SET_CONTROLLER_MANIFEST_VERSION}`); + } + const groupName = validateSafeName(requiredString(manifest, 'groupName', 'manifest'), 'manifest.groupName'); + if ( + !Array.isArray(manifest.reconcilers) || + manifest.reconcilers.length < 1 || + manifest.reconcilers.length > MAX_RECONCILERS + ) { + throw new ScaleSetConfigurationError(`manifest.reconcilers must contain between 1 and ${MAX_RECONCILERS} entries`); + } + const revision = optionalString(manifest, 'revision', 'manifest'); + if (revision !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(revision)) { + throw new ScaleSetConfigurationError('manifest.revision is invalid'); + } + const reconcilers = manifest.reconcilers.map((value, index) => + parseScaleSetReconcilerConfig(value, index, groupName), + ); + validateUniqueReconcilers(reconcilers); + return { + version: SCALE_SET_CONTROLLER_MANIFEST_VERSION, + groupName, + ...(revision === undefined ? {} : { revision }), + reconcilers, + }; +} + +export function validateUniqueReconcilers(reconcilers: readonly ScaleSetReconcilerConfig[]): void { + const names = new Set(); + const scopedScaleSets = new Set(); + for (const reconciler of reconcilers) { + if (names.has(reconciler.runnerConfigName)) { + throw new ScaleSetConfigurationError( + `runner config ${JSON.stringify(reconciler.runnerConfigName)} is duplicated`, + ); + } + const scopedScaleSet = [ + reconciler.githubConfigUrl, + reconciler.runnerGroupName ?? String(reconciler.expectedRunnerGroupId ?? ''), + reconciler.scaleSetName, + ].join('\u0000'); + if (scopedScaleSets.has(scopedScaleSet)) { + throw new ScaleSetConfigurationError( + `scale set ${JSON.stringify(reconciler.scaleSetName)} is duplicated within GitHub scope ${JSON.stringify(reconciler.githubConfigUrl)}`, + ); + } + names.add(reconciler.runnerConfigName); + scopedScaleSets.add(scopedScaleSet); + } +} diff --git a/lambdas/services/scale-set/src/controller.ts b/lambdas/services/scale-set/src/controller.ts new file mode 100644 index 0000000000..5cbd2ac229 --- /dev/null +++ b/lambdas/services/scale-set/src/controller.ts @@ -0,0 +1,51 @@ +import type { ScaleSetControllerManifest, ScaleSetServiceConfig } from './config'; +import { ScaleSetControllerHealth } from './health'; +import type { ScaleSetLogger } from './logger'; +import { ScaleSetReconciler, type ScaleSetReconcilerDependencies } from './reconciler'; + +export class ScaleSetController { + readonly health: ScaleSetControllerHealth; + + constructor( + private readonly manifest: ScaleSetControllerManifest, + private readonly serviceConfig: ScaleSetServiceConfig, + private readonly dependencies: ScaleSetReconcilerDependencies, + private readonly controllerLogger: ScaleSetLogger, + ) { + this.health = new ScaleSetControllerHealth( + manifest.groupName, + manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + serviceConfig.healthStaleAfterMs, + ); + } + + async run(signal: AbortSignal): Promise { + this.controllerLogger.debug('scale_set_reconcilers_starting', { + reconcilerCount: this.manifest.reconcilers.length, + runnerConfigNames: this.manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); + const completions = this.manifest.reconcilers.map(async (config) => { + const status = this.health.reporter(config.runnerConfigName); + try { + await new ScaleSetReconciler(config, this.serviceConfig, this.dependencies).run(signal, status); + } catch (error) { + status.markFailed(error); + this.controllerLogger.error('scale_set_reconciler_uncaught_failure', { + runnerConfigName: config.runnerConfigName, + scaleSetId: config.scaleSetId, + error, + }); + } + }); + + await Promise.race([Promise.all(completions), waitForAbort(signal)]); + if (!signal.aborted) await waitForAbort(signal); + this.health.markStopping(); + await Promise.all(completions); + } +} + +async function waitForAbort(signal: AbortSignal): Promise { + if (signal.aborted) return; + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); +} diff --git a/lambdas/services/scale-set/src/credentials.test.ts b/lambdas/services/scale-set/src/credentials.test.ts new file mode 100644 index 0000000000..4536a8e582 --- /dev/null +++ b/lambdas/services/scale-set/src/credentials.test.ts @@ -0,0 +1,159 @@ +const authMocks = vi.hoisted(() => ({ createAppAuth: vi.fn(), requestDefaults: vi.fn() })); +vi.mock('@octokit/auth-app', () => ({ createAppAuth: authMocks.createAppAuth })); +vi.mock('@octokit/request', () => ({ request: { defaults: authMocks.requestDefaults } })); + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +import { createGitHubAppAccessTokenProvider, loadGitHubAppCredentials, type ParameterStore } from './credentials'; + +const references = { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', +}; + +function encodedKey(body: string): string { + return Buffer.from(`-----BEGIN PRIVATE KEY-----\n${body}\n-----END PRIVATE KEY-----\n`).toString('base64'); +} + +describe('GitHub App credentials', () => { + beforeEach(() => { + vi.clearAllMocks(); + authMocks.requestDefaults.mockReturnValue(vi.fn()); + }); + + it('validates and decodes referenced SSM values', async () => { + const store: ParameterStore = { + get: vi.fn().mockResolvedValue( + new Map([ + ['/app/id', '123'], + ['/app/installation', '456'], + ['/app/key', encodedKey('abc')], + ]), + ), + put: vi.fn(), + }; + await expect(loadGitHubAppCredentials(references, store)).resolves.toMatchObject({ + appId: '123', + installationId: 456, + privateKey: expect.stringContaining('BEGIN PRIVATE KEY'), + }); + }); + + it('reuses auth for unchanged credentials and recreates it after rotation', async () => { + const values = [encodedKey('first'), encodedKey('first'), encodedKey('second')]; + const store: ParameterStore = { + get: vi.fn( + async () => + new Map([ + ['/app/id', '123'], + ['/app/installation', '456'], + ['/app/key', values.shift() as string], + ]), + ), + }; + const firstAuth = vi.fn().mockResolvedValue({ token: 'token-one', expiresAt: '2099-01-01T00:00:00Z' }); + const secondAuth = vi.fn().mockResolvedValue({ token: 'token-two', expiresAt: '2099-01-01T00:00:00Z' }); + const fetchImplementation = vi.fn(); + authMocks.createAppAuth.mockReturnValueOnce(firstAuth).mockReturnValueOnce(secondAuth); + + const provider = await createGitHubAppAccessTokenProvider( + references, + 'https://github.com/example', + false, + store, + fetchImplementation, + ); + await provider(); + await provider(); + await provider(); + + expect(store.get).toHaveBeenCalledTimes(3); + expect(authMocks.createAppAuth).toHaveBeenCalledTimes(2); + expect(authMocks.requestDefaults).toHaveBeenCalledWith({ + baseUrl: 'https://api.github.com', + request: { fetch: fetchImplementation }, + }); + expect(firstAuth).toHaveBeenCalledTimes(2); + expect(secondAuth).toHaveBeenCalledTimes(1); + }); + + it('discovers the installation ID from the configured organization when SSM does not provide one', async () => { + const store: ParameterStore = { + get: vi.fn().mockResolvedValue( + new Map([ + ['/app/id', '123'], + ['/app/key', encodedKey('abc')], + ]), + ), + put: vi.fn(), + }; + const appAuth = vi.fn().mockResolvedValue({ token: 'app-jwt' }); + const installationAuth = vi + .fn() + .mockResolvedValue({ token: 'installation-token', expiresAt: '2099-01-01T00:00:00Z' }); + const fetchImplementation = vi.fn().mockResolvedValue( + new Response(JSON.stringify([{ id: 456, account: { login: 'example' } }]), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + authMocks.createAppAuth.mockReturnValueOnce(appAuth).mockReturnValueOnce(installationAuth); + + const provider = await createGitHubAppAccessTokenProvider( + references, + 'https://github.com/example', + false, + store, + fetchImplementation, + ); + + await expect(provider()).resolves.toMatchObject({ token: 'installation-token' }); + await expect(provider()).resolves.toMatchObject({ token: 'installation-token' }); + expect(appAuth).toHaveBeenCalledWith({ type: 'app' }); + expect(installationAuth).toHaveBeenCalledWith({ type: 'installation', installationId: 456 }); + expect(appAuth).toHaveBeenCalledTimes(1); + expect(fetchImplementation).toHaveBeenCalledTimes(1); + expect(store.put).toHaveBeenCalledWith('/app/installation', '456'); + expect(fetchImplementation).toHaveBeenCalledWith( + expect.objectContaining({ + href: 'https://api.github.com/app/installations?per_page=100&page=1', + }), + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer app-jwt' }), + }), + ); + }); + + it.each([ + [new Map([['/app/id', '123']]), 'was not returned'], + [ + new Map([ + ['/app/id', 'bad id'], + ['/app/installation', '1'], + ['/app/key', encodedKey('abc')], + ]), + 'App ID', + ], + [ + new Map([ + ['/app/id', '1'], + ['/app/installation', 'zero'], + ['/app/key', encodedKey('abc')], + ]), + 'positive integer', + ], + [ + new Map([ + ['/app/id', '1'], + ['/app/installation', '2'], + ['/app/key', 'not-base64'], + ]), + 'canonical base64', + ], + ])('rejects malformed credential parameters', async (values, message) => { + await expect(loadGitHubAppCredentials(references, { get: vi.fn().mockResolvedValue(values) })).rejects.toThrow( + message, + ); + }); +}); diff --git a/lambdas/services/scale-set/src/credentials.ts b/lambdas/services/scale-set/src/credentials.ts new file mode 100644 index 0000000000..f1178efff2 --- /dev/null +++ b/lambdas/services/scale-set/src/credentials.ts @@ -0,0 +1,209 @@ +import { createAppAuth } from '@octokit/auth-app'; +import { request } from '@octokit/request'; +import { createHash } from 'node:crypto'; + +import { + githubApiUrl, + parseGitHubConfigUrl, + type AccessToken, + type ScaleSetFetch, +} from '@aws-github-runner/github-actions-scale-set'; + +import { ScaleSetConfigurationError, type GitHubAppParameterReferences } from './config'; + +export interface ParameterStore { + get(names: readonly string[]): Promise>; + put?(name: string, value: string): Promise; +} + +interface GitHubAppCredentials { + appId: string; + installationId?: number; + privateKey: string; +} + +interface GitHubAppInstallation { + id?: unknown; + account?: { login?: unknown }; +} + +const MAX_PRIVATE_KEY_BYTES = 64 * 1024; + +function requiredParameter(values: ReadonlyMap, name: string): string { + const value = values.get(name); + if (value === undefined || value === '') { + throw new ScaleSetConfigurationError(`required SSM parameter ${JSON.stringify(name)} was not returned`); + } + return value; +} + +function decodePrivateKey(encoded: string): string { + if ( + encoded.length === 0 || + encoded.length > Math.ceil((MAX_PRIVATE_KEY_BYTES * 4) / 3) + 4 || + encoded.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded) + ) { + throw new ScaleSetConfigurationError('GitHub App private key parameter must contain canonical base64'); + } + const decoded = Buffer.from(encoded, 'base64').toString('utf8').replace(/\\n/g, '\n'); + if (Buffer.byteLength(decoded, 'utf8') > MAX_PRIVATE_KEY_BYTES) { + throw new ScaleSetConfigurationError('GitHub App private key is too large'); + } + if (!/^-----BEGIN (?:RSA )?PRIVATE KEY-----\n[\s\S]+\n-----END (?:RSA )?PRIVATE KEY-----\n?$/.test(decoded)) { + throw new ScaleSetConfigurationError('GitHub App private key parameter is not a supported PEM private key'); + } + return decoded; +} + +export async function loadGitHubAppCredentials( + references: GitHubAppParameterReferences, + parameterStore: ParameterStore, +): Promise { + const names = [references.appIdParameterName, references.privateKeyParameterName]; + if (references.installationIdParameterName !== undefined) names.splice(1, 0, references.installationIdParameterName); + const values = await parameterStore.get(names); + const appId = requiredParameter(values, references.appIdParameterName).trim(); + if (!/^[A-Za-z0-9_-]{1,128}$/.test(appId)) { + throw new ScaleSetConfigurationError('GitHub App ID parameter is invalid'); + } + let installationId: number | undefined; + if (references.installationIdParameterName !== undefined) { + const installationIdRaw = values.get(references.installationIdParameterName)?.trim(); + if (installationIdRaw !== undefined && installationIdRaw !== '') { + if (!/^\d+$/.test(installationIdRaw)) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + installationId = Number(installationIdRaw); + if (!Number.isSafeInteger(installationId) || installationId <= 0) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + } + } + return { + appId, + installationId, + privateKey: decodePrivateKey(requiredParameter(values, references.privateKeyParameterName)), + }; +} + +async function discoverGitHubAppInstallationId( + credentials: Pick, + target: string, + apiBaseUrl: string, + fetchImplementation: ScaleSetFetch, +): Promise { + const appRequest = request.defaults({ baseUrl: apiBaseUrl, request: { fetch: fetchImplementation } }); + const appAuth = createAppAuth({ + appId: credentials.appId, + privateKey: credentials.privateKey, + request: appRequest, + }); + const appAuthentication = await appAuth({ type: 'app' }); + for (let page = 1; page <= 100; page += 1) { + const url = new URL('/app/installations', `${apiBaseUrl}/`); + url.searchParams.set('per_page', '100'); + url.searchParams.set('page', String(page)); + const response = await fetchImplementation(url, { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${appAuthentication.token}`, + 'User-Agent': 'github-aws-runners/scale-set-controller', + }, + }); + if (!response.ok) { + throw new ScaleSetConfigurationError(`GitHub App installation discovery failed with HTTP ${response.status}`); + } + let payload: unknown; + try { + payload = await response.json(); + } catch (error) { + throw new ScaleSetConfigurationError('GitHub App installation discovery returned invalid JSON', { cause: error }); + } + if (!Array.isArray(payload)) { + throw new ScaleSetConfigurationError('GitHub App installation discovery returned an invalid response'); + } + for (const value of payload as unknown[]) { + if (typeof value !== 'object' || value === null || Array.isArray(value)) continue; + const candidate = value as GitHubAppInstallation; + if ( + Number.isSafeInteger(candidate.id) && + typeof candidate.account?.login === 'string' && + candidate.account.login.toLowerCase() === target.toLowerCase() + ) { + return candidate.id as number; + } + } + if (payload.length < 100) break; + } + throw new ScaleSetConfigurationError(`GitHub App is not installed for ${JSON.stringify(target)}`); +} + +export async function createGitHubAppAccessTokenProvider( + references: GitHubAppParameterReferences, + githubConfigUrl: string, + forceGhes: boolean, + parameterStore: ParameterStore, + fetchImplementation: ScaleSetFetch = globalThis.fetch, +): Promise<() => Promise> { + const parsedConfig = parseGitHubConfigUrl(githubConfigUrl, forceGhes); + const apiBaseUrl = githubApiUrl(parsedConfig, '/').toString().replace(/\/$/, ''); + let cached: + | { + fingerprint: string; + installationId: number; + auth: ReturnType; + } + | undefined; + let discovered: + | { + fingerprint: string; + installationId: number; + } + | undefined; + + return async () => { + // Reload references for rotation visibility, but preserve the Octokit auth + // instance while credentials are unchanged so its installation-token cache + // remains effective. + const credentials = await loadGitHubAppCredentials(references, parameterStore); + const credentialFingerprint = createHash('sha256') + .update(credentials.appId) + .update('\u0000') + .update(credentials.privateKey) + .digest('base64url'); + const target = parsedConfig.organization ?? parsedConfig.enterprise; + if (credentials.installationId === undefined && target === undefined) { + throw new ScaleSetConfigurationError( + 'GitHub App installation discovery requires an organization or enterprise URL', + ); + } + let installationId = credentials.installationId; + if (installationId === undefined) { + if (discovered?.fingerprint === credentialFingerprint) { + installationId = discovered.installationId; + } else { + installationId = await discoverGitHubAppInstallationId(credentials, target!, apiBaseUrl, fetchImplementation); + discovered = { fingerprint: credentialFingerprint, installationId }; + if (references.installationIdParameterName !== undefined && parameterStore.put !== undefined) { + await parameterStore.put(references.installationIdParameterName, String(installationId)); + } + } + } + const fingerprint = `${credentialFingerprint}\u0000${installationId}`; + if (cached?.fingerprint !== fingerprint) { + cached = { + fingerprint, + installationId, + auth: createAppAuth({ + appId: credentials.appId, + installationId, + privateKey: credentials.privateKey, + request: request.defaults({ baseUrl: apiBaseUrl, request: { fetch: fetchImplementation } }), + }), + }; + } + const installation = await cached.auth({ type: 'installation', installationId: cached.installationId }); + return { token: installation.token, expiresAt: installation.expiresAt }; + }; +} diff --git a/lambdas/services/scale-set/src/github-http.test.ts b/lambdas/services/scale-set/src/github-http.test.ts new file mode 100644 index 0000000000..870399ac84 --- /dev/null +++ b/lambdas/services/scale-set/src/github-http.test.ts @@ -0,0 +1,58 @@ +const undiciMocks = vi.hoisted(() => ({ + close: vi.fn().mockResolvedValue(undefined), + createAgent: vi.fn(), +})); + +vi.mock('undici', () => ({ + Agent: class MockAgent { + constructor(options: unknown) { + undiciMocks.createAgent(options); + } + + close = undiciMocks.close; + }, +})); + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +import { createScaleSetGitHubHttp } from './github-http'; + +describe('scale-set GitHub HTTP isolation', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('uses the supplied verified fetch without changing process TLS settings', async () => { + const original = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + const fetchImplementation = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + const http = createScaleSetGitHubHttp(fetchImplementation); + + await http.fetch(true)('https://github.example/_apis/runtime/runnerscalesets'); + await http.close(); + + expect(fetchImplementation).toHaveBeenCalledWith('https://github.example/_apis/runtime/runnerscalesets'); + expect(undiciMocks.createAgent).not.toHaveBeenCalled(); + expect(process.env.NODE_TLS_REJECT_UNAUTHORIZED).toBe(original); + }); + + it('uses one scoped insecure dispatcher and closes it without mutating global TLS state', async () => { + const original = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + const fetchImplementation = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + const http = createScaleSetGitHubHttp(fetchImplementation); + const first = http.fetch(false); + const second = http.fetch(false); + + await first('https://github.example/_apis/runtime/runnerscalesets', { method: 'GET' }); + await http.close(); + + expect(first).toBe(second); + expect(undiciMocks.createAgent).toHaveBeenCalledOnce(); + expect(undiciMocks.createAgent).toHaveBeenCalledWith({ connect: { rejectUnauthorized: false } }); + expect(fetchImplementation).toHaveBeenCalledWith( + 'https://github.example/_apis/runtime/runnerscalesets', + expect.objectContaining({ method: 'GET', dispatcher: expect.anything() }), + ); + expect(undiciMocks.close).toHaveBeenCalledOnce(); + expect(process.env.NODE_TLS_REJECT_UNAUTHORIZED).toBe(original); + }); +}); diff --git a/lambdas/services/scale-set/src/github-http.ts b/lambdas/services/scale-set/src/github-http.ts new file mode 100644 index 0000000000..85a60626b1 --- /dev/null +++ b/lambdas/services/scale-set/src/github-http.ts @@ -0,0 +1,34 @@ +import { Agent, type Dispatcher } from 'undici'; + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +type DispatcherRequestInit = RequestInit & { dispatcher: Dispatcher }; + +export interface ScaleSetGitHubHttp { + fetch(sslVerify: boolean): ScaleSetFetch; + close(): Promise; +} + +/** + * Creates fetch implementations whose TLS policy is scoped to one controller + * process. Disabling verification never mutates NODE_TLS_REJECT_UNAUTHORIZED + * or the global Undici dispatcher, so verified and unverified GHES runner + * configurations may safely share one grouped task. + */ +export function createScaleSetGitHubHttp(fetchImplementation: ScaleSetFetch = globalThis.fetch): ScaleSetGitHubHttp { + let insecureAgent: Agent | undefined; + let insecureFetch: ScaleSetFetch | undefined; + + return { + fetch(sslVerify) { + if (sslVerify) return fetchImplementation; + insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }); + insecureFetch ??= async (input, init = {}) => + await fetchImplementation(input, { ...init, dispatcher: insecureAgent } as DispatcherRequestInit); + return insecureFetch; + }, + async close() { + await insecureAgent?.close(); + }, + }; +} diff --git a/lambdas/services/scale-set/src/health-server.test.ts b/lambdas/services/scale-set/src/health-server.test.ts new file mode 100644 index 0000000000..16e65ab953 --- /dev/null +++ b/lambdas/services/scale-set/src/health-server.test.ts @@ -0,0 +1,15 @@ +import { startScaleSetHealthServer } from './health-server'; + +describe('health server', () => { + it('separates liveness and readiness on loopback', async () => { + const health = { snapshot: vi.fn(() => ({ live: true, ready: false, state: 'degraded' })) }; + const server = await startScaleSetHealthServer(health, 0); + try { + await expect(fetch(`http://127.0.0.1:${server.port}/healthz`)).resolves.toMatchObject({ status: 200 }); + await expect(fetch(`http://127.0.0.1:${server.port}/readyz`)).resolves.toMatchObject({ status: 503 }); + await expect(fetch(`http://127.0.0.1:${server.port}/other`)).resolves.toMatchObject({ status: 404 }); + } finally { + await server.close(); + } + }); +}); diff --git a/lambdas/services/scale-set/src/health-server.ts b/lambdas/services/scale-set/src/health-server.ts new file mode 100644 index 0000000000..f9b967aaec --- /dev/null +++ b/lambdas/services/scale-set/src/health-server.ts @@ -0,0 +1,53 @@ +import { createServer, type Server } from 'node:http'; + +import type { ScaleSetControllerHealth } from './health'; + +export interface ScaleSetHealthServer { + port: number; + close(): Promise; +} + +export async function startScaleSetHealthServer( + health: Pick, + port: number, +): Promise { + const server = createServer((request, response) => { + response.setHeader('Cache-Control', 'no-store'); + response.setHeader('Connection', 'close'); + response.setHeader('Content-Type', 'application/json; charset=utf-8'); + response.setHeader('X-Content-Type-Options', 'nosniff'); + + if (request.method !== 'GET' || (request.url !== '/healthz' && request.url !== '/readyz')) { + response.statusCode = 404; + response.end(JSON.stringify({ status: 'not-found' })); + return; + } + + const snapshot = health.snapshot(); + const healthy = request.url === '/readyz' ? snapshot.ready : snapshot.live; + response.statusCode = healthy ? 200 : 503; + response.end(JSON.stringify(snapshot)); + }); + await listen(server, port); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('health server did not bind a TCP address'); + return { + port: address.port, + close: async () => { + server.closeAllConnections(); + if (!server.listening) return; + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + }, + }; +} + +async function listen(server: Server, port: number): Promise { + await new Promise((resolve, reject) => { + const onError = (error: Error) => reject(error); + server.once('error', onError); + server.listen(port, '127.0.0.1', () => { + server.removeListener('error', onError); + resolve(); + }); + }); +} diff --git a/lambdas/services/scale-set/src/health.test.ts b/lambdas/services/scale-set/src/health.test.ts new file mode 100644 index 0000000000..b2d5294e11 --- /dev/null +++ b/lambdas/services/scale-set/src/health.test.ts @@ -0,0 +1,46 @@ +import { ScaleSetControllerHealth } from './health'; + +describe('ScaleSetControllerHealth', () => { + it('aggregates independent readiness while reconnect heartbeats stay live', () => { + let now = 0; + const health = new ScaleSetControllerHealth('group', ['a', 'b'], 100, () => now); + const a = health.reporter('a'); + const b = health.reporter('b'); + a.markSessionReady(); + b.markSessionReady(); + a.markProgress(); + b.markProgress(); + expect(health.snapshot()).toMatchObject({ state: 'ready', live: true, ready: true }); + + now = 200; + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'ready', live: true, ready: false }); + + a.markReconnecting(new Error('outage')); + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'reconnecting', live: true, ready: false }); + }); + + it('contains one terminal reconciler failure while another stays ready', () => { + const health = new ScaleSetControllerHealth('group', ['a', 'b'], 100); + health.reporter('a').markFailed(new TypeError('bad config')); + health.reporter('b').markProgress(); + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'failed', lastErrorName: 'TypeError' }); + }); + + it('marks all reporters stopping without reviving failures', () => { + const health = new ScaleSetControllerHealth('group', ['a'], 100); + health.reporter('a').markFailed(); + health.markStopping(); + expect(health.snapshot()).toMatchObject({ state: 'stopping', live: true, ready: false }); + expect(health.snapshot().reconcilers.a.state).toBe('failed'); + }); + + it('rejects duplicate and unknown reporter names', () => { + expect(() => new ScaleSetControllerHealth('g', [], 1)).toThrow('at least one'); + expect(() => new ScaleSetControllerHealth('g', ['a', 'a'], 1)).toThrow('duplicate'); + const health = new ScaleSetControllerHealth('g', ['a'], 1); + expect(() => health.reporter('b')).toThrow('unknown runner config'); + }); +}); diff --git a/lambdas/services/scale-set/src/health.ts b/lambdas/services/scale-set/src/health.ts new file mode 100644 index 0000000000..6cad43d10d --- /dev/null +++ b/lambdas/services/scale-set/src/health.ts @@ -0,0 +1,137 @@ +export type ScaleSetReconcilerState = 'starting' | 'ready' | 'reconnecting' | 'failed' | 'stopping'; +export type ScaleSetControllerState = 'starting' | 'ready' | 'degraded' | 'failed' | 'stopping'; + +export interface ScaleSetReconcilerHealthSnapshot { + state: ScaleSetReconcilerState; + live: boolean; + ready: boolean; + lastActivityAt: string; + consecutiveFailures: number; + lastErrorName?: string; +} + +export interface ScaleSetControllerHealthSnapshot { + groupName: string; + state: ScaleSetControllerState; + live: boolean; + ready: boolean; + reconcilers: Readonly>; +} + +export interface ScaleSetReconcilerStatusReporter { + markSessionReady(): void; + markProgress(): void; + markReconnecting(error?: unknown): void; + markFailed(error?: unknown): void; + markStopping(): void; +} + +interface MutableHealth { + state: ScaleSetReconcilerState; + lastActivityAt: number; + consecutiveFailures: number; + lastErrorName?: string; +} + +function errorName(error: unknown): string | undefined { + if (error === undefined) return undefined; + return error instanceof Error ? error.name : typeof error; +} + +export class ScaleSetControllerHealth { + private readonly states = new Map(); + private stopping = false; + + constructor( + readonly groupName: string, + runnerConfigNames: readonly string[], + private readonly staleAfterMs: number, + private readonly now: () => number = Date.now, + ) { + const startedAt = now(); + for (const name of runnerConfigNames) { + if (this.states.has(name)) throw new Error(`duplicate health reporter for ${JSON.stringify(name)}`); + this.states.set(name, { state: 'starting', lastActivityAt: startedAt, consecutiveFailures: 0 }); + } + if (this.states.size === 0) throw new Error('at least one reconciler health reporter is required'); + } + + reporter(runnerConfigName: string): ScaleSetReconcilerStatusReporter { + const state = this.states.get(runnerConfigName); + if (!state) throw new Error(`unknown runner config ${JSON.stringify(runnerConfigName)}`); + return { + markSessionReady: () => { + if (state.state === 'starting') { + state.state = 'ready'; + state.lastActivityAt = this.now(); + } else if (state.state === 'reconnecting') { + state.state = 'ready'; + } + }, + markProgress: () => { + if (state.state === 'failed' || state.state === 'stopping') return; + state.state = 'ready'; + state.lastActivityAt = this.now(); + state.consecutiveFailures = 0; + state.lastErrorName = undefined; + }, + markReconnecting: (error) => { + if (state.state === 'failed' || state.state === 'stopping') return; + state.state = 'reconnecting'; + state.lastActivityAt = this.now(); + state.consecutiveFailures += 1; + state.lastErrorName = errorName(error); + }, + markFailed: (error) => { + if (state.state === 'stopping') return; + state.state = 'failed'; + state.consecutiveFailures += 1; + state.lastErrorName = errorName(error); + }, + markStopping: () => { + if (state.state !== 'failed') state.state = 'stopping'; + }, + }; + } + + markStopping(): void { + this.stopping = true; + for (const name of this.states.keys()) this.reporter(name).markStopping(); + } + + snapshot(): ScaleSetControllerHealthSnapshot { + const now = this.now(); + const reconcilers: Record = Object.create(null) as Record< + string, + ScaleSetReconcilerHealthSnapshot + >; + for (const [name, state] of this.states) { + const stale = now - state.lastActivityAt > this.staleAfterMs; + // Staleness means the reconciler is not ready, but it is not a process + // liveness failure. A single bounded GitHub/AWS request can legitimately + // outlive the readiness window; restarting the task would only churn its + // message sessions and reset the provider retry policy. + const live = state.state === 'stopping' || state.state !== 'failed'; + reconcilers[name] = { + state: state.state, + live, + ready: state.state === 'ready' && !stale, + lastActivityAt: new Date(state.lastActivityAt).toISOString(), + consecutiveFailures: state.consecutiveFailures, + ...(state.lastErrorName === undefined ? {} : { lastErrorName: state.lastErrorName }), + }; + } + const values = Object.values(reconcilers); + const liveCount = values.filter(({ live }) => live).length; + const readyCount = values.filter(({ ready }) => ready).length; + const live = this.stopping || liveCount > 0; + const ready = !this.stopping && readyCount === values.length; + let state: ScaleSetControllerState; + if (this.stopping) state = 'stopping'; + else if (ready) state = 'ready'; + else if (liveCount === 0) state = 'failed'; + else if (values.every((value) => value.state === 'starting')) state = 'starting'; + else state = 'degraded'; + return { groupName: this.groupName, state, live, ready, reconcilers }; + } +} diff --git a/lambdas/services/scale-set/src/index.ts b/lambdas/services/scale-set/src/index.ts new file mode 100644 index 0000000000..6074b49499 --- /dev/null +++ b/lambdas/services/scale-set/src/index.ts @@ -0,0 +1,9 @@ +export * from './config'; +export * from './controller'; +export * from './credentials'; +export * from './health'; +export * from './health-server'; +export * from './lifecycle'; +export * from './logger'; +export * from './parameter-store'; +export * from './reconciler'; diff --git a/lambdas/services/scale-set/src/lifecycle.test.ts b/lambdas/services/scale-set/src/lifecycle.test.ts new file mode 100644 index 0000000000..a43c31fe5c --- /dev/null +++ b/lambdas/services/scale-set/src/lifecycle.test.ts @@ -0,0 +1,45 @@ +import { ScaleSetServiceRuntime } from './lifecycle'; + +describe('ScaleSetServiceRuntime', () => { + it('starts once and performs idempotent bounded shutdown', async () => { + const health = { markStopping: vi.fn(), snapshot: vi.fn() }; + const run = vi.fn(async (signal: AbortSignal) => { + if (!signal.aborted) + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); + }); + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 100 }, { run, health } as never); + const completion = runtime.run(); + expect(() => runtime.run()).toThrow('already started'); + const shutdown = runtime.shutdown(); + expect(runtime.shutdown()).toBe(shutdown); + await shutdown; + await completion; + expect(health.markStopping).toHaveBeenCalledOnce(); + }); + + it('allows shutdown before start and prevents a later start', async () => { + const health = { markStopping: vi.fn() }; + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 100 }, { run: vi.fn(), health } as never); + await runtime.shutdown(); + expect(() => runtime.run()).toThrow('already stopping'); + }); + + it('rejects when a controller ignores cancellation past the timeout', async () => { + vi.useFakeTimers(); + try { + const health = { markStopping: vi.fn() }; + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 10 }, { + run: vi.fn(() => new Promise(() => undefined)), + health, + } as never); + void runtime.run(); + await Promise.resolve(); + const shutdown = runtime.shutdown(); + const expectation = expect(shutdown).rejects.toThrow('did not stop within 10ms'); + await vi.advanceTimersByTimeAsync(10); + await expectation; + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/lambdas/services/scale-set/src/lifecycle.ts b/lambdas/services/scale-set/src/lifecycle.ts new file mode 100644 index 0000000000..d4c4a600c0 --- /dev/null +++ b/lambdas/services/scale-set/src/lifecycle.ts @@ -0,0 +1,51 @@ +import type { ScaleSetServiceConfig } from './config'; +import type { ScaleSetController } from './controller'; + +export class ScaleSetServiceRuntime { + private readonly abortController = new AbortController(); + private completion: Promise | undefined; + private shutdownCompletion: Promise | undefined; + + constructor( + private readonly config: Pick, + private readonly controller: Pick, + ) {} + + get health() { + return this.controller.health; + } + + run(): Promise { + if (this.shutdownCompletion !== undefined) throw new Error('Scale-set service runtime is already stopping'); + if (this.completion !== undefined) throw new Error('Scale-set service runtime has already started'); + this.completion = Promise.resolve().then(async () => { + if (!this.abortController.signal.aborted) await this.controller.run(this.abortController.signal); + }); + return this.completion; + } + + shutdown(reason: unknown = new Error('Scale-set service shutdown requested')): Promise { + this.shutdownCompletion ??= this.shutdownOnce(reason); + return this.shutdownCompletion; + } + + private async shutdownOnce(reason: unknown): Promise { + this.controller.health.markStopping(); + this.abortController.abort(reason); + if (this.completion === undefined) return; + let timeout: ReturnType | undefined; + try { + await Promise.race([ + this.completion, + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`Scale-set controller did not stop within ${this.config.shutdownTimeoutMs}ms`)), + this.config.shutdownTimeoutMs, + ); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } + } +} diff --git a/lambdas/services/scale-set/src/logger.test.ts b/lambdas/services/scale-set/src/logger.test.ts new file mode 100644 index 0000000000..5e58b47d57 --- /dev/null +++ b/lambdas/services/scale-set/src/logger.test.ts @@ -0,0 +1,56 @@ +import { createScaleSetLogger, logger, sanitizeLogAttributes } from './logger'; +import { ScaleSetConfigurationError } from './config'; + +describe('redacted structured logging', () => { + it('emits debug records when LOG_LEVEL is debug', () => { + const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined); + createScaleSetLogger({ LOG_LEVEL: 'debug' }).debug('debug_event', { reconcilerCount: 2 }); + expect(spy).toHaveBeenCalledWith(expect.stringContaining('"event":"debug_event"')); + spy.mockRestore(); + }); + + it('does not emit debug records at the default info level', () => { + const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined); + createScaleSetLogger({}).debug('hidden_debug_event'); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); + }); + + it('redacts nested secrets and strips log-injection characters', () => { + expect( + sanitizeLogAttributes({ + runnerConfig: 'linux\nforged', + privateKey: 'secret', + nested: { authorization: 'Bearer secret', safe: 'ok' }, + }), + ).toEqual({ + runnerConfig: 'linux forged', + privateKey: '[REDACTED]', + nested: { authorization: '[REDACTED]', safe: 'ok' }, + }); + }); + + it('logs errors without their potentially sensitive message', () => { + const spy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + logger.error('failed', { error: new Error('token=secret') }); + expect(spy).toHaveBeenCalledOnce(); + expect(spy.mock.calls[0][0]).not.toContain('token=secret'); + expect(JSON.parse(spy.mock.calls[0][0] as string)).toMatchObject({ level: 'error', event: 'failed' }); + spy.mockRestore(); + }); + + it('includes safe configuration error messages for diagnosis', () => { + expect( + sanitizeLogAttributes({ + error: new ScaleSetConfigurationError( + 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + ), + }), + ).toEqual({ + error: { + name: 'ScaleSetConfigurationError', + message: 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + }, + }); + }); +}); diff --git a/lambdas/services/scale-set/src/logger.ts b/lambdas/services/scale-set/src/logger.ts new file mode 100644 index 0000000000..639f8a2177 --- /dev/null +++ b/lambdas/services/scale-set/src/logger.ts @@ -0,0 +1,87 @@ +const REDACTED = '[REDACTED]'; +const SENSITIVE_KEY = /(authorization|credential|encodedjit|jitconfig|password|private.?key|secret|sessionid|token)/i; +const SAFE_ERROR_MESSAGE_NAMES = new Set(['ScaleSetConfigurationError']); +const MAX_LOG_STRING_LENGTH = 1024; +const MAX_LOG_DEPTH = 4; +const LOG_LEVEL_PRIORITY = { debug: 10, info: 20, warn: 30, error: 40 } as const; + +export type ScaleSetLogLevel = keyof typeof LOG_LEVEL_PRIORITY; + +export interface ScaleSetLogger { + debug(event: string, attributes?: Readonly>): void; + info(event: string, attributes?: Readonly>): void; + warn(event: string, attributes?: Readonly>): void; + error(event: string, attributes?: Readonly>): void; +} + +function sanitizeString(value: string): string { + return value.replace(/[\r\n\u2028\u2029]/g, ' ').slice(0, MAX_LOG_STRING_LENGTH); +} + +function sanitize(value: unknown, key: string, depth: number): unknown { + if (SENSITIVE_KEY.test(key)) return REDACTED; + if (depth > MAX_LOG_DEPTH) return '[TRUNCATED]'; + if (value === null || typeof value === 'boolean' || typeof value === 'number') return value; + if (typeof value === 'string') return sanitizeString(value); + if (value instanceof Error) { + const status = 'status' in value && typeof value.status === 'number' ? value.status : undefined; + const code = 'code' in value && typeof value.code === 'string' ? sanitizeString(value.code) : undefined; + const message = SAFE_ERROR_MESSAGE_NAMES.has(value.name) ? sanitizeString(value.message) : undefined; + return { + name: sanitizeString(value.name), + ...(message ? { message } : {}), + ...(status === undefined ? {} : { status }), + ...(code ? { code } : {}), + }; + } + if (Array.isArray(value)) return value.slice(0, 100).map((item) => sanitize(item, key, depth + 1)); + if (typeof value === 'object') { + const result: Record = Object.create(null) as Record; + for (const [childKey, childValue] of Object.entries(value).slice(0, 100)) { + result[sanitizeString(childKey)] = sanitize(childValue, childKey, depth + 1); + } + return result; + } + return sanitizeString(typeof value); +} + +export function sanitizeLogAttributes(attributes: Readonly> = {}): Record { + return sanitize(attributes, '', 0) as Record; +} + +function parseLogLevel(value: string | undefined): ScaleSetLogLevel { + return value !== undefined && value in LOG_LEVEL_PRIORITY ? (value as ScaleSetLogLevel) : 'info'; +} + +function write( + level: ScaleSetLogLevel, + minimumLevel: ScaleSetLogLevel, + event: string, + attributes?: Readonly>, +): void { + if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[minimumLevel]) return; + const record = JSON.stringify({ + timestamp: new Date().toISOString(), + level, + event: sanitizeString(event), + ...sanitizeLogAttributes(attributes), + }); + if (level === 'error') console.error(record); + else if (level === 'warn') console.warn(record); + else if (level === 'info') console.info(record); + else console.debug(record); +} + +export function createScaleSetLogger( + environment: Readonly> = process.env, +): ScaleSetLogger { + const minimumLevel = parseLogLevel(environment.LOG_LEVEL); + return { + debug: (event, attributes) => write('debug', minimumLevel, event, attributes), + info: (event, attributes) => write('info', minimumLevel, event, attributes), + warn: (event, attributes) => write('warn', minimumLevel, event, attributes), + error: (event, attributes) => write('error', minimumLevel, event, attributes), + }; +} + +export const logger = createScaleSetLogger(); diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts new file mode 100644 index 0000000000..b97cb112e3 --- /dev/null +++ b/lambdas/services/scale-set/src/main.ts @@ -0,0 +1,100 @@ +import { GitHubActionsScaleSetClient } from '@aws-github-runner/github-actions-scale-set'; +import { createScaleSetComputeProviderRegistry } from '@aws-github-runner/compute-providers/scale-set'; + +import { parseScaleSetServiceConfig } from './config'; +import { ScaleSetController } from './controller'; +import { createGitHubAppAccessTokenProvider } from './credentials'; +import { createScaleSetGitHubHttp } from './github-http'; +import { startScaleSetHealthServer, type ScaleSetHealthServer } from './health-server'; +import { ScaleSetServiceRuntime } from './lifecycle'; +import { logger } from './logger'; +import { createDefaultControllerManifestLoader, defaultParameterStore } from './parameter-store'; +import { abortableSleep, type ScaleSetReconcilerDependencies } from './reconciler'; + +async function main(): Promise { + logger.info('scale_set_controller_configuration_loading', { + manifestConfigured: Boolean(process.env.SCALE_SET_CONTROLLER_MANIFEST?.trim()), + groupConfigConfigured: Boolean(process.env.SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH?.trim()), + }); + const serviceConfig = parseScaleSetServiceConfig(process.env); + const manifest = await createDefaultControllerManifestLoader().load(serviceConfig); + logger.info('scale_set_controller_manifest_loaded', { + groupName: manifest.groupName, + revision: manifest.revision, + reconcilerCount: manifest.reconcilers.length, + runnerConfigNames: manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); + logger.debug('scale_set_controller_reconcilers_loaded', { + groupName: manifest.groupName, + reconcilerCount: manifest.reconcilers.length, + runnerConfigNames: manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); + const computeProviders = createScaleSetComputeProviderRegistry(); + const githubHttp = createScaleSetGitHubHttp(); + const dependencies: ScaleSetReconcilerDependencies = { + computeProviders, + createAccessTokenProvider: async (config) => + await createGitHubAppAccessTokenProvider( + config.githubApp, + config.githubConfigUrl, + config.forceGhes, + defaultParameterStore, + githubHttp.fetch(config.sslVerify), + ), + createClient: (config, accessTokenProvider) => + new GitHubActionsScaleSetClient({ + gitHubConfigUrl: config.githubConfigUrl, + accessTokenProvider, + fetch: githubHttp.fetch(config.sslVerify), + forceGhes: config.forceGhes, + systemInfo: { + system: config.userAgent ?? 'github-aws-runners', + version: '1', + scaleSetId: config.scaleSetId ?? 0, + subsystem: 'scale-set-controller', + }, + }), + logger, + parameterStore: defaultParameterStore, + sleep: abortableSleep, + random: Math.random, + closeSignal: AbortSignal.timeout, + }; + const controller = new ScaleSetController(manifest, serviceConfig, dependencies, logger); + const runtime = new ScaleSetServiceRuntime(serviceConfig, controller); + let healthServer: ScaleSetHealthServer | undefined; + + const shutdown = (signal: NodeJS.Signals) => { + logger.info('scale_set_controller_shutdown_requested', { signal, groupName: manifest.groupName }); + void runtime.shutdown(new Error(`received ${signal}`)).catch((error) => { + logger.error('scale_set_controller_shutdown_failed', { error, groupName: manifest.groupName }); + process.exitCode = 1; + }); + }; + const onSigterm = () => shutdown('SIGTERM'); + const onSigint = () => shutdown('SIGINT'); + process.once('SIGTERM', onSigterm); + process.once('SIGINT', onSigint); + + try { + healthServer = await startScaleSetHealthServer(runtime.health, serviceConfig.healthPort); + logger.info('scale_set_controller_started', { + groupName: manifest.groupName, + revision: manifest.revision, + reconcilerCount: manifest.reconcilers.length, + healthPort: healthServer.port, + }); + await runtime.run(); + } finally { + await runtime.shutdown().catch(() => undefined); + await healthServer?.close(); + await githubHttp.close(); + process.removeListener('SIGTERM', onSigterm); + process.removeListener('SIGINT', onSigint); + } +} + +void main().catch((error) => { + logger.error('scale_set_controller_fatal_failure', { error }); + process.exitCode = 1; +}); diff --git a/lambdas/services/scale-set/src/parameter-store.test.ts b/lambdas/services/scale-set/src/parameter-store.test.ts new file mode 100644 index 0000000000..f18cc2ef7e --- /dev/null +++ b/lambdas/services/scale-set/src/parameter-store.test.ts @@ -0,0 +1,78 @@ +import { createControllerManifestLoader, type ParametersByPathClient } from './parameter-store'; +import type { ScaleSetServiceConfig } from './config'; + +function leaf(name: string, id: number): string { + return JSON.stringify({ + schemaVersion: 1, + runnerConfigName: name, + githubConfigUrl: 'https://github.com/example', + scaleSetId: id, + expectedScaleSetName: name, + expectedRunnerGroupId: null, + minRunners: 0, + maxRunners: 10, + sslVerify: true, + githubApp: { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', + }, + computeProvider: { type: 'ec2', configuration: {} }, + }); +} + +const config: ScaleSetServiceConfig = { + groupName: 'group', + groupConfigPath: '/groups/group', + groupRevision: 'rev-1', + healthPort: 8080, + healthStaleAfterMs: 1000, + shutdownTimeoutMs: 1000, + sessionCloseTimeoutMs: 1000, + reconnectInitialBackoffMs: 100, + reconnectMaxBackoffMs: 1000, +}; + +describe('createControllerManifestLoader', () => { + it('paginates direct children, sorts them, and returns a versioned group', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/b', Value: leaf('b', 2) }], NextToken: 'next' }) + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/a', Value: leaf('a', 1) }] }); + const manifest = await createControllerManifestLoader({ send } as ParametersByPathClient).load(config); + expect(manifest).toMatchObject({ version: 1, groupName: 'group', revision: 'rev-1' }); + expect(manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName)).toEqual(['a', 'b']); + expect(send).toHaveBeenCalledTimes(2); + }); + + it('uses the injected inline manifest loader path for local tests', async () => { + const inline = JSON.stringify({ version: 1, groupName: 'local', reconcilers: [JSON.parse(leaf('a', 1))] }); + const send = vi.fn(); + await expect( + createControllerManifestLoader({ send } as ParametersByPathClient).load({ ...config, manifest: inline }), + ).resolves.toMatchObject({ + groupName: 'local', + }); + expect(send).not.toHaveBeenCalled(); + }); + + it.each([ + [{ Parameters: [] }, 'contains no runner configs'], + [{ Parameters: [{ Name: '/groups/group/nested/a', Value: leaf('a', 1) }] }, 'outside the direct group path'], + [{ Parameters: [{ Name: '/groups/group/wrong', Value: leaf('a', 1) }] }, 'must match runnerConfigName'], + [{ Parameters: [{ Name: '/groups/group/a', Value: '{' }] }, 'contains invalid JSON'], + [{ Parameters: [{ Name: undefined, Value: leaf('a', 1) }] }, 'incomplete parameter'], + ])('rejects malformed SSM group pages', async (page, message) => { + await expect( + createControllerManifestLoader({ send: vi.fn().mockResolvedValue(page) }).load(config), + ).rejects.toThrow(message); + }); + + it('rejects repeated pagination tokens', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/a', Value: leaf('a', 1) }], NextToken: 'same' }) + .mockResolvedValueOnce({ Parameters: [], NextToken: 'same' }); + await expect(createControllerManifestLoader({ send }).load(config)).rejects.toThrow('repeated token'); + }); +}); diff --git a/lambdas/services/scale-set/src/parameter-store.ts b/lambdas/services/scale-set/src/parameter-store.ts new file mode 100644 index 0000000000..3df49818b8 --- /dev/null +++ b/lambdas/services/scale-set/src/parameter-store.ts @@ -0,0 +1,141 @@ +import { GetParametersByPathCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; + +import { getParameters, ssmClient } from '@aws-github-runner/aws-ssm-util'; + +import { + MAX_MANIFEST_BYTES, + SCALE_SET_CONTROLLER_MANIFEST_VERSION, + ScaleSetConfigurationError, + parseScaleSetControllerManifest, + parseScaleSetReconcilerConfig, + validateUniqueReconcilers, + type ScaleSetControllerManifest, + type ScaleSetServiceConfig, +} from './config'; +import type { ParameterStore } from './credentials'; + +const MAX_GROUP_PARAMETERS = 1000; +const MAX_PARAMETER_BYTES = 64 * 1024; +const MAX_GROUP_BYTES = 4 * 1024 * 1024; + +export const defaultParameterStore: ParameterStore = { + get: async (names) => await getParameters([...names]), + put: async (name, value) => { + await ssmClient().send( + new PutParameterCommand({ + Name: name, + Value: value, + Type: 'String', + Overwrite: true, + }), + ); + }, +}; + +export interface ControllerManifestLoader { + load(config: ScaleSetServiceConfig): Promise; +} + +export interface ParametersByPathClient { + send(command: GetParametersByPathCommand): Promise<{ + Parameters?: Array<{ Name?: string; Value?: string }>; + NextToken?: string; + }>; +} + +export function createControllerManifestLoader(client: ParametersByPathClient): ControllerManifestLoader { + return { + load: async (config) => { + if (config.manifest !== undefined) return parseScaleSetControllerManifest(config.manifest); + if (!config.groupConfigPath || !config.groupName || !config.groupRevision) { + throw new ScaleSetConfigurationError('SSM group configuration source is incomplete'); + } + const prefix = `${config.groupConfigPath.replace(/\/$/, '')}/`; + const parameters: Array<{ name: string; value: string }> = []; + const seenTokens = new Set(); + let nextToken: string | undefined; + let totalBytes = 0; + do { + if (nextToken !== undefined && seenTokens.has(nextToken)) { + throw new ScaleSetConfigurationError('SSM pagination returned a repeated token'); + } + if (nextToken !== undefined) seenTokens.add(nextToken); + const response = await client.send( + new GetParametersByPathCommand({ + Path: config.groupConfigPath, + Recursive: false, + WithDecryption: false, + MaxResults: 10, + ...(nextToken === undefined ? {} : { NextToken: nextToken }), + }), + ); + for (const parameter of response.Parameters ?? []) { + if (!parameter.Name || parameter.Value === undefined) { + throw new ScaleSetConfigurationError('SSM group configuration returned an incomplete parameter'); + } + if (!parameter.Name.startsWith(prefix) || parameter.Name.slice(prefix.length).includes('/')) { + throw new ScaleSetConfigurationError( + 'SSM group configuration returned a parameter outside the direct group path', + ); + } + const size = Buffer.byteLength(parameter.Value, 'utf8'); + if (size > MAX_PARAMETER_BYTES || size > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(parameter.Name)} is too large`, + ); + } + totalBytes += size; + if (totalBytes > MAX_GROUP_BYTES) + throw new ScaleSetConfigurationError('SSM controller group configuration is too large'); + parameters.push({ name: parameter.Name, value: parameter.Value }); + if (parameters.length > MAX_GROUP_PARAMETERS) { + throw new ScaleSetConfigurationError(`SSM controller group exceeds ${MAX_GROUP_PARAMETERS} runner configs`); + } + } + nextToken = response.NextToken; + } while (nextToken !== undefined); + + if (parameters.length === 0) + throw new ScaleSetConfigurationError('SSM controller group contains no runner configs'); + parameters.sort((left, right) => left.name.localeCompare(right.name)); + const reconcilers = parameters.map(({ name, value }, index) => { + let parsed: unknown; + try { + parsed = JSON.parse(value) as unknown; + } catch (error) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(name)} contains invalid JSON`, + { + cause: error, + }, + ); + } + const reconciler = parseScaleSetReconcilerConfig(parsed, index, config.groupName as string, 'ssmRunnerConfigs'); + const leafName = name.slice(prefix.length); + if (leafName !== reconciler.runnerConfigName) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(name)} must match runnerConfigName ${JSON.stringify(reconciler.runnerConfigName)}`, + ); + } + return reconciler; + }); + validateUniqueReconcilers(reconcilers); + return { + version: SCALE_SET_CONTROLLER_MANIFEST_VERSION, + groupName: config.groupName, + revision: config.groupRevision, + reconcilers, + }; + }, + }; +} + +export function createDefaultControllerManifestLoader(): ControllerManifestLoader { + return createControllerManifestLoader( + new SSMClient({ + region: process.env.AWS_REGION, + maxAttempts: 10, + retryMode: 'adaptive', + }), + ); +} diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts new file mode 100644 index 0000000000..e9de83afaf --- /dev/null +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -0,0 +1,441 @@ +import type { MessageSessionClient, RunnerScaleSetMessage } from '@aws-github-runner/github-actions-scale-set'; +import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '@aws-github-runner/compute-providers/scale-set'; +import { describe, expect, it, vi } from 'vitest'; + +import type { ScaleSetReconcilerConfig, ScaleSetServiceConfig } from './config'; +import type { ScaleSetReconcilerStatusReporter } from './health'; +import { + ScaleSetReconciler, + calculateDesiredRunners, + validateProviderResult, + type ScaleSetReconcilerClient, + type ScaleSetReconcilerDependencies, +} from './reconciler'; + +const config: ScaleSetReconcilerConfig = { + schemaVersion: 1, + runnerConfigName: 'linux', + scaleSetId: 42, + scaleSetName: 'linux', + githubConfigUrl: 'https://github.com/example', + githubApp: { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', + }, + computeProvider: { type: 'ec2', configuration: {} }, + minRunners: 0, + maxRunners: 10, + bootTimeoutMinutes: 10, + sessionOwner: 'group.linux', + workFolder: '_work', + forceGhes: false, +}; + +const serviceConfig: Pick< + ScaleSetServiceConfig, + 'sessionCloseTimeoutMs' | 'reconnectInitialBackoffMs' | 'reconnectMaxBackoffMs' +> = { sessionCloseTimeoutMs: 100, reconnectInitialBackoffMs: 1, reconnectMaxBackoffMs: 10 }; + +function result(overrides: Partial = {}): ScaleSetReconcileResult { + return { + status: 'converged', + desiredRunners: 1, + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + ...overrides, + }; +} + +function reporter(): ScaleSetReconcilerStatusReporter { + return { + markSessionReady: vi.fn(), + markProgress: vi.fn(), + markReconnecting: vi.fn(), + markFailed: vi.fn(), + markStopping: vi.fn(), + }; +} + +function message(): RunnerScaleSetMessage { + return { + messageId: 7, + statistics: { + totalAvailableJobs: 1, + totalAcquiredJobs: 0, + totalAssignedJobs: 1, + totalRunningJobs: 0, + totalRegisteredRunners: 1, + totalBusyRunners: 0, + totalIdleRunners: 1, + }, + jobAvailableMessages: [{ runnerRequestId: 99 } as RunnerScaleSetMessage['jobAvailableMessages'][number]], + jobAssignedMessages: [], + jobStartedMessages: [ + { runnerId: 5, runnerName: 'runner-5' } as RunnerScaleSetMessage['jobStartedMessages'][number], + ], + jobCompletedMessages: [], + }; +} + +function fixture(options: { + session: Partial & { session: MessageSessionClient['session'] }; + reconcile?: ScaleSetComputeProvider['reconcile']; +}) { + const computeProvider: ScaleSetComputeProvider = { + reconcile: options.reconcile ?? vi.fn().mockResolvedValue(result()), + }; + const client: ScaleSetReconcilerClient = { + getRunnerScaleSetById: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + getRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + createRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + getRunnerGroupByName: vi.fn().mockResolvedValue({ + id: 7, + name: 'runner-group', + size: 0, + isDefaultGroup: false, + }), + createMessageSessionClient: vi.fn().mockResolvedValue(options.session as MessageSessionClient), + generateJitRunnerConfig: vi.fn(), + getRunnerByName: vi.fn(), + removeRunner: vi.fn(), + systemInfo: { scaleSetId: 42 }, + setSystemInfo: vi.fn(), + }; + const dependencies: ScaleSetReconcilerDependencies = { + createAccessTokenProvider: vi.fn().mockResolvedValue(async () => ({ token: 'not-a-real-token' })), + createClient: vi.fn().mockReturnValue(client), + computeProviders: { create: vi.fn().mockReturnValue(computeProvider) }, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + sleep: vi.fn(async (_delay, signal) => { + if (!signal.aborted) + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); + }), + random: () => 0, + closeSignal: () => new AbortController().signal, + parameterStore: { get: vi.fn().mockResolvedValue(new Map()), put: vi.fn() }, + }; + return { client, computeProvider, dependencies }; +} + +describe('ScaleSetReconciler', () => { + it('resolves the GitHub runner-group and scale-set IDs from their names', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + deleteMessage: vi.fn(), + acquireJobs: vi.fn(), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ + session, + reconcile: vi.fn(async () => { + abort.abort(); + return result(); + }), + }); + vi.mocked(client.getRunnerScaleSet).mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }); + + await new ScaleSetReconciler( + { + ...config, + scaleSetId: undefined, + runnerGroupName: 'runner-group', + runnerGroupIdParameterName: '/runner/group-id', + scaleSetName: 'linux', + }, + serviceConfig, + dependencies, + ).run(abort.signal, reporter()); + + expect(client.getRunnerGroupByName).toHaveBeenCalledWith('runner-group', { signal: abort.signal }); + expect(client.getRunnerScaleSet).toHaveBeenCalledWith(7, 'linux', { signal: abort.signal }); + expect(client.setSystemInfo).toHaveBeenCalledWith(expect.objectContaining({ scaleSetId: 42 })); + expect(dependencies.parameterStore.put).toHaveBeenCalledWith('/runner/group-id', '7'); + expect(dependencies.logger.info).toHaveBeenCalledWith( + 'scale_set_compute_provider_created', + expect.objectContaining({ computeProviderType: 'ec2' }), + ); + expect(dependencies.logger.info).toHaveBeenCalledWith( + 'scale_set_compute_provider_reconcile_started', + expect.objectContaining({ computeProviderType: 'ec2', desiredRunners: 1, busyRunners: 0 }), + ); + }); + + it('registers a missing scale set in the resolved runner group', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + deleteMessage: vi.fn(), + acquireJobs: vi.fn(), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ + session, + reconcile: vi.fn(async () => { + abort.abort(); + return result(); + }), + }); + vi.mocked(client.getRunnerScaleSet).mockResolvedValueOnce(null); + + await new ScaleSetReconciler( + { ...config, scaleSetId: undefined, runnerGroupName: 'runner-group', scaleSetName: 'linux' }, + serviceConfig, + dependencies, + ).run(abort.signal, reporter()); + + expect(client.createRunnerScaleSet).toHaveBeenCalledWith( + { + name: 'linux', + runnerGroupId: 7, + labels: [{ name: 'linux' }], + runnerSetting: {}, + }, + { signal: abort.signal }, + ); + }); + + it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { + const order: string[] = []; + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(async () => { + order.push('acquire'); + return [99]; + }), + deleteMessage: vi.fn(async () => { + order.push('delete'); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async (request) => { + order.push('reconcile'); + expect(request.busyRunners).toBe(0); + expect(request.bootTimeoutMinutes).toBe(10); + expect(request.runnerStates).toContainEqual( + expect.objectContaining({ runnerId: 5, runnerName: 'runner-5', lifecycle: 'started' }), + ); + abort.abort(); + return result(); + }); + const { dependencies } = fixture({ session, reconcile }); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + expect(order).toEqual(['delete', 'acquire', 'reconcile']); + expect(session.deleteMessage).toHaveBeenCalledWith(7, { signal: abort.signal }); + expect(dependencies.computeProviders.create).toHaveBeenCalledWith('ec2', { + runnerConfigName: 'linux', + scaleSetId: 42, + githubScope: 'https://github.com/example', + configuration: {}, + }); + }); + + it('acknowledges and stops when reconciliation rejects', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const { dependencies } = fixture({ session, reconcile: vi.fn().mockRejectedValue(new Error('provider failed')) }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + expect(status.markReconnecting).not.toHaveBeenCalled(); + expect(dependencies.sleep).not.toHaveBeenCalled(); + }); + + it('does not process a message when acknowledgement fails', async () => { + const abort = new AbortController(); + const reconcile = vi.fn(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(), + deleteMessage: vi.fn().mockRejectedValue(new Error('acknowledgement failed')), + close: vi.fn(), + }; + const { dependencies } = fixture({ session, reconcile }); + dependencies.sleep = vi.fn(async () => abort.abort()); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(session.acquireJobs).not.toHaveBeenCalled(); + expect(reconcile).not.toHaveBeenCalled(); + expect(status.markReconnecting).toHaveBeenCalledOnce(); + }); + + it('acknowledges and stops when the provider returns an error result', async () => { + const abort = new AbortController(); + const order: string[] = []; + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(async () => { + order.push('delete'); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async () => { + order.push('reconcile'); + return result({ + status: 'error', + currentRunners: 0, + errors: [{ operation: 'launch', code: 'ThrottlingException' }], + }); + }); + const { dependencies } = fixture({ session, reconcile }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(order).toEqual(['delete', 'reconcile']); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + expect(status.markReconnecting).not.toHaveBeenCalled(); + expect(dependencies.sleep).not.toHaveBeenCalled(); + }); + + it('does not call public GitHub runner inventory after a provider error result', async () => { + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const reconcile = vi.fn().mockResolvedValue( + result({ + status: 'error', + currentRunners: 0, + errors: [{ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }], + }), + ); + const { dependencies } = fixture({ session, reconcile }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(new AbortController().signal, status); + + expect(reconcile).toHaveBeenCalledOnce(); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + }); + + it('uses the Actions-service identity check without public runner verification', async () => { + const abort = new AbortController(); + const order: string[] = []; + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(async () => { + order.push('acquire'); + return [99]; + }), + deleteMessage: vi.fn(async () => { + order.push('delete'); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async (request) => { + order.push('reconcile'); + await expect(request.removeRunner({ runnerId: 5, runnerName: 'runner-5', scaleSetId: 42 })).resolves.toEqual({ + status: 'removed', + }); + abort.abort(); + return result({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, + }); + }); + const { client, dependencies } = fixture({ session, reconcile }); + vi.mocked(client.getRunnerByName).mockImplementation(async () => { + order.push('actions-refetch'); + return { id: 5, name: 'runner-5', runnerScaleSetId: 42 }; + }); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + + expect(order).toEqual(['delete', 'acquire', 'reconcile', 'actions-refetch']); + expect(client.removeRunner).toHaveBeenCalledWith(5, { signal: abort.signal }); + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(session.acquireJobs).toHaveBeenCalledTimes(1); + }); + + it('bounds the lifecycle cache per reconciler', () => { + const { dependencies } = fixture({ session: { session: {}, close: vi.fn() } }); + const reconciler = new ScaleSetReconciler(config, serviceConfig, dependencies) as unknown as { + rememberLifecycle(id: number, name: string, lifecycle: 'started'): void; + lifecycle: Map; + resolvedScaleSetId: number; + }; + reconciler.resolvedScaleSetId = 42; + for (let index = 0; index < 1100; index += 1) reconciler.rememberLifecycle(index + 1, `runner-${index}`, 'started'); + expect(reconciler.lifecycle.size).toBe(1000); + expect(reconciler.lifecycle.has('runner-0')).toBe(false); + }); +}); + +describe('reconciler helpers', () => { + it('calculates bounded desired capacity', () => { + expect(calculateDesiredRunners(5, 2, 6)).toBe(6); + expect(calculateDesiredRunners(8, 2, 5)).toBe(8); + expect(() => calculateDesiredRunners(-1, 0, 1)).toThrow('non-negative integer'); + }); + + it.each([ + { status: 'unexpected' }, + { status: 'retryable_error' }, + { status: 'non_retryable_error' }, + { retryable: true }, + { actions: { launched: 0, terminated: 0, retainedBusy: -1, retainedUnknown: 0 } }, + { actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0, retryable: true } }, + { status: 'converged', errors: [{ operation: 'list', code: 'UNEXPECTED_ERROR' }] }, + { status: 'retained', errors: [{ operation: 'list', code: 'UNEXPECTED_ERROR' }] }, + { status: 'error', errors: [] }, + { currentRunners: 0 }, + { currentRunners: 2 }, + { status: 'retained' }, + { errors: [{ operation: 'shell', code: 'BAD' }] }, + { errors: [{ operation: 'list', code: 'contains spaces' }] }, + { errors: [{ operation: 'list', code: 'BAD!CODE' }] }, + { errors: [{ operation: 'list', code: 'BAD\nCODE' }] }, + { errors: [{ operation: 'list', code: 'BAD', retryable: true }] }, + ])('rejects malformed compute-provider result metadata: %o', (overrides) => { + expect(() => validateProviderResult({ ...result(), ...overrides } as ScaleSetReconcileResult, 1)).toThrow( + /scale-set compute provider returned (?:an? )?invalid/, + ); + }); + + it('accepts bounded provider and AWS error codes', () => { + expect(() => + validateProviderResult( + result({ + status: 'error', + errors: [ + { operation: 'list', code: 'AccessDeniedException' }, + { operation: 'launch', code: 'ThrottlingException' }, + ], + }), + 1, + ), + ).not.toThrow(); + }); +}); diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts new file mode 100644 index 0000000000..69c56e9cda --- /dev/null +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -0,0 +1,668 @@ +import { + GitHubActionsScaleSetClient, + isScaleSetHttpError, + ScaleSetProtocolError, + type AccessToken, + type MessageSessionClient, + type RunnerScaleSetMessage, + type RunnerScaleSetStatistic, +} from '@aws-github-runner/github-actions-scale-set'; +import type { + ScaleSetComputeProvider, + ScaleSetComputeProviderFactoryInput, + ScaleSetReconcileRequest, + ScaleSetReconcileResult, + ScaleSetRunnerLifecycle, + ScaleSetRunnerState, +} from '@aws-github-runner/compute-providers/scale-set'; + +import { ScaleSetConfigurationError, type ScaleSetReconcilerConfig, type ScaleSetServiceConfig } from './config'; +import type { ParameterStore } from './credentials'; +import type { ScaleSetReconcilerStatusReporter } from './health'; +import type { ScaleSetLogger } from './logger'; + +const MAX_JIT_CONFIGURATION_BYTES = 1024 * 1024; + +export interface ScaleSetComputeProviderFactory { + create(type: string, input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider; +} + +export type ScaleSetReconcilerClient = Pick< + GitHubActionsScaleSetClient, + | 'createMessageSessionClient' + | 'generateJitRunnerConfig' + | 'getRunnerGroupByName' + | 'getRunnerScaleSet' + | 'getRunnerScaleSetById' + | 'getRunnerByName' + | 'removeRunner' + | 'createRunnerScaleSet' + | 'setSystemInfo' + | 'systemInfo' +>; + +export interface ScaleSetReconcilerDependencies { + createAccessTokenProvider(config: ScaleSetReconcilerConfig): Promise<() => Promise>; + createClient(config: ScaleSetReconcilerConfig, provider: () => Promise): ScaleSetReconcilerClient; + computeProviders: ScaleSetComputeProviderFactory; + logger: ScaleSetLogger; + sleep(delayMs: number, signal: AbortSignal): Promise; + random(): number; + closeSignal(timeoutMs: number): AbortSignal; + parameterStore: ParameterStore; +} + +interface LifecycleObservation { + runnerId: number; + runnerName: string; + scaleSetId: number; + lifecycle: ScaleSetRunnerLifecycle; +} + +export class ScaleSetProviderReconciliationError extends Error { + constructor( + readonly result?: ScaleSetReconcileResult, + options?: ErrorOptions, + ) { + super( + result === undefined + ? 'scale-set compute provider reconciliation failed' + : `scale-set compute provider returned ${result.status}`, + options, + ); + this.name = 'ScaleSetProviderReconciliationError'; + } +} + +export class ScaleSetReconciler { + private readonly lifecycle = new Map(); + private readonly lifecycleLimit: number; + private resolvedScaleSetId?: number; + private resolvedRunnerGroupId?: number; + + constructor( + private readonly config: ScaleSetReconcilerConfig, + private readonly serviceConfig: Pick< + ScaleSetServiceConfig, + 'sessionCloseTimeoutMs' | 'reconnectInitialBackoffMs' | 'reconnectMaxBackoffMs' + >, + private readonly dependencies: ScaleSetReconcilerDependencies, + ) { + this.lifecycleLimit = Math.min(20_000, Math.max(1000, config.maxRunners * 4)); + } + + async run(signal: AbortSignal, status: ScaleSetReconcilerStatusReporter): Promise { + let provider: ScaleSetComputeProvider; + let client: ScaleSetReconcilerClient; + try { + const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); + client = this.dependencies.createClient(this.config, accessTokenProvider); + const resolved = await this.resolveScaleSet(client, signal); + this.resolvedScaleSetId = resolved.scaleSetId; + this.resolvedRunnerGroupId = resolved.runnerGroupId; + client.setSystemInfo({ ...client.systemInfo, scaleSetId: resolved.scaleSetId }); + this.log('debug', 'scale_set_compute_provider_loading', { + computeProviderType: this.config.computeProvider.type, + }); + provider = this.dependencies.computeProviders.create(this.config.computeProvider.type, { + runnerConfigName: this.config.runnerConfigName, + scaleSetId: resolved.scaleSetId, + githubScope: this.config.githubConfigUrl, + configuration: this.config.computeProvider.configuration, + }); + this.log('info', 'scale_set_compute_provider_created', { + computeProviderType: this.config.computeProvider.type, + }); + this.log('debug', 'scale_set_compute_provider_loaded', { + computeProviderType: this.config.computeProvider.type, + }); + } catch (error) { + status.markFailed(error); + this.log('error', 'scale_set_reconciler_initialization_failed', { + computeProviderType: this.config.computeProvider.type, + ...httpErrorLogAttributes(error), + error, + }); + return; + } + + let consecutiveFailures = 0; + while (!signal.aborted) { + let session: MessageSessionClient | undefined; + let madeProgress = false; + try { + const configuredScaleSet = await client.getRunnerScaleSetById(this.scaleSetId, { signal }); + if ( + configuredScaleSet === null || + configuredScaleSet.id !== this.scaleSetId || + configuredScaleSet.name !== this.config.scaleSetName || + (this.resolvedRunnerGroupId !== undefined && configuredScaleSet.runnerGroupId !== this.resolvedRunnerGroupId) + ) { + throw new ScaleSetConfigurationError('configured GitHub runner scale set identity does not match'); + } + session = await client.createMessageSessionClient(this.scaleSetId, this.config.sessionOwner, { signal }); + status.markSessionReady(); + this.log('info', 'scale_set_session_created'); + this.log('debug', 'scale_set_session_scale_set_loaded', { + scaleSetName: session.session.runnerScaleSet?.name, + scaleSetLabels: session.session.runnerScaleSet?.labels?.map(({ name, type }) => ({ name, type })), + }); + let latestStatistics = session.session.statistics ?? undefined; + let lastMessageId = 0; + if (latestStatistics !== undefined) { + await this.reconcile(client, provider, latestStatistics, signal); + madeProgress = true; + consecutiveFailures = 0; + status.markProgress(); + } + + while (!signal.aborted) { + const message = await session.getMessage(lastMessageId, this.config.maxRunners, { signal }); + if (message === null) { + if (latestStatistics === undefined) { + throw new ScaleSetProtocolError('message session returned no message and no statistics snapshot'); + } + await this.reconcile(client, provider, latestStatistics, signal); + } else { + if (message.statistics === null) { + throw new ScaleSetProtocolError(`scale-set message ${message.messageId} contains no statistics`); + } + latestStatistics = message.statistics; + lastMessageId = message.messageId; + await session.deleteMessage(message.messageId, { signal }); + const requestIds = uniqueRequestIds(message); + if (requestIds.length > 0) await session.acquireJobs(requestIds, { signal }); + this.observeLifecycle(message); + await this.reconcile(client, provider, latestStatistics, signal); + this.pruneCompletedLifecycle(message); + } + madeProgress = true; + consecutiveFailures = 0; + status.markProgress(); + } + } catch (error) { + if (signal.aborted) break; + if (isFatalReconcilerError(error)) { + status.markFailed(error); + this.log('error', 'scale_set_reconciler_failed', { + ...httpErrorLogAttributes(error), + error, + }); + return; + } + consecutiveFailures = madeProgress ? 1 : consecutiveFailures + 1; + status.markReconnecting(error); + this.log('warn', 'scale_set_reconciler_reconnecting', { consecutiveFailures, error }); + } finally { + if (session !== undefined) await this.closeSession(session); + } + + if (!signal.aborted) { + await this.dependencies.sleep( + calculateReconnectDelay( + consecutiveFailures, + this.serviceConfig.reconnectInitialBackoffMs, + this.serviceConfig.reconnectMaxBackoffMs, + this.dependencies.random, + ), + signal, + ); + } + } + status.markStopping(); + } + + private async resolveScaleSet( + client: ScaleSetReconcilerClient, + signal: AbortSignal, + ): Promise<{ scaleSetId: number; runnerGroupId?: number }> { + let runnerGroupId = this.config.expectedRunnerGroupId; + if (this.config.runnerGroupName !== undefined) { + const cachedRunnerGroupId = await this.loadCachedRunnerGroupId(); + const runnerGroup = + cachedRunnerGroupId === undefined + ? await client.getRunnerGroupByName(this.config.runnerGroupName, { signal }) + : { id: cachedRunnerGroupId }; + if (runnerGroupId !== undefined && runnerGroupId !== runnerGroup.id) { + throw new ScaleSetConfigurationError( + `runner group ${JSON.stringify(this.config.runnerGroupName)} resolved to ID ${runnerGroup.id}, expected ${runnerGroupId}`, + ); + } + runnerGroupId = runnerGroup.id; + if (cachedRunnerGroupId === undefined && this.config.runnerGroupIdParameterName !== undefined) { + await this.dependencies.parameterStore.put?.(this.config.runnerGroupIdParameterName, String(runnerGroupId)); + } + this.log('info', 'scale_set_runner_group_resolved', { + runnerConfigName: this.config.runnerConfigName, + runnerGroupName: this.config.runnerGroupName, + runnerGroupId, + }); + } + + if (this.config.scaleSetId === undefined && runnerGroupId === undefined) { + throw new ScaleSetConfigurationError('runner group ID was not resolved'); + } + let configuredScaleSet = + this.config.scaleSetId === undefined + ? await client.getRunnerScaleSet(runnerGroupId as number, this.config.scaleSetName, { signal }) + : await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); + if (configuredScaleSet === null && runnerGroupId !== undefined && this.config.scaleSetId === undefined) { + this.log('info', 'scale_set_registering', { + runnerConfigName: this.config.runnerConfigName, + scaleSetName: this.config.scaleSetName, + runnerGroupId, + }); + try { + configuredScaleSet = await client.createRunnerScaleSet( + { + name: this.config.scaleSetName, + runnerGroupId, + labels: [{ name: this.config.scaleSetName }], + runnerSetting: {}, + }, + { signal }, + ); + } catch (error) { + const existingScaleSet = await client.getRunnerScaleSet(runnerGroupId, this.config.scaleSetName, { signal }); + if (existingScaleSet === null) throw error; + configuredScaleSet = existingScaleSet; + } + } + if (configuredScaleSet === null || configuredScaleSet.id === undefined) { + throw new ScaleSetConfigurationError( + `GitHub runner scale set ${JSON.stringify(this.config.scaleSetName)} was not found`, + ); + } + if ( + configuredScaleSet.name !== this.config.scaleSetName || + (runnerGroupId !== undefined && configuredScaleSet.runnerGroupId !== runnerGroupId) || + (this.config.scaleSetId !== undefined && configuredScaleSet.id !== this.config.scaleSetId) + ) { + throw new ScaleSetConfigurationError('configured GitHub runner scale set identity does not match'); + } + this.log('info', 'scale_set_resolved', { + runnerConfigName: this.config.runnerConfigName, + scaleSetName: this.config.scaleSetName, + scaleSetId: configuredScaleSet.id, + runnerGroupId, + }); + this.log('debug', 'scale_set_labels_resolved', { + scaleSetName: configuredScaleSet.name, + scaleSetLabels: configuredScaleSet.labels?.map(({ name, type }) => ({ name, type })), + }); + return { scaleSetId: configuredScaleSet.id, runnerGroupId }; + } + + private async loadCachedRunnerGroupId(): Promise { + const parameterName = this.config.runnerGroupIdParameterName; + if (parameterName === undefined) return undefined; + const values = await this.dependencies.parameterStore.get([parameterName]); + const raw = values.get(parameterName)?.trim(); + if (raw === undefined || raw === '') return undefined; + if (!/^\d+$/.test(raw)) { + throw new ScaleSetConfigurationError( + `runner group ID parameter ${JSON.stringify(parameterName)} must contain a positive integer`, + ); + } + const id = Number(raw); + if (!Number.isSafeInteger(id) || id <= 0) { + throw new ScaleSetConfigurationError( + `runner group ID parameter ${JSON.stringify(parameterName)} must contain a positive integer`, + ); + } + return id; + } + + private get scaleSetId(): number { + if (this.resolvedScaleSetId === undefined) { + throw new ScaleSetConfigurationError('scale set ID was not resolved'); + } + return this.resolvedScaleSetId; + } + + private async reconcile( + client: ScaleSetReconcilerClient, + provider: ScaleSetComputeProvider, + statistics: RunnerScaleSetStatistic, + signal: AbortSignal, + ): Promise { + const desiredRunners = calculateDesiredRunners( + statistics.totalAssignedJobs, + this.config.minRunners, + this.config.maxRunners, + ); + const callbacks = this.createReconcileCallbacks(client, signal); + const result = await this.reconcileProvider(provider, { + desiredRunners, + busyRunners: statistics.totalBusyRunners, + bootTimeoutMinutes: this.config.bootTimeoutMinutes, + runnerStates: this.lifecycleStates(), + ...callbacks, + }); + validateProviderResult(result, desiredRunners); + throwIfProviderError(result); + this.logReconciliationResult(result, desiredRunners); + } + + private createReconcileCallbacks(client: ScaleSetReconcilerClient, signal: AbortSignal) { + return { + signal, + generateJitConfiguration: async ({ + runnerName, + signal: callbackSignal, + }: { + runnerName: string; + signal?: AbortSignal; + }) => { + const jit = await client.generateJitRunnerConfig( + { name: runnerName, workFolder: this.config.workFolder }, + this.scaleSetId, + { signal: callbackSignal ?? signal }, + ); + if ( + jit.runner === null || + jit.runner.name !== runnerName || + jit.runner.runnerScaleSetId !== this.scaleSetId || + !Number.isSafeInteger(jit.runner.id) || + jit.runner.id <= 0 + ) { + throw new ScaleSetProtocolError('GitHub returned a mismatched runner identity for JIT configuration'); + } + if ( + typeof jit.encodedJITConfig !== 'string' || + jit.encodedJITConfig === '' || + Buffer.byteLength(jit.encodedJITConfig, 'utf8') > MAX_JIT_CONFIGURATION_BYTES + ) { + throw new ScaleSetProtocolError('GitHub returned an invalid JIT configuration'); + } + return { + encodedJitConfiguration: jit.encodedJITConfig, + runnerId: jit.runner.id, + runnerName: jit.runner.name, + scaleSetId: jit.runner.runnerScaleSetId, + }; + }, + removeRunner: async (expected: { + runnerId: number; + runnerName: string; + scaleSetId: number; + signal?: AbortSignal; + }) => { + const callbackSignal = expected.signal ?? signal; + const runner = await client.getRunnerByName(expected.runnerName, { signal: callbackSignal }); + if (runner === null) return { status: 'retained_unknown' as const }; + if ( + runner.id !== expected.runnerId || + runner.name !== expected.runnerName || + runner.runnerScaleSetId !== expected.scaleSetId || + expected.scaleSetId !== this.scaleSetId + ) { + return { status: 'retained_unknown' as const }; + } + // Busy state comes from the aggregate scale-set statistics. The + // Actions-service runner reference above remains the exact identity + // check; no public GitHub REST runner call is required. + try { + await client.removeRunner(runner.id, { signal: callbackSignal }); + } catch (error) { + if (isScaleSetHttpError(error) && error.status === 404) return { status: 'removed' as const }; + throw error; + } + return { status: 'removed' as const }; + }, + }; + } + + private logReconciliationResult(result: ScaleSetReconcileResult, desiredRunners: number): void { + this.log('info', 'scale_set_reconciled', { + computeProviderType: this.config.computeProvider.type, + desiredRunners, + currentRunners: result.currentRunners, + status: result.status, + actions: result.actions, + errorCount: result.errors.length, + }); + if (result.status === 'retained') { + this.log('warn', 'scale_set_capacity_retained', { + computeProviderType: this.config.computeProvider.type, + desiredRunners, + currentRunners: result.currentRunners, + retainedBusy: result.actions.retainedBusy, + retainedUnknown: result.actions.retainedUnknown, + }); + } + } + + private async reconcileProvider( + provider: ScaleSetComputeProvider, + request: ScaleSetReconcileRequest, + ): Promise { + this.log('info', 'scale_set_compute_provider_reconcile_started', { + computeProviderType: this.config.computeProvider.type, + desiredRunners: request.desiredRunners, + busyRunners: request.busyRunners, + }); + try { + return await provider.reconcile(request); + } catch (error) { + request.signal.throwIfAborted(); + this.log('error', 'scale_set_compute_provider_reconcile_failed', { + computeProviderType: this.config.computeProvider.type, + desiredRunners: request.desiredRunners, + busyRunners: request.busyRunners, + error, + }); + throw new ScaleSetProviderReconciliationError(undefined, { cause: error }); + } + } + + private observeLifecycle(message: RunnerScaleSetMessage): void { + for (const runner of message.jobStartedMessages) + this.rememberLifecycle(runner.runnerId, runner.runnerName, 'started'); + for (const runner of message.jobCompletedMessages) { + this.rememberLifecycle(runner.runnerId, runner.runnerName, 'completed'); + } + } + + private rememberLifecycle(runnerId: number, runnerName: string, lifecycle: ScaleSetRunnerLifecycle): void { + if (!Number.isSafeInteger(runnerId) || runnerId <= 0 || runnerName === '') return; + const current = this.lifecycle.get(runnerName); + if (current !== undefined && current.runnerId !== runnerId) { + this.lifecycle.delete(runnerName); + return; + } + this.lifecycle.set(runnerName, { runnerId, runnerName, scaleSetId: this.scaleSetId, lifecycle }); + while (this.lifecycle.size > this.lifecycleLimit) { + const oldest = this.lifecycle.keys().next().value as string | undefined; + if (oldest === undefined) break; + this.lifecycle.delete(oldest); + } + } + + private pruneCompletedLifecycle(message: RunnerScaleSetMessage): void { + for (const runner of message.jobCompletedMessages) { + const observation = this.lifecycle.get(runner.runnerName); + if (observation?.runnerId === runner.runnerId && observation.lifecycle === 'completed') { + this.lifecycle.delete(runner.runnerName); + } + } + } + + private lifecycleStates(): ScaleSetRunnerState[] { + return [...this.lifecycle.values()].map((observation) => ({ + ...observation, + status: 'unknown', + busy: undefined, + })); + } + + private async closeSession(session: Pick): Promise { + try { + await session.close({ signal: this.dependencies.closeSignal(this.serviceConfig.sessionCloseTimeoutMs) }); + } catch (error) { + this.log('warn', 'scale_set_session_close_failed', { error }); + } + } + + private log( + level: 'debug' | 'info' | 'warn' | 'error', + event: string, + attributes: Record = {}, + ): void { + this.dependencies.logger[level](event, { + groupRunnerConfig: this.config.runnerConfigName, + scaleSetId: this.resolvedScaleSetId, + ...attributes, + }); + } +} + +export function calculateDesiredRunners(totalAssignedJobs: number, minRunners: number, maxRunners: number): number { + if (!Number.isSafeInteger(totalAssignedJobs) || totalAssignedJobs < 0) { + throw new ScaleSetProtocolError('statistics.totalAssignedJobs must be a non-negative integer'); + } + // maxRunners bounds newly requested idle capacity, but an operator reducing + // it must never make already-assigned work a scale-down target. + return Math.max(totalAssignedJobs, Math.min(maxRunners, minRunners + totalAssignedJobs)); +} + +export function calculateReconnectDelay( + attempt: number, + initialBackoffMs: number, + maxBackoffMs: number, + random: () => number = Math.random, +): number { + if (!Number.isSafeInteger(attempt) || attempt <= 0) throw new Error('attempt must be a positive integer'); + const ceiling = Math.min(maxBackoffMs, initialBackoffMs * 2 ** Math.min(attempt - 1, 30)); + const value = Math.max(0, Math.min(1, random())); + return Math.floor(ceiling / 2 + (ceiling / 2) * value); +} + +export async function abortableSleep(delayMs: number, signal: AbortSignal): Promise { + if (signal.aborted || delayMs <= 0) return; + await new Promise((resolve) => { + const done = () => { + clearTimeout(timeout); + signal.removeEventListener('abort', done); + resolve(); + }; + const timeout = setTimeout(done, delayMs); + signal.addEventListener('abort', done, { once: true }); + }); +} + +function uniqueRequestIds(message: RunnerScaleSetMessage): number[] { + return [...new Set(message.jobAvailableMessages.map(({ runnerRequestId }) => runnerRequestId))]; +} + +export function validateProviderResult(result: ScaleSetReconcileResult, desiredRunners: number): void { + const value = result as unknown; + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); + } + const record = value as Record; + const resultFields = new Set(['status', 'desiredRunners', 'currentRunners', 'actions', 'errors']); + if (Object.keys(record).some((key) => !resultFields.has(key))) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); + } + const statuses = new Set(['converged', 'retained', 'error']); + if (!statuses.has(record.status as string)) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid status'); + } + if (record.desiredRunners !== desiredRunners || !boundedCount(record.currentRunners)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid capacity counts'); + } + const actions = record.actions; + if (typeof actions !== 'object' || actions === null || Array.isArray(actions)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid actions'); + } + const actionFields = new Set(['launched', 'terminated', 'retainedBusy', 'retainedUnknown']); + if (Object.keys(actions).some((key) => !actionFields.has(key))) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid action counts'); + } + for (const key of ['launched', 'terminated', 'retainedBusy', 'retainedUnknown']) { + if (!boundedCount((actions as Record)[key])) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid action counts'); + } + } + if (!Array.isArray(record.errors) || record.errors.length > 1000) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid errors'); + } + const operations = new Set([ + 'validate', + 'reconcile', + 'list', + 'launch', + 'generate_jit_configuration', + 'publish_jit_configuration', + 'remove_runner', + 'terminate', + ]); + const errorFields = new Set(['operation', 'code', 'runnerName', 'resourceId']); + for (const error of record.errors) { + if (typeof error !== 'object' || error === null || Array.isArray(error)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); + } + const metadata = error as Record; + if ( + Object.keys(metadata).some((key) => !errorFields.has(key)) || + !operations.has(metadata.operation as string) || + typeof metadata.code !== 'string' || + !/^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(metadata.code) || + !optionalBoundedMetadata(metadata.runnerName) || + !optionalBoundedMetadata(metadata.resourceId) + ) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); + } + } + if ((record.status === 'error') !== record.errors.length > 0) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error status'); + } + let expectedStatus: ScaleSetReconcileResult['status'] = 'converged'; + if (record.errors.length > 0 || (record.currentRunners as number) < desiredRunners) { + expectedStatus = 'error'; + } else if ((record.currentRunners as number) > desiredRunners) { + expectedStatus = 'retained'; + } + if (record.status !== expectedStatus) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid reconciliation status'); + } +} + +function throwIfProviderError(result: ScaleSetReconcileResult): void { + if (result.status === 'error') { + throw new ScaleSetProviderReconciliationError(result); + } +} + +function boundedCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0 && (value as number) <= 2_147_483_647; +} + +function optionalBoundedMetadata(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && value.length <= 256 && !hasAsciiControlCharacter(value)); +} + +function hasAsciiControlCharacter(value: string): boolean { + return [...value].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }); +} + +function isFatalReconcilerError(error: unknown): boolean { + if (error instanceof ScaleSetConfigurationError || error instanceof ScaleSetProtocolError) return true; + if (error instanceof ScaleSetProviderReconciliationError) return true; + if (!isScaleSetHttpError(error)) return false; + return error.status >= 400 && error.status < 500 && ![408, 409, 425, 429].includes(error.status); +} + +function httpErrorLogAttributes(error: unknown): Record { + if (!isScaleSetHttpError(error)) return {}; + return { + requestMethod: error.method, + requestUrl: error.url, + requestStatus: error.status, + requestCode: error.code, + }; +} diff --git a/lambdas/services/scale-set/tsconfig.json b/lambdas/services/scale-set/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/services/scale-set/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/services/scale-set/vitest.config.ts b/lambdas/services/scale-set/vitest.config.ts new file mode 100644 index 0000000000..28a41aa2aa --- /dev/null +++ b/lambdas/services/scale-set/vitest.config.ts @@ -0,0 +1,18 @@ +import { mergeConfig } from 'vitest/config'; +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + root: __dirname, + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/index.ts', 'src/main.ts'], + thresholds: { + statements: 80, + branches: 70, + functions: 80, + lines: 80, + }, + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index d9cd4eccc4..45b8a14249 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -138,6 +138,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-ssm": "npm:^3.1009.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -199,6 +200,32 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/github-actions-scale-set@npm:*, @aws-github-runner/github-actions-scale-set@workspace:libs/github-actions-scale-set": + version: 0.0.0-use.local + resolution: "@aws-github-runner/github-actions-scale-set@workspace:libs/github-actions-scale-set" + dependencies: + "@types/node": "npm:^22.19.3" + typescript: "npm:^5.9.3" + languageName: unknown + linkType: soft + +"@aws-github-runner/scale-set-service@workspace:services/scale-set": + version: 0.0.0-use.local + resolution: "@aws-github-runner/scale-set-service@workspace:services/scale-set" + dependencies: + "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-github-runner/compute-providers": "npm:*" + "@aws-github-runner/github-actions-scale-set": "npm:*" + "@aws-sdk/client-ssm": "npm:^3.1009.0" + "@octokit/auth-app": "npm:8.2.0" + "@octokit/request": "npm:^9.2.2" + "@types/node": "npm:^22.19.3" + "@vercel/ncc": "npm:0.38.4" + typescript: "npm:^5.9.3" + undici: "npm:^6.19.2" + languageName: unknown + linkType: soft + "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" @@ -11025,6 +11052,13 @@ __metadata: languageName: node linkType: hard +"undici@npm:^6.19.2": + version: 6.28.0 + resolution: "undici@npm:6.28.0" + checksum: 10c0/3029a70df06b38b5b2f30732932a1e92544c753cd82c8abdf0d35afad48e0ba91612e79fe3a442dbbb9434d6a9eba2b714d5ea28984c903dda2b5d5444f38354 + languageName: node + linkType: hard + "unicode-canonical-property-names-ecmascript@npm:^2.0.0": version: 2.0.0 resolution: "unicode-canonical-property-names-ecmascript@npm:2.0.0"