diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 568403c3be..ee41d77b41 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -247,8 +247,8 @@ describe('Test simple pool.', () => { }); it('Rejects unsupported pool provider types.', async () => { - await expect(adjust({ poolSize: 10, type: 'microvm' })).rejects.toThrow( - "Unsupported compute provider type 'microvm'", + await expect(adjust({ poolSize: 10, type: 'unsupported-provider' })).rejects.toThrow( + "Unsupported compute provider type 'unsupported-provider'", ); expect(mockListRunners).not.toHaveBeenCalled(); }); diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts new file mode 100644 index 0000000000..4f687b1644 --- /dev/null +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.test.ts @@ -0,0 +1,72 @@ +import { putParameter } from '@aws-github-runner/aws-ssm-util'; +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createStartRunnerConfig } from './github-runner'; +import type { CreateGitHubRunnerConfig } from './types'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), + putParameter: vi.fn(), +})); + +const githubRunnerConfig: CreateGitHubRunnerConfig = { + disableAutoUpdate: true, + enableJitConfig: true, + ephemeral: true, + runnerGroup: 'Default', + runnerLabels: 'self-hosted,linux', + runnerNamePrefix: 'runner-', + runnerOwner: 'octocat/runner', + runnerType: 'Repo', + ssmConfigPath: '/github-action-runners/test/config', + ssmParameterStoreTags: [], + ssmTokenPath: '/github-action-runners/test/tokens', +}; + +const generateRunnerJitconfigForRepo = vi.fn(); +const githubClient = { + actions: { generateRunnerJitconfigForRepo }, +} as unknown as Octokit; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(putParameter).mockResolvedValue(); + generateRunnerJitconfigForRepo.mockResolvedValue({ + data: { + encoded_jit_config: 'encoded-jit-config', + runner: { id: 42 }, + }, + headers: {}, + }); +}); + +describe('createStartRunnerConfig', () => { + it('persists JIT configuration before notifying the provider', async () => { + const onJitConfigCreated = vi.fn(async () => { + expect(putParameter).toHaveBeenCalledWith( + '/github-action-runners/test/tokens/microvm-1', + 'encoded-jit-config', + true, + { tags: [] }, + ); + }); + + await expect( + createStartRunnerConfig(githubRunnerConfig, ['microvm-1'], githubClient, { onJitConfigCreated }), + ).resolves.toEqual([]); + expect(onJitConfigCreated).toHaveBeenCalledWith('microvm-1', { + githubRunnerId: '42', + runnerLabels: ['self-hosted', 'linux'], + }); + }); + + it('reports provider post-write fencing failures while leaving cleanup to the provider', async () => { + const onJitConfigCreated = vi.fn().mockRejectedValue(new Error('cleanup already requested')); + + await expect( + createStartRunnerConfig(githubRunnerConfig, ['microvm-1'], githubClient, { onJitConfigCreated }), + ).resolves.toEqual(['microvm-1']); + expect(putParameter).toHaveBeenCalledOnce(); + }); +}); diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index 745c66770a..db7eb2b5ca 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -264,6 +264,15 @@ function addDelay(runnerIds: string[]) { return { isDelay, delay }; } +function mergeSsmParameterTags( + configuredTags: CreateGitHubRunnerConfig['ssmParameterStoreTags'], + providerTags: CreateGitHubRunnerConfig['ssmParameterStoreTags'], +): CreateGitHubRunnerConfig['ssmParameterStoreTags'] { + const tagsByKey = new Map(configuredTags.map(({ Key, Value }) => [Key, Value])); + for (const { Key, Value } of providerTags) tagsByKey.set(Key, Value); + return [...tagsByKey].map(([Key, Value]) => ({ Key, Value })); +} + /** * Creates registration token configuration for non-ephemeral runners. * @@ -285,7 +294,10 @@ async function createRegistrationTokenConfig( for (const runnerId of runnerIds) { await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], + tags: mergeSsmParameterTags( + githubRunnerConfig.ssmParameterStoreTags, + options.getSsmParameterTags?.(runnerId) ?? [], + ), }); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit @@ -342,17 +354,19 @@ async function createJitConfig( metricGitHubAppRateLimit(runnerConfig.headers, githubRunnerConfig.appIndex); - await options.onJitConfigCreated?.(runnerId, { - githubRunnerId: runnerConfig.data.runner.id.toString(), - runnerLabels, - }); - // store jit config in ssm parameter store logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, }); await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], + tags: mergeSsmParameterTags( + githubRunnerConfig.ssmParameterStoreTags, + options.getSsmParameterTags?.(runnerId) ?? [], + ), + }); + await options.onJitConfigCreated?.(runnerId, { + githubRunnerId: runnerConfig.data.runner.id.toString(), + runnerLabels, }); if (isDelay) { // Delay to prevent AWS ssm rate limits by being within the max throughput limit diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index eb6899ff79..803eb6a659 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -409,6 +409,25 @@ describe('scaleUp with GHES', () => { }); }); + it.each([true, false])( + 'keeps the provider runner identity tag authoritative for ephemeral=%s', + async (ephemeral) => { + process.env.ENABLE_EPHEMERAL_RUNNERS = String(ephemeral); + process.env.RUNNERS_MAXIMUM_COUNT = '2'; + process.env.SSM_PARAMETER_STORE_TAGS = JSON.stringify([ + { Key: 'RunnerId', Value: 'configured-value-cannot-win' }, + { Key: 'CostCenter', Value: '1234' }, + ]); + + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockSSMClient.commandCalls(PutParameterCommand)[0].args[0].input.Tags).toEqual([ + { Key: 'RunnerId', Value: 'i-12345' }, + { Key: 'CostCenter', Value: '1234' }, + ]); + }, + ); + it('quotes runner labels with semicolon separators in non-ephemeral runner config', async () => { process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; process.env.RUNNERS_MAXIMUM_COUNT = '2'; @@ -2157,9 +2176,11 @@ describe('compute provider selection', () => { }); it('rejects unsupported scale-up provider types', async () => { - process.env.COMPUTE_PROVIDER_TYPE = 'microvm'; + process.env.COMPUTE_PROVIDER_TYPE = 'unsupported-provider'; - await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow("Unsupported compute provider type 'microvm'"); + await expect(scaleUpModule.scaleUp(TEST_DATA)).rejects.toThrow( + "Unsupported compute provider type 'unsupported-provider'", + ); expect(mockedAppAuth).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts deleted file mode 100644 index 98bba55b30..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels-policy.ts +++ /dev/null @@ -1 +0,0 @@ -export type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from '@aws-github-runner/compute-providers'; diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts deleted file mode 100644 index 790d4c2989..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { ComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; -import { describe, expect, it } from 'vitest'; - -import type { RunnerMatcherConfig } from '../sqs'; -import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; - -describe('selectAwsDynamicLabelQueue', () => { - it('defaults queues without a provider to EC2 dynamic label handling', () => { - const queue = runnerQueue('default-ec2'); - - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('normalizes compute provider casing and surrounding whitespace', () => { - const queue = runnerQueue('normalized-ec2'); - (queue as unknown as { computeProvider: string }).computeProvider = ' EC2 '; - - expect(selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('skips an unsupported provider strategy and selects the next supported queue', () => { - const unsupportedQueue = runnerQueue('unsupported-provider'); - (unsupportedQueue as unknown as { computeProvider: string }).computeProvider = 'unsupported'; - const ec2Queue = runnerQueue('ec2'); - - expect( - selectAwsDynamicLabelQueue( - [unsupportedQueue, ec2Queue], - ['self-hosted', 'linux'], - ['ghr-ec2-instance-type:t3.large'], - ), - ).toEqual({ - queue: ec2Queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); - }); - - it('rejects a malformed non-string compute provider without throwing', () => { - const queue = runnerQueue('malformed-provider'); - (queue as unknown as { computeProvider: number }).computeProvider = 42; - - expect( - selectAwsDynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); - }); -}); - -function runnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { - return { - id, - arn: `arn:${id}`, - computeProvider, - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - }, - }; -} diff --git a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts b/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts deleted file mode 100644 index 418697398a..0000000000 --- a/lambdas/functions/webhook/src/runners/aws-dynamic-labels.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import type { DynamicLabelDispatchTarget } from '@aws-github-runner/compute-providers'; -import { normalizeComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; -import { webhookProviderRegistry } from '@aws-github-runner/compute-providers/webhook'; - -import type { RunnerMatcherConfig } from '../sqs'; - -const logger = createChildLogger('handler'); - -export function selectAwsDynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - const provider = normalizeComputeProviderType(queue.computeProvider); - const dynamicLabels = provider ? webhookProviderRegistry.capability(provider, 'dynamicLabels') : undefined; - - if (!dynamicLabels) { - logger.warn(`Queue ${queue.id} has unsupported compute provider '${provider ?? String(queue.computeProvider)}'`); - continue; - } - - const target = dynamicLabels.selectQueue({ queue, nonGhrLabels, sanitizedGhrLabels }); - if (target) return target; - } - - return undefined; -} diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index ae571da9d8..bb2cdc7cce 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -1,4 +1,5 @@ import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; import nock from 'nock'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; @@ -14,6 +15,9 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; vi.mock('../sqs'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/compute-providers/webhook', () => ({ + selectDynamicLabelQueue: vi.fn(), +})); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; @@ -246,7 +250,14 @@ describe('Dispatcher', () => { describe('per-matcher dynamic labels handling', () => { const baseRunner = runnerConfig[0]; - it('strips invalid ghr- labels (too long, bad chars) before policy and dispatch', async () => { + beforeEach(() => { + vi.mocked(selectDynamicLabelQueue).mockImplementation((matches, nonGhrLabels, sanitizedGhrLabels) => ({ + queue: matches[0], + labels: [...nonGhrLabels, ...sanitizedGhrLabels], + })); + }); + + it('strips invalid ghr- labels before provider selection and dispatch', async () => { const longLabel = 'ghr-' + 'a'.repeat(125); // 129 chars config = await createConfig(undefined, [ { @@ -276,19 +287,25 @@ describe('Dispatcher', () => { } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); + expect(selectDynamicLabelQueue).toHaveBeenCalledWith( + [expect.objectContaining({ id: baseRunner.id })], + ['self-hosted', 'linux'], + ['ghr-valid:value', 'ghr-list:value;another'], + ); expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ labels: ['self-hosted', 'linux', 'ghr-valid:value', 'ghr-list:value;another'] }), ); }); - it('rejects the job (202) when the only matching runner has enableDynamicLabels=false', async () => { + it('rejects the job when no provider accepts the dynamic labels', async () => { + vi.mocked(selectDynamicLabelQueue).mockReturnValue(undefined); config = await createConfig(undefined, [ { ...baseRunner, matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, - enableDynamicLabels: false, + enableDynamicLabels: true, }, }, ]); @@ -296,7 +313,7 @@ describe('Dispatcher', () => { ...workFlowJobEvent, workflow_job: { ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:value'], }, } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); @@ -304,50 +321,20 @@ describe('Dispatcher', () => { expect(sendActionRequest).not.toHaveBeenCalled(); }); - it('keeps dynamic labels when the matched runner enables them and has no policy', async () => { + it('dispatches to the queue and labels returned by the provider selector', async () => { config = await createConfig(undefined, [ { ...baseRunner, + id: 'first', matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, }, }, - ]); - const event = { - ...workFlowJobEvent, - workflow_job: { - ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }, - } as unknown as WorkflowJobEvent; - const resp = await dispatch(event, 'workflow_job', config); - expect(resp.statusCode).toBe(201); - expect(sendActionRequest).toHaveBeenCalledWith( - expect.objectContaining({ labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'] }), - ); - }); - - it('skips a matching runner whose policy rejects the dynamic labels and uses the next compliant one', async () => { - config = await createConfig(undefined, [ - { - ...baseRunner, - id: 'strict', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - awsDynamicLabelsPolicy: { - restricted_keys: { - 'instance-type': { allowed: ['m5.*'] }, - }, - }, - }, - }, { ...baseRunner, - id: 'permissive', + id: 'selected', matcherConfig: { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, @@ -355,61 +342,29 @@ describe('Dispatcher', () => { }, }, ]); + + vi.mocked(selectDynamicLabelQueue).mockImplementation((matches) => ({ + queue: matches[1], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:normalized'], + })); + const event = { ...workFlowJobEvent, workflow_job: { ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + labels: ['self-hosted', 'linux', 'ghr-provider-setting:requested'], }, } as unknown as WorkflowJobEvent; const resp = await dispatch(event, 'workflow_job', config); expect(resp.statusCode).toBe(201); expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ - queueId: 'permissive', - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], + queueId: 'selected', + labels: ['self-hosted', 'linux', 'ghr-provider-setting:normalized'], }), ); }); - it('rejects the job (202) when no runner accepts the policy', async () => { - config = await createConfig(undefined, [ - { - ...baseRunner, - id: 'first', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: true, - awsDynamicLabelsPolicy: { - restricted_keys: { - 'instance-type': { allowed: ['m5.*'] }, - }, - }, - }, - }, - { - ...baseRunner, - id: 'second', - matcherConfig: { - labelMatchers: [['self-hosted', 'linux']], - exactMatch: true, - enableDynamicLabels: false, - }, - }, - ]); - const event = { - ...workFlowJobEvent, - workflow_job: { - ...workFlowJobEvent.workflow_job, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }, - } as unknown as WorkflowJobEvent; - const resp = await dispatch(event, 'workflow_job', config); - expect(resp.statusCode).toBe(202); - expect(sendActionRequest).not.toHaveBeenCalled(); - }); - it('forwards non-dynamic jobs as-is to the first match', async () => { config = await createConfig(undefined, [ { @@ -419,7 +374,6 @@ describe('Dispatcher', () => { labelMatchers: [['self-hosted', 'linux']], exactMatch: true, enableDynamicLabels: true, - awsDynamicLabelsPolicy: {}, }, }, ]); @@ -435,6 +389,7 @@ describe('Dispatcher', () => { expect(sendActionRequest).toHaveBeenCalledWith( expect.objectContaining({ queueId: 'first', labels: ['self-hosted', 'linux'] }), ); + expect(selectDynamicLabelQueue).not.toHaveBeenCalled(); }); }); }); diff --git a/lambdas/functions/webhook/src/runners/dispatch.ts b/lambdas/functions/webhook/src/runners/dispatch.ts index 47c1f1bfc0..da6dc01221 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.ts @@ -1,11 +1,11 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { Response } from '../lambda'; import { RunnerMatcherConfig, sendActionRequest } from '../sqs'; import ValidationError from '../ValidationError'; import { ConfigDispatcher, ConfigWebhook, QueueSelectionStrategy } from '../ConfigLoader'; -import { selectAwsDynamicLabelQueue } from './aws-dynamic-labels'; import { canRunJob, splitWorkflowJobLabels } from './labels'; const logger = createChildLogger('handler'); @@ -84,7 +84,7 @@ async function handleWorkflowJob( // Dynamic labels present: prefer the first provider-compliant queue. The // queue selection strategy applies to standard jobs only; dynamic-label jobs // always use the first compliant queue. - const dynamicTarget = selectAwsDynamicLabelQueue(matches, nonGhrLabels, sanitizedGhrLabels); + const dynamicTarget = selectDynamicLabelQueue(matches, nonGhrLabels, sanitizedGhrLabels); if (dynamicTarget) { targets = [dynamicTarget.queue]; diff --git a/lambdas/libs/aws-ssm-util/src/index.test.ts b/lambdas/libs/aws-ssm-util/src/index.test.ts index 8a1d8d3864..2f6080bb8a 100644 --- a/lambdas/libs/aws-ssm-util/src/index.test.ts +++ b/lambdas/libs/aws-ssm-util/src/index.test.ts @@ -1,6 +1,9 @@ import { + AddTagsToResourceCommand, + DeleteParameterCommand, GetParameterCommand, GetParameterCommandOutput, + GetParametersByPathCommand, GetParametersCommand, PutParameterCommand, PutParameterCommandOutput, @@ -10,7 +13,17 @@ import 'aws-sdk-client-mock-jest/vitest'; import { mockClient } from 'aws-sdk-client-mock'; import nock from 'nock'; -import { getParameter, getParameters, putParameter, resetSSMClient, ssmClient, SSM_ADVANCED_TIER_THRESHOLD } from '.'; +import { + addParameterTags, + deleteParameter, + getParameter, + getParameters, + getParametersByPath, + putParameter, + resetSSMClient, + ssmClient, + SSM_ADVANCED_TIER_THRESHOLD, +} from '.'; import { describe, it, expect, beforeEach, vi } from 'vitest'; const mockSSMClient = mockClient(SSMClient); @@ -104,6 +117,30 @@ describe('Test getParameter and putParameter', () => { }); }); + it('overwrites a parameter only when explicitly requested', async () => { + mockSSMClient.on(PutParameterCommand).resolves({}); + + await putParameter('testParam', 'updated', false, { overwrite: true }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: 'testParam', + Value: 'updated', + Type: 'String', + Overwrite: true, + }); + }); + + it('rejects tags when overwriting an existing parameter', async () => { + mockSSMClient.resetHistory(); + await expect( + putParameter('testParam', 'updated', false, { + overwrite: true, + tags: [{ Key: 'owner', Value: 'runner' }], + } as never), + ).rejects.toThrow('tags cannot be supplied when overwriting'); + expect(mockSSMClient).not.toHaveReceivedCommand(PutParameterCommand); + }); + it('Puts parameters as SecureString', async () => { // Arrange const parameterValue = 'test'; @@ -256,6 +293,70 @@ describe('Test getParameters (batch)', () => { }); }); +describe('Test direct parameter path operations', () => { + beforeEach(() => { + mockSSMClient.reset(); + }); + + it('paginates direct, non-secret children of a parameter path', async () => { + mockSSMClient + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: undefined, + }) + .resolves({ Parameters: [{ Name: '/metadata/one', Value: '1' }], NextToken: 'page-2' }) + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: 'page-2', + }) + .resolves({ Parameters: [{ Name: '/metadata/two', Value: '2' }] }); + + await expect(getParametersByPath('/metadata')).resolves.toEqual( + new Map([ + ['/metadata/one', '1'], + ['/metadata/two', '2'], + ]), + ); + expect(mockSSMClient).toHaveReceivedCommandTimes(GetParametersByPathCommand, 2); + }); + + it('deletes an exact parameter name', async () => { + mockSSMClient.on(DeleteParameterCommand).resolves({}); + + await deleteParameter('/metadata/one'); + + expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: '/metadata/one' }); + }); + + it('adds tags to an exact parameter name', async () => { + mockSSMClient.on(AddTagsToResourceCommand).resolves({}); + + await addParameterTags('/metadata/one', [{ Key: 'ghr:environment', Value: 'unit-test' }]); + + expect(mockSSMClient).toHaveReceivedCommandWith(AddTagsToResourceCommand, { + ResourceType: 'Parameter', + ResourceId: '/metadata/one', + Tags: [{ Key: 'ghr:environment', Value: 'unit-test' }], + }); + }); + + it('does not call SSM when there are no parameter tags to add', async () => { + await addParameterTags('/metadata/one', []); + + expect(mockSSMClient).not.toHaveReceivedCommand(AddTagsToResourceCommand); + }); + + it('propagates failures when adding parameter tags', async () => { + mockSSMClient.on(AddTagsToResourceCommand).rejects(new Error('AccessDenied')); + + await expect(addParameterTags('/metadata/one', [{ Key: 'Name', Value: 'runner' }])).rejects.toThrow('AccessDenied'); + }); +}); + describe('SSM client configuration', () => { it('configures adaptive retry with a raised attempt cap', async () => { const config = ssmClient().config; diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index 71b33cbf41..ad448b57ac 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -1,4 +1,12 @@ -import { GetParametersCommand, PutParameterCommand, SSMClient, Tag } from '@aws-sdk/client-ssm'; +import { + AddTagsToResourceCommand, + DeleteParameterCommand, + GetParametersByPathCommand, + GetParametersCommand, + PutParameterCommand, + SSMClient, + Tag, +} from '@aws-sdk/client-ssm'; import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; import { SSMProvider } from '@aws-lambda-powertools/parameters/ssm'; @@ -103,14 +111,68 @@ export async function getParameters(parameter_names: string[]): Promise> { + const result = new Map(); + let nextToken: string | undefined; + + do { + const response = await ssmClient().send( + new GetParametersByPathCommand({ + Path: parameter_path, + Recursive: false, + WithDecryption: false, + NextToken: nextToken, + }), + ); + + for (const parameter of response.Parameters ?? []) { + if (parameter.Name && parameter.Value) { + result.set(parameter.Name, parameter.Value); + } + } + nextToken = response.NextToken; + } while (nextToken); + + return result; +} + +export async function deleteParameter(parameter_name: string): Promise { + await ssmClient().send(new DeleteParameterCommand({ Name: parameter_name })); +} + +export async function addParameterTags(parameter_name: string, tags: Tag[]): Promise { + if (tags.length === 0) return; + + await ssmClient().send( + new AddTagsToResourceCommand({ + ResourceType: 'Parameter', + ResourceId: parameter_name, + Tags: tags, + }), + ); +} + export const SSM_ADVANCED_TIER_THRESHOLD = 4000; +type PutParameterOptions = { overwrite: true; tags?: never } | { overwrite?: false | undefined; tags?: Tag[] }; + export async function putParameter( parameter_name: string, parameter_value: string, secure: boolean, - options: { tags?: Tag[] } = {}, + options: PutParameterOptions = {}, ): Promise { + if (options.overwrite && options.tags !== undefined) { + throw new Error('SSM parameter tags cannot be supplied when overwriting an existing parameter'); + } + const client = ssmClient(); // Determine tier based on parameter_value size @@ -121,6 +183,7 @@ export async function putParameter( Name: parameter_name, Value: parameter_value, Type: secure ? 'SecureString' : 'String', + Overwrite: options.overwrite, Tags: options.tags, Tier: valueSizeBytes >= SSM_ADVANCED_TIER_THRESHOLD ? 'Advanced' : 'Standard', }), diff --git a/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts new file mode 100644 index 0000000000..64b7507add --- /dev/null +++ b/lambdas/libs/compute-providers/aws/dynamic-labels-policy.ts @@ -0,0 +1,61 @@ +import type { AwsDynamicLabelsPolicy } from '../contracts'; + +function globToRegExp(glob: string): RegExp { + const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); + return new RegExp(`^${pattern}$`); +} + +function matchesAny(value: string, patterns: string[] | undefined): boolean { + if (!patterns || patterns.length === 0) return false; + return patterns.some((pattern) => globToRegExp(pattern).test(value)); +} + +function evaluateLabel(label: string, policy: AwsDynamicLabelsPolicy, labelPrefix: string): string | null { + const stripped = label.slice(labelPrefix.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? undefined : stripped.slice(colonIndex + 1); + + if (policy.blocked_keys?.includes(key)) { + return `key '${key}' is in blocked_keys`; + } + + const rule = policy.restricted_keys?.[key]; + if (!rule || value === undefined) return null; + + if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { + return `value '${value}' not in allowed list`; + } + if (rule.denied && matchesAny(value, rule.denied)) { + return `value '${value}' in denied list`; + } + if (rule.max !== undefined && rule.max !== null) { + const valueNumber = Number(value); + const maximum = Number(rule.max); + if (!Number.isFinite(valueNumber) || !Number.isFinite(maximum)) { + return `max set but value '${value}' or max '${rule.max}' is not numeric`; + } + if (valueNumber > maximum) { + return `value '${value}' exceeds max '${rule.max}'`; + } + } + + return null; +} + +export function violationsAgainstAwsDynamicLabelsPolicy( + labels: string[], + policy: AwsDynamicLabelsPolicy | null | undefined, + labelPrefix: string, +): { label: string; reason: string }[] { + if (!policy) return []; + + const violations: { label: string; reason: string }[] = []; + for (const label of labels) { + if (!label.startsWith(labelPrefix)) continue; + const reason = evaluateLabel(label, policy, labelPrefix); + if (reason) violations.push({ label, reason }); + } + return violations; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts index a9b919c7bd..8babbadd55 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels-policy.ts @@ -1,4 +1,5 @@ import type { AwsDynamicLabelsPolicy, AwsDynamicLabelsValueRule } from '../../../../contracts'; +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; export type Ec2DynamicLabelsValueRule = AwsDynamicLabelsValueRule; @@ -10,50 +11,6 @@ export type Ec2DynamicLabelsValueRule = AwsDynamicLabelsValueRule; */ export type Ec2DynamicLabelsPolicy = AwsDynamicLabelsPolicy; -function globToRegExp(glob: string): RegExp { - const escaped = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&'); - const pattern = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); - return new RegExp(`^${pattern}$`); -} - -function matchesAny(value: string, patterns: string[] | undefined): boolean { - if (!patterns || patterns.length === 0) return false; - return patterns.some((p) => globToRegExp(p).test(value)); -} - -function evaluateLabel(label: string, policy: Ec2DynamicLabelsPolicy): string | null { - const stripped = label.replace(/^ghr-ec2-/, ''); - const colonIdx = stripped.indexOf(':'); - const key = colonIdx === -1 ? stripped : stripped.slice(0, colonIdx); - const value = colonIdx === -1 ? undefined : stripped.slice(colonIdx + 1); - - if (policy.blocked_keys?.includes(key)) { - return `key '${key}' is in blocked_keys`; - } - - const rule = policy.restricted_keys?.[key]; - if (!rule) return null; - if (value === undefined) return null; - - if (rule.allowed && rule.allowed.length > 0 && !matchesAny(value, rule.allowed)) { - return `value '${value}' not in allowed list`; - } - if (rule.denied && matchesAny(value, rule.denied)) { - return `value '${value}' in denied list`; - } - if (rule.max !== undefined && rule.max !== null) { - const valueNum = Number(value); - const maxNum = Number(rule.max); - if (!Number.isFinite(valueNum) || !Number.isFinite(maxNum)) { - return `max set but value '${value}' or max '${rule.max}' is not numeric`; - } - if (valueNum > maxNum) { - return `value '${value}' exceeds max '${rule.max}'`; - } - } - return null; -} - /** * Inspects the labels and returns the rejection reasons for any `ghr-ec2-*` * label that violates the policy. Non-`ghr-ec2-*` labels are ignored. @@ -62,12 +19,5 @@ export function violationsAgainstPolicy( labels: string[], policy: Ec2DynamicLabelsPolicy | null | undefined, ): { label: string; reason: string }[] { - if (!policy) return []; - const violations: { label: string; reason: string }[] = []; - for (const label of labels) { - if (!label.startsWith('ghr-ec2-')) continue; - const reason = evaluateLabel(label, policy); - if (reason) violations.push({ label, reason }); - } - return violations; + return violationsAgainstAwsDynamicLabelsPolicy(labels, policy, 'ghr-ec2-'); } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts index 400807554f..99c1844a5f 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.test.ts @@ -1,18 +1,38 @@ import { describe, expect, it } from 'vitest'; import type { RunnerMatcherConfig } from '../../../../contracts'; -import { selectEc2DynamicLabelQueue } from './dynamic-labels'; +import { ec2DynamicLabelProvider } from './dynamic-labels'; + +describe('ec2DynamicLabelProvider', () => { + it('returns no violations when the queue has no policy', () => { + const queue = runnerQueue('no-policy'); + + expect(getViolations(queue)).toEqual([]); + }); + + it('returns violations for labels rejected by the policy', () => { + const strictQueue = runnerQueue('strict'); + strictQueue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'instance-type': { allowed: ['m5.*'] }, + }, + }; + + expect(getViolations(strictQueue)).toEqual([ + { + label: 'ghr-ec2-instance-type:t3.large', + reason: "value 't3.large' not in allowed list", + }, + ]); + }); -describe('selectEc2DynamicLabelQueue', () => { it('enforces a legacy EC2 dynamic labels policy when the new key is absent', () => { const queue = runnerQueue('legacy-ec2-policy'); queue.matcherConfig.ec2DynamicLabelsPolicy = { blocked_keys: ['instance-type'], }; - expect( - selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + expect(getViolations(queue)).toHaveLength(1); }); it('falls back to the legacy EC2 dynamic labels policy when the new policy is null', () => { @@ -22,9 +42,7 @@ describe('selectEc2DynamicLabelQueue', () => { }; queue.matcherConfig.awsDynamicLabelsPolicy = null; - expect( - selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large']), - ).toBeUndefined(); + expect(getViolations(queue)).toHaveLength(1); }); it('prefers a configured AWS dynamic labels policy over the legacy policy', () => { @@ -36,13 +54,17 @@ describe('selectEc2DynamicLabelQueue', () => { blocked_keys: [], }; - expect(selectEc2DynamicLabelQueue([queue], ['self-hosted', 'linux'], ['ghr-ec2-instance-type:t3.large'])).toEqual({ - queue, - labels: ['self-hosted', 'linux', 'ghr-ec2-instance-type:t3.large'], - }); + expect(getViolations(queue)).toEqual([]); }); }); +function getViolations(queue: RunnerMatcherConfig) { + return ec2DynamicLabelProvider.getViolations({ + queue, + labels: ['ghr-ec2-instance-type:t3.large'], + }); +} + function runnerQueue(id: string): RunnerMatcherConfig { return { id, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts index 6ddf5b8fbb..5e671da189 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/webhook/dynamic-labels.ts @@ -1,12 +1,10 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import type { DynamicLabelDispatchTarget, DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; +import type { DynamicLabelProvider, RunnerMatcherConfig } from '../../../../contracts'; import { violationsAgainstPolicy } from './dynamic-labels-policy'; const logger = createChildLogger('handler'); -export type Ec2DynamicLabelDispatchTarget = DynamicLabelDispatchTarget; - function resolveEc2DynamicLabelsPolicy(queue: RunnerMatcherConfig) { const hasLegacyEc2DynamicLabelsPolicy = Object.prototype.hasOwnProperty.call( queue.matcherConfig, @@ -23,36 +21,6 @@ function resolveEc2DynamicLabelsPolicy(queue: RunnerMatcherConfig) { return queue.matcherConfig.awsDynamicLabelsPolicy; } -export function selectEc2DynamicLabelQueue( - matches: RunnerMatcherConfig[], - nonGhrLabels: string[], - sanitizedGhrLabels: string[], -): Ec2DynamicLabelDispatchTarget | undefined { - for (const queue of matches) { - if (!queue.matcherConfig.enableDynamicLabels) { - logger.warn(`Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`); - continue; - } - - const violations = violationsAgainstPolicy(sanitizedGhrLabels, resolveEc2DynamicLabelsPolicy(queue)); - if (violations.length === 0) { - return { - queue, - labels: [...nonGhrLabels, ...sanitizedGhrLabels], - }; - } - - for (const violation of violations) { - logger.warn( - `Queue ${queue.id}: dynamic label '${violation.label}' does not match policy (${violation.reason}); trying next match`, - ); - } - } - - return undefined; -} - export const ec2DynamicLabelProvider: DynamicLabelProvider = { - selectQueue: ({ queue, nonGhrLabels, sanitizedGhrLabels }) => - selectEc2DynamicLabelQueue([queue], nonGhrLabels, sanitizedGhrLabels), + getViolations: ({ queue, labels }) => violationsAgainstPolicy(labels, resolveEc2DynamicLabelsPolicy(queue)), }; diff --git a/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts new file mode 100644 index 0000000000..755831fb91 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/webhook.test.ts @@ -0,0 +1,27 @@ +import { defineWebhookProviderContractTests } from '../../test/webhook-provider-contract'; +import { provider } from './webhook'; + +defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels: ['ghr-ec2-instance-type:t3.large'], + rejectingPolicies: [ + { + name: 'blocked keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + blocked_keys: ['instance-type'], + }; + }, + }, + { + name: 'restricted keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'instance-type': { allowed: ['m5.*'] }, + }, + }; + }, + }, + ], +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md new file mode 100644 index 0000000000..bc672dd2a2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -0,0 +1,149 @@ +# Lambda MicroVM compute provider + +This provider manages a compatible AWS Lambda MicroVM image through the control-plane Lambda. It currently supports ephemeral JIT runners only. + +The MicroVM image `/run` hook receives this `runHookPayload`: + +```json +{ + "version": 1, + "imageArn": "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner", + "imageVersion": "12.0", + "runnerConfigSsmPath": "/github-action-runners/example/runners/config", + "runnerTokenSsmPath": "/github-action-runners/example/runners/tokens" +} +``` + +Lambda adds `microvmId` beside that payload. `imageArn` and `imageVersion` are the requested launch values and are included together when an explicit image version is selected. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. The image separately polls its complete non-secret tag map at `/microvm-metadata/.tags`. The control plane stores the JIT parameter before the provider callback writes the tag map, preventing cleanup from deleting an absent JIT that could otherwise be recreated later. Neither metadata record contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. + +Runner ownership and lifecycle state are stored separately as non-secret `String` +parameters under `/`. The immutable base +record and independent state parameters prevent concurrent GitHub ID, orphan, +and cleanup updates from overwriting one another. Deleting the JIT SecureString +does not delete this metadata. Use a dedicated metadata prefix that does not +overlap the JIT path, and grant the MicroVM execution role only the exact +value-read access described below, without path-listing permissions. The control +plane retries pending cleanup, removes metadata after termination, and reconciles +expired records during inventory. + +The immutable base metadata parameter carries the same AWS resource tags that +are serialized as a JSON object in the `.tags` parameter. The tag +set starts with `SSM_PARAMETER_STORE_TAGS`, omits `Name`, and derives +`ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` from +the existing `ENVIRONMENT`, `SSM_CONFIG_PATH`, and `RUNNER_NAME_PREFIX` +settings. The Lambda then adds authoritative runtime tags: +`ghr:Application`, `ghr:created_by`, `ghr:environment`, `ghr:Owner`, +`ghr:Type`, `ghr:microvm_id`, `ghr:microvm_image_arn`, and, when available, +`ghr:microvm_image_version`. After JIT registration, the control plane adds +`ghr:github_runner_id` and base64url-encoded runner-label groups under +`ghr:runner_labels` through `ghr:runner_labels:5`. Runtime-owned values override +configured collisions. The `aws:` tag prefix is reserved and cannot be used for +these SSM parameters. The `.tags` value may use the Parameter Store advanced +tier when its UTF-8 representation is at least 4,000 bytes and is rejected if +the complete value could exceed the 8 KiB Parameter Store limit. + +Final cleanup deletes `/`, the +`.github-runner-id`, `.orphan`, and `.tags` companions, the base ownership +record, and `.cleanup-requested-at` last. The tombstone keeps its original +timestamp through a five-minute grace window so cleanup can repeatedly revoke a +late JIT write before removing every record. Missing parameters are treated as +already cleaned. + +The runner configuration publishes `/enable_cloudwatch` +and, when enabled, `/cloudwatch_agent_config_runner`. +The generated agent configuration reads these image-owned files by default: + +- `/var/log/microvm/internal-services.log` +- `/var/log/microvm/run.log` +- `/opt/actions-runner/_diag/Runner_**.log` + +Their default log-group suffixes are `internal_service`, `run`, and `runner`, +and `{microvm_id}` is an image-expanded log-stream placeholder. The first two +files are part of the MicroVM image contract; the portable lifecycle hook does +not create CloudWatch-specific files. Native RunMicrovm stdout and stderr stay +enabled independently as the early-startup and failure backstop. + +The control-plane Lambda requires these provider environment variables: + +- `MICROVM_IMAGE_ARN` +- `MICROVM_EXECUTION_ROLE_ARN` +- `MICROVM_IMAGE_VERSION` (optional) +- `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) +- `MICROVM_LOG_GROUP` (optional) +- `SSM_TOKEN_PATH` (lane-scoped JIT parameter path) + +Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). + +The control-plane role requires `ssm:GetParametersByPath`, `ssm:GetParameters`, +`ssm:PutParameter`, `ssm:AddTagsToResource`, and `ssm:DeleteParameter` on the +dedicated metadata prefix, plus a separate `ssm:DeleteParameter` grant on the +lane-scoped JIT prefix, and `lambda:ListMicrovms`, `lambda:RunMicrovm`, and +`lambda:TerminateMicrovm` for inventory and lifecycle reconciliation. Restrict +`lambda:RunMicrovm` and `lambda:TerminateMicrovm` to approved image resources; +`lambda:ListMicrovms` does not support resource-level permissions. + +The MicroVM execution role must trust `lambda.amazonaws.com` for both +`sts:AssumeRole` and `sts:TagSession`. Restrict `iam:PassRole` to that exact role +ARN. Network connectors also require `lambda:PassNetworkConnector`; because +that action does not currently support resource-level permissions, enforce the +connector boundary with the explicit dynamic-label allowlist described below. + +All MicroVMs using one execution role, JIT prefix, and metadata prefix share a +trust boundary. Grant that role only `ssm:GetParameter` on the Parameter Store +ARN corresponding to `/microvm-metadata/*` and the exact +CloudWatch configuration parameters, `ssm:GetParameter` and +`ssm:DeleteParameter` on the lane-scoped JIT prefix, and stream-write access to +the provider-managed log groups. The image must address its own metadata with +its AWS-provided `microvmId` and must not receive path-listing access. IAM cannot +bind that ID to the calling MicroVM session, so a MicroVM can read other +metadata records in the same lane if it learns their IDs. Only allow trusted +images and workloads within a shared role, or isolate trust domains with +separate roles, prefixes, and provider deployments. + +## Dynamic labels + +When a runner matcher enables dynamic labels, workflow jobs can override the +following `RunMicrovm` inputs: + +| Label | Override | +| --------------------------------------------- | -------------------------------- | +| `ghr-microvm-egress-network-connectors:` | One egress network connector ARN | +| `ghr-microvm-image-arn:` | MicroVM image ARN | +| `ghr-microvm-image-version:` | MicroVM image version | + +Repeat `ghr-microvm-egress-network-connectors:` to attach multiple +connectors. Specify one ARN per label; `RunMicrovm` accepts at most 10. These +labels replace the compute provider's configured +`MICROVM_EGRESS_NETWORK_CONNECTORS` value for that job. + +Lambda MicroVM does not expose CPU or memory as `RunMicrovm` inputs. Select an +image and version with the required resources instead. Labels such as +`ghr-microvm-memory` are rejected. + +Execution roles, ingress network connectors, logging, idle policy, run hook +payloads, and client tokens remain deployment-controlled. Image ARN, image +version, and egress connector overrides change executable code or the network +boundary, so they are rejected unless `awsDynamicLabelsPolicy` supplies an +explicit `allowed` list for the corresponding key. + +Use the matcher's `awsDynamicLabelsPolicy` to restrict values accepted from +workflow jobs. The MicroVM policy keys are `egress-network-connectors`, +`image-arn`, and `image-version`. For example: + +```json +{ + "restricted_keys": { + "egress-network-connectors": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-*"] + }, + "image-arn": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:microvm-image:github-runner-*"] + }, + "image-version": { + "allowed": ["3.*"] + } + } +} +``` diff --git a/lambdas/libs/compute-providers/aws/microvm/control-plane.ts b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts new file mode 100644 index 0000000000..d6287ca1e1 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts @@ -0,0 +1,25 @@ +import type { ComputeProviderPlugin, CreateStartRunnerConfig } from '../../core'; + +import type { ControlPlaneProviderCapabilities, ControlPlaneProviderModule } from '../../contracts'; +import type {} from './src/environment'; +import { createMicrovmPoolProvider } from './src/control-plane/pool'; +import { createMicrovmScaleDownProvider } from './src/control-plane/scale-down'; +import { createMicrovmScaleUpProvider } from './src/control-plane/scale-up'; + +export function createMicrovmControlPlanePlugin( + createStartRunnerConfig: CreateStartRunnerConfig, +): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { + pool: () => createMicrovmPoolProvider(createStartRunnerConfig), + scaleUp: () => createMicrovmScaleUpProvider(createStartRunnerConfig), + scaleDown: createMicrovmScaleDownProvider, + }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmControlPlanePlugin, +} satisfies ControlPlaneProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts new file mode 100644 index 0000000000..e58d73093c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; + +const cleanEnv = process.env; + +beforeEach(() => { + process.env = { ...cleanEnv }; + process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; + process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; + process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/unit-test/token/'; + delete process.env.MICROVM_IMAGE_VERSION; + delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_LOG_GROUP; +}); + +describe('loadMicrovmProviderConfig', () => { + it('loads required values and applies optional defaults', () => { + expect(loadMicrovmProviderConfig()).toEqual({ + imageIdentifier: process.env.MICROVM_IMAGE_ARN, + imageVersion: undefined, + executionRoleArn: process.env.MICROVM_EXECUTION_ROLE_ARN, + ingressNetworkConnectors: undefined, + egressNetworkConnectors: undefined, + metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', + runnerTokenSsmPath: '/github-action-runners/unit-test/token', + logging: undefined, + }); + }); + + it('loads versions, logging, and either connector list format', () => { + process.env.MICROVM_IMAGE_VERSION = ' 3.0 '; + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; + process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; + + expect(loadMicrovmProviderConfig()).toMatchObject({ + imageVersion: '3.0', + ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], + egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, + }); + }); + + it.each([ + ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], + ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], + ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], + ['SSM_TOKEN_PATH', 'SSM_TOKEN_PATH'], + ])('requires %s', (environmentVariable, expectedName) => { + delete process.env[environmentVariable]; + + expect(() => loadMicrovmProviderConfig()).toThrow( + `${expectedName} must be configured for the MicroVM compute provider`, + ); + }); + + it.each(['[not-json', '[]', '["valid", 2]', 'first,'])('rejects malformed connector lists %s', (connectors) => { + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = connectors; + + expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_EGRESS_NETWORK_CONNECTORS must/); + }); + + it.each(['metadata', '/', '/metadata//nested', '/metadata/../nested', '/metadata/has space'])( + 'rejects malformed metadata SSM path %s', + (metadataPath) => { + process.env.MICROVM_METADATA_SSM_PATH = metadataPath; + + expect(() => loadMicrovmProviderConfig()).toThrow( + 'MICROVM_METADATA_SSM_PATH must be a valid absolute SSM parameter path', + ); + }, + ); + + it.each(['token', '/', '/token//nested', '/token/../nested', '/token/has space'])( + 'rejects malformed JIT SSM path %s', + (tokenPath) => { + process.env.SSM_TOKEN_PATH = tokenPath; + + expect(() => loadMicrovmProviderConfig()).toThrow('SSM_TOKEN_PATH must be a valid absolute SSM parameter path'); + }, + ); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts new file mode 100644 index 0000000000..b86331967b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -0,0 +1,78 @@ +import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +export interface MicrovmProviderConfig { + egressNetworkConnectors?: string[]; + executionRoleArn: string; + imageIdentifier: string; + imageVersion?: string; + ingressNetworkConnectors?: string[]; + logging?: Logging; + metadataSsmPath: string; + runnerTokenSsmPath: string; +} + +function requiredEnvironmentValue(name: string, value: string | undefined): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error(`${name} must be configured for the MicroVM compute provider`); + } + return trimmed; +} + +function optionalEnvironmentValue(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function parseSsmPath(name: string, value: string | undefined): string { + const path = requiredEnvironmentValue(name, value).replace(/\/+$/, ''); + if (path === '' || !/^\/[A-Za-z0-9_.\-/]+$/.test(path) || path.includes('//') || path.split('/').includes('..')) { + throw new Error(`${name} must be a valid absolute SSM parameter path`); + } + return path; +} + +function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { + const configuredValue = optionalEnvironmentValue(value); + if (!configuredValue) return undefined; + + let connectors: unknown; + try { + connectors = configuredValue.startsWith('[') + ? JSON.parse(configuredValue) + : configuredValue.split(',').map((connector) => connector.trim()); + } catch (error) { + throw new Error(`${name} must be a JSON array or comma-separated list`, { cause: error }); + } + + if ( + !Array.isArray(connectors) || + connectors.length === 0 || + connectors.some((connector) => typeof connector !== 'string' || connector.trim().length === 0) + ) { + throw new Error(`${name} must contain one or more non-empty connector ARNs`); + } + + return connectors.map((connector) => connector.trim()); +} + +export function loadMicrovmProviderConfig(): MicrovmProviderConfig { + const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); + + return { + imageIdentifier: requiredEnvironmentValue('MICROVM_IMAGE_ARN', process.env.MICROVM_IMAGE_ARN), + imageVersion: optionalEnvironmentValue(process.env.MICROVM_IMAGE_VERSION), + executionRoleArn: requiredEnvironmentValue('MICROVM_EXECUTION_ROLE_ARN', process.env.MICROVM_EXECUTION_ROLE_ARN), + ingressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_INGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS, + ), + egressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_EGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, + ), + metadataSsmPath: parseSsmPath('MICROVM_METADATA_SSM_PATH', process.env.MICROVM_METADATA_SSM_PATH), + runnerTokenSsmPath: parseSsmPath('SSM_TOKEN_PATH', process.env.SSM_TOKEN_PATH), + logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts new file mode 100644 index 0000000000..09b6a46f8d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts @@ -0,0 +1 @@ +export const MICROVM_LIFETIME_IN_SECONDS = 28_800; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts new file mode 100644 index 0000000000..3271fd8b98 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -0,0 +1,418 @@ +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + RunMicrovmCommand, + TerminateMicrovmCommand, +} from '@aws-sdk/client-lambda-microvms'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MicrovmProviderConfig } from './config'; +import { + isRetryableMicrovmError, + listMicrovmRunners, + microvmBootTimeExceeded, + runMicrovmRunner, + terminateMicrovm, +} from './microvms'; +import { + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), + createMicrovmRunnerMetadata: vi.fn(), + deleteMicrovmRunnerJitConfig: vi.fn(), + listMicrovmRunnerMetadata: vi.fn(), + markMicrovmCleanupPending: vi.fn(), +})); + +const mockMicrovmClient = mockClient(LambdaMicrovmsClient); +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; +const config: MicrovmProviderConfig = { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + egressNetworkConnectors: ['arn:egress'], + metadataSsmPath, + runnerTokenSsmPath, + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, +}; +const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; + +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-managed', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.0', + createdAt: '2026-08-06T10:00:00.000Z', + expiresAt: '2026-08-06T11:00:00.000Z', + ...overrides, + }; +} + +beforeEach(() => { + mockMicrovmClient.reset(); + vi.clearAllMocks(); + vi.useRealTimers(); + delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; + vi.mocked(createMicrovmRunnerMetadata).mockResolvedValue(ssmParameterStoreTags); + vi.mocked(deleteMicrovmRunnerJitConfig).mockResolvedValue(); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ cleanupMicrovmIds: [], metadataById: new Map() }); + vi.mocked(markMicrovmCleanupPending).mockResolvedValue(); +}); + +describe('runMicrovmRunner', () => { + it('launches a runner for the fixed lifetime and records durable ownership metadata', async () => { + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123', imageArn, imageVersion: '3.1' }); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{"version":1}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags, + source: 'scale-up-lambda', + }), + ).resolves.toEqual({ microvmId: 'mvm-123', metadataTags: ssmParameterStoreTags }); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(RunMicrovmCommand, { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: config.executionRoleArn, + egressNetworkConnectors: ['arn:egress'], + maximumDurationInSeconds: 28_800, + logging: config.logging, + runHookPayload: '{"version":1}', + clientToken: expect.any(String), + }); + expect(createMicrovmRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, { + microvmId: 'mvm-123', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.1', + ssmParameterStoreTags, + }); + }); + + it('rejects invalid metadata tags before launching a MicroVM', async () => { + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: imageArn }], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + expect(mockMicrovmClient).not.toHaveReceivedCommand(RunMicrovmCommand); + }); + + it('rejects a launch response without an ID', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'pool-lambda', + }), + ).rejects.toThrow('RunMicrovm returned no microvmId'); + }); + + it('terminates a new runner when required metadata cannot be recorded', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('metadata failed'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-untracked', + }); + }); + + it('preserves the metadata error when termination also fails', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('metadata failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-untracked'); + }); +}); + +describe('listMicrovmRunners', () => { + it('paginates active MicroVMs and filters them by durable metadata', async () => { + const startedAt = new Date('2026-08-06T10:00:00.000Z'); + mockMicrovmClient + .on(ListMicrovmsCommand) + .resolvesOnce({ + nextToken: 'page-2', + items: [ + { microvmId: 'mvm-managed', imageArn, imageVersion: '3.0', startedAt, state: 'RUNNING' }, + { microvmId: 'mvm-terminated', imageArn, imageVersion: '3.0', startedAt, state: 'TERMINATED' }, + ], + }) + .resolvesOnce({ + items: [{ microvmId: 'mvm-other', imageArn, imageVersion: '3.0', startedAt, state: 'PENDING' }], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-managed', metadata({ githubRunnerId: '42', bypassRemoval: true })], + ['mvm-other', metadata({ microvmId: 'mvm-other', runnerOwner: 'Other' })], + ]), + }); + + await expect( + listMicrovmRunners( + { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }, + ssmPaths, + ), + ).resolves.toEqual([ + { + id: 'mvm-managed', + imageArn, + launchTime: startedAt, + owner: 'Codertocat', + type: 'Org', + orphan: false, + githubRunnerId: '42', + bypassRemoval: true, + state: 'RUNNING', + }, + ]); + + expect(mockMicrovmClient).toHaveReceivedNthCommandWith(2, ListMicrovmsCommand, { + maxResults: 50, + nextToken: 'page-2', + }); + expect(listMicrovmRunnerMetadata).toHaveBeenCalledWith( + ssmPaths, + new Map([ + ['mvm-managed', 'RUNNING'], + ['mvm-terminated', 'TERMINATED'], + ['mvm-other', 'PENDING'], + ]), + ); + }); + + it('applies environment, owner, type, and orphan filters after loading metadata', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { + microvmId: 'mvm-filtered', + imageArn, + imageVersion: '3.0', + startedAt: new Date(), + state: 'SUSPENDED', + }, + ], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + [ + 'mvm-filtered', + metadata({ microvmId: 'mvm-filtered', environment: 'other', runnerOwner: 'Other', runnerType: 'Repo' }), + ], + ]), + }); + + await expect(listMicrovmRunners({ environment: 'unit-test' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerType: 'Org' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ orphan: true }, ssmPaths)).resolves.toEqual([]); + }); + + it('fails closed for an image mismatch while ignoring unowned MicroVMs', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { microvmId: 'mvm-missing', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-mismatch', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + ], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-mismatch', metadata({ microvmId: 'mvm-mismatch', imageArn: imageArn.replace(':runner', ':other') })], + ]), + }); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('does not match its metadata'); + }); + + it('attempts every pending cleanup and fails inventory closed when a retry fails', async () => { + const cleanupFailure = new Error('cleanup failed'); + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { microvmId: 'mvm-first', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-second', imageArn, imageVersion: '3.0', state: 'PENDING' }, + ], + }); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-first' }).rejects(cleanupFailure); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-second' }).resolves({}); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: ['mvm-first', 'mvm-second'], + metadataById: new Map(), + }); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('cleanup failed'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-first', + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-second', + }); + expect(markMicrovmCleanupPending).toHaveBeenCalledTimes(2); + }); + + it('surfaces metadata lookup failures instead of reporting zero runners', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [{ microvmId: 'mvm-error', imageArn, imageVersion: '3.0', state: 'RUNNING' }], + }); + vi.mocked(listMicrovmRunnerMetadata).mockRejectedValue(new Error('AccessDenied')); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('AccessDenied'); + }); +}); + +describe('MicroVM lifecycle helpers', () => { + it('retains metadata until inventory observes a terminated MicroVM', async () => { + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await terminateMicrovm('mvm-123', ssmPaths); + + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('retains the tombstone when the MicroVM is already terminated so a late JIT write can be revoked', async () => { + const notFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(notFound); + + await expect(terminateMicrovm('mvm-gone', ssmPaths)).resolves.toBeUndefined(); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-gone'); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-gone'); + }); + + it('retains metadata and marks cleanup pending when termination fails', async () => { + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toThrow('terminate failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('retains the cleanup marker and reports a JIT deletion failure after termination succeeds', async () => { + const error = new Error('JIT cleanup failed'); + vi.mocked(deleteMicrovmRunnerJitConfig).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('still terminates and reports a cleanup-marker failure for retry', async () => { + const error = new Error('metadata cleanup marker failed'); + vi.mocked(markMicrovmCleanupPending).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-123', + }); + }); + + it('evaluates the configured boot window', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-06T10:10:00.000Z')); + + expect(microvmBootTimeExceeded({})).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:06:00.000Z') })).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:04:00.000Z') })).toBe(true); + }); +}); + +describe('isRetryableMicrovmError', () => { + it.each([ + 'ConflictException', + 'InternalServerException', + 'ServiceQuotaExceededException', + 'ThrottlingException', + 'TooManyUpdates', + ])('classifies %s as retryable', (name) => { + expect(isRetryableMicrovmError(Object.assign(new Error(name), { name }))).toBe(true); + }); + + it('classifies server, throttling, network, and nested failures as retryable', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('server'), { $fault: 'server' }))).toBe(true); + expect(isRetryableMicrovmError(Object.assign(new Error('throttle'), { $metadata: { httpStatusCode: 429 } }))).toBe( + true, + ); + expect(isRetryableMicrovmError(Object.assign(new Error('network'), { code: 'ECONNRESET' }))).toBe(true); + expect( + isRetryableMicrovmError( + Object.assign(new Error('outer'), { cause: Object.assign(new Error(), { code: 'ETIMEDOUT' }) }), + ), + ).toBe(true); + }); + + it('does not retry configuration, unknown, or non-error failures', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('invalid'), { name: 'ValidationException' }))).toBe(false); + expect(isRetryableMicrovmError(new Error('unknown'))).toBe(false); + expect(isRetryableMicrovmError('failure')).toBe(false); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts new file mode 100644 index 0000000000..5ffd6d74b5 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -0,0 +1,275 @@ +import { randomUUID } from 'node:crypto'; + +import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + RunMicrovmCommand, + TerminateMicrovmCommand, +} from '@aws-sdk/client-lambda-microvms'; +import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +import type { + CreateGitHubRunnerConfig, + LambdaRunnerSource, + ListRunnerFilters, + RunnerInfo, + RunnerType, +} from '../../../../core'; +import { loadMicrovmProviderConfig, type MicrovmProviderConfig } from './config'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; +import { + assertValidMicrovmMetadataTags, + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + type MicrovmSsmPaths, +} from './runner-metadata'; + +const logger = createChildLogger('microvm-runners'); + +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); + +export interface MicrovmRunnerInfo extends RunnerInfo { + imageArn?: string; + state?: MicrovmState; +} + +export interface RunMicrovmRunnerInput { + config: MicrovmProviderConfig; + environment: string; + runHookPayload: string; + runnerOwner: string; + runnerType: RunnerType; + ssmParameterStoreTags: CreateGitHubRunnerConfig['ssmParameterStoreTags']; + source: LambdaRunnerSource; +} + +export interface RunMicrovmRunnerResult { + metadataTags: CreateGitHubRunnerConfig['ssmParameterStoreTags']; + microvmId: string; +} + +interface AwsErrorLike extends Error { + cause?: unknown; + code?: string; + $fault?: 'client' | 'server'; + $metadata?: { httpStatusCode?: number }; +} + +const RETRYABLE_ERROR_NAMES = new Set([ + 'ConflictException', + 'InternalServerException', + 'RequestTimeout', + 'RequestTimeoutException', + 'ResourceConflictException', + 'ServiceException', + 'ServiceQuotaExceededException', + 'Throttling', + 'ThrottlingException', + 'TooManyUpdates', + 'TooManyRequestsException', +]); + +const RETRYABLE_NETWORK_ERROR_CODES = new Set([ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'ENETUNREACH', + 'ENOTFOUND', + 'ETIMEDOUT', +]); + +function microvmClient(): LambdaMicrovmsClient { + return getTracedAWSV3Client(new LambdaMicrovmsClient({ region: process.env.AWS_REGION })); +} + +export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { + assertValidMicrovmMetadataTags({ + microvmId: 'microvm-validation', + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.config.imageIdentifier, + imageVersion: input.config.imageVersion ?? 'version-validation', + ssmParameterStoreTags: input.ssmParameterStoreTags, + }); + + const commandInput: RunMicrovmCommandInput = { + imageIdentifier: input.config.imageIdentifier, + imageVersion: input.config.imageVersion, + executionRoleArn: input.config.executionRoleArn, + ingressNetworkConnectors: input.config.ingressNetworkConnectors, + egressNetworkConnectors: input.config.egressNetworkConnectors, + maximumDurationInSeconds: MICROVM_LIFETIME_IN_SECONDS, + logging: input.config.logging, + runHookPayload: input.runHookPayload, + clientToken: randomUUID(), + }; + + logger.debug('Launching Lambda MicroVM runner', { + imageIdentifier: commandInput.imageIdentifier, + imageVersion: commandInput.imageVersion, + maximumDurationInSeconds: commandInput.maximumDurationInSeconds, + }); + + const response = await microvmClient().send(new RunMicrovmCommand(commandInput)); + if (!response.microvmId) { + throw new Error('RunMicrovm returned no microvmId'); + } + + const imageArn = response.imageArn ?? input.config.imageIdentifier; + const imageVersion = response.imageVersion ?? input.config.imageVersion; + + try { + const metadataTags = await createMicrovmRunnerMetadata(input.config.metadataSsmPath, { + microvmId: response.microvmId, + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn, + imageVersion, + ssmParameterStoreTags: input.ssmParameterStoreTags, + }); + return { microvmId: response.microvmId, metadataTags }; + } catch (error) { + logger.error(`Failed to record metadata for new MicroVM runner '${response.microvmId}', terminating it`, { + error, + }); + await terminateMicrovm(response.microvmId, input.config).catch((terminationError) => { + logger.error(`Failed to terminate untracked MicroVM runner '${response.microvmId}'`, { + error: terminationError, + }); + }); + throw error; + } +} + +export async function listMicrovmRunners( + filters: ListRunnerFilters = {}, + paths: MicrovmSsmPaths = loadMicrovmProviderConfig(), +): Promise { + const client = microvmClient(); + const items: MicrovmItem[] = []; + let nextToken: string | undefined; + + do { + const response = await client.send( + new ListMicrovmsCommand({ + maxResults: 50, + nextToken, + }), + ); + items.push(...(response.items ?? [])); + nextToken = response.nextToken; + } while (nextToken); + + const activeItems = items.filter( + (item): item is MicrovmItem & { imageArn: string; microvmId: string; state: MicrovmState } => + Boolean(item.microvmId && item.imageArn && item.state && ACTIVE_STATES.has(item.state)), + ); + const microvmStates = new Map( + items.flatMap((item) => (item.microvmId && item.state ? [[item.microvmId, item.state] as const] : [])), + ); + const { cleanupMicrovmIds, metadataById } = await listMicrovmRunnerMetadata(paths, microvmStates); + + let cleanupError: unknown; + for (const microvmId of cleanupMicrovmIds) { + logger.warn(`Retrying cleanup of MicroVM runner '${microvmId}'`); + try { + await terminateMicrovm(microvmId, paths); + } catch (error) { + cleanupError ??= error; + logger.error(`Failed to retry cleanup of MicroVM runner '${microvmId}'`, { error }); + } + } + if (cleanupError !== undefined) throw cleanupError; + + const runners: MicrovmRunnerInfo[] = []; + for (const item of activeItems) { + const metadata = metadataById.get(item.microvmId); + if (!metadata) continue; + if (metadata.imageArn !== item.imageArn) { + throw new Error(`Active MicroVM runner '${item.microvmId}' has an image that does not match its metadata`); + } + + const orphan = Boolean(metadata.orphan); + if (filters.environment !== undefined && metadata.environment !== filters.environment) continue; + if (filters.runnerType !== undefined && metadata.runnerType !== filters.runnerType) continue; + if (filters.runnerOwner !== undefined && metadata.runnerOwner !== filters.runnerOwner) continue; + if (filters.orphan && !orphan) continue; + + runners.push({ + id: item.microvmId, + imageArn: item.imageArn, + launchTime: item.startedAt, + owner: metadata.runnerOwner, + type: metadata.runnerType, + orphan, + githubRunnerId: metadata.githubRunnerId, + bypassRemoval: metadata.bypassRemoval ?? false, + state: item.state, + }); + } + + return runners; +} + +export async function terminateMicrovm(microvmId: string, paths: MicrovmSsmPaths): Promise { + let cleanupPreparationError: unknown; + try { + await markMicrovmCleanupPending(paths.metadataSsmPath, microvmId); + } catch (error) { + cleanupPreparationError = error; + logger.error(`Failed to mark MicroVM runner '${microvmId}' for cleanup`, { error }); + } + + try { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + } catch (error) { + cleanupPreparationError ??= error; + logger.error(`Failed to delete JIT configuration for MicroVM runner '${microvmId}'`, { error }); + } + + try { + await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); + } catch (error) { + if (error instanceof Error && error.name === 'ResourceNotFoundException') { + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; + return; + } + + throw error; + } + + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; +} + +export function microvmBootTimeExceeded(runner: { launchTime?: Date }): boolean { + if (!runner.launchTime) return false; + + const bootTimeInMinutes = Number(process.env.RUNNER_BOOT_TIME_IN_MINUTES || '5'); + return runner.launchTime.getTime() + bootTimeInMinutes * 60_000 < Date.now(); +} + +export function isRetryableMicrovmError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + + const awsError = error as AwsErrorLike; + if (RETRYABLE_ERROR_NAMES.has(awsError.name)) return true; + + const statusCode = awsError.$metadata?.httpStatusCode; + if ( + awsError.$fault === 'server' || + statusCode === 429 || + (statusCode !== undefined && statusCode >= 500) || + (awsError.code !== undefined && RETRYABLE_NETWORK_ERROR_CODES.has(awsError.code)) + ) { + return true; + } + + return awsError.cause !== undefined && awsError.cause !== error ? isRetryableMicrovmError(awsError.cause) : false; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts new file mode 100644 index 0000000000..8f46818b50 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts @@ -0,0 +1,112 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import type { MicrovmRunnerInfo } from './microvms'; +import { calculateMicrovmPoolSize, createMicrovmPoolProvider } from './pool'; +import { createMicrovmRunners } from './runner-config'; + +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), +})); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +function runner(id: string, state: MicrovmRunnerInfo['state']): MicrovmRunnerInfo { + return { id, state, owner: 'Codertocat', type: 'Org' }; +} + +function githubRunnerConfig(): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/runner/token', + ssmConfigPath: '/runner/config', + ssmParameterStoreTags: [], + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-1'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('calculateMicrovmPoolSize', () => { + it('counts online idle running runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-idle', 'RUNNING')], + new Map([['mvm-idle', { busy: false, status: 'online' }]]), + ), + ).toBe(1); + }); + + it('optionally counts online busy runners', () => { + const runners = [runner('mvm-busy', 'RUNNING')]; + const statuses = new Map([['mvm-busy', { busy: true, status: 'online' }]]); + + expect(calculateMicrovmPoolSize(runners, statuses)).toBe(0); + expect(calculateMicrovmPoolSize(runners, statuses, true)).toBe(1); + }); + + it('counts pending runners only during their boot window', () => { + const runners = [runner('mvm-pending', 'PENDING')]; + vi.mocked(microvmBootTimeExceeded).mockReturnValueOnce(false).mockReturnValueOnce(true); + + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(1); + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(0); + }); + + it('does not count suspended or offline runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-suspended', 'SUSPENDED'), runner('mvm-offline', 'RUNNING')], + new Map([['mvm-offline', { busy: false, status: 'offline' }]]), + ), + ).toBe(0); + }); +}); + +describe('createMicrovmPoolProvider', () => { + it('lists managed MicroVMs and returns successfully created IDs', async () => { + const provider = createMicrovmPoolProvider(createStartRunnerConfig); + const input = { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + }; + + await expect(provider.listRunners(input)).resolves.toEqual([]); + expect(listMicrovmRunners).toHaveBeenCalledWith(input); + + await expect( + provider.createRunners({ + githubRunnerConfig: githubRunnerConfig(), + numberOfRunners: 1, + githubInstallationClient: githubClient, + }), + ).resolves.toEqual(['mvm-1']); + expect(createMicrovmRunners).toHaveBeenCalledWith( + expect.any(Object), + 1, + githubClient, + createStartRunnerConfig, + 'pool-lambda', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts new file mode 100644 index 0000000000..8deed5562d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts @@ -0,0 +1,65 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import type { + CreatePoolRunnersInput, + CreateStartRunnerConfig, + ListPoolRunnersInput, + PoolComputeProvider, + RunnerStatus, +} from '../../../../core'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +const logger = createChildLogger('microvm-pool'); + +async function listMicrovmPoolRunners(input: ListPoolRunnersInput): Promise { + return await listMicrovmRunners(input); +} + +async function createMicrovmPoolRunners( + { githubRunnerConfig, numberOfRunners, githubInstallationClient }: CreatePoolRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + const result = await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'pool-lambda', + ); + return result.instances; +} + +export function calculateMicrovmPoolSize( + runners: MicrovmRunnerInfo[], + runnerStatus: Map, + includeBusyRunners = false, +): number { + let availableRunners = 0; + + for (const runner of runners) { + const status = runnerStatus.get(runner.id); + if (runner.state === 'RUNNING' && status?.status === 'online' && (!status.busy || includeBusyRunners)) { + availableRunners++; + logger.debug(`MicroVM runner ${runner.id} is online and counted as part of the pool`); + } else if (runner.state === 'PENDING' && !microvmBootTimeExceeded(runner)) { + availableRunners++; + logger.info(`MicroVM runner ${runner.id} is still booting and counted as part of the pool`); + } else { + logger.debug(`MicroVM runner ${runner.id} is not available and is not counted as part of the pool`); + } + } + + return availableRunners; +} + +export function createMicrovmPoolProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + listRunners: listMicrovmPoolRunners, + countAvailableRunners: calculateMicrovmPoolSize, + createRunners: (input) => createMicrovmPoolRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts new file mode 100644 index 0000000000..586d7f6c84 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -0,0 +1,341 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; +import { createMicrovmRunHookPayload, createMicrovmRunners } from './runner-config'; +import { setMicrovmGithubRunnerMetadata } from './runner-metadata'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + isRetryableMicrovmError: vi.fn(), + runMicrovmRunner: vi.fn(), + terminateMicrovm: vi.fn(), +})); +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), + setMicrovmGithubRunnerMetadata: vi.fn(), +})); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerConfigSsmPath = '/github-action-runners/unit-test/config'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const githubClient = {} as Octokit; +const createStartRunnerConfig = vi.fn(); +const ssmParameterStoreTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:environment', Value: 'caller-cannot-override' }, + { Key: 'ghr:runner_name_prefix', Value: 'caller-cannot-override' }, + { Key: 'ghr:ssm_config_path', Value: 'caller-cannot-override' }, +]; +const microvmMetadataTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: runnerConfigSsmPath }, +]; +const canonicalMetadataTags = [ + ...microvmMetadataTags, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; +const providerConfig = { + imageIdentifier: imageArn, + imageVersion: '2.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, +}; + +function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: 'unit-test-', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: runnerTokenSsmPath, + ssmConfigPath: '/github-action-runners/unit-test/config', + ssmParameterStoreTags, + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(runMicrovmRunner).mockResolvedValue({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }); + vi.mocked(setMicrovmGithubRunnerMetadata).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); + vi.mocked(isRetryableMicrovmError).mockReturnValue(false); + createStartRunnerConfig.mockResolvedValue([]); +}); + +describe('createMicrovmRunHookPayload', () => { + it('contains the image and versioned runner paths', () => { + expect( + JSON.parse( + createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath: '/runner/token', + }), + ), + ).toEqual({ + imageArn, + imageVersion: '2.0', + version: 1, + runnerConfigSsmPath, + runnerTokenSsmPath: '/runner/token', + }); + }); + + it('requires the image ARN and version to be provided together', () => { + expect(() => + createMicrovmRunHookPayload({ + imageArn, + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ).toThrow('MicroVM hook payload image ARN and version must be provided together'); + }); + + it('omits image metadata when no explicit image version is selected', () => { + expect(JSON.parse(createMicrovmRunHookPayload({ runnerConfigSsmPath, runnerTokenSsmPath }))).toEqual({ + version: 1, + runnerConfigSsmPath, + runnerTokenSsmPath, + }); + }); +}); + +describe('createMicrovmRunners', () => { + it.each([{ ephemeral: false }, { enableJitConfig: false }])( + 'rejects unsupported runner configuration %j', + async (overrides) => { + await expect( + createMicrovmRunners(runnerConfig(overrides), 2, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 2 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }, + ); + + it('requires an SSM token path', async () => { + await expect( + createMicrovmRunners( + runnerConfig({ ssmTokenPath: '' }), + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + ), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('requires an SSM config path', async () => { + await expect( + createMicrovmRunners( + runnerConfig({ ssmConfigPath: '' }), + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + ), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('rejects a metadata path that overlaps the JIT token path', async () => { + vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath: '/github-action-runners/unit-test/token/metadata', + runnerTokenSsmPath, + }); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('canonicalizes the configuration and token paths before launching or writing JIT configuration', async () => { + await expect( + createMicrovmRunners( + runnerConfig({ ssmConfigPath: `${runnerConfigSsmPath}/`, ssmTokenPath: `${runnerTokenSsmPath}/` }), + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + ), + ).resolves.toEqual({ instances: ['mvm-1'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenCalledWith( + expect.objectContaining({ + runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ssmParameterStoreTags: microvmMetadataTags, + }), + ); + expect(createStartRunnerConfig).toHaveBeenCalledWith( + expect.objectContaining({ ssmConfigPath: runnerConfigSsmPath, ssmTokenPath: runnerTokenSsmPath }), + ['mvm-1'], + githubClient, + expect.any(Object), + ); + }); + + it('classifies invalid provider configuration as non-retryable', async () => { + vi.mocked(loadMicrovmProviderConfig).mockImplementation(() => { + throw new Error('missing image'); + }); + + await expect( + createMicrovmRunners(runnerConfig(), 3, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 3 }); + }); + + it('launches each MicroVM and delivers its JIT configuration', async () => { + vi.mocked(runMicrovmRunner) + .mockResolvedValueOnce({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }) + .mockResolvedValueOnce({ microvmId: 'mvm-2', metadataTags: canonicalMetadataTags }); + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { + githubRunnerId: `github-${runnerIds[0]}`, + runnerLabels: ['self-hosted', 'microvm'], + }); + return []; + }); + + await expect( + createMicrovmRunners(runnerConfig(), 2, githubClient, createStartRunnerConfig, 'pool-lambda'), + ).resolves.toEqual({ instances: ['mvm-1', 'mvm-2'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenNthCalledWith(1, { + config: expect.objectContaining({ imageIdentifier: imageArn }), + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: microvmMetadataTags, + source: 'pool-lambda', + }); + expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); + const options = createStartRunnerConfig.mock.calls[0][3]; + expect(options?.getSsmParameterTags?.('mvm-1')).toEqual([{ Key: 'MicrovmId', Value: 'mvm-1' }]); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenNthCalledWith( + 1, + providerConfig, + 'mvm-1', + { + githubRunnerId: 'github-mvm-1', + runnerLabels: ['self-hosted', 'microvm'], + }, + canonicalMetadataTags, + ); + }); + + it('applies supported dynamic labels to the provider configuration', async () => { + const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; + const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: 'github-mvm-1', runnerLabels: [] }); + return []; + }); + + await createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda', { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }); + + expect(runMicrovmRunner).toHaveBeenCalledWith({ + config: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, + }, + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload({ + imageArn: overrideImageArn, + imageVersion: '3.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: microvmMetadataTags, + source: 'scale-up-lambda', + }); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenCalledWith( + { + ...providerConfig, + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }, + 'mvm-1', + { githubRunnerId: 'github-mvm-1', runnerLabels: [] }, + canonicalMetadataTags, + ); + }); + + it('retries a JIT setup failure even when runner cleanup fails', async () => { + createStartRunnerConfig.mockResolvedValue(['mvm-1']); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); + + it.each([ + [true, { instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }], + [false, { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }], + ])('classifies launch failures with retryable=%s', async (retryable, expected) => { + vi.mocked(runMicrovmRunner).mockRejectedValue(new Error('launch failed')); + vi.mocked(isRetryableMicrovmError).mockReturnValue(retryable); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual(expected); + }); + + it('attempts cleanup when setup throws after launch', async () => { + createStartRunnerConfig.mockRejectedValue(new Error('JIT setup failed')); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts new file mode 100644 index 0000000000..15f1fb4502 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -0,0 +1,176 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import type { Octokit } from '@octokit/rest'; + +import type { + CreateGitHubRunnerConfig, + CreateRunnerResult, + CreateStartRunnerConfig, + LambdaRunnerSource, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; +import { + assertMatchingMicrovmRunnerTokenPath, + assertSeparatedMicrovmMetadataPath, + normalizeMicrovmSsmPath, + setMicrovmGithubRunnerMetadata, +} from './runner-metadata'; + +const logger = createChildLogger('microvm-runner-config'); +const MICROVM_METADATA_CONTEXT_TAG_KEYS = new Set([ + 'Name', + 'ghr:environment', + 'ghr:runner_name_prefix', + 'ghr:ssm_config_path', +]); + +export interface MicrovmRunHookPayloadV1 { + imageArn?: string; + imageVersion?: string; + runnerConfigSsmPath: string; + runnerTokenSsmPath: string; + version: 1; +} + +export function createMicrovmRunHookPayload(payload: Omit): string { + const hasImageArn = payload.imageArn !== undefined; + const hasImageVersion = payload.imageVersion !== undefined; + if (hasImageArn !== hasImageVersion) { + throw new Error('MicroVM hook payload image ARN and version must be provided together'); + } + + return JSON.stringify({ + version: 1, + ...(hasImageArn + ? { + imageArn: payload.imageArn, + imageVersion: payload.imageVersion, + } + : {}), + runnerConfigSsmPath: payload.runnerConfigSsmPath, + runnerTokenSsmPath: payload.runnerTokenSsmPath, + } satisfies MicrovmRunHookPayloadV1); +} + +function createMicrovmMetadataTags( + config: CreateGitHubRunnerConfig, + environment: string, +): CreateGitHubRunnerConfig['ssmParameterStoreTags'] { + return [ + ...config.ssmParameterStoreTags.filter((tag) => !MICROVM_METADATA_CONTEXT_TAG_KEYS.has(tag.Key)), + { Key: 'ghr:environment', Value: environment }, + { Key: 'ghr:runner_name_prefix', Value: config.runnerNamePrefix }, + { Key: 'ghr:ssm_config_path', Value: config.ssmConfigPath }, + ]; +} + +export async function createMicrovmRunners( + githubRunnerConfig: CreateGitHubRunnerConfig, + numberOfRunners: number, + githubInstallationClient: Octokit, + createStartRunnerConfig: CreateStartRunnerConfig, + source: LambdaRunnerSource, + overrides: MicrovmDynamicLabelOverrides = {}, +): Promise { + if (!githubRunnerConfig.ephemeral || !githubRunnerConfig.enableJitConfig) { + logger.error('Lambda MicroVM runners require ephemeral runners with JIT configuration enabled'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + if (!githubRunnerConfig.ssmTokenPath?.trim()) { + logger.error('Lambda MicroVM runners require SSM_TOKEN_PATH to deliver JIT configuration'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + if (!githubRunnerConfig.ssmConfigPath?.trim()) { + logger.error('Lambda MicroVM runners require SSM_CONFIG_PATH to locate runner metadata'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + let config; + let normalizedGithubRunnerConfig: CreateGitHubRunnerConfig; + try { + config = { ...loadMicrovmProviderConfig(), ...overrides }; + assertMatchingMicrovmRunnerTokenPath(config.runnerTokenSsmPath, githubRunnerConfig.ssmTokenPath); + assertSeparatedMicrovmMetadataPath(config.metadataSsmPath, config.runnerTokenSsmPath); + normalizedGithubRunnerConfig = { + ...githubRunnerConfig, + ssmConfigPath: normalizeMicrovmSsmPath(githubRunnerConfig.ssmConfigPath), + ssmTokenPath: config.runnerTokenSsmPath, + }; + } catch (error) { + logger.error('Invalid Lambda MicroVM provider configuration', { error }); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + const result: CreateRunnerResult = { + instances: [], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }; + const runHookPayload = createMicrovmRunHookPayload({ + ...(config.imageVersion !== undefined + ? { + imageArn: config.imageIdentifier, + imageVersion: config.imageVersion, + } + : {}), + runnerConfigSsmPath: normalizedGithubRunnerConfig.ssmConfigPath, + runnerTokenSsmPath: normalizedGithubRunnerConfig.ssmTokenPath, + }); + const environment = process.env.ENVIRONMENT; + const metadataTags = createMicrovmMetadataTags(normalizedGithubRunnerConfig, environment); + + for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { + let microvmId: string | undefined; + try { + const runner = await runMicrovmRunner({ + config, + environment, + runHookPayload, + runnerOwner: normalizedGithubRunnerConfig.runnerOwner, + runnerType: normalizedGithubRunnerConfig.runnerType, + ssmParameterStoreTags: metadataTags, + source, + }); + microvmId = runner.microvmId; + + const failedRunnerIds = await createStartRunnerConfig( + normalizedGithubRunnerConfig, + [microvmId], + githubInstallationClient, + { + getSsmParameterTags: (runnerId) => [{ Key: 'MicrovmId', Value: runnerId }], + onJitConfigCreated: async (runnerId, metadata) => { + await setMicrovmGithubRunnerMetadata(config, runnerId, metadata, runner.metadataTags); + }, + }, + ); + + if (failedRunnerIds.includes(microvmId)) { + await terminateMicrovm(microvmId, config).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after JIT configuration failed`, { + error: terminationError, + }); + }); + result.retryableErrorCount++; + } else { + result.instances.push(microvmId); + } + } catch (error) { + if (microvmId) { + await terminateMicrovm(microvmId, config).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after setup failed`, { + error: terminationError, + }); + }); + } + + const retryable = isRetryableMicrovmError(error); + logger.error('Failed to create Lambda MicroVM runner', { error, retryable }); + if (retryable) result.retryableErrorCount++; + else result.nonRetryableErrorCount++; + } + } + + return result; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts new file mode 100644 index 0000000000..502fc77d05 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -0,0 +1,631 @@ +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + assertMatchingMicrovmRunnerTokenPath, + assertSeparatedMicrovmMetadataPath, + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + deleteMicrovmRunnerSsmState, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + microvmMetadataParameterName, + microvmRunnerJitParameterName, + setMicrovmGithubRunnerMetadata, + setMicrovmOrphan, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + addParameterTags: vi.fn(), + deleteParameter: vi.fn(), + getParameters: vi.fn(), + getParametersByPath: vi.fn(), + putParameter: vi.fn(), +})); + +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; +const launchTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; + +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + createdAt: '2026-08-19T10:00:00.000Z', + expiresAt: '2026-08-19T11:00:00.000Z', + ...overrides, + }; +} + +function states(entries: [string, MicrovmState][]): Map { + return new Map(entries); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useRealTimers(); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(addParameterTags).mockResolvedValue(); + vi.mocked(getParameters).mockImplementation(async (names) => new Map([[names[0], '{}']])); + vi.mocked(getParametersByPath).mockResolvedValue(new Map()); + vi.mocked(putParameter).mockResolvedValue(); +}); + +describe('MicroVM metadata paths', () => { + it('uses one base parameter per validated MicroVM ID', () => { + expect(microvmMetadataParameterName(`${metadataSsmPath}/`, 'microvm-123')).toBe(`${metadataSsmPath}/microvm-123`); + expect(() => microvmMetadataParameterName(metadataSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + expect(microvmRunnerJitParameterName(`${runnerTokenSsmPath}/`, 'microvm-123')).toBe( + `${runnerTokenSsmPath}/microvm-123`, + ); + expect(() => microvmRunnerJitParameterName(runnerTokenSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + }); + + it('requires metadata to use a prefix separate from JIT configuration', () => { + expect(() => + assertSeparatedMicrovmMetadataPath(metadataSsmPath, '/github-action-runners/unit-test/token'), + ).not.toThrow(); + expect(() => assertSeparatedMicrovmMetadataPath('/runner/token/metadata', '/runner/token')).toThrow( + 'must be separate', + ); + expect(() => assertSeparatedMicrovmMetadataPath('/runner', '/runner/token')).toThrow('must be separate'); + expect(() => assertMatchingMicrovmRunnerTokenPath(`${runnerTokenSsmPath}/`, runnerTokenSsmPath)).not.toThrow(); + expect(() => assertMatchingMicrovmRunnerTokenPath('/runner/other-token', runnerTokenSsmPath)).toThrow( + 'must match the runner JIT token path', + ); + }); +}); + +describe('MicroVM metadata lifecycle', () => { + it('creates non-secret, expiring ownership metadata without overwrite', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T10:00:00.000Z')); + + const createdTags = await createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:Owner', Value: 'configured-owner-cannot-win' }, + { Key: 'ghr:created_by', Value: 'configured-source-cannot-win' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:github_runner_id', Value: 'configured-id-is-not-launch-metadata' }, + { Key: 'ghr:runner_labels', Value: 'configured-labels-are-not-launch-metadata' }, + ], + }); + + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1`, + JSON.stringify(metadata({ expiresAt: '2026-08-19T18:05:00.000Z' })), + false, + { + tags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:Owner', Value: 'Codertocat' }, + { Key: 'ghr:created_by', Value: 'scale-up-lambda' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:Type', Value: 'Org' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, + { + Key: 'ghr:microvm_image_arn', + Value: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + }, + { Key: 'ghr:microvm_image_version', Value: '3.0' }, + ], + }, + ); + expect(createdTags).toEqual(vi.mocked(putParameter).mock.calls[0][3]?.tags); + }); + + it('rejects reserved tag keys and preserves room for late GitHub metadata', async () => { + const input = { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + source: 'scale-up-lambda' as const, + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: [], + }; + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: input.imageArn }], + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + ssmParameterStoreTags: Array.from({ length: 37 }, (_, index) => ({ + Key: `Custom${index}`, + Value: 'value', + })), + }), + ).rejects.toThrow('cannot have more than 44 launch tags'); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('rejects launch tags whose complete serialized metadata could exceed the Parameter Store value limit', async () => { + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: Array.from({ length: 20 }, (_, index) => ({ + Key: `Custom${index}${'k'.repeat(100)}`, + Value: 'v'.repeat(256), + })), + }), + ).rejects.toThrow('cannot exceed 8192 bytes when serialized'); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('loads active metadata and schedules expired or invalid inactive records for two-phase cleanup', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + const active = metadata({ expiresAt: '2026-08-19T12:30:00.000Z' }); + const expiredInactive = metadata({ microvmId: 'mvm-old', expiresAt: '2026-08-19T11:00:00.000Z' }); + const unexpiredInactive = metadata({ microvmId: 'mvm-new', expiresAt: '2026-08-19T12:30:00.000Z' }); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(active)], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.orphan`, 'true'], + [`${metadataSsmPath}/mvm-old`, JSON.stringify(expiredInactive)], + [`${metadataSsmPath}/mvm-new`, JSON.stringify(unexpiredInactive)], + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-old', 'mvm-invalid'], + metadataById: new Map([['mvm-1', { ...active, githubRunnerId: 'github-42', orphan: true }]]), + }); + expect(getParametersByPath).toHaveBeenCalledWith(metadataSsmPath); + expect(deleteParameter).not.toHaveBeenCalled(); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-new`); + }); + + it('fails closed for invalid ownership metadata belonging to an active MicroVM', async () => { + vi.mocked(getParametersByPath).mockResolvedValue(new Map([[`${metadataSsmPath}/mvm-1`, '{not-json']])); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'invalid ownership metadata', + ); + }); + + it('schedules provider-owned metadata with invalid orphan state for two-phase cleanup', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.orphan`, 'invalid'], + ]), + ); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); + }); + + it('propagates metadata path lookup errors so inventory fails closed', async () => { + vi.mocked(getParametersByPath).mockRejectedValue(new Error('AccessDenied')); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow('AccessDenied'); + }); + + it('updates GitHub state and adds late GitHub metadata tags to the base parameter', async () => { + const runnerLabels = ['self-hosted', 'linux', 'env:unit-test']; + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.tags`, expect.any(String), false, { + overwrite: true, + }); + const tagsValue = vi.mocked(putParameter).mock.calls.find(([name]) => name.endsWith('.tags'))?.[1]; + expect(JSON.parse(tagsValue ?? '{}')).toEqual({ + CostCenter: '1234', + 'ghr:Application': 'github-action-runner', + 'ghr:github_runner_id': 'github-42', + 'ghr:microvm_id': 'mvm-1', + 'ghr:runner_labels': `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }); + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('revokes JIT configuration when cleanup starts before late metadata is recorded', async () => { + vi.mocked(getParameters).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, '{}'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when ownership metadata is already absent', async () => { + vi.mocked(getParameters).mockResolvedValue(new Map()); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when the post-write ownership fence cannot be read', async () => { + vi.mocked(getParameters).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('splits encoded runner labels into SSM-safe tag values', async () => { + const runnerLabels = [`label-${'a'.repeat(140)}`, `label-${'b'.repeat(140)}`]; + + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[0]]), 'utf8').toString('base64url')}`, + }, + { + Key: 'ghr:runner_labels:2', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[1]]), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('keeps the GitHub runner ID tag when a runner label is too large', async () => { + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: ['x'.repeat(300)], + }, + launchTags, + ); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + ]); + }); + + it('keeps the durable GitHub runner ID when late metadata tagging fails', async () => { + vi.mocked(addParameterTags).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: [], + }, + launchTags, + ), + ).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + }); + + it('fails JIT setup when the canonical tag-value parameter cannot be written', async () => { + vi.mocked(putParameter).mockImplementation(async (name) => { + if (name.endsWith('.tags')) throw new Error('AccessDenied'); + }); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(addParameterTags).not.toHaveBeenCalled(); + }); + + it('updates orphan state without a shared read-modify-write record', async () => { + await setMicrovmOrphan(metadataSsmPath, 'mvm-1', true); + expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.orphan`, 'true', false, { + overwrite: true, + }); + }); + + it('marks cleanup independently and deletes JIT plus metadata while retaining the tombstone until last', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + ); + + await deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1'); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.tags`, + `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + ]); + }); + + it('does not reset the cleanup grace window when its tombstone already exists', async () => { + vi.mocked(putParameter).mockRejectedValueOnce( + Object.assign(new Error('ParameterAlreadyExists'), { __type: 'ParameterAlreadyExists' }), + ); + + await expect(markMicrovmCleanupPending(metadataSsmPath, 'mvm-1')).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledOnce(); + }); + + it('continues deleting metadata when optional parameters are already absent', async () => { + vi.mocked(deleteParameter) + .mockRejectedValueOnce( + Object.assign(new Error('ParameterNotFound'), { + __type: 'ParameterNotFound', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }), + ) + .mockRejectedValueOnce(Object.assign(new Error('missing parameter'), { name: 'ParameterNotFound' })); + + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).resolves.toBeUndefined(); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.tags`, + `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + ]); + }); + + it('propagates metadata deletion failures other than missing parameters', async () => { + const error = Object.assign(new Error('AccessDeniedException'), { + __type: 'AccessDeniedException', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }); + vi.mocked(deleteParameter).mockRejectedValueOnce(error); + + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).rejects.toBe(error); + expect(deleteParameter).toHaveBeenCalledTimes(1); + }); + + it('deletes only the lane JIT parameter when revoking pending runner configuration', async () => { + await deleteMicrovmRunnerJitConfig(runnerTokenSsmPath, 'mvm-1'); + + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + }); + + it('returns tracked and state-only active cleanup requests for termination retry', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-untracked.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-terminating.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + ]), + ); + + await expect( + listMicrovmRunnerMetadata( + ssmPaths, + states([ + ['mvm-1', 'RUNNING'], + ['mvm-untracked', 'PENDING'], + ['mvm-terminating', 'TERMINATING'], + ]), + ), + ).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1', 'mvm-untracked'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + + it('does not starve cleanup requests when more than one reconciliation batch is pending', async () => { + const cleanupIds = Array.from({ length: 11 }, (_, index) => `mvm-cleanup-${index}`); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map( + cleanupIds.map((microvmId) => [ + `${metadataSsmPath}/${microvmId}.cleanup-requested-at`, + '2026-08-19T10:15:00.000Z', + ]), + ), + ); + + await expect( + listMicrovmRunnerMetadata( + ssmPaths, + states(cleanupIds.map((microvmId): [string, MicrovmState] => [microvmId, 'RUNNING'])), + ), + ).resolves.toEqual({ cleanupMicrovmIds: cleanupIds, metadataById: new Map() }); + }); + + it('keeps cleanup discoverable through the grace window before deleting JIT and every metadata record', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-terminal.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-missing.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + [`${metadataSsmPath}/mvm-missing.tags`, '{"ghr:microvm_id":"mvm-missing"}'], + [`${metadataSsmPath}/mvm-recent.cleanup-requested-at`, '2026-08-19T11:59:00.000Z'], + [`${metadataSsmPath}/mvm-recent.tags`, '{"ghr:microvm_id":"mvm-recent"}'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-terminal', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-terminal', 'mvm-recent'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-missing`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing.tags`); + expect(deleteParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-missing.cleanup-requested-at`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-terminal`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-recent`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-recent`); + }); + + it('deletes invalid ownership metadata after its valid cleanup tombstone ages', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + [`${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, new Map())).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.github-runner-id`, + `${metadataSsmPath}/mvm-invalid.orphan`, + `${metadataSsmPath}/mvm-invalid.tags`, + `${metadataSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, + ]); + }); + + it('repairs an invalid cleanup timestamp before recreating the two-phase cleanup marker', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, 'not-a-timestamp'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.cleanup-requested-at`); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + ); + + vi.clearAllMocks(); + vi.setSystemTime(new Date('2026-08-19T12:06:00.000Z')); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + }); + + it('marks a terminal tags-only companion for two-phase cleanup instead of deleting it immediately', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-tags-only.tags`, '{"ghr:microvm_id":"mvm-tags-only"}']]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-tags-only', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-tags-only'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + + it('fails closed for active state metadata without ownership or a cleanup request', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42']]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'state metadata but no ownership metadata', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts new file mode 100644 index 0000000000..f447ae135f --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -0,0 +1,591 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; + +import type { CreateGitHubRunnerConfig, GitHubRunnerMetadata, LambdaRunnerSource, RunnerType } from '../../../../core'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; + +const logger = createChildLogger('microvm-runner-metadata'); + +const METADATA_VERSION = 1; +const EXPIRATION_GRACE_IN_SECONDS = 300; +const MAX_RECONCILED_RUNNERS = 10; +const MAX_PARAMETER_TAGS = 50; +const MAX_RUNNER_LABEL_TAGS = 5; +const MAX_BASE_PARAMETER_TAGS = MAX_PARAMETER_TAGS - MAX_RUNNER_LABEL_TAGS - 1; +const MAX_TAG_KEY_LENGTH = 128; +const MAX_TAG_VALUE_LENGTH = 256; +const MAX_PARAMETER_VALUE_SIZE_IN_BYTES = 8 * 1024; +const SSM_TAG_VALUE_PATTERN = /^[\p{L}\p{Z}\p{N}_.:/=+\-@]*$/u; +const MICROVM_ID_PATTERN = /^[A-Za-z0-9_-]+$/; +const GITHUB_RUNNER_ID_SUFFIX = '.github-runner-id'; +const ORPHAN_SUFFIX = '.orphan'; +const CLEANUP_REQUESTED_AT_SUFFIX = '.cleanup-requested-at'; +const TAGS_SUFFIX = '.tags'; +const METADATA_COMPANION_SUFFIXES = [ + GITHUB_RUNNER_ID_SUFFIX, + ORPHAN_SUFFIX, + CLEANUP_REQUESTED_AT_SUFFIX, + TAGS_SUFFIX, +] as const; +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); +type MicrovmMetadataTag = CreateGitHubRunnerConfig['ssmParameterStoreTags'][number]; + +export interface MicrovmSsmPaths { + metadataSsmPath: string; + runnerTokenSsmPath: string; +} + +export interface MicrovmRunnerMetadata { + bypassRemoval?: boolean; + createdAt: string; + environment: string; + expiresAt: string; + githubRunnerId?: string; + imageArn: string; + imageVersion?: string; + microvmId: string; + orphan?: boolean; + runnerOwner: string; + runnerType: RunnerType; + source: LambdaRunnerSource; + version: 1; +} + +export interface MicrovmRunnerMetadataInventory { + cleanupMicrovmIds: string[]; + metadataById: Map; +} + +export interface CreateMicrovmRunnerMetadataInput { + environment: string; + imageArn: string; + imageVersion?: string; + microvmId: string; + runnerOwner: string; + runnerType: RunnerType; + ssmParameterStoreTags: MicrovmMetadataTag[]; + source: LambdaRunnerSource; +} + +function isProviderOwnedLateTag(key: string): boolean { + return key === 'ghr:github_runner_id' || key === 'ghr:runner_labels' || key.startsWith('ghr:runner_labels:'); +} + +function assertValidParameterTags(tags: MicrovmMetadataTag[]): void { + if (tags.length > MAX_PARAMETER_TAGS) { + throw new Error(`MicroVM metadata cannot have more than ${MAX_PARAMETER_TAGS} tags`); + } + + for (const tag of tags) { + if ( + Array.from(tag.Key).length === 0 || + Array.from(tag.Key).length > MAX_TAG_KEY_LENGTH || + Array.from(tag.Value).length > MAX_TAG_VALUE_LENGTH || + !SSM_TAG_VALUE_PATTERN.test(tag.Key) || + !SSM_TAG_VALUE_PATTERN.test(tag.Value) + ) { + throw new Error(`MicroVM metadata tag '${tag.Key}' does not satisfy SSM tag constraints`); + } + if (tag.Key.toLowerCase().startsWith('aws:')) { + throw new Error(`MicroVM metadata tag '${tag.Key}' uses the AWS-reserved tag prefix`); + } + } +} + +function mergeParameterTags(...tagSets: MicrovmMetadataTag[][]): MicrovmMetadataTag[] { + const tagsByKey = new Map(); + for (const tags of tagSets) { + for (const tag of tags) tagsByKey.set(tag.Key, tag.Value); + } + + return [...tagsByKey].map(([Key, Value]) => ({ Key, Value })); +} + +function serializeParameterTags(tags: MicrovmMetadataTag[]): string { + assertValidParameterTags(tags); + const tagValues: Record = Object.create(null) as Record; + for (const { Key, Value } of [...tags].sort((left, right) => + left.Key < right.Key ? -1 : left.Key > right.Key ? 1 : 0, + )) { + tagValues[Key] = Value; + } + + const value = JSON.stringify(tagValues); + if (Buffer.byteLength(value, 'utf8') > MAX_PARAMETER_VALUE_SIZE_IN_BYTES) { + throw new Error(`MicroVM metadata tags cannot exceed ${MAX_PARAMETER_VALUE_SIZE_IN_BYTES} bytes when serialized`); + } + return value; +} + +function maximumGitHubRunnerMetadataTags(): MicrovmMetadataTag[] { + return [ + { Key: 'ghr:github_runner_id', Value: '0'.repeat(MAX_TAG_VALUE_LENGTH) }, + ...Array.from({ length: MAX_RUNNER_LABEL_TAGS }, (_, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value: '0'.repeat(MAX_TAG_VALUE_LENGTH), + })), + ]; +} + +function createMetadataParameterTags(input: CreateMicrovmRunnerMetadataInput): MicrovmMetadataTag[] { + const configuredTags = mergeParameterTags(input.ssmParameterStoreTags).filter( + (tag) => !isProviderOwnedLateTag(tag.Key) && tag.Key !== 'ghr:microvm_image_version' && tag.Key !== 'Name', + ); + const providerTags: MicrovmMetadataTag[] = [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: input.source }, + { Key: 'ghr:environment', Value: input.environment }, + { Key: 'ghr:Owner', Value: input.runnerOwner }, + { Key: 'ghr:Type', Value: input.runnerType }, + { Key: 'ghr:microvm_id', Value: input.microvmId }, + { Key: 'ghr:microvm_image_arn', Value: input.imageArn }, + ]; + if (input.imageVersion !== undefined) { + providerTags.push({ Key: 'ghr:microvm_image_version', Value: input.imageVersion }); + } + + const tags = mergeParameterTags(configuredTags, providerTags); + assertValidParameterTags(tags); + if (tags.length > MAX_BASE_PARAMETER_TAGS) { + throw new Error( + `MicroVM metadata cannot have more than ${MAX_BASE_PARAMETER_TAGS} launch tags because ${MAX_RUNNER_LABEL_TAGS + 1} tags are reserved for GitHub runner metadata`, + ); + } + serializeParameterTags(mergeParameterTags(tags, maximumGitHubRunnerMetadataTags())); + return tags; +} + +export function assertValidMicrovmMetadataTags(input: CreateMicrovmRunnerMetadataInput): void { + createMetadataParameterTags(input); +} + +function encodeRunnerLabelGroups(labels: string[]): string[] { + const encodedGroups: string[] = []; + let group: string[] = []; + const encode = (values: string[]) => `base64url:${Buffer.from(JSON.stringify(values), 'utf8').toString('base64url')}`; + + for (const label of labels) { + const candidate = [...group, label]; + if (Array.from(encode(candidate)).length <= MAX_TAG_VALUE_LENGTH) { + group = candidate; + continue; + } + if (group.length === 0) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + continue; + } + encodedGroups.push(encode(group)); + group = [label]; + if (Array.from(encode(group)).length > MAX_TAG_VALUE_LENGTH) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + group = []; + } + } + if (group.length > 0) encodedGroups.push(encode(group)); + + if (encodedGroups.length > MAX_RUNNER_LABEL_TAGS) { + logger.warn('GitHub runner label SSM tags were truncated to avoid exceeding the metadata tag budget', { + maxRunnerLabelsTagCount: MAX_RUNNER_LABEL_TAGS, + }); + } + return encodedGroups.slice(0, MAX_RUNNER_LABEL_TAGS); +} + +function createGitHubRunnerMetadataTags(metadata: GitHubRunnerMetadata): MicrovmMetadataTag[] { + const tags: MicrovmMetadataTag[] = [{ Key: 'ghr:github_runner_id', Value: metadata.githubRunnerId }]; + tags.push( + ...encodeRunnerLabelGroups(metadata.runnerLabels).map((Value, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value, + })), + ); + assertValidParameterTags(tags); + return tags; +} + +export function normalizeMicrovmSsmPath(path: string): string { + const normalized = path.trim().replace(/\/+$/, ''); + if (!/^\/[A-Za-z0-9_.\-/]+$/.test(normalized) || normalized.includes('//') || normalized.split('/').includes('..')) { + throw new Error(`Invalid SSM parameter path '${path}'`); + } + return normalized; +} + +export function microvmMetadataParameterName(metadataSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizeMicrovmSsmPath(metadataSsmPath)}/${microvmId}`; +} + +export function microvmRunnerJitParameterName(runnerTokenSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizeMicrovmSsmPath(runnerTokenSsmPath)}/${microvmId}`; +} + +function stateParameterName(metadataSsmPath: string, microvmId: string, suffix: string): string { + return `${microvmMetadataParameterName(metadataSsmPath, microvmId)}${suffix}`; +} + +function metadataParameterNames(metadataSsmPath: string, microvmId: string): string[] { + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + return [ + `${baseName}${GITHUB_RUNNER_ID_SUFFIX}`, + `${baseName}${ORPHAN_SUFFIX}`, + `${baseName}${TAGS_SUFFIX}`, + baseName, + `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`, + ]; +} + +export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runnerTokenSsmPath: string): void { + const metadataPath = normalizeMicrovmSsmPath(metadataSsmPath); + const runnerTokenPath = normalizeMicrovmSsmPath(runnerTokenSsmPath); + if ( + metadataPath === runnerTokenPath || + metadataPath.startsWith(`${runnerTokenPath}/`) || + runnerTokenPath.startsWith(`${metadataPath}/`) + ) { + throw new Error('MICROVM_METADATA_SSM_PATH must be separate from the runner JIT token path'); + } +} + +export function assertMatchingMicrovmRunnerTokenPath( + configuredRunnerTokenSsmPath: string, + runnerTokenSsmPath: string, +): void { + if (normalizeMicrovmSsmPath(configuredRunnerTokenSsmPath) !== normalizeMicrovmSsmPath(runnerTokenSsmPath)) { + throw new Error('MicroVM provider SSM_TOKEN_PATH must match the runner JIT token path'); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isParameterError(error: unknown, type: string): boolean { + return error instanceof Error && (error.name === type || ('__type' in error && error.__type === type)); +} + +function isParameterNotFound(error: unknown): boolean { + return isParameterError(error, 'ParameterNotFound'); +} + +function optionalString(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && value.length > 0); +} + +function optionalBoolean(value: unknown): value is boolean | undefined { + return value === undefined || typeof value === 'boolean'; +} + +function parseMetadata(value: string, expectedMicrovmId: string): MicrovmRunnerMetadata | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + + if (!isRecord(parsed)) return undefined; + + const createdAt = typeof parsed.createdAt === 'string' ? Date.parse(parsed.createdAt) : Number.NaN; + const expiresAt = typeof parsed.expiresAt === 'string' ? Date.parse(parsed.expiresAt) : Number.NaN; + if ( + parsed.version !== METADATA_VERSION || + parsed.microvmId !== expectedMicrovmId || + typeof parsed.environment !== 'string' || + parsed.environment.length === 0 || + typeof parsed.runnerOwner !== 'string' || + parsed.runnerOwner.length === 0 || + (parsed.runnerType !== 'Org' && parsed.runnerType !== 'Repo') || + (parsed.source !== 'scale-up-lambda' && parsed.source !== 'pool-lambda') || + typeof parsed.imageArn !== 'string' || + parsed.imageArn.length === 0 || + !optionalString(parsed.imageVersion) || + !optionalBoolean(parsed.bypassRemoval) || + !Number.isFinite(createdAt) || + !Number.isFinite(expiresAt) || + expiresAt <= createdAt + ) { + return undefined; + } + + return { + version: METADATA_VERSION, + microvmId: expectedMicrovmId, + environment: parsed.environment, + runnerOwner: parsed.runnerOwner, + runnerType: parsed.runnerType, + source: parsed.source, + imageArn: parsed.imageArn, + imageVersion: parsed.imageVersion, + bypassRemoval: parsed.bypassRemoval, + createdAt: parsed.createdAt as string, + expiresAt: parsed.expiresAt as string, + }; +} + +export async function createMicrovmRunnerMetadata( + metadataSsmPath: string, + input: CreateMicrovmRunnerMetadataInput, +): Promise { + const createdAt = new Date(); + const metadata: MicrovmRunnerMetadata = { + version: METADATA_VERSION, + microvmId: input.microvmId, + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.imageArn, + imageVersion: input.imageVersion, + createdAt: createdAt.toISOString(), + expiresAt: new Date( + createdAt.getTime() + (MICROVM_LIFETIME_IN_SECONDS + EXPIRATION_GRACE_IN_SECONDS) * 1000, + ).toISOString(), + }; + + const metadataTags = createMetadataParameterTags(input); + await putParameter(microvmMetadataParameterName(metadataSsmPath, input.microvmId), JSON.stringify(metadata), false, { + tags: metadataTags, + }); + return metadataTags; +} + +function invalidOrphanState(parameters: Map, baseName: string): boolean { + const orphan = parameters.get(`${baseName}${ORPHAN_SUFFIX}`); + return orphan !== undefined && orphan !== 'true' && orphan !== 'false'; +} + +type CleanupRequestStatus = 'absent' | 'elapsed' | 'invalid' | 'pending'; + +function cleanupRequestStatus(parameters: Map, baseName: string, now: number): CleanupRequestStatus { + const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + if (cleanupRequestedAt === undefined) return 'absent'; + const requestedAt = Date.parse(cleanupRequestedAt); + if (!Number.isFinite(requestedAt)) return 'invalid'; + return requestedAt + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now ? 'elapsed' : 'pending'; +} + +export async function listMicrovmRunnerMetadata( + paths: MicrovmSsmPaths, + microvmStates: ReadonlyMap, +): Promise { + const { metadataSsmPath } = paths; + const metadataById = new Map(); + const cleanupMicrovmIds = new Set(); + const parameters = await getParametersByPath(normalizeMicrovmSsmPath(metadataSsmPath)); + const parameterPrefix = `${normalizeMicrovmSsmPath(metadataSsmPath)}/`; + const now = Date.now(); + const metadataBaseIds = new Set(); + const stateParameterIds = new Set(); + const runnersToDelete = new Set(); + + for (const parameterName of parameters.keys()) { + if (!parameterName.startsWith(parameterPrefix)) continue; + for (const suffix of METADATA_COMPANION_SUFFIXES) { + if (!parameterName.endsWith(suffix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length, -suffix.length); + if (MICROVM_ID_PATTERN.test(microvmId)) stateParameterIds.add(microvmId); + break; + } + } + + for (const [parameterName, value] of parameters) { + if (!parameterName.startsWith(parameterPrefix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length); + if (!MICROVM_ID_PATTERN.test(microvmId)) continue; + metadataBaseIds.add(microvmId); + + const state = microvmStates.get(microvmId); + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else { + cleanupMicrovmIds.add(microvmId); + } + continue; + } + + const metadata = parseMetadata(value, microvmId); + if (!metadata) { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has invalid ownership metadata`); + } + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + } + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling invalid MicroVM runner metadata for '${microvmId}' for cleanup`); + continue; + } + + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + cleanupMicrovmIds.add(microvmId); + logger.warn(`Repairing invalid cleanup request metadata for '${microvmId}'`); + continue; + } + + if (invalidOrphanState(parameters, baseName)) { + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling MicroVM runner metadata for '${microvmId}' with invalid orphan state for cleanup`); + continue; + } + + if (state === 'TERMINATED') { + cleanupMicrovmIds.add(microvmId); + continue; + } + if (state === undefined) { + if (Date.parse(metadata.expiresAt) <= now) cleanupMicrovmIds.add(microvmId); + continue; + } + if (!ACTIVE_STATES.has(state)) continue; + + metadataById.set(microvmId, { + ...metadata, + githubRunnerId: parameters.get(`${baseName}${GITHUB_RUNNER_ID_SUFFIX}`), + orphan: parameters.get(`${baseName}${ORPHAN_SUFFIX}`) === 'true', + }); + } + + for (const microvmId of stateParameterIds) { + if (metadataBaseIds.has(microvmId)) continue; + + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const state = microvmStates.get(microvmId); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); + + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else if (state === undefined || state === 'TERMINATED' || ACTIVE_STATES.has(state)) { + cleanupMicrovmIds.add(microvmId); + } + continue; + } + + if (cleanupStatus === 'invalid') { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has an invalid cleanup request timestamp`); + } + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + cleanupMicrovmIds.add(microvmId); + continue; + } + + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has state metadata but no ownership metadata`); + } + if (state === 'TERMINATED' || state === undefined) { + cleanupMicrovmIds.add(microvmId); + } + } + + for (const microvmId of [...runnersToDelete].slice(0, MAX_RECONCILED_RUNNERS)) { + try { + await deleteMicrovmRunnerSsmState(paths, microvmId); + } catch (error) { + logger.warn(`Failed to delete reconciled MicroVM runner metadata '${microvmId}'`, { error }); + } + } + + return { + cleanupMicrovmIds: [...cleanupMicrovmIds], + metadataById, + }; +} + +export async function setMicrovmGithubRunnerMetadata( + paths: MicrovmSsmPaths, + microvmId: string, + metadata: GitHubRunnerMetadata, + launchTags: MicrovmMetadataTag[], +): Promise { + if (!metadata.githubRunnerId) throw new Error('GitHub runner ID must not be empty'); + const baseName = microvmMetadataParameterName(paths.metadataSsmPath, microvmId); + const cleanupMarkerName = `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`; + try { + const parameters = await getParameters([baseName, cleanupMarkerName]); + if (!parameters.has(baseName) || parameters.has(cleanupMarkerName)) { + throw new Error(`MicroVM runner '${microvmId}' is no longer accepting JIT configuration`); + } + } catch (error) { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + throw error; + } + + const githubRunnerTags = createGitHubRunnerMetadataTags(metadata); + const tags = mergeParameterTags(launchTags, githubRunnerTags); + const serializedTags = serializeParameterTags(tags); + await putParameter( + stateParameterName(paths.metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), + metadata.githubRunnerId, + false, + { + overwrite: true, + }, + ); + await putParameter(stateParameterName(paths.metadataSsmPath, microvmId, TAGS_SUFFIX), serializedTags, false, { + overwrite: true, + }); + try { + await addParameterTags(baseName, githubRunnerTags); + } catch (error) { + logger.error(`Failed to tag MicroVM runner '${microvmId}' with GitHub runner metadata`, { error }); + } +} + +export async function setMicrovmOrphan(metadataSsmPath: string, microvmId: string, orphan: boolean): Promise { + await putParameter(stateParameterName(metadataSsmPath, microvmId, ORPHAN_SUFFIX), String(orphan), false, { + overwrite: true, + }); +} + +export async function markMicrovmCleanupPending(metadataSsmPath: string, microvmId: string): Promise { + try { + await putParameter( + stateParameterName(metadataSsmPath, microvmId, CLEANUP_REQUESTED_AT_SUFFIX), + new Date().toISOString(), + false, + ); + } catch (error) { + if (!isParameterError(error, 'ParameterAlreadyExists')) throw error; + } +} + +async function deleteParameterIfPresent(parameterName: string): Promise { + try { + await deleteParameter(parameterName); + } catch (error) { + if (!isParameterNotFound(error)) throw error; + } +} + +export async function deleteMicrovmRunnerJitConfig(runnerTokenSsmPath: string, microvmId: string): Promise { + await deleteParameterIfPresent(microvmRunnerJitParameterName(runnerTokenSsmPath, microvmId)); +} + +export async function deleteMicrovmRunnerSsmState(paths: MicrovmSsmPaths, microvmId: string): Promise { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + for (const parameterName of metadataParameterNames(paths.metadataSsmPath, microvmId)) { + await deleteParameterIfPresent(parameterName); + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts new file mode 100644 index 0000000000..f98eb88628 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; +import { createMicrovmScaleDownProvider } from './scale-down'; +import { setMicrovmOrphan } from './runner-metadata'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), + terminateMicrovm: vi.fn(), +})); +vi.mock('./runner-metadata', () => ({ setMicrovmOrphan: vi.fn() })); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const providerConfig = { + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, +}; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(setMicrovmOrphan).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); +}); + +describe('createMicrovmScaleDownProvider', () => { + it('lists active and orphan runners through provider filters', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.list('unit-test'); + await provider.list('unit-test', true); + + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 1, + { + environment: 'unit-test', + orphan: undefined, + }, + providerConfig, + ); + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 2, + { + environment: 'unit-test', + orphan: true, + }, + providerConfig, + ); + }); + + it('uses durable metadata when marking, unmarking, and terminating runners', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.markOrphan('mvm-1'); + await provider.unmarkOrphan('mvm-1'); + await provider.terminate('mvm-1'); + + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', true); + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(2, metadataSsmPath, 'mvm-1', false); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); + + it('uses the MicroVM boot-time policy', () => { + const provider = createMicrovmScaleDownProvider(); + const runner = { id: 'mvm-1', owner: 'Codertocat', type: 'Org' as const }; + + expect(provider.bootTimeExceeded(runner)).toBe(false); + expect(microvmBootTimeExceeded).toHaveBeenCalledWith(runner); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts new file mode 100644 index 0000000000..82038711df --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts @@ -0,0 +1,21 @@ +import type { ScaleDownComputeProvider } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; +import { setMicrovmOrphan } from './runner-metadata'; + +export function createMicrovmScaleDownProvider(): Omit { + const ssmPaths = () => loadMicrovmProviderConfig(); + + async function list(environment: string, orphan?: boolean): Promise { + return await listMicrovmRunners({ environment, orphan }, ssmPaths()); + } + + return { + list, + bootTimeExceeded: microvmBootTimeExceeded, + markOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, true), + unmarkOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, false), + terminate: async (id) => await terminateMicrovm(id, ssmPaths()), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts new file mode 100644 index 0000000000..bd10efd41e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts @@ -0,0 +1,115 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; +import { createMicrovmScaleUpProvider } from './scale-up'; + +vi.mock('./microvms', () => ({ listMicrovmRunners: vi.fn() })); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const githubRunnerConfig: CreateGitHubRunnerConfig = { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ssmTokenPath: '/runner/token', + ssmConfigPath: '/runner/config', + ssmParameterStoreTags: [], +}; + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(listMicrovmRunners).mockResolvedValue([ + { id: 'mvm-current', owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, + ]); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-new'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('createMicrovmScaleUpProvider', () => { + it('resolves supported resource override labels and registers them on the runner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.resolveLabelsForRunners([ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).resolves.toEqual({ + runnerLabels: [ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + ], + state: { + overrides: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }, + }, + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('rejects unsupported MicroVM override label %s at the control-plane boundary', async (label, reason) => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect(provider.resolveLabelsForRunners([label])).rejects.toThrow(reason); + }); + + it('counts managed MicroVMs for the runner owner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.getCurrentRunners({ overrides: {} }, { runnerOwner: 'Codertocat', runnerType: 'Org' }), + ).resolves.toBe(1); + expect(listMicrovmRunners).toHaveBeenCalledWith({ + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }); + }); + + it('delegates runner creation to the shared MicroVM lifecycle', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.createRunners({ + githubRunnerConfig, + numberOfRunners: 1, + githubInstallationClient: githubClient, + state: { overrides: { imageVersion: '3.0' } }, + }), + ).resolves.toEqual({ instances: ['mvm-new'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + expect(createMicrovmRunners).toHaveBeenCalledWith( + githubRunnerConfig, + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + { imageVersion: '3.0' }, + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts new file mode 100644 index 0000000000..a3dcf1219e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts @@ -0,0 +1,77 @@ +import type { + CreateRunnerResult, + CreateScaleUpRunnersInput, + CreateStartRunnerConfig, + CurrentRunnersInput, + RunnerLabelResolution, + ScaleUpComputeProvider, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { parseMicrovmDynamicLabels } from '../dynamic-labels'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +interface MicrovmScaleUpState { + overrides: MicrovmDynamicLabelOverrides; +} + +async function resolveMicrovmLabelsForRunners( + messageLabels: string[], +): Promise> { + const trimmedLabels = messageLabels.map((label) => label.trim()); + const parsed = parseMicrovmDynamicLabels(trimmedLabels); + if (parsed.violations.length > 0) { + throw new Error( + `Invalid MicroVM dynamic labels: ${parsed.violations + .map((violation) => `${violation.label} (${violation.reason})`) + .join(', ')}`, + ); + } + + return { + runnerLabels: trimmedLabels.filter((label) => label.startsWith('ghr-')), + state: { overrides: parsed.overrides }, + }; +} + +async function getCurrentMicrovmRunners( + _state: MicrovmScaleUpState, + { runnerType, runnerOwner }: CurrentRunnersInput, +): Promise { + return ( + await listMicrovmRunners({ + environment: process.env.ENVIRONMENT, + runnerType, + runnerOwner, + }) + ).length; +} + +async function createMicrovmScaleUpRunners( + { + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + state, + }: CreateScaleUpRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + return await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'scale-up-lambda', + state.overrides, + ); +} + +export function createMicrovmScaleUpProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + resolveLabelsForRunners: resolveMicrovmLabelsForRunners, + getCurrentRunners: getCurrentMicrovmRunners, + createRunners: (input) => createMicrovmScaleUpRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts new file mode 100644 index 0000000000..442986a0f8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; + +import { parseMicrovmDynamicLabels } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const internetEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS'; + +describe('parseMicrovmDynamicLabels', () => { + it('parses every supported RunMicrovm override', () => { + expect( + parseMicrovmDynamicLabels([ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-egress-network-connectors:${internetEgressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).toEqual({ + overrides: { + egressNetworkConnectors: [egressConnectorArn, internetEgressConnectorArn], + imageIdentifier: imageArn, + imageVersion: '3.0', + }, + violations: [], + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-egress-network-connectors:not-an-arn', + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn};${internetEgressConnectorArn}`, + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + ['ghr-microvm-image-arn:not-an-arn', 'is not a valid customer MicroVM image ARN'], + ['ghr-microvm-image-version:', "key 'image-version' requires a value"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('rejects invalid override %s', (label, reason) => { + const result = parseMicrovmDynamicLabels([label]); + + expect(result.overrides).toEqual({}); + expect(result.violations).toEqual([{ label, reason: expect.stringContaining(reason) }]); + }); + + it('ignores generic dynamic labels', () => { + expect(parseMicrovmDynamicLabels(['ghr-team:platform'])).toEqual({ overrides: {}, violations: [] }); + }); + + it('rejects more than ten egress network connectors', () => { + const labels = Array.from( + { length: 11 }, + (_, index) => + `ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:connector-${index}`, + ); + + const result = parseMicrovmDynamicLabels(labels); + + expect(result.overrides.egressNetworkConnectors).toHaveLength(10); + expect(result.violations).toEqual([ + { + label: labels[10], + reason: 'at most 10 egress network connector labels are supported', + }, + ]); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts new file mode 100644 index 0000000000..2851716149 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts @@ -0,0 +1,76 @@ +export const MICROVM_DYNAMIC_LABEL_PREFIX = 'ghr-microvm-'; + +const MAXIMUM_EGRESS_NETWORK_CONNECTORS = 10; +const MICROVM_IMAGE_ARN_PATTERN = /^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$/; +const MICROVM_NETWORK_CONNECTOR_ARN_PATTERN = + /^arn:aws[a-zA-Z-]*:lambda:[a-z0-9-]+:(?:[0-9]{12}|aws):network-connector:[a-zA-Z0-9_-]+(?::[a-zA-Z0-9_-]+)?$/; + +export interface MicrovmDynamicLabelOverrides { + egressNetworkConnectors?: string[]; + imageIdentifier?: string; + imageVersion?: string; +} + +export interface MicrovmDynamicLabelViolation { + label: string; + reason: string; +} + +export function parseMicrovmDynamicLabels(labels: string[]): { + overrides: MicrovmDynamicLabelOverrides; + violations: MicrovmDynamicLabelViolation[]; +} { + const overrides: MicrovmDynamicLabelOverrides = {}; + const violations: MicrovmDynamicLabelViolation[] = []; + + for (const label of labels) { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) continue; + + const stripped = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? '' : stripped.slice(colonIndex + 1).trim(); + + if (!value) { + violations.push({ label, reason: `key '${key}' requires a value` }); + continue; + } + + switch (key) { + case 'egress-network-connectors': { + if (!MICROVM_NETWORK_CONNECTOR_ARN_PATTERN.test(value)) { + violations.push({ + label, + reason: `'${value}' is not a valid Lambda network connector ARN; specify one ARN per label`, + }); + break; + } + + const connectors = overrides.egressNetworkConnectors ?? []; + if (connectors.length >= MAXIMUM_EGRESS_NETWORK_CONNECTORS) { + violations.push({ + label, + reason: `at most ${MAXIMUM_EGRESS_NETWORK_CONNECTORS} egress network connector labels are supported`, + }); + } else { + overrides.egressNetworkConnectors = [...connectors, value]; + } + break; + } + case 'image-arn': + if (!MICROVM_IMAGE_ARN_PATTERN.test(value)) { + violations.push({ label, reason: `'${value}' is not a valid customer MicroVM image ARN` }); + } else { + overrides.imageIdentifier = value; + } + break; + case 'image-version': + overrides.imageVersion = value; + break; + default: + violations.push({ label, reason: `key '${key}' is not a supported MicroVM override` }); + } + } + + return { overrides, violations }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts new file mode 100644 index 0000000000..0111373247 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -0,0 +1,16 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + MICROVM_EGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_EXECUTION_ROLE_ARN: string; + MICROVM_IMAGE_ARN: string; + MICROVM_IMAGE_VERSION: string | undefined; + MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_LOG_GROUP: string | undefined; + MICROVM_METADATA_SSM_PATH: string; + SSM_TOKEN_PATH: string; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts new file mode 100644 index 0000000000..2b7b85d74e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig } from '../../../../contracts'; +import { microvmDynamicLabelProvider } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + +describe('microvmDynamicLabelProvider', () => { + it('accepts supported MicroVM overrides', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { allowed: [egressConnectorArn] }, + 'image-arn': { allowed: [imageArn] }, + 'image-version': { allowed: ['3.0'] }, + }, + }; + const dynamicLabels = [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]; + + expect(getViolations(queue, dynamicLabels)).toEqual([]); + }); + + it('requires explicit allowlists for image code and network-boundary overrides', () => { + expect( + getViolations(microvmQueue(), [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).toEqual([ + { + label: `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + reason: "key 'egress-network-connectors' requires an explicit allowed list", + }, + { + label: `ghr-microvm-image-arn:${imageArn}`, + reason: "key 'image-arn' requires an explicit allowed list", + }, + { + label: 'ghr-microvm-image-version:3.0', + reason: "key 'image-version' requires an explicit allowed list", + }, + ]); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('preserves the parser violation for %s', (label, reason) => { + expect(getViolations(microvmQueue(), [label])).toEqual([ + { + label, + reason, + }, + ]); + }); + + it('enforces the AWS dynamic-label policy', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { 'image-version': { allowed: ['2.*'] } }, + }; + + expect(getViolations(queue, ['ghr-microvm-image-version:3.0'])).toEqual([ + { + label: 'ghr-microvm-image-version:3.0', + reason: "value '3.0' not in allowed list", + }, + ]); + }); + + it('applies allowed patterns to the complete image ARN', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-arn': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-large', + ]), + ).toEqual([]); + expect( + getViolations(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:unapproved']), + ).toHaveLength(1); + }); + + it('applies the policy to each egress connector label', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-private', + ]), + ).toEqual([]); + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:unapproved', + ]), + ).toHaveLength(1); + }); +}); + +function getViolations(queue: RunnerMatcherConfig, labels: string[]) { + return microvmDynamicLabelProvider.getViolations({ + queue, + labels, + }); +} + +function microvmQueue(): RunnerMatcherConfig { + return { + id: 'microvm', + arn: 'arn:aws:sqs:eu-west-1:123456789012:microvm', + computeProvider: 'microvm', + matcherConfig: { + labelMatchers: [['self-hosted', 'linux', 'arm64', 'microvm']], + exactMatch: false, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts new file mode 100644 index 0000000000..e7c5485617 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts @@ -0,0 +1,32 @@ +import type { DynamicLabelProvider } from '../../../../contracts'; +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; +import { MICROVM_DYNAMIC_LABEL_PREFIX, parseMicrovmDynamicLabels } from '../dynamic-labels'; + +const RESOURCE_BOUNDARY_KEYS = new Set(['egress-network-connectors', 'image-arn', 'image-version']); + +function resourceBoundaryViolations( + labels: string[], + policy: Parameters[1], +) { + return labels.flatMap((label) => { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) return []; + + const key = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length).split(':', 1)[0]; + if (!RESOURCE_BOUNDARY_KEYS.has(key) || policy?.blocked_keys?.includes(key)) return []; + + const allowed = policy?.restricted_keys?.[key]?.allowed; + return allowed && allowed.length > 0 ? [] : [{ label, reason: `key '${key}' requires an explicit allowed list` }]; + }); +} + +export const microvmDynamicLabelProvider: DynamicLabelProvider = { + getViolations: ({ queue, labels }) => [ + ...parseMicrovmDynamicLabels(labels).violations, + ...resourceBoundaryViolations(labels, queue.matcherConfig.awsDynamicLabelsPolicy), + ...violationsAgainstAwsDynamicLabelsPolicy( + labels, + queue.matcherConfig.awsDynamicLabelsPolicy, + MICROVM_DYNAMIC_LABEL_PREFIX, + ), + ], +}; diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts new file mode 100644 index 0000000000..ad3baed78b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts @@ -0,0 +1,34 @@ +import { defineWebhookProviderContractTests } from '../../test/webhook-provider-contract'; +import { provider } from './webhook'; + +defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels: ['ghr-microvm-image-version:3.0'], + configureQueue: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-version': { allowed: ['3.0'] }, + }, + }; + }, + rejectingPolicies: [ + { + name: 'blocked keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + blocked_keys: ['image-version'], + }; + }, + }, + { + name: 'restricted keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-version': { allowed: ['2.*'] }, + }, + }; + }, + }, + ], +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.ts new file mode 100644 index 0000000000..48d603e476 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.ts @@ -0,0 +1,16 @@ +import type { ComputeProviderPlugin } from '../../core'; + +import type { WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; +import { microvmDynamicLabelProvider } from './src/webhook/dynamic-labels'; + +export function createMicrovmWebhookPlugin(): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { dynamicLabels: microvmDynamicLabelProvider }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmWebhookPlugin, +} satisfies WebhookProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/contracts.ts b/lambdas/libs/compute-providers/contracts.ts index 617789ec10..85e99f3949 100644 --- a/lambdas/libs/compute-providers/contracts.ts +++ b/lambdas/libs/compute-providers/contracts.ts @@ -43,12 +43,13 @@ export interface DynamicLabelDispatchTarget { labels: string[]; } +export interface DynamicLabelViolation { + label: string; + reason: string; +} + export interface DynamicLabelProvider { - selectQueue(input: { - queue: RunnerMatcherConfig; - nonGhrLabels: string[]; - sanitizedGhrLabels: string[]; - }): DynamicLabelDispatchTarget | undefined; + getViolations(input: { queue: RunnerMatcherConfig; labels: string[] }): DynamicLabelViolation[]; } export interface ControlPlaneProviderCapabilities { diff --git a/lambdas/libs/compute-providers/dynamic-labels.test.ts b/lambdas/libs/compute-providers/dynamic-labels.test.ts new file mode 100644 index 0000000000..88aa39689c --- /dev/null +++ b/lambdas/libs/compute-providers/dynamic-labels.test.ts @@ -0,0 +1,13 @@ +import { expect, it } from 'vitest'; + +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; + +const providerTypes = ['alpha', 'beta'] as const; + +it.each(providerTypes)('returns labels belonging to providers other than %s', (provider) => { + const providerLabels = providerTypes.map((type) => `ghr-${type}-size:large`); + + expect(dynamicLabelsForOtherProvider(providerLabels, provider, providerTypes)).toEqual( + providerLabels.filter((label) => !label.startsWith(`ghr-${provider}-`)), + ); +}); diff --git a/lambdas/libs/compute-providers/dynamic-labels.ts b/lambdas/libs/compute-providers/dynamic-labels.ts new file mode 100644 index 0000000000..3c72d77966 --- /dev/null +++ b/lambdas/libs/compute-providers/dynamic-labels.ts @@ -0,0 +1,11 @@ +import { computeProviderTypes } from './provider-types'; + +export function dynamicLabelsForOtherProvider( + labels: string[], + provider: string, + providerTypes: readonly string[] = computeProviderTypes, +): string[] { + return labels.filter((label) => + providerTypes.some((candidate) => candidate !== provider && label.startsWith(`ghr-${candidate}-`)), + ); +} diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index 9d39fd294a..cd03f897f0 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -11,7 +11,9 @@ "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", "./aws/ec2/control-plane/runners": "./aws/ec2/src/control-plane/runners.ts", - "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts" + "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts", + "./aws/microvm/webhook": "./aws/microvm/webhook.ts", + "./aws/microvm/control-plane": "./aws/microvm/control-plane.ts" }, "type": "module", "license": "MIT", @@ -27,6 +29,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-lambda-microvms": "^3.1074.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/provider-types.test.ts b/lambdas/libs/compute-providers/provider-types.test.ts index 76111897ab..9f6f4a981e 100644 --- a/lambdas/libs/compute-providers/provider-types.test.ts +++ b/lambdas/libs/compute-providers/provider-types.test.ts @@ -1,11 +1,15 @@ import { describe, expect, it } from 'vitest'; -import { - defaultComputeProvider, - normalizeComputeProviderType, - resolveComputeProviderType, - computeProviderTypes, -} from './provider-types'; +import { computeProviderTypes, defaultComputeProvider, resolveComputeProviderType } from './provider-types'; + +const defaultProviderInputs = [undefined, '', ' '] as const; +const supportedProviderCases = computeProviderTypes.flatMap( + (provider) => + [ + [provider, provider], + [` ${provider.toUpperCase()} `, provider], + ] as const, +); describe('compute provider configuration', () => { it('defines an explicit default provider', () => { @@ -13,32 +17,16 @@ describe('compute provider configuration', () => { }); }); -describe('compute provider normalization', () => { - it.each([ - [undefined, 'ec2'], - ['', 'ec2'], - [' ', 'ec2'], - [' EC2 ', 'ec2'], - ])('normalizes provider type %j to %j', (type, expected) => { - expect(normalizeComputeProviderType(type)).toBe(expected); - }); - - it.each([[' Unknown '], ['microvm'], [null], [1]])('returns undefined for unsupported provider type %j', (type) => { - expect(normalizeComputeProviderType(type)).toBeUndefined(); +describe('compute provider resolution', () => { + it.each(defaultProviderInputs)('resolves default provider input %j', (type) => { + expect(resolveComputeProviderType(type)).toBe(defaultComputeProvider); }); -}); -describe('compute provider resolution', () => { - it.each([ - [undefined, 'ec2'], - ['', 'ec2'], - [' ', 'ec2'], - [' EC2 ', 'ec2'], - ])('resolves provider type %j to %j', (type, expected) => { + it.each(supportedProviderCases)('resolves provider type %j to %j', (type, expected) => { expect(resolveComputeProviderType(type)).toBe(expected); }); - it.each([[' Unknown '], ['microvm'], [null], [1]])('rejects unsupported provider type %j', (type) => { + it.each([[' Unknown '], ['unsupported-provider'], [null], [1]])('rejects unsupported provider type %j', (type) => { expect(() => resolveComputeProviderType(type)).toThrow(`Unsupported compute provider type '${String(type)}'`); }); }); diff --git a/lambdas/libs/compute-providers/provider-types.ts b/lambdas/libs/compute-providers/provider-types.ts index dcac6c5769..087f61de71 100644 --- a/lambdas/libs/compute-providers/provider-types.ts +++ b/lambdas/libs/compute-providers/provider-types.ts @@ -1,24 +1,22 @@ -export const computeProviderTypes = ['ec2'] as const; +export const computeProviderTypes = ['ec2', 'microvm'] as const; export type ComputeProviderType = (typeof computeProviderTypes)[number]; export const defaultComputeProvider = 'ec2' satisfies ComputeProviderType; -export function normalizeComputeProviderType(type: unknown): ComputeProviderType | undefined { +export function resolveComputeProviderType(type: unknown): ComputeProviderType { if (type === undefined) return defaultComputeProvider; - if (typeof type !== 'string') return undefined; + if (typeof type !== 'string') { + throw new Error(`Unsupported compute provider type '${String(type)}'`); + } const normalizedType = type.trim().toLowerCase(); if (!normalizedType) return defaultComputeProvider; - return computeProviderTypes.find((computeProviderType) => computeProviderType === normalizedType); -} - -export function resolveComputeProviderType(type: unknown): ComputeProviderType { - const normalizedType = normalizeComputeProviderType(type); - if (!normalizedType) { + const computeProviderType = computeProviderTypes.find((provider) => provider === normalizedType); + if (!computeProviderType) { throw new Error(`Unsupported compute provider type '${String(type)}'`); } - return normalizedType; + return computeProviderType; } diff --git a/lambdas/libs/compute-providers/providers.config.control-plane.ts b/lambdas/libs/compute-providers/providers.config.control-plane.ts index 55ebaca95e..45a584bc06 100644 --- a/lambdas/libs/compute-providers/providers.config.control-plane.ts +++ b/lambdas/libs/compute-providers/providers.config.control-plane.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/control-plane'; +import { provider as microvm } from './aws/microvm/control-plane'; import type { ControlPlaneProviderModule } from './contracts'; /** Provider plugins included in the control-plane bundle. */ -export const enabledControlPlaneProviders = [ec2] as const satisfies readonly ControlPlaneProviderModule[]; +export const enabledControlPlaneProviders = [ec2, microvm] as const satisfies readonly ControlPlaneProviderModule[]; diff --git a/lambdas/libs/compute-providers/providers.config.webhook.ts b/lambdas/libs/compute-providers/providers.config.webhook.ts index 19c92734da..a4aec0853a 100644 --- a/lambdas/libs/compute-providers/providers.config.webhook.ts +++ b/lambdas/libs/compute-providers/providers.config.webhook.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/webhook'; +import { provider as microvm } from './aws/microvm/webhook'; import type { WebhookProviderModule } from './contracts'; /** Provider plugins included in the webhook bundle. */ -export const enabledWebhookProviders = [ec2] as const satisfies readonly WebhookProviderModule[]; +export const enabledWebhookProviders = [ec2, microvm] as const satisfies readonly WebhookProviderModule[]; diff --git a/lambdas/libs/compute-providers/registry.test.ts b/lambdas/libs/compute-providers/registry.test.ts index 3c95dcaca4..93227831cd 100644 --- a/lambdas/libs/compute-providers/registry.test.ts +++ b/lambdas/libs/compute-providers/registry.test.ts @@ -33,6 +33,6 @@ it('exposes every configured provider through both capability registries', () => unmarkOrphan: expect.any(Function), terminate: expect.any(Function), }); - expect(webhookProviderRegistry.capability(type, 'dynamicLabels').selectQueue).toEqual(expect.any(Function)); + expect(webhookProviderRegistry.capability(type, 'dynamicLabels').getViolations).toEqual(expect.any(Function)); } }); diff --git a/lambdas/libs/compute-providers/templates/provider/provider.test.ts b/lambdas/libs/compute-providers/templates/provider/provider.test.ts index 816b2f9cfc..2644fc4f2a 100644 --- a/lambdas/libs/compute-providers/templates/provider/provider.test.ts +++ b/lambdas/libs/compute-providers/templates/provider/provider.test.ts @@ -29,5 +29,5 @@ it('exposes every compute provider capability from its compute-provider entry po terminate: expect.any(Function), }); expect(webhookPlugin.type).toBe(webhookProvider.type); - expect(webhookPlugin.capabilities.dynamicLabels.selectQueue).toEqual(expect.any(Function)); + expect(webhookPlugin.capabilities.dynamicLabels.getViolations).toEqual(expect.any(Function)); }); diff --git a/lambdas/libs/compute-providers/templates/provider/webhook.ts b/lambdas/libs/compute-providers/templates/provider/webhook.ts index 86e59da7b3..31c522c588 100644 --- a/lambdas/libs/compute-providers/templates/provider/webhook.ts +++ b/lambdas/libs/compute-providers/templates/provider/webhook.ts @@ -3,10 +3,10 @@ import type { ComputeProviderPlugin } from '../../core'; import type { DynamicLabelProvider, WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; export const templateDynamicLabelProvider: DynamicLabelProvider = { - selectQueue: (input) => { + getViolations: (input) => { void input; - // Return a dispatch target when this provider accepts the requested dynamic labels. - return undefined; + // Return violations for dynamic labels this provider does not accept. + return []; }, }; diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts new file mode 100644 index 0000000000..a6d01b7e6e --- /dev/null +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig, WebhookProviderModule } from '../contracts'; +import { defaultComputeProvider } from '../provider-types'; +import type { ComputeProviderType } from '../provider-types'; +import { selectDynamicLabelQueue } from '../webhook'; + +interface RejectingPolicyCase { + name: string; + apply(queue: RunnerMatcherConfig): void; +} + +interface WebhookProviderContractOptions { + provider: WebhookProviderModule; + acceptedDynamicLabels: readonly [string, ...string[]]; + configureQueue?(queue: RunnerMatcherConfig): void; + rejectingPolicies: readonly [RejectingPolicyCase, ...RejectingPolicyCase[]]; +} + +export function defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels, + configureQueue, + rejectingPolicies, +}: WebhookProviderContractOptions): void { + const nonGhrLabels = ['self-hosted', 'linux']; + const dynamicLabels = [...acceptedDynamicLabels]; + + function configuredRunnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { + const queue = runnerQueue(id, computeProvider); + configureQueue?.(queue); + return queue; + } + + function expectProviderSelected(queue: RunnerMatcherConfig) { + expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toEqual({ + queue, + labels: [...nonGhrLabels, ...dynamicLabels], + }); + } + + describe(`${provider.type} webhook provider contract`, () => { + it('selects an explicitly configured provider through the production registry', () => { + expectProviderSelected(configuredRunnerQueue(`${provider.type}-configured`, provider.type)); + }); + + it('skips the provider when dynamic labels are disabled', () => { + const queue = configuredRunnerQueue(`${provider.type}-disabled`, provider.type); + queue.matcherConfig.enableDynamicLabels = false; + + expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); + }); + + for (const policy of rejectingPolicies) { + it(`skips the provider when its ${policy.name} policy rejects the labels`, () => { + const queue = configuredRunnerQueue(`${provider.type}-policy-rejected`, provider.type); + policy.apply(queue); + + expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); + }); + } + + it('normalizes provider configuration before registry selection', () => { + const queue = configuredRunnerQueue(`${provider.type}-normalized`); + (queue as unknown as { computeProvider: string }).computeProvider = ` ${provider.type.toUpperCase()} `; + + expectProviderSelected(queue); + }); + + if (provider.type === defaultComputeProvider) { + it('selects the default provider when the queue omits provider configuration', () => { + expectProviderSelected(configuredRunnerQueue(`${provider.type}-default`)); + }); + } + }); +} + +function runnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { + return { + id, + arn: `arn:${id}`, + computeProvider, + matcherConfig: { + labelMatchers: [['self-hosted', 'linux']], + exactMatch: true, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/tsconfig.json b/lambdas/libs/compute-providers/tsconfig.json index 52d55867fe..51beb73b87 100644 --- a/lambdas/libs/compute-providers/tsconfig.json +++ b/lambdas/libs/compute-providers/tsconfig.json @@ -1,5 +1,5 @@ { "extends": "../../tsconfig.json", - "include": ["*.ts", "core/**/*", "aws/**/*", "templates/**/*"], + "include": ["*.ts", "core/**/*", "aws/**/*", "templates/**/*", "test/**/*"], "exclude": ["aws/**/*.test.ts"] } diff --git a/lambdas/libs/compute-providers/webhook.test.ts b/lambdas/libs/compute-providers/webhook.test.ts new file mode 100644 index 0000000000..7ec7343f97 --- /dev/null +++ b/lambdas/libs/compute-providers/webhook.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it, vi } from 'vitest'; + +import type { DynamicLabelProvider, DynamicLabelViolation, RunnerMatcherConfig } from './contracts'; +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; +import { createDynamicLabelQueueSelector, selectDynamicLabelQueue } from './webhook'; + +const testProviderTypes = ['alpha', 'beta'] as const; +type TestProviderType = (typeof testProviderTypes)[number]; + +describe('selectDynamicLabelQueue', () => { + it.each([ + ['unsupported string', 'unsupported-provider'], + ['non-string', 42], + ])('strictly rejects an %s compute provider', (_description, computeProvider) => { + const invalidQueue = runnerQueue('invalid-provider'); + (invalidQueue as unknown as { computeProvider: unknown }).computeProvider = computeProvider; + + expect(() => selectDynamicLabelQueue([invalidQueue], [], [])).toThrow( + `Unsupported compute provider type '${String(computeProvider)}'`, + ); + }); +}); + +describe('createDynamicLabelQueueSelector', () => { + it('returns the first queue accepted by its provider', () => { + const queue = runnerQueue('accepted'); + const { selectQueue } = selector(); + + expect(selectQueue([queue], ['self-hosted', 'linux'], ['ghr-test-size:large'])).toEqual({ + queue, + labels: ['self-hosted', 'linux', 'ghr-test-size:large'], + }); + }); + + it('skips queues that disable dynamic labels', () => { + const disabledQueue = runnerQueue('disabled'); + disabledQueue.matcherConfig.enableDynamicLabels = false; + const enabledQueue = runnerQueue('enabled'); + const { getViolations, selectQueue } = selector(); + + expect(selectQueue([disabledQueue, enabledQueue], ['self-hosted'], ['ghr-test-size:large'])).toEqual({ + queue: enabledQueue, + labels: ['self-hosted', 'ghr-test-size:large'], + }); + expect(getViolations).toHaveBeenCalledOnce(); + expect(getViolations).toHaveBeenCalledWith({ queue: enabledQueue, labels: ['ghr-test-size:large'] }); + }); + + it('skips queues whose provider reports violations', () => { + const rejectedQueue = runnerQueue('rejected'); + const acceptedQueue = runnerQueue('accepted'); + const { selectQueue } = selector({ + violationsByQueue: { + rejected: [{ label: 'ghr-test-size:large', reason: 'size is unavailable' }], + }, + }); + + expect(selectQueue([rejectedQueue, acceptedQueue], ['self-hosted'], ['ghr-test-size:large'])).toEqual({ + queue: acceptedQueue, + labels: ['self-hosted', 'ghr-test-size:large'], + }); + }); + + it('returns undefined when every provider reports violations', () => { + const queue = runnerQueue('rejected'); + const { selectQueue } = selector({ + violationsByQueue: { + rejected: [{ label: 'ghr-test-size:large', reason: 'size is unavailable' }], + }, + }); + + expect(selectQueue([queue], ['self-hosted'], ['ghr-test-size:large'])).toBeUndefined(); + }); + + it('selects the queue targeted by provider-specific labels', () => { + const alphaQueue = runnerQueue('alpha'); + const betaQueue = runnerQueue('beta'); + const betaLabel = 'ghr-beta-size:large'; + const { getViolations, selectQueue } = selector({ + providerByQueue: { alpha: 'alpha', beta: 'beta' }, + }); + + expect(selectQueue([alphaQueue, betaQueue], ['self-hosted', 'linux'], [betaLabel])).toEqual({ + queue: betaQueue, + labels: ['self-hosted', 'linux', betaLabel], + }); + expect(getViolations).toHaveBeenCalledOnce(); + expect(getViolations).toHaveBeenCalledWith({ queue: betaQueue, labels: [betaLabel] }); + }); +}); + +function selector(options?: { + providerByQueue?: Record; + violationsByQueue?: Record; +}) { + const getViolations = vi.fn(({ queue }) => { + return options?.violationsByQueue?.[queue.id] ?? []; + }); + + return { + getViolations, + selectQueue: createDynamicLabelQueueSelector({ + resolveProvider: (queue) => ({ + type: options?.providerByQueue?.[queue.id] ?? 'alpha', + dynamicLabels: { getViolations }, + }), + dynamicLabelsForOtherProvider: (labels, provider) => + dynamicLabelsForOtherProvider(labels, provider, testProviderTypes), + }), + }; +} + +function runnerQueue(id: string): RunnerMatcherConfig { + return { + id, + arn: `arn:${id}`, + matcherConfig: { + labelMatchers: [['self-hosted', 'linux']], + exactMatch: true, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/webhook.ts b/lambdas/libs/compute-providers/webhook.ts index ee80a54203..1aa0a7b5e6 100644 --- a/lambdas/libs/compute-providers/webhook.ts +++ b/lambdas/libs/compute-providers/webhook.ts @@ -1,8 +1,70 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + import { createComputeProviderRegistry } from './core'; -import type { WebhookProviderCapabilities } from './contracts'; +import type { + DynamicLabelDispatchTarget, + DynamicLabelProvider, + RunnerMatcherConfig, + WebhookProviderCapabilities, +} from './contracts'; +import { dynamicLabelsForOtherProvider } from './dynamic-labels'; +import { resolveComputeProviderType } from './provider-types'; import { enabledWebhookProviders } from './providers.config.webhook'; +const logger = createChildLogger('handler'); + export const webhookProviderRegistry = createComputeProviderRegistry( enabledWebhookProviders.map((provider) => provider.createPlugin()), ); + +export function createDynamicLabelQueueSelector(dependencies: { + resolveProvider(queue: RunnerMatcherConfig): { type: TProvider; dynamicLabels: DynamicLabelProvider }; + dynamicLabelsForOtherProvider(labels: string[], provider: TProvider): string[]; +}) { + return ( + matches: RunnerMatcherConfig[], + nonGhrLabels: string[], + sanitizedGhrLabels: string[], + ): DynamicLabelDispatchTarget | undefined => { + for (const queue of matches) { + const { type: provider, dynamicLabels } = dependencies.resolveProvider(queue); + + if (!queue.matcherConfig.enableDynamicLabels) { + logger.warn( + `Queue ${queue.id} matches non-dynamic labels but does not allow dynamic labels; trying next match`, + ); + continue; + } + + const labelsForOtherProvider = dependencies.dynamicLabelsForOtherProvider(sanitizedGhrLabels, provider); + if (labelsForOtherProvider.length > 0) { + logger.warn(`Queue ${queue.id}: dynamic labels target another compute provider; trying next match`, { + dynamicLabels: labelsForOtherProvider, + }); + continue; + } + + const violations = dynamicLabels.getViolations({ queue, labels: sanitizedGhrLabels }); + if (violations.length === 0) { + return { queue, labels: [...nonGhrLabels, ...sanitizedGhrLabels] }; + } + + for (const violation of violations) { + logger.warn( + `Queue ${queue.id}: dynamic label '${violation.label}' is not accepted (${violation.reason}); trying next match`, + ); + } + } + + return undefined; + }; +} + +export const selectDynamicLabelQueue = createDynamicLabelQueueSelector({ + resolveProvider: (queue) => { + const type = resolveComputeProviderType(queue.computeProvider); + return { type, dynamicLabels: webhookProviderRegistry.capability(type, 'dynamicLabels') }; + }, + dynamicLabelsForOtherProvider, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 56ae435c2c..4f60fc7f3a 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -148,6 +148,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-lambda-microvms": "npm:^3.1074.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -439,6 +440,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-lambda-microvms@npm:^3.1074.0": + version: 3.1104.0 + resolution: "@aws-sdk/client-lambda-microvms@npm:3.1104.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-node": "npm:^3.972.78" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/219ad52f822def4caa4a20d8d91d46a1b78e6726363a145be86c37cdeef4e4c13653e8a59ada67154146c6c2554e2c12944efad35c689850bf6f72f2d55246f4 + languageName: node + linkType: hard + "@aws-sdk/client-s3@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-s3@npm:3.1014.0" @@ -620,6 +637,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.6": + version: 3.977.6 + resolution: "@aws-sdk/core@npm:3.977.6" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@aws-sdk/xml-builder": "npm:^3.972.37" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/4d743603bb41aeed426e2928be0947202191c341f9fbefe9ea347b0b4b7154b1ea94189d01c8abf3b03b9635449e2e7c268bd67379ea2294b9f49a61b909b9af + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -643,6 +676,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/547bcac01ac0912d0e42bb11f7d51bafcf2eaab1db35a098bea2be322211a86457ea60455a5294e58081c32376c240b07e66f81946a9be30e9722f723c6eaac2 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -661,6 +707,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.69": + version: 3.972.69 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.69" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/6e4cf9628919163a2a9784bf8618bc85a8c0ba7056813bedb9758c04eb3b36663f5099cfad329f89ac86c4e408bb3d0698ee7cff7e4a61c8a0335ab98078d678 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -683,6 +744,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.12": + version: 3.973.12 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.12" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-login": "npm:^3.972.74" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/84646fee1c61e31b2052d902559ecf163c1d00558ecdc21d77b396250527348b9ebba324d0bf8ffee4b3e45476c691de502e6faad3d39d1f7420eee5d326c7c5 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -699,6 +781,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.74": + version: 3.972.74 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.74" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/1ab9996accb61bccbdaefae023e9befab9e5062435370a37f672089457dc13c485d9d2fee6926381672bdb32143c00266b1aa5913b18ef4873584907835b3a92 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -719,6 +815,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.78": + version: 3.972.78 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.78" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-ini": "npm:^3.973.12" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2b6e5bd455a3c2b530a884a0c5919bb7d2d91941b655a56351b957de038d318c1d42b86674e20893ee7dab6db6ea32c4653bce9b1d3ca98ac803d6b49a948343 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -733,6 +848,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/0381c39f171df2119791b03647545ab5084f6a8d2c227c5d3c5bfa9db027d0566102b6322bc09075c0779b0f0aa88ae1ca7bbdc773d8415d8dd8f163e69e45ea + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -749,6 +877,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.11": + version: 3.973.11 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.11" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/token-providers": "npm:3.1103.0" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d6df0ae72009c2f74f1c7f12e41c0a7b395ba1860d4f9f1554fd8fbc3b5f0c1be64c83aacd2b329561e27844520ae0b57bb268265f5e7850b1d0fb769455d7a1 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -764,6 +907,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.73": + version: 3.972.73 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.73" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a7bee06b4200ff04141d4ce07d49d69f24b55b57f3aab929b445a9d9ea70f043d0b09fca8b95872d2b92df6837b771aeb14b03ca784d17ce1ee871b14173558c + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -1002,6 +1159,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.41": + version: 3.997.41 + resolution: "@aws-sdk/nested-clients@npm:3.997.41" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.43" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/fe1a84bb58675a24ecd0ce3b7bcaf1a456494f10c1a9dd5b55bd268be6713f83bd3c3d3dadee6bb51a53bc24ee18b2b0882d6741bcbabc224feeae97454fdb4a + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1029,6 +1202,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.43": + version: 3.996.43 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.43" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/268608dd5624c6377243903d588b9c13b8de3f3f3e6bea68fc684d125bc92a991fd15a67cb178d1a7a599d0415ce5283f44ba6b96d14909b185d7ff26a9d979b + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1044,6 +1229,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1103.0": + version: 3.1103.0 + resolution: "@aws-sdk/token-providers@npm:3.1103.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/5f86aa221e537b8a3fd11ed76ac025935f8859cc62b3af293abd759b8ca3aa390c17f6716723c08056b3373997a17bbdee2ff568eab5049bf0a372d35893b48d + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1054,6 +1253,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.2": + version: 3.974.2 + resolution: "@aws-sdk/types@npm:3.974.2" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b5ce05e8a4160c545edce1e8527e8ac490be7a6651c736f6811190b5d31d5682699889d51186ab0600df756679bebd2df9d650a17f577523441df803c4fb5777 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1139,6 +1348,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.37": + version: 3.972.37 + resolution: "@aws-sdk/xml-builder@npm:3.972.37" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/738f9302f495b3b95602641166a4182244add6e9e079201dba7e8994657dd442df0e4cea3355aa8c7d7f08efb385decaaf0b543f03efdb291c118536f36ac1a1 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1146,6 +1365,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.23.5, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4404,6 +4630,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.31.1": + version: 3.31.1 + resolution: "@smithy/core@npm:3.31.1" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b953c792dea2c13249b58c1799e4d6aaf21eb1a61e203b83e8e3a9156bebe14ca0585f0ca1ffdf65a193294dddff92a06fbe5c3fbd63ff0c174c88130b47a128 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4417,6 +4653,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.4.16 + resolution: "@smithy/credential-provider-imds@npm:4.4.16" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d03687efbbd1f95e77b7dcb639f24f1600671929627cd743f7acf9640238746664e91f955026f22e235603e10537d46e31fa60f231adbdf37457e53720bc80f9 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4485,6 +4732,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.13": + version: 5.6.13 + resolution: "@smithy/fetch-http-handler@npm:5.6.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/028ba8794a6c487ebefae7f40d0124f70e51a1f4e0e465457845c1a44fd607320cd3c64d4a961f159aef59470f0fd43f0d2011b44ee5ef753b7e1dccbdf32ca3 + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4650,6 +4908,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.13": + version: 4.9.13 + resolution: "@smithy/node-http-handler@npm:4.9.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2f1cdef7a300ad49c3bb698c2ca4773af5e9202d291cfcd855c1b21ab08b3c4ddf56f3722d3251db4e9b7ac39ec1ebc551b156abf3fa70f74c5491bec421f6b5 + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4735,6 +5004,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.6.12 + resolution: "@smithy/signature-v4@npm:5.6.12" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/33656a41ad61dee16209703cb96b46b29014b3c4fad23bfbb90cdb5415ac06c6577b2bfff958ef9e6c19091364945135a0370b12ddc2daed557c903846e81fe7 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4768,6 +5048,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.16.1": + version: 4.16.1 + resolution: "@smithy/types@npm:4.16.1" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/e024d9d148deca7bd21d032a9316db109bbe7cf256ffbb8d3981655b9f4f7695c08ec9b87f5a8cf1442e783ba26cb27e4f09603c5bfa3ba1e526c41b1b3e94d2 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12"