refactor(ec2): decouple creation result from control plane

lambdas/libs/compute-providers/aws/ec2/src/control-plane/create-result.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+
+import type { Ec2RunnerFailureCode } from '../runner-create-result';
+import { toControlPlaneCreateRunnerResult } from './create-result';
+
+function result(failureCodes: Ec2RunnerFailureCode[] = []) {
+  return {
+    instances: [],
+    failedInstanceCount: 2,
+    failureCodes,
+  };
+}
+
+describe('control-plane EC2 create result', () => {
+  it.each([
+    ['configured AWS name', ['aws-name:InsufficientInstanceCapacity'], ['InsufficientInstanceCapacity']],
+    ['built-in AWS name', ['aws-name:ThrottlingException'], []],
+    ['AWS server fault', ['aws-fault:server'], []],
+    ['HTTP throttling status', ['aws-http:429'], []],
+    ['HTTP server status', ['aws-http:503'], []],
+    ['network code', ['aws-code:ECONNRESET'], []],
+  ] as const)('classifies %s as retryable', (_name, failureCodes, configuredErrors) => {
+    expect(toControlPlaneCreateRunnerResult(result([...failureCodes]), configuredErrors)).toEqual({
+      instances: [],
+      retryableErrorCount: 2,
+      nonRetryableErrorCount: 0,
+    });
+  });
+
+  it('classifies missing capacity without retry evidence as non-retryable', () => {
+    expect(toControlPlaneCreateRunnerResult(result(['aws-name:InvalidParameterValue']), [])).toEqual({
+      instances: [],
+      retryableErrorCount: 0,
+      nonRetryableErrorCount: 2,
+    });
+  });
+
+  it('classifies every missing instance as retryable when any Fleet code is retryable', () => {
+    expect(
+      toControlPlaneCreateRunnerResult(result(['aws-name:InvalidParameterValue', 'aws-name:InternalError']), []),
+    ).toEqual({ instances: [], retryableErrorCount: 2, nonRetryableErrorCount: 0 });
+  });
+
+  it('preserves created instances while converting failed instances', () => {
+    expect(
+      toControlPlaneCreateRunnerResult(
+        {
+          instances: ['i-created'],
+          failedInstanceCount: 1,
+          failureCodes: ['aws-name:InvalidParameterValue'],
+        },
+        [],
+      ),
+    ).toEqual({ instances: ['i-created'], retryableErrorCount: 0, nonRetryableErrorCount: 1 });
+  });
+});

lambdas/libs/compute-providers/aws/ec2/src/control-plane/create-result.ts

@@ -0,0 +1,56 @@
+import type { CreateRunnerResult } from '../../../../core';
+
+import type { Ec2RunnerCreateResult, Ec2RunnerFailureCode } from '../runner-create-result';
+
+const RETRYABLE_AWS_ERROR_NAMES = new Set([
+  'EC2ThrottledException',
+  'InternalError',
+  'RequestLimitExceeded',
+  'RequestTimeout',
+  'RequestTimeoutException',
+  'ServiceUnavailable',
+  'Throttling',
+  'ThrottlingException',
+]);
+
+const RETRYABLE_NETWORK_ERROR_CODES = new Set([
+  'EAI_AGAIN',
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'ENETUNREACH',
+  'ENOTFOUND',
+  'ETIMEDOUT',
+]);
+
+function failureCodeValue(failureCode: Ec2RunnerFailureCode, prefix: string): string | undefined {
+  return failureCode.startsWith(prefix) ? failureCode.slice(prefix.length) : undefined;
+}
+
+function triggersControlPlaneRetry(failureCode: Ec2RunnerFailureCode, configuredErrors: Set<string>): boolean {
+  const errorName = failureCodeValue(failureCode, 'aws-name:');
+  if (errorName !== undefined) {
+    return configuredErrors.has(errorName) || RETRYABLE_AWS_ERROR_NAMES.has(errorName);
+  }
+
+  const errorCode = failureCodeValue(failureCode, 'aws-code:');
+  if (errorCode !== undefined) return RETRYABLE_NETWORK_ERROR_CODES.has(errorCode);
+  if (failureCode === 'aws-fault:server') return true;
+
+  const httpStatus = failureCodeValue(failureCode, 'aws-http:');
+  if (httpStatus === undefined) return false;
+  const status = Number(httpStatus);
+  return status === 429 || status >= 500;
+}
+
+export function toControlPlaneCreateRunnerResult(
+  result: Ec2RunnerCreateResult,
+  configuredRetryableErrors: readonly string[],
+): CreateRunnerResult {
+  const configuredErrors = new Set(configuredRetryableErrors);
+  const retryable = result.failureCodes.some((failureCode) => triggersControlPlaneRetry(failureCode, configuredErrors));
+  return {
+    instances: result.instances,
+    retryableErrorCount: retryable ? result.failedInstanceCount : 0,
+    nonRetryableErrorCount: retryable ? 0 : result.failedInstanceCount,
+  };
+}

lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts

@@ -13,6 +13,7 @@ import yn from 'yn';
 
 import type { Ec2RunnerResourceOperations } from '../runners';
 import type { RunnerInputParameters } from '../runners.d';
+import { toControlPlaneCreateRunnerResult } from './create-result';
 
 const logger = createChildLogger('ec2-runners');
 const RUNNER_LABELS_TAG_KEY = 'ghr:runner_labels';
@@ -71,13 +72,15 @@ export async function createRunners(
 ): Promise<CreateRunnerResult> {
   let result: CreateRunnerResult;
   try {
-    result = await ec2Operations.create({
+    const { scaleErrors, ...ec2CreateConfig } = ec2RunnerConfig;
+    const ec2Result = await ec2Operations.create({
+      ...ec2CreateConfig,
       runnerType: githubRunnerConfig.runnerType,
       runnerOwner: githubRunnerConfig.runnerOwner,
       numberOfRunners,
       source,
-      ...ec2RunnerConfig,
     });
+    result = toControlPlaneCreateRunnerResult(ec2Result, scaleErrors);
   } catch (error) {
     logger.error('Unexpected error while creating EC2 runner instances.', {
       error,

lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts

@@ -37,8 +37,8 @@ interface CreateProviderRunnersOptions {
   githubRunnerConfig?: Partial<CreateGitHubRunnerConfig>;
 }
 
-function createRunnerResult(instances: string[], retryableErrorCount = 0, nonRetryableErrorCount = 0) {
-  return { instances, retryableErrorCount, nonRetryableErrorCount };
+function createRunnerResult(instances: string[], failedInstanceCount = 0, failureCodes: string[] = []) {
+  return { instances, failedInstanceCount, failureCodes };
 }
 
 function runnerConfig(overrides: Partial<CreateGitHubRunnerConfig> = {}): CreateGitHubRunnerConfig {
@@ -77,7 +77,6 @@ function expectedRunnerParams(
     subnets: ['subnet-123'],
     tracingEnabled: false,
     onDemandFailoverOnError: [],
-    scaleErrors: ['UnfulfillableCapacity', 'MaxSpotInstanceCountExceeded', 'TargetCapacityLimitExceededException'],
     source: 'scale-up-lambda',
     useDedicatedHost: false,
     ec2OverrideConfig: undefined,
@@ -532,14 +531,15 @@ describe('scaleUp with public GH', () => {
       );
     });
 
-    it('creates a runner with correct config and labels and custom scale errors enabled.', async () => {
+    it('converts configured EC2 failures to the control-plane create result', async () => {
       process.env.SCALE_ERRORS = '["RequestLimitExceeded"]';
-      await createProviderRunners({
-        githubRunnerConfig: { runnerType: 'Repo', runnerOwner: repositoryRunnerOwner },
-      });
-      expect(mockCreateRunner).toHaveBeenCalledWith(
-        expectedRunnerParams('Repo', repositoryRunnerOwner, { scaleErrors: ['RequestLimitExceeded'] }),
-      );
+      mockCreateRunner.mockResolvedValueOnce(createRunnerResult([], 1, ['aws-name:RequestLimitExceeded']));
+      await expect(
+        createProviderRunners({
+          githubRunnerConfig: { runnerType: 'Repo', runnerOwner: repositoryRunnerOwner },
+        }),
+      ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 });
+      expect(mockCreateRunner).toHaveBeenCalledWith(expectedRunnerParams('Repo', repositoryRunnerOwner));
     });
   });
 });

lambdas/libs/compute-providers/aws/ec2/src/runner-create-result.ts

@@ -0,0 +1,11 @@
+export type Ec2RunnerFailureCode =
+  | `aws-name:${string}`
+  | `aws-code:${string}`
+  | `aws-fault:${'client' | 'server'}`
+  | `aws-http:${number}`;
+
+export interface Ec2RunnerCreateResult {
+  instances: string[];
+  failedInstanceCount: number;
+  failureCodes: Ec2RunnerFailureCode[];
+}

lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts

@@ -46,6 +46,5 @@ export interface RunnerInputParameters {
   amiIdSsmParameterName?: string;
   tracingEnabled?: boolean;
   onDemandFailoverOnError?: string[];
-  scaleErrors: string[];
   useDedicatedHost?: boolean;
 }