refactor(storage): extract SSM provider capabilities

docs/adr/003-runner-storage-provider-boundary.md

@@ -0,0 +1,124 @@
+# ADR-003: Runner Storage Provider Boundary
+
+## Status
+
+Proposed
+
+## Date
+
+2026-08-24
+
+## Context
+
+The runner control plane uses AWS Systems Manager Parameter Store for several
+unrelated purposes:
+
+- durable GitHub App credentials and webhook secrets;
+- durable webhook matcher and runner configuration;
+- short-lived registration tokens and just-in-time runner configuration;
+- a rebuildable runner-group ID cache;
+- compute-provider-specific values such as an EC2 AMI ID; and
+- EC2 Systems Manager access, which is not a storage concern.
+
+Treating all of these uses as one generic key/value provider would erase their
+different confidentiality, ownership, retention, consistency, and cleanup
+requirements. It would also make a future backend inherit operations that it
+does not need. For example, a runner-bootstrap backend must write a sensitive
+payload for one runner, while the runner-group cache stores a non-secret value
+that can be rebuilt from GitHub.
+
+The existing control-plane implementation calls the shared SSM utility
+directly for both runtime-created runner payloads and runner-group cache
+entries. That couples orchestration logic to Parameter Store and its write-rate
+behavior.
+
+## Decision
+
+Storage is divided by usage capability. This change introduces two initial
+contracts:
+
+- `RunnerBootstrapStore` writes the short-lived, sensitive registration or JIT
+  payload consumed by one runner.
+- `RunnerGroupCacheStore` reads and writes the rebuildable GitHub runner-group
+  ID cache.
+
+The contracts expose only the operations needed by their consumers. They do
+not expose a generic `get`, `put`, or `delete` API shared across all storage
+uses. Provider implementations own backend-specific path construction,
+serialization, encryption selection, tags, throughput guidance, SDK calls, and
+errors.
+
+The first registered implementation is `aws_ssm`. Scale-up and pool select it
+independently through
+`RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE` and
+`RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE`. Independent selection prevents a
+future cache or bootstrap migration from silently moving both data classes.
+
+The SSM implementation preserves the existing behavior:
+
+- runner payloads remain `SecureString` parameters at
+  `<token path>/<runner ID>`;
+- runner-group IDs remain `String` parameters at
+  `<config path>/runner-group/<group name>`;
+- the existing parameter tags and per-runner metadata tags are preserved; and
+- the existing SSM write-rate guidance continues to pace burst writes.
+
+No Terraform-managed SSM resource, resource address, path, IAM policy, KMS
+behavior, runner bootstrap reader, or housekeeper is moved by this decision.
+The stable and experimental Terraform paths only add explicit provider
+selection to the existing Lambda environments.
+
+### Package ownership
+
+The runtime implementation is organized as follows:
+
+```text
+lambdas/libs/storage-providers/
+├── core/             # usage contracts and provider registry
+├── provider-types.ts # supported provider identifiers and default resolution
+└── aws/ssm.ts        # Parameter Store implementation
+```
+
+Shared orchestration consumes the usage contracts. The SSM package owns calls
+to `aws-ssm-util`; compute- and orchestration-provider packages do not acquire
+new SSM parsing or IAM behavior.
+
+### Deferred storage uses
+
+The following SSM uses remain unchanged and are deliberately not forced behind
+the initial contracts:
+
+- GitHub credentials and webhook secrets need a secret-resolution contract.
+- Matcher configuration, persistent runner configuration, and controller
+  manifests need owner-specific durable configuration contracts.
+- EC2 AMI Parameter Store resolution remains compute-provider-specific because
+  EC2 consumes the SSM reference directly.
+- Systems Manager access to runner instances remains an EC2 capability rather
+  than a storage provider.
+- Reading and deleting the bootstrap payload remains in the existing runner
+  bootstrap implementation until that protocol can be migrated without
+  changing images or runtime compatibility.
+
+A later backend, including DynamoDB, must implement and test only the usage
+capabilities it supports. It must not broaden these contracts into an
+unrestricted storage API.
+
+## Consequences
+
+### Positive
+
+- Runtime-created runner payloads and rebuildable cache entries have separate,
+  testable semantics.
+- Existing SSM resources and operational behavior remain stable.
+- A later backend can be introduced for one usage without changing unrelated
+  credentials, configuration, or compute-provider behavior.
+- Sensitive runner payloads remain encrypted and are not added to logs or
+  provider configuration.
+
+### Negative
+
+- The initial provider boundary covers only the control-plane writer and cache;
+  runner-side consumption is still SSM-specific.
+- Two provider selectors are more verbose than one global storage selector.
+- Additional usage contracts will be needed before other SSM responsibilities
+  can migrate.

lambdas/functions/control-plane/package.json

@@ -33,6 +33,7 @@
     "@aws-github-runner/aws-powertools-util": "*",
     "@aws-github-runner/aws-ssm-util": "*",
     "@aws-github-runner/compute-providers": "*",
+    "@aws-github-runner/storage-providers": "*",
     "@aws-lambda-powertools/parameters": "^2.31.0",
     "@aws-sdk/client-ec2": "^3.1009.0",
     "@aws-sdk/client-sqs": "^3.1009.0",

lambdas/functions/control-plane/src/modules.d.ts

@@ -16,6 +16,8 @@ declare namespace NodeJS {
     PARAMETER_GITHUB_APP_ID_NAME: string;
     PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string;
     RUNNER_OWNER: string;
+    RUNNER_BOOTSTRAP_STORAGE_PROVIDER_TYPE?: string;
+    RUNNER_GROUP_CACHE_STORAGE_PROVIDER_TYPE?: string;
     COMPUTE_PROVIDER_TYPE?: string;
     SCALE_DOWN_CONFIG: string;
     SSM_TOKEN_PATH: string;

lambdas/functions/control-plane/src/scale-runners/github-runner.ts

@@ -1,5 +1,9 @@
 import { createChildLogger } from '@aws-github-runner/aws-powertools-util';
-import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util';
+import {
+  createRunnerBootstrapStoreFromEnvironment,
+  createRunnerGroupCacheStoreFromEnvironment,
+} from '@aws-github-runner/storage-providers';
+import type { RunnerBootstrapStore, RunnerGroupCacheStore } from '@aws-github-runner/storage-providers';
 import { Octokit } from '@octokit/rest';
 
 import { getStoredInstallationId } from '../github/auth';
@@ -14,7 +18,7 @@ export interface GitHubRunnerMetadata {
 }
 
 export interface StartRunnerConfigOptions {
-  getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[];
+  getStorageMetadataTags?: (runnerId: string) => { key: string; value: string }[];
   onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise<void>;
 }
 
@@ -182,39 +186,27 @@ export async function isJobQueued(
 export async function getRunnerGroupId(
   githubRunnerConfig: CreateGitHubRunnerConfig,
   ghClient: Octokit,
+  runnerGroupCacheStore: RunnerGroupCacheStore = createRunnerGroupCacheStore(githubRunnerConfig),
 ): Promise<number> {
   // if the runnerType is Repo, then runnerGroupId is default to 1
   let runnerGroupId: number | undefined = 1;
   if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) {
     let runnerGroup: string | undefined;
-    // check if runner group id is already stored in SSM Parameter Store and
-    // use it if it exists to avoid API call to GitHub
+    const identity = { kind: 'runner_group_cache' as const, groupName: githubRunnerConfig.runnerGroup };
+    // Use the cached runner-group ID when available to avoid a GitHub API call.
     try {
-      runnerGroup = await getParameter(
-        `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`,
-      );
+      runnerGroup = (await runnerGroupCacheStore.get(identity))?.payload;
     } catch (err) {
       logger.debug('Handling error:', err as Error);
-      logger.warn(
-        `SSM Parameter "${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}"
-         for Runner group ${githubRunnerConfig.runnerGroup} does not exist`,
-      );
+      logger.warn(`Cached ID for runner group ${githubRunnerConfig.runnerGroup} does not exist`);
     }
     if (runnerGroup === undefined) {
       // get runner group id from GitHub
       runnerGroupId = await getRunnerGroupByName(ghClient, githubRunnerConfig);
-      // store runner group id in SSM
       try {
-        await putParameter(
-          `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`,
-          runnerGroupId.toString(),
-          false,
-          {
-            tags: githubRunnerConfig.ssmParameterStoreTags,
-          },
-        );
+        await runnerGroupCacheStore.put({ identity, payload: runnerGroupId.toString() });
       } catch (err) {
-        logger.debug('Error storing runner group id in SSM Parameter Store', err as Error);
+        logger.debug('Error storing runner group ID in the cache', err as Error);
         throw err;
       }
     } else {
@@ -250,18 +242,45 @@ export async function createStartRunnerConfig(
   ghClient: Octokit,
   options: StartRunnerConfigOptions = {},
 ): Promise<string[]> {
+  const runnerBootstrapStore = createRunnerBootstrapStore(githubRunnerConfig);
   if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) {
-    return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, options);
+    return await createJitConfig(
+      githubRunnerConfig,
+      runnerIds,
+      ghClient,
+      options,
+      runnerBootstrapStore,
+      createRunnerGroupCacheStore(githubRunnerConfig),
+    );
   } else {
-    return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, options);
+    return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, options, runnerBootstrapStore);
   }
 }
 
-function addDelay(runnerIds: string[]) {
+function createRunnerBootstrapStore(githubRunnerConfig: CreateGitHubRunnerConfig): RunnerBootstrapStore {
+  return createRunnerBootstrapStoreFromEnvironment({
+    locator: githubRunnerConfig.ssmTokenPath,
+    metadataTags: githubRunnerConfig.ssmParameterStoreTags.map(({ Key, Value }) => ({ key: Key, value: Value })),
+  });
+}
+
+function createRunnerGroupCacheStore(githubRunnerConfig: CreateGitHubRunnerConfig): RunnerGroupCacheStore {
+  return createRunnerGroupCacheStoreFromEnvironment({
+    locator: githubRunnerConfig.ssmConfigPath,
+    metadataTags: githubRunnerConfig.ssmParameterStoreTags.map(({ Key, Value }) => ({ key: Key, value: Value })),
+  });
+}
+
+function addDelay(runnerIds: string[], runnerBootstrapStore: RunnerBootstrapStore) {
   const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
-  const ssmParameterStoreMaxThroughput = 40;
-  const isDelay = runnerIds.length >= ssmParameterStoreMaxThroughput;
-  return { isDelay, delay };
+  const maxWritesPerSecond = runnerBootstrapStore.maxWritesPerSecond;
+  const isDelay = maxWritesPerSecond !== undefined && runnerIds.length >= maxWritesPerSecond;
+  const delayMs = maxWritesPerSecond === undefined ? 0 : 1000 / maxWritesPerSecond;
+  return { isDelay, delay, delayMs };
+}
+
+function getRunnerBootstrapMetadataTags(options: StartRunnerConfigOptions, runnerId: string) {
+  return options.getStorageMetadataTags?.(runnerId) ?? [];
 }
 
 /**
@@ -274,8 +293,9 @@ async function createRegistrationTokenConfig(
   runnerIds: string[],
   ghClient: Octokit,
   options: StartRunnerConfigOptions,
+  runnerBootstrapStore: RunnerBootstrapStore,
 ): Promise<string[]> {
-  const { isDelay, delay } = addDelay(runnerIds);
+  const { isDelay, delay, delayMs } = addDelay(runnerIds, runnerBootstrapStore);
   const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient);
   const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token);
 
@@ -284,12 +304,13 @@ async function createRegistrationTokenConfig(
   });
 
   for (const runnerId of runnerIds) {
-    await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, {
-      tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags],
-    });
+    await runnerBootstrapStore.put(
+      { identity: { kind: 'runner_bootstrap', runnerId }, payload: runnerServiceConfig.join(' ') },
+      { metadataTags: getRunnerBootstrapMetadataTags(options, runnerId) },
+    );
     if (isDelay) {
       // Delay to prevent AWS ssm rate limits by being within the max throughput limit
-      await delay(25);
+      await delay(delayMs);
     }
   }
 
@@ -307,9 +328,11 @@ async function createJitConfig(
   runnerIds: string[],
   ghClient: Octokit,
   options: StartRunnerConfigOptions,
+  runnerBootstrapStore: RunnerBootstrapStore,
+  runnerGroupCacheStore: RunnerGroupCacheStore,
 ): Promise<string[]> {
-  const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient);
-  const { isDelay, delay } = addDelay(runnerIds);
+  const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient, runnerGroupCacheStore);
+  const { isDelay, delay, delayMs } = addDelay(runnerIds, runnerBootstrapStore);
   const runnerLabels = githubRunnerConfig.runnerLabels.split(',');
   const failedRunnerIds: string[] = [];
 
@@ -347,16 +370,16 @@ async function createJitConfig(
         runnerLabels,
       });
 
-      // store jit config in ssm parameter store
       logger.debug('Runner JIT config for ephemeral runner generated.', {
         instance: runnerId,
       });
-      await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, {
-        tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags],
-      });
+      await runnerBootstrapStore.put(
+        { identity: { kind: 'runner_bootstrap', runnerId }, payload: runnerConfig.data.encoded_jit_config },
+        { metadataTags: getRunnerBootstrapMetadataTags(options, runnerId) },
+      );
       if (isDelay) {
         // Delay to prevent AWS ssm rate limits by being within the max throughput limit
-        await delay(25);
+        await delay(delayMs);
       }
     } catch (error) {
       failedRunnerIds.push(runnerId);

lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts

@@ -168,7 +168,7 @@ async function createTestProviderRunners(input: CreateScaleUpRunnersInput<unknow
       result.instances,
       input.githubInstallationClient,
       {
-        getSsmParameterTags: (runnerId) => [{ Key: 'RunnerId', Value: runnerId }],
+        getStorageMetadataTags: (runnerId) => [{ key: 'RunnerId', value: runnerId }],
       },
     );
   } catch {

lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-config.ts

@@ -141,7 +141,7 @@ async function terminateFailedInstances(instanceIds: string[]): Promise<void> {
 
 function createEc2StartRunnerConfigOptions(): StartRunnerConfigOptions {
   return {
-    getSsmParameterTags: (instanceId) => [{ Key: 'InstanceId', Value: instanceId }],
+    getStorageMetadataTags: (instanceId) => [{ key: 'InstanceId', value: instanceId }],
     onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(instanceId, metadata),
   };
 }

lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts

@@ -182,7 +182,7 @@ describe('scaleUp with GHES', () => {
         { Key: 'ghr:runner_labels', Value: 'label1,label2' },
       ]);
       const [, , , options] = mockCreateStartRunnerConfig.mock.calls[0];
-      expect(options?.getSsmParameterTags?.('i-12345')).toEqual([{ Key: 'InstanceId', Value: 'i-12345' }]);
+      expect(options?.getStorageMetadataTags?.('i-12345')).toEqual([{ key: 'InstanceId', value: 'i-12345' }]);
     });
 
     it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => {

lambdas/libs/compute-providers/core/index.ts

@@ -32,7 +32,7 @@ export interface GitHubRunnerMetadata {
 }
 
 export interface StartRunnerConfigOptions {
-  getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[];
+  getStorageMetadataTags?: (runnerId: string) => { key: string; value: string }[];
   onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise<void>;
 }