diff --git a/agent/README.md b/agent/README.md index f3903a524..ecfc6bdde 100644 --- a/agent/README.md +++ b/agent/README.md @@ -218,6 +218,34 @@ Immediate response (acceptance): Final metrics (PR URL, cost, turns, build status, etc.) appear in **container logs**, in **DynamoDB** when configured, and in the **REST API** for deployed tasks (`GET /v1/tasks/{task_id}` via the `bgagent` CLI or HTTP client). +### AWS Lambda MicroVMs lifecycle hooks (ADR-021 P1) + +The same uvicorn process also serves the **Lambda MicroVMs** lifecycle hooks, on the same port (8080 — the port declared in the image's `hooks.port`). On that backend there is no `InvokeAgentRuntime` and no orchestrator→agent HTTP path at all: the task payload arrives as the `/run` hook body and nothing else dials in. + +**`POST /aws/lambda-microvms/runtime/v1/ready`** — Build hook. Returns `{"status": "ready"}` as soon as the server is up, which is the signal the service waits for before taking the snapshot. **Mandatory**, not optional: `CreateMicrovmImage` refuses an image that enables *any* lifecycle hook without `/ready`, and with the hook enabled but unserved every build fails with `Ready hook check failed: the application returned a client error (HTTP 4xx) response`. + +**`POST /aws/lambda-microvms/runtime/v1/run`** — Payload delivery. Validates the body, starts the pipeline in a background thread (the same `_extract_invocation_params` → `_spawn_background` path `/invocations` uses), and returns 200 inside the 1–60 s hook budget. Body: + +```json +{ + "microvmId": "microvm-b44b69d9-…", + "runHookPayload": "{\"agent_payload_s3_uri\": \"s3://bucket//payload.json\"}" +} +``` + +`runHookPayload` is an opaque **string** the service passes through from `RunMicrovm`. ABCA's contract for it is one of two shapes, mirroring the ECS container env contract (`AGENT_PAYLOAD` / `AGENT_PAYLOAD_S3_URI`): + +| Envelope | When | +|---|---| +| `{"agent_payload": {…}}` | the whole orchestrator payload inline — only when it fits | +| `{"agent_payload_s3_uri": "s3://bucket/key"}` | pointer to the payload in the platform payload bucket | + +The service caps `runHookPayload` at **4 096 bytes**, so the **pointer form is the normal one** — a hydrated payload is essentially always larger. Fetching it needs no new env var: the MicroVM execution role holds read-only access to that bucket and the URI carries bucket + key. + +Rejections are structured so they are readable in the MicroVM log group: `400 MICROVM_RUN_PAYLOAD_INVALID` (unusable envelope — retrying the same body cannot help), `500 MICROVM_RUN_PAYLOAD_UNREADABLE` (the S3 fetch failed), `400 TASK_RECORD_INCOMPLETE` (same validator and vocabulary as `/invocations`). + +`/validate` (build) and `/suspend`, `/resume`, `/terminate` (runtime) are deliberately **not** served — declaring a hook nothing answers fails the corresponding build or lifecycle transition, so the CDK construct declares exactly `/ready` + `/run`. `/terminate` and `/validate` land in P2; `/suspend` + `/resume` in P3 with the ComputeStrategy interface widening. + ### Testing Server Mode Locally Use `run.sh --server` to build and start the server locally. It handles credentials, port mapping, and resource constraints automatically: @@ -375,7 +403,7 @@ agent/ │ ├── repo.py Repository setup: clone, branch, git auth, mise trust/install/build/lint │ ├── shell.py Shell utilities: log(), run_cmd(), redact_secrets(), slugify(), truncate() │ ├── telemetry.py Metrics, disk usage, trajectory writer (_TrajectoryWriter with write_policy_decision) -│ ├── server.py FastAPI — async /invocations (background thread), /ping health check, heartbeat daemon; OTEL session correlation +│ ├── server.py FastAPI — async /invocations (background thread), /ping health check, MicroVM /ready + /run lifecycle hooks, heartbeat daemon; OTEL session correlation │ ├── task_state.py Best-effort DynamoDB task status and heartbeat writes (no-op if TASK_TABLE_NAME unset) │ ├── observability.py OpenTelemetry helpers (e.g. AgentCore session id) │ ├── memory.py Optional memory / episode integration for the agent diff --git a/agent/src/server.py b/agent/src/server.py index 4c2674c16..831696555 100644 --- a/agent/src/server.py +++ b/agent/src/server.py @@ -1,15 +1,17 @@ -"""FastAPI server for AgentCore Runtime. +"""FastAPI server for AgentCore Runtime and Lambda MicroVMs. -Exposes /invocations (POST) and /ping (GET) on port 8080, -matching the AgentCore Runtime container contract. +Exposes /invocations (POST) and /ping (GET) on port 8080, matching the AgentCore +Runtime container contract, plus the AWS Lambda MicroVMs lifecycle hooks under +``/aws/lambda-microvms/runtime/v1/`` (ADR-021 P1) on the same port. -The /invocations handler accepts the task, spawns a background thread to run -the pipeline, and returns a small JSON acceptance immediately. Task progress -is tracked in DynamoDB via ``task_state`` + ``ProgressWriter``. +Both entry paths accept the task, spawn a background thread to run the pipeline, +and return a small JSON acceptance immediately. Task progress is tracked in +DynamoDB via ``task_state`` + ``ProgressWriter``. """ import asyncio import contextlib as _ctx_for_debug +import json import logging import os import threading @@ -814,3 +816,226 @@ async def invoke_agent(request: Request, body: InvocationRequest): } } ) + + +# -------------------------------------------------------------------------- +# AWS Lambda MicroVMs lifecycle hooks (ADR-021 P1) +# -------------------------------------------------------------------------- +# The MicroVM backend has NO orchestrator→agent HTTP path: the task payload +# arrives as the ``/run`` hook's request body and nothing else dials in. The +# service calls these routes on the port declared in the image's ``hooks.port`` +# (8080 — the same uvicorn process that serves /invocations and /ping), so the +# hooks live here rather than in a sidecar. +# +# Only ``/ready`` and ``/run`` are implemented, and both are P1: +# * ``/ready`` is MANDATORY. ``CreateMicrovmImage`` refuses an image that +# enables ANY lifecycle hook without it ("The ready (/ready) MicroVM image +# hook must be enabled when any MicroVM lifecycle hook … is enabled"), and an +# image with no hooks at all cannot receive a ``runHookPayload``. So ADR-021's +# original "declare /run in P1, serve it in P2" split was not a reachable +# service state. +# * ``/run`` is the payload-delivery channel. +# ``/suspend`` and ``/resume`` are P3 (they need the ComputeStrategy interface +# widening), and ``/validate`` + ``/terminate`` are P2 polish. Declaring a hook +# the agent does not answer fails the corresponding build or lifecycle +# transition, which is why the construct declares exactly these two. +MICROVM_HOOK_PREFIX = "/aws/lambda-microvms/runtime/v1" + +#: ``s3://`` scheme prefix for the out-of-band payload pointer. +_S3_URI_SCHEME = "s3://" + + +class MicrovmRunHookRequest(BaseModel): + """Body the MicroVM service POSTs to the ``/run`` hook. + + ``runHookPayload`` is the opaque STRING the orchestrator passed to + ``RunMicrovm`` — the service does not parse it. ABCA's contract for that + string (``lambda-microvm-strategy.ts``) is one of two shapes, mirroring the + ECS container env contract (``AGENT_PAYLOAD`` / ``AGENT_PAYLOAD_S3_URI``): + + * ``{"agent_payload": {...}}`` — the whole orchestrator payload, inline. + * ``{"agent_payload_s3_uri": "s3://bucket/key"}`` — a pointer to it. + + The pointer form is the DOMINANT one: the service caps ``runHookPayload`` at + 4 096 bytes and a hydrated payload is essentially always larger. + + Both fields default to empty so a malformed call produces this module's + structured 400 rather than FastAPI's 422 — the service surfaces a 4xx as a + generic "client error" hook failure either way, and our own body is what ends + up in the MicroVM log group. + """ + + microvmId: str = "" # service field name; camelCase on the wire + runHookPayload: str = "" # service field name; camelCase on the wire + + +def _fetch_microvm_payload_from_s3(uri: str) -> dict: + """Read and parse the out-of-band ``/run`` payload from S3. + + Same fetch the ECS boot command performs for ``AGENT_PAYLOAD_S3_URI``; the + MicroVM **execution role** holds the read grant, scoped to the platform + payload bucket. Errors propagate to the caller, which turns them into a + structured 400/500 — silently starting a pipeline with no payload would + produce a task that runs with an empty prompt. + """ + remainder = uri[len(_S3_URI_SCHEME) :] + bucket, _, key = remainder.partition("/") + if not bucket or not key: + raise ValueError(f"agent_payload_s3_uri is not a bucket/key URI: {uri!r}") + + import boto3 + + region = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") + client = boto3.client("s3", region_name=region) + body = client.get_object(Bucket=bucket, Key=key)["Body"].read() + payload = json.loads(body) + if not isinstance(payload, dict): + raise ValueError(f"S3 payload at {uri!r} is {type(payload).__name__}, expected an object") + return payload + + +def _resolve_microvm_run_payload(run_hook_payload: str) -> dict: + """Turn the ``runHookPayload`` string into the orchestrator payload dict. + + Raises ``ValueError`` for every shape the agent cannot act on, so the caller + has exactly one failure branch to map onto a 400. + """ + if not run_hook_payload.strip(): + raise ValueError("runHookPayload is empty") + + try: + envelope = json.loads(run_hook_payload) + except json.JSONDecodeError as exc: + raise ValueError(f"runHookPayload is not valid JSON: {exc}") from exc + + if not isinstance(envelope, dict): + raise ValueError(f"runHookPayload must be a JSON object, got {type(envelope).__name__}") + + inline = envelope.get("agent_payload") + if inline is not None: + if not isinstance(inline, dict): + raise ValueError(f"agent_payload must be an object, got {type(inline).__name__}") + return inline + + uri = envelope.get("agent_payload_s3_uri") + if isinstance(uri, str) and uri.startswith(_S3_URI_SCHEME): + return _fetch_microvm_payload_from_s3(uri) + if uri is not None: + raise ValueError(f"agent_payload_s3_uri must be an s3:// URI, got {uri!r}") + + raise ValueError( + "runHookPayload envelope has neither agent_payload nor agent_payload_s3_uri " + f"(keys: {sorted(envelope)})" + ) + + +@app.post(f"{MICROVM_HOOK_PREFIX}/ready") +def microvm_ready(): + """MicroVM image ``/ready`` build hook — "the application has initialised". + + A 200 from this route is the signal the service waits for before taking the + snapshot, so answering it at all is what makes the image buildable: with the + hook enabled and nothing serving it, both chipset builds fail with "Ready hook + check failed: the application returned a client error (HTTP 4xx) response". + + Reaching this handler already proves everything P1 needs: uvicorn is bound on + the hook port and ``server`` imported cleanly (which pulls in ``pipeline`` → + ``runner`` → the policy engine, so a missing policy file or a broken import + fails the BUILD instead of the first task). Deeper warm-up assertions — + Bedrock reachability, Memory access, tool availability — belong to P2's + ``/validate``, which is deliberately still not declared: a hook that 404s or + reports failure fails every image build. + + Declared ``def`` rather than ``async def`` on purpose: Starlette runs sync + handlers in a threadpool, so this never competes with the event loop that has + to keep ``GET /ping`` fast. + """ + _debug_cw("/ready hook: server is up, reporting ready for snapshot") + return {"status": "ready"} + + +@app.post(f"{MICROVM_HOOK_PREFIX}/run") +def microvm_run(request: Request, body: MicrovmRunHookRequest): + """MicroVM ``/run`` lifecycle hook — accept the task and start the pipeline. + + Fast-notification contract (1-60 s hook budget): validate, spawn, return 200. + The pipeline itself must NOT run on the hook path, so this reuses the exact + mechanism ``/invocations`` uses — ``_extract_invocation_params`` → + ``_validate_required_params`` → ``_spawn_background`` — rather than a second, + drifting payload mapper. The orchestrator payload is byte-identical across + substrates (AgentCore receives it as ``input``, ECS as ``AGENT_PAYLOAD``, + MicroVMs inside this envelope), which is what makes that reuse correct. + + Session/workload headers are absent here (there is no AgentCore Runtime in + front of this call), so ``_extract_invocation_params`` resolves an empty + ``session_id`` / workload token — the same posture the ECS backend already + has, per ADR-021 sub-decision 3's identity delta. + + Sync ``def`` for the same reason as ``/ready``, and additionally because the + S3 payload fetch is a blocking boto3 call: in a threadpool it cannot stall + the event loop. + """ + _debug_cw(f"/run hook received: microvm_id={body.microvmId!r} bytes={len(body.runHookPayload)}") + + try: + payload = _resolve_microvm_run_payload(body.runHookPayload) + except ValueError as exc: + # Bad envelope — the orchestrator built something this agent cannot act + # on. 400 (not 500) because retrying an identical body cannot help. + _warn_cw(f"/run hook rejected: {exc}") + return JSONResponse( + status_code=400, + content={ + "code": "MICROVM_RUN_PAYLOAD_INVALID", + "message": str(exc), + }, + ) + except Exception as exc: + # Payload fetch failed (S3 AccessDenied / NoSuchKey / transient). 500 so + # the failure is distinguishable from a malformed body, and loud enough to + # find in the MicroVM log group. + _debug_cw_exc("/run hook payload fetch FAILED", exc) + return JSONResponse( + status_code=500, + content={ + "code": "MICROVM_RUN_PAYLOAD_UNREADABLE", + "message": f"{type(exc).__name__}: {exc}", + }, + ) + + task_id_log = str(payload.get("task_id", "")) + try: + params = _extract_invocation_params(payload, request) + except Exception as exc: + _debug_cw_exc( + "/run hook _extract_invocation_params FAILED", exc, task_id=task_id_log or None + ) + raise + + missing = _validate_required_params(params) + if missing: + _debug_cw( + f"/run hook rejected: missing required params {missing!r}", + task_id=task_id_log or None, + ) + return JSONResponse( + status_code=400, + content={ + "code": "TASK_RECORD_INCOMPLETE", + "message": ( + "Task record is missing required fields. The orchestrator " + "should have populated these before starting the MicroVM." + ), + "missing": missing, + }, + ) + + _spawn_background(params) + task_id = params["task_id"] + _debug_cw(f"/run hook accepted task_id={task_id!r}", task_id=task_id or None) + return { + "status": "accepted", + "task_id": task_id, + "microvm_id": body.microvmId, + "timestamp": datetime.now(UTC).isoformat(), + } diff --git a/agent/tests/test_server.py b/agent/tests/test_server.py index 32383703d..dcec9de59 100644 --- a/agent/tests/test_server.py +++ b/agent/tests/test_server.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import threading import time from typing import Any @@ -831,6 +832,373 @@ def test_none_stays_none(self): assert params["approval_gate_cap"] is None +# -------------------------------------------------------------------------- +# AWS Lambda MicroVMs lifecycle hooks (ADR-021 P1) +# -------------------------------------------------------------------------- + + +READY_HOOK = f"{server.MICROVM_HOOK_PREFIX}/ready" +RUN_HOOK = f"{server.MICROVM_HOOK_PREFIX}/run" + + +def _run_hook_body(envelope: dict, microvm_id: str = "microvm-abc") -> dict: + """Wrap an ABCA payload envelope in the service's ``/run`` request body. + + The service passes ``runHookPayload`` through as an opaque STRING (it never + parses it), so the double encoding here is the real wire shape, not a test + artifact. + """ + return {"microvmId": microvm_id, "runHookPayload": json.dumps(envelope)} + + +class TestMicrovmReadyHook: + """``/ready`` is what makes a MicroVM image buildable at all. + + ``CreateMicrovmImage`` refuses an image that enables any lifecycle hook + without ``/ready``, and with the hook enabled but unserved both chipset + builds fail ("Ready hook check failed: the application returned a client + error (HTTP 4xx) response"). A 200 from a booted server is the whole + contract in P1 — deeper warm-up checks are P2's ``/validate``. + """ + + def test_ready_returns_200_once_the_server_is_up(self, client): + r = client.post(READY_HOOK) + assert r.status_code == 200 + assert r.json() == {"status": "ready"} + + def test_ready_is_mounted_under_the_service_hook_prefix(self): + assert server.MICROVM_HOOK_PREFIX == "/aws/lambda-microvms/runtime/v1" + routes = {getattr(r, "path", None) for r in server.app.routes} + assert READY_HOOK in routes + assert RUN_HOOK in routes + + def test_ready_does_not_start_a_pipeline(self, client, monkeypatch): + # A build hook must never run task work: the snapshot is taken right + # after it answers, so anything it starts would be frozen into the image. + monkeypatch.setattr(server, "run_task", MagicMock()) + client.post(READY_HOOK) + with server._threads_lock: + assert server._active_threads == [] + + def test_validate_and_suspend_resume_terminate_are_NOT_served(self, client): + # Declaring a hook nothing answers fails the corresponding build or + # lifecycle transition, so the construct declares exactly /ready + /run. + # This asserts the agent side of that: the others must 404. + for hook in ("validate", "suspend", "resume", "terminate"): + assert client.post(f"{server.MICROVM_HOOK_PREFIX}/{hook}").status_code == 404 + + +class TestMicrovmRunHookInlinePayload: + """Inline envelope: ``{"agent_payload": {...}}``. + + The exception rather than the rule — the service caps ``runHookPayload`` at + 4 096 bytes and a hydrated payload is larger — but it is the branch that + proves the payload→pipeline mapping without any S3 involvement. + """ + + def test_accepts_the_payload_and_starts_the_pipeline_asynchronously(self, client, monkeypatch): + started = threading.Event() + seen: dict = {} + + def fake_run_task(**kwargs): + seen.update(kwargs) + started.set() + + monkeypatch.setattr(server, "run_task", fake_run_task) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + r = client.post( + RUN_HOOK, + json=_run_hook_body( + { + "agent_payload": { + "task_id": "t-microvm-1", + "repo_url": "org/repo", + "prompt": "Fix the bug", + "github_token": "ghp_x", + "aws_region": "us-east-1", + } + }, + microvm_id="microvm-inline", + ), + ) + + assert r.status_code == 200 + body = r.json() + assert body["status"] == "accepted" + assert body["task_id"] == "t-microvm-1" + # Echoed so a MicroVM log line can be joined to the control-plane id. + assert body["microvm_id"] == "microvm-inline" + + assert started.wait(timeout=5.0), "pipeline thread did not start" + # Same mapping the /invocations path performs: prompt→task_description, + # model_id→anthropic_model, etc. — one mapper, not two. + assert seen["task_id"] == "t-microvm-1" + assert seen["repo_url"] == "org/repo" + assert seen["task_description"] == "Fix the bug" + + def test_returns_before_the_pipeline_finishes(self, client, monkeypatch): + release = threading.Event() + entered = threading.Event() + + def slow_run_task(**_kwargs): + entered.set() + release.wait(timeout=10.0) + + monkeypatch.setattr(server, "run_task", slow_run_task) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + try: + r = client.post( + RUN_HOOK, + json=_run_hook_body( + {"agent_payload": {"task_id": "t-async", "repo_url": "o/r", "prompt": "x"}} + ), + ) + # The hook budget is 1-60 s and the pipeline runs for minutes, so the + # 200 must land while the pipeline is still executing. + assert r.status_code == 200 + assert entered.wait(timeout=5.0) + with server._threads_lock: + assert any(t.is_alive() for t in server._active_threads) + finally: + release.set() + + def test_uses_the_same_model_id_and_prompt_aliases_as_invocations(self, client, monkeypatch): + seen: dict = {} + started = threading.Event() + + def fake_run_task(**kwargs): + seen.update(kwargs) + started.set() + + monkeypatch.setattr(server, "run_task", fake_run_task) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + client.post( + RUN_HOOK, + json=_run_hook_body( + { + "agent_payload": { + "task_id": "t-alias", + "repo_url": "o/r", + "prompt": "do it", + "model_id": "anthropic.claude-x", + "cedar_policies": ["p1"], + "channel_source": "linear", + } + } + ), + ) + assert started.wait(timeout=5.0) + assert seen["anthropic_model"] == "anthropic.claude-x" + assert seen["cedar_policies"] == ["p1"] + assert seen["channel_source"] == "linear" + + +class TestMicrovmRunHookS3Payload: + """S3-pointer envelope: ``{"agent_payload_s3_uri": "s3://bucket/key"}``. + + The DOMINANT path on this backend: with a 4 096-byte ``runHookPayload`` cap, + any hydrated payload is offloaded to the platform payload bucket and only the + pointer travels in the hook body. + """ + + def test_fetches_the_payload_from_s3_and_starts_the_pipeline(self, client, monkeypatch): + seen: dict = {} + started = threading.Event() + + def fake_run_task(**kwargs): + seen.update(kwargs) + started.set() + + fetched: dict = {} + + def fake_fetch(uri): + fetched["uri"] = uri + return {"task_id": "t-s3", "repo_url": "org/repo", "prompt": "from s3"} + + monkeypatch.setattr(server, "_fetch_microvm_payload_from_s3", fake_fetch) + monkeypatch.setattr(server, "run_task", fake_run_task) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + r = client.post( + RUN_HOOK, + json=_run_hook_body({"agent_payload_s3_uri": "s3://payload-bucket/t-s3/payload.json"}), + ) + + assert r.status_code == 200 + assert r.json()["task_id"] == "t-s3" + assert fetched["uri"] == "s3://payload-bucket/t-s3/payload.json" + assert started.wait(timeout=5.0) + assert seen["task_description"] == "from s3" + + def test_parses_bucket_and_key_out_of_the_uri(self, monkeypatch): + captured: dict = {} + + class _Body: + @staticmethod + def read(): + return b'{"task_id": "t-1", "repo_url": "o/r"}' + + class _S3: + @staticmethod + def get_object(**kwargs): + captured.update(kwargs) + return {"Body": _Body} + + import boto3 + + monkeypatch.setattr(boto3, "client", lambda *_a, **_k: _S3) + + payload = server._fetch_microvm_payload_from_s3("s3://my-bucket/prefix/t-1/payload.json") + + # Key keeps every slash after the bucket — a naive split would truncate it. + assert captured == {"Bucket": "my-bucket", "Key": "prefix/t-1/payload.json"} + assert payload == {"task_id": "t-1", "repo_url": "o/r"} + + def test_rejects_a_uri_with_no_key(self, monkeypatch): + with pytest.raises(ValueError, match="not a bucket/key URI"): + server._fetch_microvm_payload_from_s3("s3://bucket-only") + + def test_rejects_a_non_object_s3_body(self, monkeypatch): + class _Body: + @staticmethod + def read(): + return b"[1, 2, 3]" + + class _S3: + @staticmethod + def get_object(**_kwargs): + return {"Body": _Body} + + import boto3 + + monkeypatch.setattr(boto3, "client", lambda *_a, **_k: _S3) + + with pytest.raises(ValueError, match="expected an object"): + server._fetch_microvm_payload_from_s3("s3://b/k") + + def test_s3_failure_returns_500_and_starts_nothing(self, client, monkeypatch): + def boom(_uri): + raise RuntimeError("AccessDenied") + + monkeypatch.setattr(server, "_fetch_microvm_payload_from_s3", boom) + monkeypatch.setattr(server, "run_task", MagicMock()) + + r = client.post(RUN_HOOK, json=_run_hook_body({"agent_payload_s3_uri": "s3://bucket/key"})) + + # 500, not 400: the body was well-formed, the fetch was not. Retrying an + # identical body CAN help here, unlike a malformed envelope. + assert r.status_code == 500 + assert r.json()["code"] == "MICROVM_RUN_PAYLOAD_UNREADABLE" + assert "AccessDenied" in r.json()["message"] + with server._threads_lock: + assert server._active_threads == [] + + +class TestMicrovmRunHookRejections: + """Every shape the agent cannot act on must fail LOUDLY, before spawning. + + A hook that 200s on a payload it could not read would start a pipeline with + an empty prompt and burn a full task before anyone noticed. + """ + + @pytest.mark.parametrize( + "run_hook_payload,expected_fragment", + [ + ("", "runHookPayload is empty"), + (" ", "runHookPayload is empty"), + ("not json at all", "not valid JSON"), + ('"a string"', "must be a JSON object"), + ("[1,2,3]", "must be a JSON object"), + ('{"agent_payload": "not-an-object"}', "agent_payload must be an object"), + ('{"agent_payload_s3_uri": "https://example.com/x"}', "must be an s3:// URI"), + ('{"agent_payload_s3_uri": 42}', "must be an s3:// URI"), + ('{"something_else": 1}', "neither agent_payload nor agent_payload_s3_uri"), + ], + ) + def test_returns_400_with_a_named_code( + self, client, monkeypatch, run_hook_payload, expected_fragment + ): + monkeypatch.setattr(server, "run_task", MagicMock()) + + r = client.post( + RUN_HOOK, json={"microvmId": "microvm-x", "runHookPayload": run_hook_payload} + ) + + assert r.status_code == 400 + body = r.json() + assert body["code"] == "MICROVM_RUN_PAYLOAD_INVALID" + assert expected_fragment in body["message"] + with server._threads_lock: + assert server._active_threads == [] + + def test_a_missing_body_field_is_a_400_not_a_422(self, client, monkeypatch): + # Both fields default to "", so an empty body reaches our own structured + # rejection instead of FastAPI's 422 — the message ends up in the MicroVM + # log group, so it has to be ours. + monkeypatch.setattr(server, "run_task", MagicMock()) + r = client.post(RUN_HOOK, json={}) + assert r.status_code == 400 + assert r.json()["code"] == "MICROVM_RUN_PAYLOAD_INVALID" + + def test_incomplete_task_record_reuses_the_invocations_rejection_shape( + self, client, monkeypatch + ): + monkeypatch.setattr(server, "run_task", MagicMock()) + + r = client.post( + RUN_HOOK, + json=_run_hook_body({"agent_payload": {"task_id": "t-bad"}}), + ) + + assert r.status_code == 400 + body = r.json() + assert body["code"] == "TASK_RECORD_INCOMPLETE" + # Same validator as /invocations, so the same missing-field vocabulary. + assert "repo_url" in body["missing"] + with server._threads_lock: + assert server._active_threads == [] + + +class TestMicrovmRunHookHeaderPosture: + """No AgentCore Runtime sits in front of this call. + + So there is no session-id header and no workload access token — the same + env-var identity posture the ECS backend already has (ADR-021 sub-decision 3, + identity delta). Asserted so a future reader does not mistake the empty + values for a bug. + """ + + def test_session_id_and_workload_token_resolve_empty(self, client, monkeypatch): + seen: dict = {} + started = threading.Event() + + def fake_run_task(**kwargs): + seen.update(kwargs) + started.set() + + monkeypatch.setattr(server, "run_task", fake_run_task) + monkeypatch.setattr(server.task_state, "write_terminal", MagicMock()) + + client.post( + RUN_HOOK, + json=_run_hook_body( + {"agent_payload": {"task_id": "t-hdr", "repo_url": "o/r", "prompt": "x"}} + ), + ) + assert started.wait(timeout=5.0) + # run_task never receives the token/session (they are consumed by + # _run_task_background), so assert via the extractor instead. + fake_request: Any = _FakeRequest() + params = server._extract_invocation_params( + {"task_id": "t-hdr", "repo_url": "o/r", "prompt": "x"}, fake_request + ) + assert params["session_id"] == "" + assert params["workload_access_token"] == "" + + class TestInvocationParamContract: """The invocation boundary is wired as: diff --git a/cdk/bootstrap/BOOTSTRAP_HASH b/cdk/bootstrap/BOOTSTRAP_HASH index bc08e5ced..35b70e2bd 100644 --- a/cdk/bootstrap/BOOTSTRAP_HASH +++ b/cdk/bootstrap/BOOTSTRAP_HASH @@ -1 +1 @@ -b0501c8a57f20e4b5bf50d6fe8bbb934310adc4b64480228565d8392a19d1503 +40d0a8b2343663084f614423fc3e6210761377dedf0bc079956ba7d4cea84c5e diff --git a/cdk/bootstrap/BOOTSTRAP_VERSION b/cdk/bootstrap/BOOTSTRAP_VERSION index 26aaba0e8..f0bb29e76 100644 --- a/cdk/bootstrap/BOOTSTRAP_VERSION +++ b/cdk/bootstrap/BOOTSTRAP_VERSION @@ -1 +1 @@ -1.2.0 +1.3.0 diff --git a/cdk/bootstrap/bootstrap-template.yaml b/cdk/bootstrap/bootstrap-template.yaml index 8026647ce..a2dcd08b4 100644 --- a/cdk/bootstrap/bootstrap-template.yaml +++ b/cdk/bootstrap/bootstrap-template.yaml @@ -1,13 +1,13 @@ # GENERATED FILE - DO NOT EDIT DIRECTLY # This template is generated by: npx tsx scripts/generate-bootstrap-template.ts -# ABCA Bootstrap Policy Version: 1.2.0 -# ABCA Bootstrap Policy Hash: b0501c8a57f20e4b5bf50d6fe8bbb934310adc4b64480228565d8392a19d1503 +# ABCA Bootstrap Policy Version: 1.3.0 +# ABCA Bootstrap Policy Hash: 40d0a8b2343663084f614423fc3e6210761377dedf0bc079956ba7d4cea84c5e # # Based on the default CDK bootstrap template with the following modifications: # - BootstrapVariant set to "ABCA: Least-Privilege Bootstrap" # - ComputeTypes parameter added for compute-variant selection -# - IncludeComputeEcs condition added -# - 5 inline AWS::IAM::ManagedPolicy resources replace AdministratorAccess +# - IncludeComputeEcs / IncludeComputeLambdaMicrovms conditions added +# - 6 inline AWS::IAM::ManagedPolicy resources replace AdministratorAccess # - CloudFormationExecutionRole references our least-privilege policies # - BootstrapPolicyVersion, BootstrapPolicyHash, BootstrapPolicySet outputs added Description: This stack includes resources needed to deploy AWS CDK apps into this environment @@ -77,7 +77,7 @@ Parameters: ComputeTypes: Type: CommaDelimitedList Default: agentcore - Description: 'Comma-separated list of compute backends to enable. Valid values: agentcore, ecs.' + Description: 'Comma-separated list of compute backends to enable. Valid values: agentcore, ecs, lambda-microvm.' Conditions: HasTrustedAccounts: Fn::Not: @@ -148,6 +148,19 @@ Conditions: - Fn::Join: - '' - Ref: ComputeTypes + IncludeComputeLambdaMicrovms: + Fn::Not: + - Fn::Equals: + - Fn::Select: + - 0 + - Fn::Split: + - lambda-microvm + - Fn::Join: + - '' + - Ref: ComputeTypes + - Fn::Join: + - '' + - Ref: ComputeTypes Resources: FileAssetsBucketEncryptionKey: Type: AWS::KMS::Key @@ -728,6 +741,10 @@ Resources: - IncludeComputeEcs - Ref: IaCRoleABCAComputeEcs - Ref: AWS::NoValue + - Fn::If: + - IncludeComputeLambdaMicrovms + - Ref: IaCRoleABCAComputeLambdaMicrovms + - Ref: AWS::NoValue RoleName: Fn::Sub: cdk-${Qualifier}-cfn-exec-role-${AWS::AccountId}-${AWS::Region} PermissionsBoundary: @@ -1342,6 +1359,39 @@ Resources: Version: '2012-10-17' Description: 'ABCA Bootstrap: IaCRole-ABCA-Compute-ECS permissions for CloudFormation execution role' Condition: IncludeComputeEcs + IaCRoleABCAComputeLambdaMicrovms: + Type: AWS::IAM::ManagedPolicy + Properties: + ManagedPolicyName: + Fn::Sub: cdk-${Qualifier}-IaCRole-ABCA-Compute-LambdaMicrovms-${AWS::AccountId}-${AWS::Region} + PolicyDocument: + Statement: + - Action: + - lambda:CreateMicrovmImage + - lambda:GetMicrovmImage + - lambda:UpdateMicrovmImage + - lambda:DeleteMicrovmImage + - lambda:ListMicrovmImages + - lambda:GetMicrovmImageVersion + - lambda:UpdateMicrovmImageVersion + - lambda:DeleteMicrovmImageVersion + - lambda:ListMicrovmImageVersions + - lambda:GetMicrovmImageBuild + - lambda:ListMicrovmImageBuilds + - lambda:ListManagedMicrovmImages + - lambda:ListManagedMicrovmImageVersions + - lambda:CreateNetworkConnector + - lambda:GetNetworkConnector + - lambda:UpdateNetworkConnector + - lambda:DeleteNetworkConnector + - lambda:ListNetworkConnectors + - lambda:PassNetworkConnector + Effect: Allow + Resource: '*' + Sid: LambdaMicrovms + Version: '2012-10-17' + Description: 'ABCA Bootstrap: IaCRole-ABCA-Compute-LambdaMicrovms permissions for CloudFormation execution role' + Condition: IncludeComputeLambdaMicrovms Outputs: BucketName: Description: The name of the S3 bucket owned by the CDK toolkit stack @@ -1370,10 +1420,10 @@ Outputs: Value: '32' BootstrapPolicyVersion: Description: The version of the ABCA bootstrap policy bundle - Value: 1.2.0 + Value: 1.3.0 BootstrapPolicyHash: Description: SHA-256 hash of the ABCA bootstrap policy bundle for drift detection - Value: b0501c8a57f20e4b5bf50d6fe8bbb934310adc4b64480228565d8392a19d1503 + Value: 40d0a8b2343663084f614423fc3e6210761377dedf0bc079956ba7d4cea84c5e BootstrapPolicySet: Description: Comma-separated list of active ABCA bootstrap policy names Value: @@ -1387,3 +1437,7 @@ Outputs: - IncludeComputeEcs - Compute-ECS - Ref: AWS::NoValue + - Fn::If: + - IncludeComputeLambdaMicrovms + - Compute-LambdaMicrovms + - Ref: AWS::NoValue diff --git a/cdk/bootstrap/policies/compute-lambda-microvm.json b/cdk/bootstrap/policies/compute-lambda-microvm.json new file mode 100644 index 000000000..302a385a8 --- /dev/null +++ b/cdk/bootstrap/policies/compute-lambda-microvm.json @@ -0,0 +1,31 @@ +{ + "Statement": [ + { + "Action": [ + "lambda:CreateMicrovmImage", + "lambda:GetMicrovmImage", + "lambda:UpdateMicrovmImage", + "lambda:DeleteMicrovmImage", + "lambda:ListMicrovmImages", + "lambda:GetMicrovmImageVersion", + "lambda:UpdateMicrovmImageVersion", + "lambda:DeleteMicrovmImageVersion", + "lambda:ListMicrovmImageVersions", + "lambda:GetMicrovmImageBuild", + "lambda:ListMicrovmImageBuilds", + "lambda:ListManagedMicrovmImages", + "lambda:ListManagedMicrovmImageVersions", + "lambda:CreateNetworkConnector", + "lambda:GetNetworkConnector", + "lambda:UpdateNetworkConnector", + "lambda:DeleteNetworkConnector", + "lambda:ListNetworkConnectors", + "lambda:PassNetworkConnector" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "LambdaMicrovms" + } + ], + "Version": "2012-10-17" +} diff --git a/cdk/mise.toml b/cdk/mise.toml index 8e479e40f..22ceddee4 100644 --- a/cdk/mise.toml +++ b/cdk/mise.toml @@ -87,7 +87,7 @@ run = "npx cdk deploy" # parameter directly on the CDKToolkit stack: # aws cloudformation update-stack --stack-name CDKToolkit --use-previous-template \ # --capabilities CAPABILITY_NAMED_IAM \ -# --parameters ParameterKey=ComputeTypes,ParameterValue=agentcore\,ecs +# --parameters ParameterKey=ComputeTypes,ParameterValue=agentcore\,ecs\,lambda-microvm # # verify: aws cloudformation describe-stacks --stack-name CDKToolkit --query 'Stacks[0].Parameters' [tasks.bootstrap] description = "Bootstrap CDK with least-privilege policies (ComputeTypes=agentcore; for ECS set the ComputeTypes CFN param on CDKToolkit — see comment above the task)" diff --git a/cdk/package.json b/cdk/package.json index 1809861b2..619c994d2 100644 --- a/cdk/package.json +++ b/cdk/package.json @@ -21,6 +21,7 @@ "@aws-sdk/client-dynamodb": "^3.1078.0", "@aws-sdk/client-ecs": "^3.1078.0", "@aws-sdk/client-lambda": "^3.1078.0", + "@aws-sdk/client-lambda-microvms": "^3.1098.0", "@aws-sdk/client-s3": "^3.1078.0", "@aws-sdk/client-secrets-manager": "^3.1078.0", "@aws-sdk/credential-provider-node": "^3.972.61", diff --git a/cdk/scripts/README.md b/cdk/scripts/README.md index ab37decd5..548533c34 100644 --- a/cdk/scripts/README.md +++ b/cdk/scripts/README.md @@ -1,3 +1,11 @@ # CDK helper scripts Bundling for Lambda assets is handled at synth time; the **`bundle`** task in **`cdk/mise.toml`** is a no-op placeholder for **`cdk/cdk.json`**. Prefer **`mise //cdk:*`** tasks from the repository root (`MISE_EXPERIMENTAL=1`). + +| Script | Purpose | Invoke via | +|--------|---------|------------| +| `generate-bootstrap-artifacts.ts` | Regenerates `cdk/bootstrap/policies/*.json`, `BOOTSTRAP_VERSION`, `BOOTSTRAP_HASH` from the typed policies in `src/bootstrap/policies/` | `mise //cdk:bootstrap:generate` | +| `generate-bootstrap-template.ts` | Regenerates `cdk/bootstrap/bootstrap-template.yaml` (least-privilege CDK bootstrap, `ComputeTypes`-gated compute policies) | `mise //cdk:bootstrap:generate` | +| `package-microvm-artifact.sh` | Packages `agent/` + `contracts/` + `Dockerfile` into the zip artifact an `AWS::Lambda::MicrovmImage` builds from, and uploads it to the CDK-created artifact bucket (ADR-021) | run directly — see the script header for the full bootstrap sequence | + +`package-microvm-artifact.sh` exists because CloudFormation cannot produce its own MicroVM `codeArtifact`: the image resource consumes a zip that must already be in S3, and there is no CDK asset type for "zip + Dockerfile a MicroVM image builds from". Everything else on that backend (buckets, roles, network connector, log group, the image resource itself) is CDK-managed in `src/constructs/lambda-microvm-compute.ts`. diff --git a/cdk/scripts/generate-bootstrap-artifacts.ts b/cdk/scripts/generate-bootstrap-artifacts.ts index 7d08935d0..ca4a32dd8 100644 --- a/cdk/scripts/generate-bootstrap-artifacts.ts +++ b/cdk/scripts/generate-bootstrap-artifacts.ts @@ -24,6 +24,7 @@ import { applicationPolicy, computeAgentcorePolicy, computeEcsPolicy, + computeLambdaMicrovmPolicy, infrastructurePolicy, observabilityPolicy, } from '../src/bootstrap/policies'; @@ -38,6 +39,7 @@ const policies = [ { name: 'observability', fn: observabilityPolicy }, { name: 'compute-agentcore', fn: computeAgentcorePolicy }, { name: 'compute-ecs', fn: computeEcsPolicy }, + { name: 'compute-lambda-microvm', fn: computeLambdaMicrovmPolicy }, ]; for (const { name, fn } of policies) { diff --git a/cdk/scripts/generate-bootstrap-template.ts b/cdk/scripts/generate-bootstrap-template.ts index 732e7cb05..75a741302 100644 --- a/cdk/scripts/generate-bootstrap-template.ts +++ b/cdk/scripts/generate-bootstrap-template.ts @@ -35,6 +35,7 @@ import { applicationPolicy, computeAgentcorePolicy, computeEcsPolicy, + computeLambdaMicrovmPolicy, infrastructurePolicy, observabilityPolicy, } from '../src/bootstrap/policies'; @@ -66,30 +67,36 @@ template.Parameters.ComputeTypes = { Type: 'CommaDelimitedList', Default: 'agentcore', Description: - 'Comma-separated list of compute backends to enable. Valid values: agentcore, ecs.', + 'Comma-separated list of compute backends to enable. Valid values: agentcore, ecs, lambda-microvm.', }; // --- Step 3: Add conditions --- -template.Conditions.IncludeComputeEcs = { - 'Fn::Not': [ - { - 'Fn::Equals': [ - { - 'Fn::Select': [ - 0, - { - 'Fn::Split': [ - 'ecs', - { 'Fn::Join': ['', { Ref: 'ComputeTypes' }] }, - ], - }, - ], - }, - { 'Fn::Join': ['', { Ref: 'ComputeTypes' }] }, - ], - }, - ], -}; +/** + * "The joined ComputeTypes list contains `needle`" as a CloudFormation + * condition. There is no `Fn::Contains`, so the trick is: split the joined list + * on the needle and compare element 0 against the whole string — they differ + * only if the needle was actually present. Extracted into a helper (it was + * inline for ECS) so a third backend cannot introduce a subtly different + * variant of the same expression. + */ +function includesComputeType(needle: string): Record { + const joined = { 'Fn::Join': ['', { Ref: 'ComputeTypes' }] }; + return { + 'Fn::Not': [ + { + 'Fn::Equals': [ + { 'Fn::Select': [0, { 'Fn::Split': [needle, joined] }] }, + joined, + ], + }, + ], + }; +} + +template.Conditions.IncludeComputeEcs = includesComputeType('ecs'); +// ADR-021: matched on the full `lambda-microvm` token, NOT a prefix — a bare +// `Fn::Split` on `lambda` would also fire for any future `lambda*` backend name. +template.Conditions.IncludeComputeLambdaMicrovms = includesComputeType('lambda-microvm'); // --- Step 4: Add managed policy resources --- interface PolicyDef { @@ -127,6 +134,12 @@ const policyDefs: PolicyDef[] = [ policyFn: computeEcsPolicy, condition: 'IncludeComputeEcs', }, + { + logicalId: 'IaCRoleABCAComputeLambdaMicrovms', + policyName: 'IaCRole-ABCA-Compute-LambdaMicrovms', + policyFn: computeLambdaMicrovmPolicy, + condition: 'IncludeComputeLambdaMicrovms', + }, ]; for (const { logicalId, policyName, policyFn, condition } of policyDefs) { @@ -164,6 +177,13 @@ const coreRefs = [ { Ref: 'IaCRoleABCAObservability' }, { Ref: 'IaCRoleABCAComputeAgentcore' }, { 'Fn::If': ['IncludeComputeEcs', { Ref: 'IaCRoleABCAComputeEcs' }, { Ref: 'AWS::NoValue' }] }, + { + 'Fn::If': [ + 'IncludeComputeLambdaMicrovms', + { Ref: 'IaCRoleABCAComputeLambdaMicrovms' }, + { Ref: 'AWS::NoValue' }, + ], + }, ]; template.Resources.CloudFormationExecutionRole.Properties.ManagedPolicyArns = { @@ -196,6 +216,13 @@ template.Outputs.BootstrapPolicySet = { 'Observability', 'Compute-Agentcore', { 'Fn::If': ['IncludeComputeEcs', 'Compute-ECS', { Ref: 'AWS::NoValue' }] }, + { + 'Fn::If': [ + 'IncludeComputeLambdaMicrovms', + 'Compute-LambdaMicrovms', + { Ref: 'AWS::NoValue' }, + ], + }, ], ], }, @@ -221,8 +248,8 @@ const header = [ '# Based on the default CDK bootstrap template with the following modifications:', '# - BootstrapVariant set to "ABCA: Least-Privilege Bootstrap"', '# - ComputeTypes parameter added for compute-variant selection', - '# - IncludeComputeEcs condition added', - '# - 5 inline AWS::IAM::ManagedPolicy resources replace AdministratorAccess', + '# - IncludeComputeEcs / IncludeComputeLambdaMicrovms conditions added', + '# - 6 inline AWS::IAM::ManagedPolicy resources replace AdministratorAccess', '# - CloudFormationExecutionRole references our least-privilege policies', '# - BootstrapPolicyVersion, BootstrapPolicyHash, BootstrapPolicySet outputs added', '', diff --git a/cdk/scripts/package-microvm-artifact.sh b/cdk/scripts/package-microvm-artifact.sh new file mode 100755 index 000000000..526e40f70 --- /dev/null +++ b/cdk/scripts/package-microvm-artifact.sh @@ -0,0 +1,420 @@ +#!/usr/bin/env bash +# +# package-microvm-artifact.sh — build and upload the AWS Lambda MicroVMs image +# artifact for the ABCA agent (ADR-021, sub-decision 3). +# +# --------------------------------------------------------------------------- +# WHY THIS SCRIPT EXISTS +# --------------------------------------------------------------------------- +# `AWS::Lambda::MicrovmImage` *is* wired in CDK (see +# `cdk/src/constructs/lambda-microvm-compute.ts`), but CloudFormation cannot +# produce its own `codeArtifact`: the resource consumes a zip that already sits +# in S3, and that zip has to be assembled from the `agent/` tree. Unlike an ECR +# image there is no CDK asset type for "zip + Dockerfile a MicroVM image builds +# from", so packaging + upload is one out-of-band step, run once per agent +# change. Everything else — buckets, roles, connector, log group, the image +# resource itself — is CDK-managed. +# +# --------------------------------------------------------------------------- +# BOOTSTRAP SEQUENCE (first time) +# --------------------------------------------------------------------------- +# 1. Deploy the MicroVM substrate WITHOUT an image. Synth warns that no image +# is configured; that is expected — the artifact bucket must exist before +# you can upload to it. +# +# MISE_EXPERIMENTAL=1 mise //cdk:deploy -- --context compute_type=lambda-microvm +# +# 2. Package + upload the artifact (this script). It reads the bucket name and +# object key straight from the stack outputs, so there is nothing to copy +# by hand: +# +# cdk/scripts/package-microvm-artifact.sh --stack-name backgroundagent-dev +# +# 3. Pick a managed base image (ADR-021's Region list applies — Lambda MicroVMs +# exist in 5 Regions at launch). NOTE the version list comes back NEWEST +# FIRST, so the latest version is `items[0]`, not `items[-1]`: +# +# aws lambda-microvms list-managed-microvm-images +# aws lambda-microvms list-managed-microvm-image-versions \ +# --image-identifier \ +# --query 'items[0].imageVersion' --output text +# +# 4. Redeploy with the base image pinned. THIS is the step that creates the +# image resource and injects MICROVM_IMAGE_IDENTIFIER into the orchestrator: +# +# MISE_EXPERIMENTAL=1 mise //cdk:deploy -- \ +# --context compute_type=lambda-microvm \ +# --context microvm_base_image_arn= \ +# --context microvm_base_image_version= +# +# On subsequent agent changes only step 2 is needed, followed by a CloudFormation +# update of the image resource (the service builds a NEW image version from the +# refreshed artifact). +# +# --------------------------------------------------------------------------- +# THE OUT-OF-BAND ALTERNATIVE (--create-image) +# --------------------------------------------------------------------------- +# Snapshot builds fail for ordinary Dockerfile reasons and take minutes, so +# iterating through CloudFormation is painful. With `--create-image` the script +# calls `aws lambda-microvms create-microvm-image` itself, using the CDK-created +# build role, BUILD-time egress connector and log group. Hand the resulting image +# to the orchestrator without CDK owning it (pass the image ARN the script prints +# — `run-microvm` rejects a bare name): +# +# MISE_EXPERIMENTAL=1 mise //cdk:deploy -- \ +# --context compute_type=lambda-microvm \ +# --context microvm_image_identifier= +# +# --------------------------------------------------------------------------- +# WHAT GOES IN THE ZIP +# --------------------------------------------------------------------------- +# The service builds the zip's `Dockerfile` with the zip root as build context, +# so the layout must mirror the paths `agent/Dockerfile` COPYs from — which are +# repo-root-relative (`COPY agent/src/ …`, `COPY contracts/ …`). The staged tree +# is therefore: +# +# Dockerfile <- verbatim copy of agent/Dockerfile (MicroVM needs it at the root) +# agent/ <- minus .venv/, __pycache__, test caches +# contracts/ <- cross-language constants the agent reads at runtime +# +# --------------------------------------------------------------------------- +# !! A P1 IMAGE IS RUNNABLE, BUT NOT SMOKE-VERIFIED !! +# --------------------------------------------------------------------------- +# This script packages and uploads a real artifact, and the image the service +# builds from it will reach ACTIVE, accept a `runHookPayload`, and launch. What +# it does NOT have is any smoke-parity guarantee. +# +# ADR-021 sub-decision 3's hook-phasing table (corrected after the live P1 +# verification run) is now: +# +# /ready, /run declared by the CDK construct AND served by +# the agent in P1 (agent/src/server.py). +# /ready is MANDATORY: create-microvm-image +# refuses any lifecycle hook without it, and an +# image with no hooks at all cannot receive a +# runHookPayload — so "declare /run in P1, +# serve it in P2" was never a reachable state. +# /validate P2 (a /validate that 404s fails every build) +# /terminate P2; /suspend, /resume P3 +# +# Still unverified and owned by P2: AgentCore Memory grants + MEMORY_ID delivery, +# the agent's non-secret env parity inside the snapshot, egress specifics from a +# running MicroVM, and heartbeat/progress behaviour end to end. So clone → change +# → PR on this backend is untested. Keep production repos on +# compute_type=agentcore or ecs until P2 (smoke parity) lands. The Dockerfile is +# copied unmodified deliberately — adapting it to a MicroVM base image is P2 work. +# +# Requires: awscli v2, zip, rsync, python3 (none of which are installed by this script). + +set -euo pipefail + +# Make a failure impossible to miss even when the caller pipes us through `tee`. +# `foo.sh 2>&1 | tee log` reports tee's status, not ours (the live P1 run recorded +# EXIT=0 for a run that had actually failed at create-microvm-image), so print an +# explicit marker the teed log carries. Callers should ALSO set `set -o pipefail` +# or check `${PIPESTATUS[0]}`. +# shellcheck disable=SC2154 # $? inside the trap string is evaluated at trap time +trap 'rc=$?; [[ $rc -ne 0 ]] && echo "!! package-microvm-artifact.sh FAILED (exit ${rc}) !!" >&2; exit $rc' ERR + +STACK_NAME="${STACK_NAME:-backgroundagent-dev}" +CREATE_IMAGE=0 +BASE_IMAGE_ARN="${MICROVM_BASE_IMAGE_ARN:-}" +BASE_IMAGE_VERSION="${MICROVM_BASE_IMAGE_VERSION:-}" +IMAGE_NAME="${MICROVM_IMAGE_NAME:-}" +KEEP_STAGE=0 + +# BASELINE memory sizes the service accepts, and the default. NOT a range: the +# service enumerates the allowed baselines per base image and rejects anything +# else — 32768 (which this script used to send) fails with +# "The requested memory size of 32768 MiB is not supported by base MicroVM +# image ...al2023-1. Supported memory sizes in MiB are: [512, 1024, 2048, +# 4096, 8192]." +# This value is where the MicroVM STARTS, not a cap: the service scales a running +# MicroVM vertically to a 32 GiB / 16 vCPU peak on its own, and there is no field +# to request that. So a bigger build does not need a bigger number here. +# Keep in lockstep with MICROVM_SUPPORTED_MEMORY_MIB / DEFAULT_MINIMUM_MEMORY_MIB +# in cdk/src/constructs/lambda-microvm-compute.ts. +SUPPORTED_MEMORY_MIB="512 1024 2048 4096 8192" +MEMORY_MIB="${MICROVM_MEMORY_MIB:-8192}" + +usage() { + cat <<'USAGE' +Usage: package-microvm-artifact.sh [options] + + --stack-name NAME ABCA stack to read outputs from (default: backgroundagent-dev, + or $STACK_NAME) + --create-image After uploading, call `aws lambda-microvms create-microvm-image` + using the CDK-created build role / BUILD connector / log group + --base-image-arn ARN Managed base image ARN (required with --create-image) + --base-image-version V Managed base image version (required with --create-image) + --image-name NAME Image name for --create-image (default: -abca-agent) + --memory-mib MIB BASELINE memory for the image; one of 512 1024 2048 4096 8192 + (default: 8192 — the largest accepted baseline, or + $MICROVM_MEMORY_MIB). The service scales a running MicroVM + vertically to a 32 GiB / 16 vCPU peak on its own, so this sets + the starting size, not a ceiling. + --keep-stage Leave the staging directory in place for inspection + -h, --help This message +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --stack-name) STACK_NAME="$2"; shift 2 ;; + --create-image) CREATE_IMAGE=1; shift ;; + --base-image-arn) BASE_IMAGE_ARN="$2"; shift 2 ;; + --base-image-version) BASE_IMAGE_VERSION="$2"; shift 2 ;; + --image-name) IMAGE_NAME="$2"; shift 2 ;; + --memory-mib) MEMORY_MIB="$2"; shift 2 ;; + --keep-stage) KEEP_STAGE=1; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;; + esac +done + +case " ${SUPPORTED_MEMORY_MIB} " in + *" ${MEMORY_MIB} "*) ;; + *) + echo "error: --memory-mib ${MEMORY_MIB} is not a baseline Lambda MicroVMs accepts." >&2 + echo " Supported baselines (MiB): ${SUPPORTED_MEMORY_MIB}" >&2 + echo " (This is a BASELINE, not a cap - the service bursts to 32 GiB / 16 vCPU.)" >&2 + exit 2 + ;; +esac + +for tool in aws zip python3 rsync; do + command -v "$tool" >/dev/null 2>&1 || { echo "error: '$tool' is required but not on PATH" >&2; exit 1; } +done + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# --- Stack outputs ------------------------------------------------------------ +# One describe-stacks call, then pull each output by key. Failing loudly here +# (rather than uploading to a guessed bucket) is deliberate: a typo'd bucket +# name would silently produce an artifact the build role cannot read, and the +# only symptom would be a failed snapshot build. +echo "==> Reading outputs from stack '${STACK_NAME}'" +STACK_JSON="$(aws cloudformation describe-stacks --stack-name "${STACK_NAME}" --output json)" + +stack_output() { + STACK_JSON="${STACK_JSON}" python3 -c ' +import json, os, sys +key = sys.argv[1] +stacks = json.loads(os.environ["STACK_JSON"])["Stacks"] +for output in stacks[0].get("Outputs", []): + if output["OutputKey"] == key: + print(output["OutputValue"]) + break +' "$1" +} + +ARTIFACT_BUCKET="$(stack_output MicrovmArtifactBucketName)" +ARTIFACT_KEY="$(stack_output MicrovmArtifactObjectKey)" +BUILD_ROLE_ARN="$(stack_output MicrovmBuildRoleArn)" +EGRESS_CONNECTORS="$(stack_output MicrovmEgressConnectorArns)" +# BUILD-time connectors (TCP 443 + 80). `agent/Dockerfile` runs `apt-get`, which +# fetches over plain HTTP — with the 443-only RUNTIME connector every snapshot +# build failed ("Could not connect to deb.debian.org:80 … E: Unable to locate +# package curl", exit 100). Fall back to the runtime connectors only so an older +# stack still produces a comprehensible failure rather than an empty flag value. +BUILD_EGRESS_CONNECTORS="$(stack_output MicrovmBuildEgressConnectorArns)" +if [[ -z "${BUILD_EGRESS_CONNECTORS}" ]]; then + BUILD_EGRESS_CONNECTORS="${EGRESS_CONNECTORS}" + echo " WARNING: stack has no MicrovmBuildEgressConnectorArns output; falling back to the" >&2 + echo " 443-only runtime connector. apt-get needs port 80 — the snapshot build will" >&2 + echo " FAIL. Redeploy the stack to create the build-time egress connector." >&2 +fi +LOG_GROUP="$(stack_output MicrovmLogGroupName)" + +if [[ -z "${ARTIFACT_BUCKET}" || -z "${ARTIFACT_KEY}" ]]; then + cat >&2 < change -> PR on this backend is + untested. Keep production repos on compute_type=agentcore or ecs until P2 + (smoke parity) lands. CDK synth emits the same warning + (abca:microvm-image-p1-smoke-unverified) on every deploy that configures an image. +EOF +} + +# --- Stage + zip -------------------------------------------------------------- +STAGE_DIR="$(mktemp -d "${TMPDIR:-/tmp}/abca-microvm-artifact.XXXXXX")" +cleanup() { + if [[ "${KEEP_STAGE}" -eq 0 ]]; then + rm -rf "${STAGE_DIR}" + else + echo "==> Staging directory kept at ${STAGE_DIR}" + fi +} +trap cleanup EXIT + +echo "==> Staging agent tree in ${STAGE_DIR}" +# The MicroVM build context is the zip root, and agent/Dockerfile COPYs +# repo-root-relative paths — so the staged tree mirrors the repo, with the +# Dockerfile additionally promoted to the root where the service looks for it. +cp "${REPO_ROOT}/agent/Dockerfile" "${STAGE_DIR}/Dockerfile" + +# Excludes match the build inputs the Dockerfile never COPYs but which dominate +# the zip size: the local virtualenv, Python/pytest caches, and node_modules. +rsync -a \ + --exclude '.venv/' \ + --exclude '__pycache__/' \ + --exclude '.pytest_cache/' \ + --exclude '.ruff_cache/' \ + --exclude '.mypy_cache/' \ + --exclude 'node_modules/' \ + --exclude '*.pyc' \ + "${REPO_ROOT}/agent" "${STAGE_DIR}/" +rsync -a --exclude 'node_modules/' "${REPO_ROOT}/contracts" "${STAGE_DIR}/" + +ARTIFACT_ZIP="${STAGE_DIR}.zip" +rm -f "${ARTIFACT_ZIP}" +echo "==> Zipping to ${ARTIFACT_ZIP}" +( cd "${STAGE_DIR}" && zip -q -r "${ARTIFACT_ZIP}" . ) +echo " $(du -h "${ARTIFACT_ZIP}" | cut -f1) artifact" + +# --- Upload ------------------------------------------------------------------- +echo "==> Uploading to s3://${ARTIFACT_BUCKET}/${ARTIFACT_KEY}" +aws s3 cp "${ARTIFACT_ZIP}" "s3://${ARTIFACT_BUCKET}/${ARTIFACT_KEY}" +rm -f "${ARTIFACT_ZIP}" + +if [[ "${CREATE_IMAGE}" -eq 0 ]]; then + cat < Artifact uploaded. Next: create (or update) the image. + + CDK-managed (recommended) — redeploy with the base image pinned: + + aws lambda-microvms list-managed-microvm-images + MISE_EXPERIMENTAL=1 mise //cdk:deploy -- \\ + --context compute_type=lambda-microvm \\ + --context microvm_base_image_arn= \\ + --context microvm_base_image_version= + + Out of band — re-run this script with: + + --create-image --base-image-arn --base-image-version +EOF + print_p1_reminder + exit 0 +fi + +# --- Optional: create the image out of band ----------------------------------- +if [[ -z "${BASE_IMAGE_ARN}" || -z "${BASE_IMAGE_VERSION}" ]]; then + echo "error: --create-image requires --base-image-arn and --base-image-version." >&2 + echo " Discover them with: aws lambda-microvms list-managed-microvm-images" >&2 + echo " NOTE: list-managed-microvm-image-versions returns NEWEST FIRST, so the latest" >&2 + echo " version is items[0].imageVersion — items[-1] picks the OLDEST." >&2 + exit 2 +fi + +IMAGE_NAME="${IMAGE_NAME:-${STACK_NAME}-abca-agent}" + +# Printed BEFORE the create call, so the reminder survives a create that fails +# service-side validation (which is exactly what happened on the first live run). +print_p1_reminder + +echo "==> Creating MicroVM image '${IMAGE_NAME}' (${MEMORY_MIB} MiB baseline)" +# Flags use the Lambda MicroVMs service API shape (which differs from the +# CloudFormation shape used by CfnMicrovmImage), all confirmed against the live +# CLI/SDK model on 2026-07-31: +# * `ARM_64` is the only documented architecture value. +# * hooks are ENABLED/DISABLED with timeouts, NOT paths. +# * `/ready` is MANDATORY whenever any lifecycle hook is enabled: +# "The ready (/ready) MicroVM image hook must be enabled when any MicroVM +# lifecycle hook (run, resume, suspend, or terminate) is enabled." +# `/validate` stays disabled — the agent serves no validation endpoint, and a +# 404 there fails every build. +# * the BUILD connector (443 + 80) is used here, not the runtime one. +CREATE_RESPONSE="$(aws lambda-microvms create-microvm-image \ + --name "${IMAGE_NAME}" \ + --description "ABCA agent snapshot for ${STACK_NAME} (ADR-021, out-of-band build)" \ + --base-image-arn "${BASE_IMAGE_ARN}" \ + --base-image-version "${BASE_IMAGE_VERSION}" \ + --build-role-arn "${BUILD_ROLE_ARN}" \ + --code-artifact "{\"uri\":\"s3://${ARTIFACT_BUCKET}/${ARTIFACT_KEY}\"}" \ + --cpu-configurations '[{"architecture":"ARM_64"}]' \ + --resources "[{\"minimumMemoryInMiB\":${MEMORY_MIB}}]" \ + --egress-network-connectors "${BUILD_EGRESS_CONNECTORS}" \ + --logging "{\"cloudWatch\":{\"logGroup\":\"${LOG_GROUP}\"}}" \ + --hooks '{"port":8080,"microvmHooks":{"run":"ENABLED","runTimeoutInSeconds":60},"microvmImageHooks":{"ready":"ENABLED","readyTimeoutInSeconds":60}}' \ + --tags "abca:compute-backend=lambda-microvm" \ + --output json)" + +echo "${CREATE_RESPONSE}" + +# Every downstream call needs the image ARN, not the name: +# list-microvm-image-builds --image-identifier +# -> ValidationException: Invalid ARN format: +# and RunMicrovm likewise rejects a bare name ("Malformed ARN"). Pull the ARN and +# the real version string out of the response instead of telling the operator to +# retype the name. +IMAGE_ARN="$(CREATE_RESPONSE="${CREATE_RESPONSE}" python3 -c ' +import json, os +print(json.loads(os.environ["CREATE_RESPONSE"]).get("imageArn", "")) +')" +IMAGE_VERSION="$(CREATE_RESPONSE="${CREATE_RESPONSE}" python3 -c ' +import json, os +print(json.loads(os.environ["CREATE_RESPONSE"]).get("imageVersion", "")) +')" + +cat < Image creation started. Poll the build (snapshot builds take ~6 minutes): + + # NOTE: an ARN is required — a bare image name is rejected with + # "ValidationException: Invalid ARN format". The version is "1.0", not "1". + # NOTE: there are TWO builds per version (one per chipsetGeneration), so inspect + # every entry in items[], not just items[0]. + aws lambda-microvms list-microvm-image-builds \\ + --image-identifier ${IMAGE_ARN:-} \\ + --image-version ${IMAGE_VERSION:-1.0} + + # snapshotBuild sizes live on get-microvm-image-build; get-microvm-image-version + # returns snapshotBuild: null. + aws lambda-microvms get-microvm-image-build \\ + --image-identifier ${IMAGE_ARN:-} \\ + --image-version ${IMAGE_VERSION:-1.0} --build-id + + aws logs tail ${LOG_GROUP} --since 20m --follow + +Once the version reports status ACTIVE, point the orchestrator at it: + + MISE_EXPERIMENTAL=1 mise //cdk:deploy -- \\ + --context compute_type=lambda-microvm \\ + --context microvm_image_identifier=${IMAGE_ARN:-${IMAGE_NAME}} \\ + --context microvm_image_version=${IMAGE_VERSION:-1.0} + +Cleanup, when you are done with an image: delete every version EXCEPT the last +(the last one cannot be deleted individually — "This is the last version. Please +delete the entire image"), then delete the image, which reaps the final version. +Failed builds still create versions that must be reaped. +EOF + +print_p1_reminder diff --git a/cdk/src/bootstrap/policies/compute-lambda-microvm.ts b/cdk/src/bootstrap/policies/compute-lambda-microvm.ts new file mode 100644 index 000000000..8e1c120d6 --- /dev/null +++ b/cdk/src/bootstrap/policies/compute-lambda-microvm.ts @@ -0,0 +1,92 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { aws_iam as iam } from 'aws-cdk-lib'; + +/** + * Returns the IAM PolicyDocument for the IaCRole-ABCA-Compute-LambdaMicrovms role. + * + * Covers: MicroVM image and network-connector management for the AWS Lambda + * MicroVMs compute backend (ADR-021 sub-decision 4). + * + * This is the *deploy-time* (CloudFormation execution role) policy — the + * `AWS::Lambda::MicrovmImage` and `AWS::Lambda::NetworkConnector` resources the + * `LambdaMicrovmCompute` construct synthesizes. It deliberately does NOT include + * the *runtime* lifecycle actions (`lambda:RunMicrovm` / `GetMicrovm` / + * `TerminateMicrovm`): those belong to the orchestrator's own role, are granted + * per-deployment in `task-orchestrator.ts`, and would be a privilege escalation + * on a deploy role that never starts a session. + * + * `Resource: '*'` matches the `compute-ecs` precedent and is load-bearing here + * rather than lazy: several of these actions (`CreateMicrovmImage`, + * `PassNetworkConnector`, the `List*` reads) support no resource-level + * permissions at all in the Service Authorization Reference, so a narrowed ARN + * would silently never match and the deploy would fail with AccessDenied. + * + * Dependent actions that are already covered elsewhere in the bundle and are + * therefore not repeated: `iam:PassRole` (the build role, → + * `lambda.amazonaws.com`) and `iam:CreateServiceLinkedRole` (network-connector + * ENI management) live in the `infrastructure` policy; `lambda:TagResource` / + * `lambda:UntagResource` live in `application`. + */ +export function computeLambdaMicrovmPolicy(): iam.PolicyDocument { + return new iam.PolicyDocument({ + statements: [ + new iam.PolicyStatement({ + sid: 'LambdaMicrovms', + effect: iam.Effect.ALLOW, + actions: [ + // MicroVM images — the snapshot CloudFormation creates, updates, + // replaces and deletes. `Get*`/`List*` are required by the CFN + // resource handlers for read/stabilization, not just by humans. + 'lambda:CreateMicrovmImage', + 'lambda:GetMicrovmImage', + 'lambda:UpdateMicrovmImage', + 'lambda:DeleteMicrovmImage', + 'lambda:ListMicrovmImages', + // Image *versions* and their builds: every snapshot build creates a + // new version, and CFN must be able to read build state (a failed + // build otherwise stalls the stack with no diagnosis) and delete + // versions on replace/rollback — the ADR calls out that versions are + // billed storage and need lifecycle cleanup. + 'lambda:GetMicrovmImageVersion', + 'lambda:UpdateMicrovmImageVersion', + 'lambda:DeleteMicrovmImageVersion', + 'lambda:ListMicrovmImageVersions', + 'lambda:GetMicrovmImageBuild', + 'lambda:ListMicrovmImageBuilds', + // Base images: resolving/validating the managed base image the + // snapshot is built on. + 'lambda:ListManagedMicrovmImages', + 'lambda:ListManagedMicrovmImageVersions', + // Network connectors — the platform-VPC egress path. + 'lambda:CreateNetworkConnector', + 'lambda:GetNetworkConnector', + 'lambda:UpdateNetworkConnector', + 'lambda:DeleteNetworkConnector', + 'lambda:ListNetworkConnectors', + // Documented dependent of CreateMicrovmImage/UpdateMicrovmImage: the + // image declares its egress connectors, so creating it *passes* them. + 'lambda:PassNetworkConnector', + ], + resources: ['*'], + }), + ], + }); +} diff --git a/cdk/src/bootstrap/policies/index.ts b/cdk/src/bootstrap/policies/index.ts index ef89b5bb8..cb775bab8 100644 --- a/cdk/src/bootstrap/policies/index.ts +++ b/cdk/src/bootstrap/policies/index.ts @@ -22,12 +22,14 @@ import { aws_iam as iam } from 'aws-cdk-lib'; import { applicationPolicy } from './application'; import { computeAgentcorePolicy } from './compute-agentcore'; import { computeEcsPolicy } from './compute-ecs'; +import { computeLambdaMicrovmPolicy } from './compute-lambda-microvm'; import { infrastructurePolicy } from './infrastructure'; import { observabilityPolicy } from './observability'; export { applicationPolicy } from './application'; export { computeAgentcorePolicy } from './compute-agentcore'; export { computeEcsPolicy } from './compute-ecs'; +export { computeLambdaMicrovmPolicy } from './compute-lambda-microvm'; export { infrastructurePolicy } from './infrastructure'; export { observabilityPolicy } from './observability'; @@ -41,5 +43,6 @@ export function allPolicies(): iam.PolicyDocument[] { observabilityPolicy(), computeAgentcorePolicy(), computeEcsPolicy(), + computeLambdaMicrovmPolicy(), ]; } diff --git a/cdk/src/bootstrap/resource-action-map.ts b/cdk/src/bootstrap/resource-action-map.ts index f27394961..cb117290f 100644 --- a/cdk/src/bootstrap/resource-action-map.ts +++ b/cdk/src/bootstrap/resource-action-map.ts @@ -81,6 +81,12 @@ export const RESOURCE_ACTION_MAP: Record = { 'AWS::Lambda::EventSourceMapping': ['lambda:CreateEventSourceMapping'], 'AWS::Lambda::Function': ['lambda:CreateFunction'], 'AWS::Lambda::LayerVersion': ['lambda:PublishLayerVersion'], + // ADR-021 lambda-microvm backend. Only synthesized under + // `--context compute_type=lambda-microvm`, so the default-context + // synth-coverage test never sees these — they are mapped anyway so the map + // stays a complete statement of what the bootstrap bundle must cover. + 'AWS::Lambda::MicrovmImage': ['lambda:CreateMicrovmImage'], + 'AWS::Lambda::NetworkConnector': ['lambda:CreateNetworkConnector'], 'AWS::Logs::Delivery': ['logs:CreateDelivery'], 'AWS::Logs::DeliveryDestination': ['logs:PutDeliveryDestination'], 'AWS::Logs::DeliverySource': ['logs:PutDeliverySource'], @@ -104,7 +110,7 @@ export const RESOURCE_ACTION_MAP: Record = { * Returns true when {@link allowedAction} covers {@link requiredAction}. * Supports service-level wildcards (e.g. `bedrock-agentcore:*`). */ -export function actionIsAllowed(requiredAction: string, allowedAction: string): boolean { +function actionIsAllowed(requiredAction: string, allowedAction: string): boolean { if (allowedAction === requiredAction) { return true; } diff --git a/cdk/src/bootstrap/version.ts b/cdk/src/bootstrap/version.ts index d81d32326..17ad37ae7 100644 --- a/cdk/src/bootstrap/version.ts +++ b/cdk/src/bootstrap/version.ts @@ -21,8 +21,15 @@ import { createHash } from 'node:crypto'; import { allPolicies } from './policies'; -/** Semantic version of the bootstrap policy bundle. */ -export const BOOTSTRAP_VERSION = '1.2.0'; +/** + * Semantic version of the bootstrap policy bundle. + * + * Bump history: 1.0.0 → 1.1.0 added the `compute-ecs` policy (#162), 1.1.0 → + * 1.2.0 refreshed policies for a full deploy (#350), 1.2.0 → 1.3.0 adds the + * `compute-lambda-microvm` policy (#645 / ADR-021). Adding a policy to the + * bundle is a minor bump — that is the precedent `compute-ecs` set. + */ +export const BOOTSTRAP_VERSION = '1.3.0'; /** * Computes a SHA-256 hash over all bootstrap policies. diff --git a/cdk/src/constructs/lambda-microvm-compute.ts b/cdk/src/constructs/lambda-microvm-compute.ts new file mode 100644 index 000000000..d7590a2c4 --- /dev/null +++ b/cdk/src/constructs/lambda-microvm-compute.ts @@ -0,0 +1,1134 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Annotations, ArnFormat, Duration, RemovalPolicy, Stack, Tags, Token } from 'aws-cdk-lib'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import * as logs from 'aws-cdk-lib/aws-logs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import { NagSuppressions } from 'cdk-nag'; +import { Construct } from 'constructs'; +// Single source of truth for the supported-Region list. ADR-021's +// `microvm-regions.ts` header is explicit that the list "is the ONLY place the +// list is declared — do not copy it", so the synth-time gate IMPORTS it rather +// than duplicating it. The module is a dependency-free pair of pure constants +// (no AWS SDK, no Lambda-runtime code), so pulling it into the CDK app tree +// costs nothing and cannot drift. +import { AgentSessionRole } from './agent-session-role'; +import { LAMBDA_MICROVM_SUPPORTED_REGIONS, isLambdaMicrovmRegionSupported } from '../handlers/shared/microvm-regions'; + +/** + * Lifecycle expiry for MicroVM `/run` hook payloads, in days. + * + * Mirrors {@link ECS_PAYLOAD_TTL_DAYS} in shape but NOT in role: on ECS the + * orchestrator deletes the payload at finalize and the rule is only a crash + * backstop, whereas the MicroVM strategy never deletes (ADR-021 sub-decision 3 + * / `microvmPayloadKey`) — so on this backend the lifecycle rule is the ONLY + * reaper. Payloads carry the hydrated prompt context, so the TTL stays as tight + * as the read-once-at-`/run` access pattern allows. + */ +export const MICROVM_PAYLOAD_TTL_DAYS = 1; + +/** + * Cost-allocation tag applied to every resource this construct creates + * (ADR-021 sub-decision 4, "Cost attribution"). + * + * The stack-level `compute_type` tag in `main.ts` carries a single value and is + * already imprecise with two backends; these per-resource tags make MicroVM + * spend attributable regardless of what the stack-level tag says. + */ +export const MICROVM_BACKEND_TAG_KEY = 'abca:compute-backend'; + +/** Tag value identifying the Lambda MicroVMs backend. */ +export const MICROVM_BACKEND_TAG_VALUE = 'lambda-microvm'; + +/** + * CloudWatch Logs namespace Lambda MicroVMs writes build- and run-time logs + * under. Both the build role and the execution role are scoped to this prefix. + */ +export const MICROVM_LOG_GROUP_PREFIX = '/aws/lambda-microvms'; + +/** + * Default S3 key the packaging helper (`cdk/scripts/package-microvm-artifact.sh`) + * uploads the zip + Dockerfile artifact to, inside the artifact bucket this + * construct creates. Kept in one place so the script, the `CfnMicrovmImage` + * `codeArtifact.uri`, and the build role's `s3:GetObject` scope cannot drift. + */ +export const MICROVM_ARTIFACT_OBJECT_KEY = 'microvm-images/agent-artifact.zip'; + +/** + * TCP port the agent's FastAPI server listens on inside the snapshot + * (`agent/Dockerfile` → `EXPOSE 8080`), and therefore the port the MicroVM + * lifecycle-hook listener is configured for. + */ +const AGENT_HOOK_PORT = 8080; + +/** + * `/run` hook path, as SERVED by `agent/src/server.py`. + * + * Load-bearing in P1: it is how the task payload reaches the agent + * (`runHookPayload`, ADR-021 sub-decision 3) — there is no other + * orchestrator→agent channel on this backend. + * + * The value is the service's fixed lifecycle-hook route, not a path of our + * choosing. Live verification (2026-07-31, issue #645) found that the + * `CreateMicrovmImage` **API** model has no hook-path field at all — it takes + * only `ENABLED`/`DISABLED` plus a timeout — while the generated + * CloudFormation type still types these as strings. Setting the string to the + * route the agent actually answers is therefore correct under either reading: + * if CloudFormation genuinely routes on it, it points at the real endpoint; if + * it is ignored (as the API model implies), the value is inert. Keep in lockstep + * with `MICROVM_HOOK_PREFIX` in `agent/src/server.py`. + */ +const RUN_HOOK_PATH = '/aws/lambda-microvms/runtime/v1/run'; + +/** `/run` hook budget (seconds). The hook only validates + starts the pipeline + * asynchronously, so it stays well inside the service's 1–60 s hook window. */ +const RUN_HOOK_TIMEOUT_SECONDS = 60; + +/** + * `/ready` build hook path, as SERVED by `agent/src/server.py` (see + * {@link RUN_HOOK_PATH} for why the string is the real route). + * + * **Mandatory, not optional.** `CreateMicrovmImage` rejects an image that + * enables ANY lifecycle hook without `/ready` (live 2026-07-31): + * + * > The ready (/ready) MicroVM image hook must be enabled when any MicroVM + * > lifecycle hook (run, resume, suspend, or terminate) is enabled. The ready + * > hook signals when the application has finished initializing so the snapshot + * > is taken in a ready state. + * + * So ADR-021's original "declare `/run` in P1, serve it in P2" plan was not a + * reachable service state: `/ready` + `/run` now land together in P1. + */ +const READY_HOOK_PATH = '/aws/lambda-microvms/runtime/v1/ready'; + +/** `/ready` build-hook budget (seconds). The agent answers as soon as uvicorn is + * bound, so the snapshot is taken with a warm server. */ +const READY_HOOK_TIMEOUT_SECONDS = 60; + +/** + * BASELINE memory sizes (MiB) the service accepts for a MicroVM image. + * + * NOT a range and NOT a per-VM ceiling: the service enumerates the allowed + * baseline values per base image and rejects anything else. Live 2026-07-31 + * against `arn:aws:lambda:us-east-1:aws:microvm-image:al2023-1`: + * + * > The requested memory size of 32768 MiB is not supported by base MicroVM + * > image …al2023-1. Supported memory sizes in MiB are: + * > [512, 1024, 2048, 4096, 8192]. + * + * The service then scales a running MicroVM VERTICALLY on demand, up to a + * 32 GiB / 16 vCPU peak (developer-guide sizing table) — so 8 GiB is the top of + * the *configurable baseline*, not the amount of memory a task can use. The + * boundary probe above establishes what the FIELD accepts; only the guide + * establishes what the field MEANS (ADR-021's source hierarchy). + * + * Kept as an exported constant so {@link LambdaMicrovmComputeProps.minimumMemoryInMiB} + * can fail at synth with the real list instead of at image-create time. The + * individually named sizes exist only to keep the list out of `no-magic-numbers` + * territory — the list itself is the contract. + */ +const MEMORY_512_MIB = 512; +const MEMORY_1_GIB_IN_MIB = 1024; +const MEMORY_2_GIB_IN_MIB = 2048; +const MEMORY_4_GIB_IN_MIB = 4096; +const MEMORY_8_GIB_IN_MIB = 8192; +export const MICROVM_SUPPORTED_MEMORY_MIB: readonly number[] = [ + MEMORY_512_MIB, + MEMORY_1_GIB_IN_MIB, + MEMORY_2_GIB_IN_MIB, + MEMORY_4_GIB_IN_MIB, + MEMORY_8_GIB_IN_MIB, +]; + +/** + * Baseline memory the image declares, in MiB — the largest baseline the service + * accepts (see {@link MICROVM_SUPPORTED_MEMORY_MIB}). + * + * The ABCA agent is a build-heavy workload, so P1 asks for the top of the + * accepted baseline list rather than a smaller one it would spend the whole task + * scaling up from. Automatic vertical scaling then supplies burst capacity to a + * 32 GiB peak; nothing here requests that, and nothing can. + * + * This was `32768` until live verification refuted it as a *baseline* value. + */ +export const DEFAULT_MINIMUM_MEMORY_MIB = MEMORY_8_GIB_IN_MIB; + +/** Retention for the MicroVM log group — parity with the ECS task log group. */ +const LOG_RETENTION = logs.RetentionDays.THREE_MONTHS; + +/** HTTPS port — the only egress allowed out of the MicroVM at RUN time. */ +const HTTPS_PORT = 443; + +/** + * HTTP port. Allowed on the **build-time** connector only. + * + * `agent/Dockerfile` installs Debian packages, and `apt-get` fetches over plain + * HTTP. With a 443-only egress path every snapshot build failed (live + * 2026-07-31): `Could not connect to deb.debian.org:80 … E: Unable to locate + * package curl` → `exit code: 100`. DNS resolved fine — the port was the sole + * cause. Opening 80 for the build path (and only the build path) made the build + * succeed immediately; see {@link LambdaMicrovmCompute.buildSecurityGroup}. + */ +const HTTP_PORT = 80; + +/** Graviton/ARM64: the agent image is ARM64 on every backend. */ +const CPU_ARCHITECTURE = 'arm64'; + +/** + * Resource-name half of the Lambda-managed **`NO_INGRESS`** network connector + * ARN, i.e. everything after `…:aws:network-connector:`. + * + * Why this exists at all: `RunMicrovm` does **not** default to "no ingress". A + * launch that omits `ingressNetworkConnectors` entirely came back (live + * 2026-07-31) with a service-attached PUBLIC connector and a public endpoint: + * + * ``` + * "ingressNetworkConnectors": ["arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:HTTP_INGRESS"] + * ``` + * + * ADR-021's "no inbound exposure" posture therefore has to be an **explicit + * control**, not an omission: the strategy passes the `NO_INGRESS` connector on + * every `RunMicrovm`. The ARN shape is taken verbatim from that observed + * `HTTP_INGRESS` ARN, with the connector name swapped. + */ +export const MICROVM_NO_INGRESS_CONNECTOR_RESOURCE = 'aws-network-connector:NO_INGRESS'; + +/** + * ARN of the Lambda-managed `NO_INGRESS` connector in {@link scope}'s + * partition/Region (see {@link MICROVM_NO_INGRESS_CONNECTOR_RESOURCE}). + * + * Account segment is the literal `aws` — these connectors are service-owned, not + * account-owned, exactly like the AWS-managed policy ARNs. + */ +export function microvmNoIngressConnectorArn(scope: Construct): string { + return Stack.of(scope).formatArn({ + service: 'lambda', + account: 'aws', + resource: 'network-connector', + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + resourceName: MICROVM_NO_INGRESS_CONNECTOR_RESOURCE, + }); +} + +/** + * CDK context flag that bypasses the synth-time Region gate. + * + * Exists because {@link LAMBDA_MICROVM_SUPPORTED_REGIONS} rots by design: when + * AWS launches Lambda MicroVMs in a new Region, an operator there must not have + * to wait for an ABCA release. The live probes (CLI onboarding, `platform + * doctor`) already accept the new Region, so this flag only unblocks synth. + */ +export const MICROVM_REGION_OVERRIDE_CONTEXT = 'microvm_region_override'; + +/** + * Fail synth when the `lambda-microvm` backend is enabled in a Region that is + * not in the statically documented support list (ADR-021 sub-decision 4, + * "Regional availability enforcement" — the synth/deploy row). + * + * Three behaviours worth knowing: + * + * - **Unresolved (token) Region → check SKIPPED.** A region-agnostic app + * (`new Stack(app, 'X')` with no `env`, or `env.region` left to the CLI) + * resolves `Stack.region` to the `AWS::Region` pseudo-parameter, whose value + * is unknowable at synth. Comparing a token against a Region list would + * reject every region-agnostic synth — including `cdk synth` on a developer + * box with no `CDK_DEFAULT_REGION` — so the static layer stands down and the + * live probes (onboarding / doctor / orchestration classification) carry the + * enforcement. This is a deliberate hole in the *static* layer only. + * - **Escape hatch.** `--context microvm_region_override=true` skips the check; + * the error message names the flag so an operator in a just-launched Region + * is never blocked on a code change. + * - **Failure is a synth-time throw, not a warning.** ADR-021 requires "synth + * fails when ComputeTypes includes lambda-microvm in an unlisted Region"; a + * warning would let a broken deploy through to a runtime AccessDenied. + */ +export function assertLambdaMicrovmRegionSupported(scope: Construct): void { + const region = Stack.of(scope).region; + + if (Token.isUnresolved(region)) { + // Region-agnostic synth — nothing to compare. See the doc comment above. + return; + } + if (isLambdaMicrovmRegionSupported(region)) { + return; + } + if (scope.node.tryGetContext(MICROVM_REGION_OVERRIDE_CONTEXT)) { + Annotations.of(scope).addWarningV2( + 'abca:microvm-region-override', + `The lambda-microvm compute backend is enabled in ${region}, which is not in the ABCA ` + + `supported-Region list (${LAMBDA_MICROVM_SUPPORTED_REGIONS.join(', ')}). Proceeding because ` + + `--context ${MICROVM_REGION_OVERRIDE_CONTEXT} is set. If AWS has launched Lambda MicroVMs in ` + + `${region}, add it to LAMBDA_MICROVM_SUPPORTED_REGIONS (cdk/src/handlers/shared/microvm-regions.ts) ` + + 'and drop the flag.', + ); + return; + } + + throw new Error( + `AWS Lambda MicroVMs are not available in ${region}. The lambda-microvm compute backend is ` + + 'enabled (--context compute_type=lambda-microvm) but the stack Region is not one of: ' + + `${LAMBDA_MICROVM_SUPPORTED_REGIONS.join(', ')}. Either deploy the stack into a supported ` + + 'Region, drop the backend (--context compute_type=agentcore or ecs), or — if AWS has since ' + + `launched Lambda MicroVMs in ${region} — bypass this static check with ` + + `--context ${MICROVM_REGION_OVERRIDE_CONTEXT}=true and add ${region} to ` + + 'LAMBDA_MICROVM_SUPPORTED_REGIONS in cdk/src/handlers/shared/microvm-regions.ts.', + ); +} + +/** + * The four operator-supplied image inputs, read from CDK context by the stack. + * + * Extracted into a type so the stack can resolve them ONCE, before `TaskApi` is + * constructed, and hand the same object to this construct — see + * {@link isLambdaMicrovmImageConfigured} for why that ordering matters. + */ +export interface LambdaMicrovmImageInputs { + readonly baseImageArn?: string; + readonly baseImageVersion?: string; + readonly externalImageIdentifier?: string; + readonly externalImageVersion?: string; +} + +/** + * True when {@link inputs} selects one of the two image-provisioning states + * (managed-base-image build, or an out-of-band image), i.e. the deployment will + * have a MicroVM image and therefore an `imageArn` to scope IAM against. + * + * Exists so the three-state decision documented on {@link LambdaMicrovmCompute} + * is made in exactly ONE place. `TaskApi` is constructed before this construct + * (the cancel Lambda's ARN is needed earlier, hence the `Lazy.string` holders in + * `stacks/agent.ts`), so the stack must know whether an image will exist *before* + * the construct that creates it runs. Without a shared predicate the stack and + * the construct would each re-derive that answer and could drift — and a drift + * here means either a missing cancel grant or a grant scoped to an image that + * does not exist. + */ +export function isLambdaMicrovmImageConfigured(inputs: LambdaMicrovmImageInputs): boolean { + return Boolean((inputs.baseImageArn && inputs.baseImageVersion) || inputs.externalImageIdentifier); +} + +/** + * Properties for {@link LambdaMicrovmCompute}. + */ +export interface LambdaMicrovmComputeProps extends LambdaMicrovmImageInputs { + /** + * Platform VPC. Egress leaves the MicroVM through a `AWS::Lambda::NetworkConnector` + * bound to this VPC's private-with-egress subnets, so the DNS Firewall / + * security-group / flow-log stack applies to MicroVM traffic unchanged + * (ADR-021 security table: "Egress ... None" delta). + */ + readonly vpc: ec2.IVpc; + + /** + * Per-task SessionRole (#209). When provided, the MicroVM **execution role** + * is admitted to the SessionRole's trust via + * {@link AgentSessionRole.admitComputeRole} — the mechanism was designed for + * exactly this (ADR-021 sub-decision 4) — so tenant-data access stays on the + * tag-scoped SessionRole instead of the execution role. Mirrors how + * `EcsAgentCluster` delegates the Fargate task role. Omitted in isolated + * construct tests, in which case NO tenant-data access is granted at all + * (this construct never grants DynamoDB directly: unlike the ECS backend + * there is no legacy direct-grant path to preserve). + */ + readonly agentSessionRole?: AgentSessionRole; + + /** + * ARN of the Lambda-managed base MicroVM image to build on + * (`aws lambda-microvms list-managed-microvm-images`), e.g. + * `arn:aws:lambda:us-east-1:aws:microvm-image:al2023-1`. + * + * Supplying this **and** {@link baseImageVersion} switches the construct into + * its primary mode: it synthesizes an `AWS::Lambda::MicrovmImage` (L1) whose + * `codeArtifact.uri` points at {@link artifactObjectKey} in the artifact + * bucket this construct creates. There is no default — base-image ARNs are + * account/Region-scoped service data that is only discoverable through a live + * API call, so hardcoding one would be a guess that rots. + */ + readonly baseImageArn?: string; + + /** + * Version of {@link baseImageArn} + * (`aws lambda-microvms list-managed-microvm-image-versions`). Required + * alongside `baseImageArn` because CloudFormation marks it required on + * `AWS::Lambda::MicrovmImage` even though the API treats it as optional. + */ + readonly baseImageVersion?: string; + + /** + * Identifier (name or ARN) of a MicroVM image built **out of band** — i.e. by + * running `cdk/scripts/package-microvm-artifact.sh` and then + * `aws lambda-microvms create-microvm-image` by hand. + * + * Only consulted when {@link baseImageArn} is absent. It exists so an + * operator can iterate on the snapshot (which takes minutes and often several + * attempts) without a stack update per attempt, then hand the finished image + * to the orchestrator. + */ + readonly externalImageIdentifier?: string; + + /** + * Version of {@link externalImageIdentifier}. Optional: the service resolves + * the latest active version when omitted, which is what a + * rebuild-in-place workflow wants. + */ + readonly externalImageVersion?: string; + + /** + * Name for the MicroVM image and its log group. Must be unique in the + * account. + * @default `-abca-agent` + */ + readonly imageName?: string; + + /** + * S3 key of the zip + Dockerfile artifact inside the artifact bucket. + * @default MICROVM_ARTIFACT_OBJECT_KEY + */ + readonly artifactObjectKey?: string; + + /** + * **Baseline** memory the image declares, in MiB. + * + * This is the size the MicroVM STARTS at, not a cap on what it may use: the + * service scales a running MicroVM vertically on demand, up to a 32 GiB / + * 16 vCPU peak, with no field to request that. So the practical effect of this + * prop is where the VM begins (and what it is baseline-priced at), not whether + * a build will fit. + * + * Must be one of {@link MICROVM_SUPPORTED_MEMORY_MIB} — the construct throws at + * synth otherwise, because the service rejects any other baseline at + * image-create time with a `ValidationException` an operator would only see + * minutes into a build. + * + * @default 8192 — the largest accepted baseline (see DEFAULT_MINIMUM_MEMORY_MIB) + */ + readonly minimumMemoryInMiB?: number; + + /** + * Non-secret environment variables baked into the snapshot at build time. + * + * Deliberately empty by default. ADR-021 sub-decision 3 forbids secrets, + * tokens, and per-task identity in the snapshot; the agent's non-secret + * configuration parity with the ECS container (table names, `MEMORY_ID`, + * `ARTIFACTS_BUCKET_NAME`, …) is P2 "smoke parity" work and is wired here + * when it lands. + * @default {} — no baked configuration + */ + readonly imageEnvironmentVariables?: Record; +} + +/** + * AWS Lambda MicroVMs compute backend — infrastructure half (ADR-021, + * sub-decision 4; P1 of the phased rollout). + * + * Provisions, in dependency order: + * + * 1. **Egress network connectors** (`AWS::Lambda::NetworkConnector`) on the + * platform VPC's private-with-egress subnets — TWO of them, sharing one + * operator role: + * - the **runtime** connector with a 443-only security group. This is what + * keeps the ADR's "Egress: no delta vs AgentCore/ECS" claim true — MicroVM + * traffic traverses the same NAT / DNS Firewall / flow logs as the other + * two backends. + * - the **build-time** connector with a 443 **and 80** security group, + * referenced only by the image resource. `agent/Dockerfile` runs + * `apt-get`, which is plain HTTP; a 443-only build path fails every + * snapshot build (see {@link HTTP_PORT}). Runtime egress stays 443-only. + * 2. **Artifact bucket** for the zip + Dockerfile the service builds the + * snapshot from, and a **payload bucket** for `/run` payloads that exceed + * the 4 KB `runHookPayload` cap (which is nearly all of them). + * 3. **Build role** — assumed by Lambda during image creation: `s3:GetObject` + * on the artifact object and CloudWatch Logs writes. Without it Lambda + * cannot emit build logs, which makes a failed snapshot build undebuggable. + * 4. **Execution role** — assumed by the running MicroVM: CloudWatch Logs, + * read-only on the payload bucket, and (when a SessionRole is wired) + * admission to the per-task SessionRole for tenant-data access. + * 5. **MicroVM image** (`AWS::Lambda::MicrovmImage`) — see {@link baseImageArn} + * for why this is conditional. + * + * ## Image provisioning: three states, one construct + * + * | Props supplied | What happens | When to use it | + * |---|---|---| + * | `baseImageArn` + `baseImageVersion` | `AWS::Lambda::MicrovmImage` L1 is synthesized from `s3:///`; {@link imageIdentifier} is its ARN | steady state | + * | `externalImageIdentifier` | no image resource; the supplied identifier is resolved to its exact ARN and handed to the orchestrator | iterating on the snapshot out of band | + * | neither | roles + buckets + connectors only; a synth-time **warning**, no image, and no `MICROVM_IMAGE_IDENTIFIER` for the orchestrator | first deploy — you cannot upload the artifact before the bucket that holds it exists | + * + * That third state is not an oversight: the artifact bucket is created by this + * stack, so the very first `--context compute_type=lambda-microvm` deploy has + * nowhere to have put the zip yet. It is a **warning rather than a throw** + * precisely so the bootstrap sequence (deploy → run the packaging script + * against the now-existing bucket → redeploy with `microvm_base_image_arn`) is + * possible at all. A `lambda-microvm` task submitted in that interim window + * fails fast with the strategy's own "stack deployed without the MicroVM + * substrate" error, which names the remedy. + * + * ## ⚠️ A P1 image is runnable, but NOT smoke-verified + * + * Reaching state 1 or 2 provisions a complete substrate, a buildable image, and + * a payload-deliverable `/run` path: P1 declares AND the agent serves `/ready` + * and `/run` (`agent/src/server.py`), because live verification proved the + * original "declare in P1, serve in P2" split was not a reachable service state + * — `CreateMicrovmImage` refuses any lifecycle hook without `/ready` (see + * {@link READY_HOOK_PATH}), and an image with no hooks at all cannot receive a + * `runHookPayload`. + * + * What is still unverified is everything P2 owns: AgentCore Memory grants + + * `MEMORY_ID` delivery, the non-secret env parity the agent needs inside the + * snapshot, egress specifics from a running MicroVM, and heartbeat/progress + * behaviour end to end. So a `lambda-microvm` task can start and receive its + * payload, but clone → change → PR is **not** covered by any test or live run + * yet. That is what the `abca:microvm-image-p1-smoke-unverified` warning below + * says, and it is repeated in `cdk/scripts/package-microvm-artifact.sh`. + * `/validate` (build) and `/suspend`, `/resume`, `/terminate` (runtime) are + * still deliberately not declared: a hook the service calls but nothing answers + * fails the corresponding build or lifecycle transition. + * + * ## Deliberately NOT here (P1 scope) + * + * The execution role gets **no** Bedrock, Secrets Manager, AgentCore Memory, or + * artifacts-bucket grants. On the ECS backend those exist because the agent + * actually runs there today; ADR-021 puts agent parity on this backend in P2 + * ("smoke parity … AgentCore Memory parity (IAM grant + MEMORY_ID)"). Adding + * them now would hand a role permissions nothing exercises, and would have to + * be reviewed twice. `lambda:SuspendMicrovm` / `lambda:ResumeMicrovm` are + * likewise absent (P3) and `lambda:CreateMicrovmAuthToken` is granted to no + * role in any phase — no JWE consumer exists (sub-decision 3). + */ +export class LambdaMicrovmCompute extends Construct { + /** S3 bucket holding the zip + Dockerfile the snapshot is built from. */ + public readonly artifactBucket: s3.Bucket; + + /** Key of the artifact object inside {@link artifactBucket}. */ + public readonly artifactObjectKey: string; + + /** S3 bucket holding oversized `/run` payloads (S3-pointer delivery). */ + public readonly payloadBucket: s3.Bucket; + + /** Role Lambda assumes while building the snapshot image. */ + public readonly buildRole: iam.Role; + + /** Role the running MicroVM (and its runtime lifecycle hooks) assumes. */ + public readonly executionRole: iam.Role; + + /** + * Role Lambda assumes to manage the connectors' ENIs in the platform VPC. + * + * REQUIRED for `VPC_EGRESS` connectors, despite the generated L1 typing + * `operatorRole` as optional — see the comment at the connector below. + */ + public readonly connectorOperatorRole: iam.Role; + + /** Runtime egress network connector bound to the platform VPC (443 only). */ + public readonly egressConnector: lambda.CfnNetworkConnector; + + /** ARNs for `MICROVM_EGRESS_CONNECTOR_ARNS` / `lambda:PassNetworkConnector`. */ + public readonly egressConnectorArns: string[]; + + /** + * Build-time egress network connector (443 **and** 80), used only by the image + * build — never by a running MicroVM. See {@link HTTP_PORT}. + */ + public readonly buildEgressConnector: lambda.CfnNetworkConnector; + + /** ARNs to pass as `create-microvm-image --egress-network-connectors`. */ + public readonly buildEgressConnectorArns: string[]; + + /** + * Ingress connectors passed on every `RunMicrovm` + * (`MICROVM_INGRESS_CONNECTOR_ARNS`). In P1–P3 this is exactly the + * Lambda-managed `NO_INGRESS` connector: the service's default is a PUBLIC + * `HTTP_INGRESS`, so "no inbound" has to be requested explicitly (see + * {@link MICROVM_NO_INGRESS_CONNECTOR_RESOURCE}). + */ + public readonly ingressConnectorArns: string[]; + + /** 443-only security group applied to the runtime connector's ENIs. */ + public readonly securityGroup: ec2.SecurityGroup; + + /** 443 + 80 security group applied to the BUILD connector's ENIs. */ + public readonly buildSecurityGroup: ec2.SecurityGroup; + + /** Log group for MicroVM build- and run-time logs. */ + public readonly logGroup: logs.LogGroup; + + /** The image resource, when this deployment builds one (see class docs). */ + public readonly image?: lambda.CfnMicrovmImage; + + /** Image name used for the image resource and the log group. */ + public readonly imageName: string; + + /** BASELINE memory (MiB) declared on the image; the service bursts above it. */ + public readonly minimumMemoryInMiB: number; + + /** + * Value for `MICROVM_IMAGE_IDENTIFIER`. **Always a full image ARN**, never a + * bare name — `RunMicrovm` rejects bare names outright + * (`ValidationException: Malformed ARN - doesn't start with 'arn:'`, live + * 2026-07-31), and so does `list-microvm-image-builds`. `undefined` in the + * neither-input-supplied bootstrap state, in which case the stack must not + * inject the MicroVM env block at all. + */ + public readonly imageIdentifier?: string; + + /** Value for `MICROVM_IMAGE_VERSION`, when pinned. */ + public readonly imageVersion?: string; + + /** + * The image's IAM resource ARN. Always set whenever {@link imageIdentifier} + * is — a bare image name is resolved to its full + * `…:microvm-image:` ARN — so the orchestrator's `lambda:RunMicrovm` / + * `GetMicrovm` / `TerminateMicrovm` grant is *always* scoped to this one + * platform-created image and never widens to an account-level wildcard. + * `undefined` only in the no-image bootstrap state, where no grant is issued + * at all. + */ + public readonly imageArn?: string; + + constructor(scope: Construct, id: string, props: LambdaMicrovmComputeProps) { + super(scope, id); + + // Regional availability is checked HERE rather than in the stack so it + // cannot be lost to a stack refactor: constructing this construct at all + // means the backend is enabled. + assertLambdaMicrovmRegionSupported(this); + + const stack = Stack.of(this); + this.artifactObjectKey = props.artifactObjectKey ?? MICROVM_ARTIFACT_OBJECT_KEY; + this.imageName = props.imageName ?? sanitizeImageName(`${stack.stackName}-abca-agent`); + + // Fail at SYNTH on an unsupported memory size. The service enumerates the + // sizes a base image accepts and rejects anything else at create time, which + // an operator otherwise discovers only after packaging + uploading. + this.minimumMemoryInMiB = props.minimumMemoryInMiB ?? DEFAULT_MINIMUM_MEMORY_MIB; + if (!MICROVM_SUPPORTED_MEMORY_MIB.includes(this.minimumMemoryInMiB)) { + throw new Error( + `minimumMemoryInMiB=${this.minimumMemoryInMiB} is not a BASELINE memory size AWS Lambda ` + + `MicroVMs accepts. Supported baselines (MiB): ${MICROVM_SUPPORTED_MEMORY_MIB.join(', ')}. ` + + `The default is ${DEFAULT_MINIMUM_MEMORY_MIB}, the largest accepted baseline. Note this is ` + + 'a BASELINE, not a cap: the service scales a running MicroVM vertically to a 32 GiB / ' + + '16 vCPU peak on its own, so asking for more here is neither possible nor necessary. A ' + + 'repo needing more than that SUSTAINED belongs on compute_type=ecs (16 vCPU / 120 GB).', + ); + } + + // Backend-identifying cost-allocation tags on every resource below + // (ADR-021: "MicroVM-specific resources shall carry backend-identifying + // cost-allocation tags"). Applied at the construct scope so a resource + // added later cannot be forgotten. L1 MicroVM resources are ITaggableV2, + // so the aspect reaches them too. + Tags.of(this).add(MICROVM_BACKEND_TAG_KEY, MICROVM_BACKEND_TAG_VALUE); + + // --- Networking: egress through the platform VPC --- + this.securityGroup = new ec2.SecurityGroup(this, 'MicrovmSG', { + vpc: props.vpc, + description: 'Lambda MicroVMs agent sessions - egress TCP 443 only', + allowAllOutbound: false, + }); + this.securityGroup.addEgressRule( + ec2.Peer.anyIpv4(), + ec2.Port.tcp(HTTPS_PORT), + 'Allow HTTPS egress (GitHub API, AWS services)', + ); + + // BUILD-TIME security group: 443 + 80. Separate from the runtime group on + // purpose — the agent at run time has no business speaking plain HTTP, but + // `apt-get` inside `agent/Dockerfile` does, and a 443-only build path fails + // every snapshot build (see HTTP_PORT). Keeping them apart means the + // narrower runtime posture is unchanged by the build's requirement. + this.buildSecurityGroup = new ec2.SecurityGroup(this, 'MicrovmBuildSG', { + vpc: props.vpc, + description: 'Lambda MicroVMs image BUILD - egress TCP 443 + 80 (apt-get)', + allowAllOutbound: false, + }); + this.buildSecurityGroup.addEgressRule( + ec2.Peer.anyIpv4(), + ec2.Port.tcp(HTTPS_PORT), + 'Allow HTTPS egress (package indexes, PyPI, npm)', + ); + this.buildSecurityGroup.addEgressRule( + ec2.Peer.anyIpv4(), + ec2.Port.tcp(HTTP_PORT), + 'Allow HTTP egress for apt-get during the snapshot build (build path only)', + ); + + // OPERATOR ROLE — required, despite the generated L1 typing it optional. + // + // `CfnNetworkConnector.operatorRole?: string` reads like an opt-in, and this + // construct originally left it unset "so Lambda manages the ENIs with its own + // service-linked role". Live deploy 2026-07-31 refuted that outright — the + // connector never reaches CREATE_COMPLETE without it: + // + // NetworkConnectorOperatorRole is required for VPC_EGRESS connector type + // (Service: Lambda, Status Code: 400) HandlerErrorCode: InvalidRequest + // + // The permission set below is the minimal recipe validated standalone in that + // run: `AWSLambdaVPCAccessExecutionRole` plus the ENI/tag/private-IP actions + // the managed policy omits. Trust mirrors the build/execution roles + // (`lambda.amazonaws.com` + `aws:SourceAccount`), so the confused-deputy + // posture is identical on all three. + // + // ONE role for BOTH connectors: they differ only in security group, both are + // created and owned by this construct in the same VPC, and a second identical + // role would double the IAM surface a reviewer has to check for no isolation + // gain (the role manages ENIs, not traffic). + const microvmAssumedBy = new iam.ServicePrincipal('lambda.amazonaws.com', { + conditions: { + StringEquals: { 'aws:SourceAccount': stack.account }, + }, + }); + this.connectorOperatorRole = new iam.Role(this, 'ConnectorOperatorRole', { + assumedBy: microvmAssumedBy, + description: + 'ABCA Lambda MicroVMs network-connector operator role: lets Lambda manage the connector ' + + 'ENIs in the platform VPC (required for VPC_EGRESS connectors).', + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaVPCAccessExecutionRole'), + ], + }); + this.connectorOperatorRole.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: [ + 'ec2:CreateNetworkInterface', + 'ec2:DeleteNetworkInterface', + 'ec2:DescribeNetworkInterfaces', + 'ec2:DescribeNetworkInterfaceAttribute', + 'ec2:ModifyNetworkInterfaceAttribute', + 'ec2:DescribeSubnets', + 'ec2:DescribeVpcs', + 'ec2:DescribeSecurityGroups', + 'ec2:AssignPrivateIpAddresses', + 'ec2:UnassignPrivateIpAddresses', + 'ec2:CreateTags', + ], + // EC2 network-interface APIs are largely non-resource-scopable (the ENI + // does not exist when CreateNetworkInterface is authorized, and the + // Describe* calls take no resource), which is exactly why the AWS-managed + // VPC-access policy uses `*` too. See the cdk-nag suppression below. + resources: ['*'], + })); + + // The connector, not the MicroVM, owns the ENIs — which is why + // `lambda:PassNetworkConnector` is required on the orchestrator even for + // AWS-managed connectors (ADR-021 sub-decision 4). + // + // `associatedComputeResourceTypes: ['MicroVm']` is the only value the + // service accepts today (CloudFormation: "Currently, only MicroVm is + // supported"). + const vpcEgressSubnetIds = props.vpc.selectSubnets({ + subnetType: ec2.SubnetType.PRIVATE_WITH_EGRESS, + }).subnetIds; + + this.egressConnector = new lambda.CfnNetworkConnector(this, 'EgressConnector', { + name: sanitizeImageName(`${stack.stackName}-microvm-egress`), + operatorRole: this.connectorOperatorRole.roleArn, + configuration: { + vpcEgressConfiguration: { + associatedComputeResourceTypes: ['MicroVm'], + networkProtocol: 'IPv4', + securityGroupIds: [this.securityGroup.securityGroupId], + subnetIds: vpcEgressSubnetIds, + }, + }, + }); + this.egressConnectorArns = [this.egressConnector.attrArn]; + + // Build-time twin of the connector above, differing ONLY in security group + // (443 + 80). Referenced by the image resource / packaging script, never by + // `RunMicrovm`, so a running agent still gets the 443-only posture. + this.buildEgressConnector = new lambda.CfnNetworkConnector(this, 'BuildEgressConnector', { + name: sanitizeImageName(`${stack.stackName}-microvm-build-egress`), + operatorRole: this.connectorOperatorRole.roleArn, + configuration: { + vpcEgressConfiguration: { + associatedComputeResourceTypes: ['MicroVm'], + networkProtocol: 'IPv4', + securityGroupIds: [this.buildSecurityGroup.securityGroupId], + subnetIds: vpcEgressSubnetIds, + }, + }, + }); + this.buildEgressConnectorArns = [this.buildEgressConnector.attrArn]; + + // Explicit NO_INGRESS (F7). NOT an empty list: omitting the field makes the + // service attach a PUBLIC HTTP_INGRESS connector and mint a public endpoint. + this.ingressConnectorArns = [microvmNoIngressConnectorArn(this)]; + + // --- Logs --- + // Explicitly named under the service's `/aws/lambda-microvms/` namespace so + // the build/execution role grants can be prefix-scoped AND so retention is + // under our control (a service-created group defaults to never expire). + this.logGroup = new logs.LogGroup(this, 'MicrovmLogGroup', { + logGroupName: `${MICROVM_LOG_GROUP_PREFIX}/${this.imageName}`, + retention: LOG_RETENTION, + removalPolicy: RemovalPolicy.DESTROY, + }); + + // --- Buckets --- + // Artifact bucket: the zip + Dockerfile the service builds the snapshot + // from. Dedicated (not a prefix on the attachments/trace bucket) for the + // same structural reason EcsPayloadBucket is dedicated — the build role's + // s3:GetObject then cannot reach tenant data. Deliberately NO expiry rule: + // the artifact must survive as long as the image versions built from it, + // which the service may re-read. + this.artifactBucket = new s3.Bucket(this, 'ArtifactBucket', { + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + encryption: s3.BucketEncryption.S3_MANAGED, + enforceSSL: true, + lifecycleRules: [ + { + id: 'microvm-artifact-mpu-abort', + enabled: true, + // The artifact is tens/hundreds of MB, so uploads are multipart; a + // failed `aws s3 cp` otherwise leaves billable parts forever. + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + // Payload bucket: mirrors EcsPayloadBucket's configuration (BLOCK_ALL + + // enforceSSL + S3_MANAGED + tight object expiry). See + // MICROVM_PAYLOAD_TTL_DAYS for the one behavioural difference. + this.payloadBucket = new s3.Bucket(this, 'PayloadBucket', { + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + encryption: s3.BucketEncryption.S3_MANAGED, + enforceSSL: true, + lifecycleRules: [ + { + id: 'microvm-payload-ttl', + enabled: true, + expiration: Duration.days(MICROVM_PAYLOAD_TTL_DAYS), + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + }); + + // --- Roles --- + // + // TRUST POLICY (verified against the AWS developer guide, "Lambda MicroVMs + // → Security and permissions"): the build role, the execution role and the + // connector operator role are all assumed by the ORDINARY Lambda service + // principal `lambda.amazonaws.com` (`microvmAssumedBy`, declared with the + // connector above because the operator role needs it first). The build and + // execution roles additionally need `sts:TagSession` alongside + // `sts:AssumeRole`. There is no `microvms.lambda.amazonaws.com` principal — + // using one is rejected at role-creation time with MalformedPolicyDocument. + // + // CONFUSED-DEPUTY: `aws:SourceAccount` is pinned to this account, which is + // the meaningful protection here — `lambda.amazonaws.com` is shared with + // every other Lambda feature, so without it any caller who could make + // Lambda act in *some* account could target these roles. + // + // `aws:SourceArn` is deliberately NOT added. Field reports of this exact + // pattern (an `ArnLike` on `…:microvm-image/*`) have the service failing to + // satisfy the condition — the image does not exist yet at build time — so + // both `create-microvm-image` and `run-microvm` fail with "unable to assume + // role". The AWS docs themselves are inconsistent about the separator in + // MicroVM image ARNs (`microvm-image:` vs `microvm-image/`), + // which is a second reason an ARN condition here is a deploy-time + // foot-gun. The live 2026-07-31 run confirmed the observed image ARN uses + // the `microvm-image:` (colon) form; narrowing to `aws:SourceArn` + // stays a P2 candidate rather than a P1 change. + + this.buildRole = new iam.Role(this, 'BuildRole', { + assumedBy: microvmAssumedBy, + description: + 'ABCA Lambda MicroVMs image-build role: reads the zip+Dockerfile artifact from S3 ' + + 'and writes snapshot build logs to CloudWatch.', + }); + grantTagSession(this.buildRole, microvmAssumedBy); + + // s3:GetObject only, scoped to the single artifact key — not the bucket. + // The build role runs the `/ready` and `/validate` build hooks, i.e. code + // from the repo under build, so it gets the narrowest possible read. + this.buildRole.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['s3:GetObject'], + resources: [this.artifactBucket.arnForObjects(this.artifactObjectKey)], + })); + this.grantMicrovmLogWrites(this.buildRole); + + this.executionRole = new iam.Role(this, 'ExecutionRole', { + assumedBy: microvmAssumedBy, + description: + 'ABCA Lambda MicroVMs execution role: assumed by the running MicroVM and its runtime ' + + 'lifecycle hooks; writes logs and reads out-of-band /run payloads.', + }); + grantTagSession(this.executionRole, microvmAssumedBy); + this.grantMicrovmLogWrites(this.executionRole); + + // READ-ONLY on the payload bucket (ADR-021: "The MicroVM execution role + // shall hold read-only access to the payload bucket, scoped to that + // bucket"). Read-only is not a nicety: the MicroVM runs untrusted repo + // code, so it must not be able to clobber another task's payload. Write + + // lifecycle stay with the trusted orchestrator. + this.payloadBucket.grantRead(this.executionRole); + + // Tenant-data access is delegated to the per-task SessionRole, exactly as + // EcsAgentCluster does for the Fargate task role. NOTE the asymmetry with + // that construct: there is no `else` branch granting DynamoDB directly — + // this backend has no legacy deployments to keep working, so a missing + // SessionRole means no tenant-data access rather than broad access. + if (props.agentSessionRole) { + props.agentSessionRole.admitComputeRole(this.executionRole); + } + + // --- Image --- + // Branch through the shared predicate's two components rather than an + // ad-hoc condition, so this construct and the stack's pre-TaskApi decision + // (isLambdaMicrovmImageConfigured) can never disagree. + if (props.baseImageArn && props.baseImageVersion) { + this.image = new lambda.CfnMicrovmImage(this, 'Image', { + name: this.imageName, + description: `ABCA agent snapshot for ${stack.stackName} (ADR-021 lambda-microvm backend)`, + baseImageArn: props.baseImageArn, + baseImageVersion: props.baseImageVersion, + buildRoleArn: this.buildRole.roleArn, + codeArtifact: { + uri: this.artifactBucket.s3UrlForObject(this.artifactObjectKey), + }, + // ARM64 everywhere — the agent image is Graviton on all three backends. + cpuConfigurations: [{ architecture: CPU_ARCHITECTURE }], + resources: [{ minimumMemoryInMiB: this.minimumMemoryInMiB }], + // Build-time egress uses the DEDICATED build connector (443 + 80), not + // the runtime one: `apt-get` in `agent/Dockerfile` speaks plain HTTP and + // a 443-only build path fails every snapshot build. Both connectors sit + // on the same private-with-egress subnets, so DNS Firewall / NAT / flow + // logs still apply to build traffic. + egressNetworkConnectors: this.buildEgressConnectorArns, + logging: { cloudWatch: { logGroup: this.logGroup.logGroupName } }, + // No extra OS capabilities: the agent runs ordinary user-space tooling. + additionalOsCapabilities: [], + // Nothing baked in — see `imageEnvironmentVariables`. + environmentVariables: Object.entries(props.imageEnvironmentVariables ?? {}) + .map(([key, value]) => ({ key, value })), + hooks: { + port: AGENT_HOOK_PORT, + microvmHooks: { + // `/run` is the payload-delivery channel, and the agent SERVES it as + // of P1 (`agent/src/server.py`). `/suspend` and `/resume` land with + // the P3 interface widening, and `/terminate` with P2: declaring a + // runtime hook the agent does not answer fails the corresponding + // lifecycle transition. P1 termination is the orchestrator's + // `TerminateMicrovm`, which needs no in-guest cooperation. + run: RUN_HOOK_PATH, + runTimeoutInSeconds: RUN_HOOK_TIMEOUT_SECONDS, + }, + microvmImageHooks: { + // `/ready` is MANDATORY whenever any lifecycle hook is enabled — the + // service refuses the create otherwise (see READY_HOOK_PATH), which + // is why it moved from P2 to P1. `/validate` stays out: the agent has + // no validation endpoint, and one that 404s fails every image build. + ready: READY_HOOK_PATH, + readyTimeoutInSeconds: READY_HOOK_TIMEOUT_SECONDS, + }, + }, + }); + // The image reads the artifact through the build role, so both must exist + // (and the artifact bucket policy be in place) before the build starts. + this.image.node.addDependency(this.buildRole, this.artifactBucket); + + this.imageIdentifier = this.image.attrImageArn; + this.imageArn = this.image.attrImageArn; + // Version intentionally unpinned: the service resolves the latest ACTIVE + // version, which is what a redeploy-after-rebuild flow wants. Pinning to + // `attrLatestActiveImageVersion` would be empty on the very first create + // (the build has not finished) and would force a stack update per rebuild. + this.imageVersion = undefined; + } else if (props.externalImageIdentifier) { + this.imageVersion = props.externalImageVersion; + // An operator may pass a bare image NAME (that is what + // `create-microvm-image --name` takes), but a bare name is useless to BOTH + // consumers: it is not an IAM resource, and — refuting this construct's + // former comment — `RunMicrovm` rejects it outright + // (`ValidationException: Malformed ARN - doesn't start with 'arn:'`, live + // 2026-07-31), as does `list-microvm-image-builds`. So the name is resolved + // to its exact ARN ONCE here and that single value feeds both + // `MICROVM_IMAGE_IDENTIFIER` and the lifecycle IAM scope. + // + // The Service Authorization Reference gives the `microvmImage` resource an + // unambiguous shape — + // `arn:${Partition}:lambda:${Region}:${Account}:microvm-image:${MicrovmImageName}` + // — matching the ARN the live run observed, so the ARN is derivable from + // the name plus this stack's partition/Region/account. `formatArn` emits + // the Aws.PARTITION / Aws.REGION / Aws.ACCOUNT_ID pseudo-parameters, so it + // is correct in a region-agnostic app too. + // + // Note the version is NOT part of the resource ARN: the SAR pattern ends + // at the image name, and `RunMicrovm` carries `imageVersion` as a separate + // request field, so pinning a version must never change the IAM scope. + this.imageArn = props.externalImageIdentifier.startsWith('arn:') + ? props.externalImageIdentifier + : stack.formatArn({ + service: 'lambda', + resource: 'microvm-image', + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + resourceName: props.externalImageIdentifier, + }); + this.imageIdentifier = this.imageArn; + } else { + Annotations.of(this).addWarningV2( + 'abca:microvm-image-not-provisioned', + 'The lambda-microvm backend is enabled but no MicroVM image is configured, so the ' + + 'orchestrator will reject lambda-microvm tasks. This is the expected state of the FIRST ' + + 'deploy (the artifact bucket must exist before the artifact can be uploaded). Next: run ' + + 'cdk/scripts/package-microvm-artifact.sh to upload the zip+Dockerfile, then redeploy with ' + + '--context microvm_base_image_arn= --context microvm_base_image_version= ' + + '(or point at an image you built by hand with --context microvm_image_identifier=).', + ); + } + + if (this.imageIdentifier) { + // Emitted on EVERY deploy that configures an image, in both image states. + // Not a throw and not suppressible: P1 now provisions a substrate, a + // buildable image AND a payload-deliverable /run path, which makes it look + // even MORE like a working backend than before — while nothing in P1 has + // exercised clone → change → PR on this substrate. The warning's job is to + // keep "deploy succeeded" from reading as "backend works". + Annotations.of(this).addWarningV2( + 'abca:microvm-image-p1-smoke-unverified', + 'A MicroVM image is configured. As of ADR-021 P1 the image IS creatable and launchable and ' + + 'the agent DOES serve the /ready and /run hooks, so a lambda-microvm task can start and ' + + 'receive its payload — but the backend has NO smoke-parity guarantee: AgentCore Memory ' + + 'grants and MEMORY_ID delivery, the agent\'s non-secret env parity inside the snapshot, ' + + 'egress specifics from a running MicroVM, and heartbeat/progress behaviour are all P2 and ' + + 'untested here. Keep production repos on compute_type=agentcore or ecs until P2 (smoke ' + + 'parity) lands. The /validate build hook and the /suspend, /resume and /terminate runtime ' + + 'hooks are deliberately still not declared: a hook the service calls but nothing answers ' + + 'fails the corresponding build or lifecycle transition.', + ); + } + + NagSuppressions.addResourceSuppressions([this.artifactBucket, this.payloadBucket], [ + { + id: 'AwsSolutions-S1', + reason: 'Artifact bucket holds a single build input (the agent zip+Dockerfile) read only by ' + + 'the Lambda MicroVMs build role; the payload bucket holds ephemeral per-task /run payloads ' + + `with a ${MICROVM_PAYLOAD_TTL_DAYS}-day TTL, written only by the orchestrator (grantPut) and ` + + 'read only by the MicroVM execution role, both scoped to the bucket. Object-level access ' + + 'logging (a second log bucket + CloudTrail data events) is not justified for a single ' + + 'build input or for transient boot payloads.', + }, + ], true); + + NagSuppressions.addResourceSuppressions([this.buildRole, this.executionRole], [ + { + id: 'AwsSolutions-IAM5', + reason: 'CloudWatch Logs wildcard is the service-owned ' + + `${MICROVM_LOG_GROUP_PREFIX}/* namespace (log stream names are minted per MicroVM, so no ` + + 'synth-time ARN exists); S3 object/* wildcard comes from CDK grantRead on the dedicated ' + + 'payload bucket (read-only, scoped to that bucket — ADR-021 sub-decision 3). The build ' + + 'role\'s s3:GetObject is scoped to a single object key, not a wildcard.', + }, + ], true); + + NagSuppressions.addResourceSuppressions([this.connectorOperatorRole], [ + { + id: 'AwsSolutions-IAM4', + reason: 'AWSLambdaVPCAccessExecutionRole is the AWS-managed policy for exactly this job — ' + + 'letting Lambda manage ENIs in a customer VPC. The Lambda MicroVMs network connector ' + + 'REQUIRES an operator role for VPC_EGRESS (live-verified: ' + + '"NetworkConnectorOperatorRole is required for VPC_EGRESS connector type"), and the ' + + 'managed policy plus the explicit ENI/tag/private-IP statement is the minimal recipe ' + + 'validated standalone against the service. Hand-rolling the managed half would drift ' + + 'from AWS as the service evolves without narrowing anything (see the IAM5 note).', + appliesTo: [ + 'Policy::arn::iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole', + ], + }, + { + id: 'AwsSolutions-IAM5', + reason: 'EC2 network-interface APIs are not meaningfully resource-scopable here: ' + + 'CreateNetworkInterface is authorized before the ENI exists, and the Describe* calls ' + + 'take no resource at all. The role is assumable ONLY by lambda.amazonaws.com with ' + + 'aws:SourceAccount pinned to this account, holds no data-plane permission, and is used ' + + 'solely to attach the two platform-owned connectors to the platform VPC. The ' + + 'AWS-managed VPC-access policy uses the same wildcard for the same reason.', + }, + ], true); + } + + /** + * CloudWatch Logs writes scoped to the MicroVM log namespace. + * + * The service documents `logs:CreateLogGroup` + `CreateLogStream` + + * `PutLogEvents` for the build role; the execution role gets the same set so + * runtime logs land in the same, retention-managed group. `CreateLogGroup` is + * included even though this construct pre-creates the group: the service + * creates per-image/per-MicroVM groups under the prefix, and a missing + * `CreateLogGroup` silently costs you the build logs — the one artifact you + * need when a snapshot build fails. + */ + private grantMicrovmLogWrites(role: iam.IRole): void { + role.addToPrincipalPolicy(new iam.PolicyStatement({ + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + resources: [ + Stack.of(this).formatArn({ + service: 'logs', + resource: 'log-group', + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + resourceName: `${MICROVM_LOG_GROUP_PREFIX}/*`, + }), + Stack.of(this).formatArn({ + service: 'logs', + resource: 'log-group', + arnFormat: ArnFormat.COLON_RESOURCE_NAME, + resourceName: `${MICROVM_LOG_GROUP_PREFIX}/*:log-stream:*`, + }), + ], + })); + } +} + +/** + * Add `sts:TagSession` alongside the `sts:AssumeRole` CDK's `assumedBy` emits. + * + * The MicroVM service needs BOTH actions (developer guide, "Trust policies"), + * but `iam.Role`'s `assumedBy` only renders `sts:AssumeRole`. Passing a second + * statement through `assumeRolePolicy` keeps the `aws:SourceAccount` condition + * identical on both actions — dropping it on the `TagSession` half would leave + * the confused-deputy hole half-open. + */ +function grantTagSession(role: iam.Role, principal: iam.ServicePrincipal): void { + role.assumeRolePolicy?.addStatements(new iam.PolicyStatement({ + actions: ['sts:TagSession'], + principals: [principal], + conditions: principal.policyFragment.conditions, + })); +} + +/** + * Reduce a candidate name to the character set MicroVM image / network + * connector names accept (alphanumerics, `-`, `_`) and cap its length. + * + * Stack names can contain characters these APIs reject, and an unresolved + * (token) stack name would otherwise produce a name containing `${Token[...]}`. + */ +function sanitizeImageName(candidate: string): string { + const MAX_NAME_LENGTH = 64; + return candidate + .replace(/[^A-Za-z0-9-_]/g, '-') + .slice(0, MAX_NAME_LENGTH) + .replace(/-+$/, ''); +} diff --git a/cdk/src/constructs/task-api.ts b/cdk/src/constructs/task-api.ts index f90c4d0ef..030166419 100644 --- a/cdk/src/constructs/task-api.ts +++ b/cdk/src/constructs/task-api.ts @@ -189,6 +189,29 @@ export interface TaskApiProps { */ readonly ecsClusterArn?: string; + /** + * IAM resource ARN of the MicroVM image this deployment provisioned (ADR-021 + * sub-decision 4). When provided, the cancel Lambda gets + * `lambda:TerminateMicrovm` **scoped to that one image**, so a cancelled + * MicroVM-backed task actually stops billing — mirroring the conditional + * AgentCore `RUNTIME_ARN` / `ecsClusterArn` wiring above. + * + * An ARN rather than an on/off boolean so the grant is exactly scoped. `TaskApi` + * is constructed before `LambdaMicrovmCompute` (the cancel Lambda's ARN is + * needed earlier), so the stack passes a `Lazy.string` that resolves after the + * image exists — the same cycle-breaking pattern `agentCoreStopSessionRuntimeArn` + * uses for the AgentCore runtime ARN. + * + * Absent ⇒ no grant. That is the correct behaviour in the "backend enabled but + * no image configured yet" bootstrap state too: with no image there can be no + * MicroVM-backed task to cancel. + * + * No matching env var: `cancel-task.ts` reads the `microvmId` from the task + * row's `compute_metadata` and — unlike the AgentCore path — never falls back + * to a stack-level identifier. + */ + readonly lambdaMicrovmImageArn?: string; + /** * S3 bucket for task attachments. When provided, the create-task Lambda * gets PutObject/DeleteObject grants and the bucket name as env var. @@ -678,6 +701,35 @@ export class TaskApi extends Construct { })); } + // ADR-021: cancelling a `lambda-microvm` task must actively terminate the + // MicroVM — leaving it to the 8-hour `maximumDurationInSeconds` cap would + // keep billing 16 vCPU and would hold account memory quota that gates + // admission of new tasks. Conditional for the same reason the AgentCore and + // ECS grants above are: a deployment without the backend gets no grant. + // + // ONLY `lambda:TerminateMicrovm`. `cancel-task.ts` sends + // `TerminateMicrovmCommand` and nothing else — it does not read MicroVM + // state first — so `lambda:GetMicrovm` would be a permission with no caller. + // (The approve/deny Lambdas get `ResumeMicrovm` + `GetMicrovm` in P3, where + // a state read is genuinely needed for the resume reconciliation.) + // + // Resource is the MicroVM *image*, not the running instance: every MicroVM + // lifecycle action authorizes against `microvm-image:` (Service + // Authorization Reference), so the per-session `microvmId` never appears in + // IAM — which is what lets this be scoped to the one platform-created image + // rather than an account-wide `microvm-image:*`. The ARN arrives as a + // `Lazy.string` because TaskApi is built before the MicroVM construct. + // + // `:*` sibling: version-suffix hedge, same rationale as the + // orchestrator's lifecycle grant in `task-orchestrator.ts` — still pinned to + // this image's name, so it can never match a different image. + if (props.lambdaMicrovmImageArn) { + cancelTaskFn.addToRolePolicy(new iam.PolicyStatement({ + actions: ['lambda:TerminateMicrovm'], + resources: [props.lambdaMicrovmImageArn, `${props.lambdaMicrovmImageArn}:*`], + })); + } + // Repo table read for onboarding gate if (props.repoTable) { props.repoTable.grantReadData(createTaskFn); @@ -1322,7 +1374,7 @@ export class TaskApi extends Construct { }, { id: 'AwsSolutions-IAM5', - reason: 'DynamoDB index/* wildcards generated by CDK grantReadWriteData/grantReadData for GSI access', + reason: 'DynamoDB index/* wildcards generated by CDK grantReadWriteData/grantReadData for GSI access; ecs:StopTask is conditioned on the cluster ARN; lambda:TerminateMicrovm is scoped to the single platform MicroVM image ARN plus a :* version-suffix sibling (delivered as a Lazy.string because TaskApi is built before the MicroVM construct) — ADR-021', }, ], true); } diff --git a/cdk/src/constructs/task-orchestrator.ts b/cdk/src/constructs/task-orchestrator.ts index 364a2e4ce..fcaf4d282 100644 --- a/cdk/src/constructs/task-orchestrator.ts +++ b/cdk/src/constructs/task-orchestrator.ts @@ -189,6 +189,82 @@ export interface TaskOrchestratorProps { * ReadWrite grants for URL fetch/screen/upload during hydration. */ readonly attachmentsBucket?: s3.IBucket; + + /** + * AWS Lambda MicroVMs compute strategy configuration (ADR-021 sub-decision 4). + * When provided, the `MICROVM_*` env vars and the MicroVM lifecycle IAM + * statements are added to the orchestrator. + * + * Grouped into one all-or-nothing object for the same reason `ecsConfig` is: + * the strategy refuses to start a session unless `imageIdentifier`, + * `executionRoleArn`, `egressConnectorArns` and `payloadBucket` are ALL + * present, so the type makes a partial configuration unrepresentable instead + * of deferring the failure to the first task on the backend. + * + * `ingressConnectorArns` is required for a different reason — it is a security + * control whose absence has a *wider* meaning than "off" (see the field). Only + * `imageVersion` is genuinely optional, and its absent state ("let the service + * resolve the latest ACTIVE version") is a real, intended configuration. + */ + readonly microvmConfig?: { + /** + * Image **ARN** passed as `imageIdentifier` on every `RunMicrovm`. + * + * Must be an ARN, not a bare name: `RunMicrovm` rejects names + * (`Malformed ARN - doesn't start with 'arn:'`). `LambdaMicrovmCompute` + * resolves a name to its exact ARN, so in practice this is the same value as + * {@link imageArn} — both fields are kept because they answer different + * questions (request field vs IAM resource) and could legitimately diverge + * if the service ever accepts a version-qualified identifier. + */ + readonly imageIdentifier: string; + /** + * The image's IAM resource ARN, used to scope every MicroVM lifecycle grant + * to that one platform-created image (ADR-021). + * + * REQUIRED, and separate from {@link imageIdentifier} for the reason above. + * Required rather than optional on purpose: an optional field here would + * silently re-introduce an account-wide `microvm-image:*` fallback, and no + * caller needs one — the `microvmImage` ARN shape is fully derivable from an + * image name plus the stack's partition/Region/account, which + * `LambdaMicrovmCompute` does. + */ + readonly imageArn: string; + /** + * Optional image version pin. Omitted ⇒ the service resolves the latest + * active version, which is what a rebuild-in-place flow wants. + */ + readonly imageVersion?: string; + /** Role the MicroVM assumes at runtime; passed on `RunMicrovm`. */ + readonly executionRoleArn: string; + /** Egress network connectors; comma-joined into the env var. */ + readonly egressConnectorArns: string[]; + /** + * Ingress network connectors. **Required** — and required for a security + * reason, not a stylistic one. + * + * `RunMicrovm` attaches a PUBLIC `HTTP_INGRESS` connector (and mints a public + * `*.lambda-microvm..on.aws` endpoint) when the request omits the + * field, so "nothing dials into the MicroVM" is an ACTIVE control that has to + * be passed on every launch. An optional field here would let a caller + * express "no ingress configured" and silently get the widest possible + * posture — which is exactly the class of bug the omitted-field rule in + * ADR-021's source-hierarchy note warns about. + * + * In P1–P3 `LambdaMicrovmCompute` always supplies the Lambda-managed + * `NO_INGRESS` connector; #391 operator shell access can widen it without a + * strategy change. + */ + readonly ingressConnectorArns: string[]; + /** + * Bucket for `/run` payloads that exceed the 4 KB `runHookPayload` cap — + * i.e. nearly all of them, since a hydrated payload is bigger than that. + * The orchestrator gets **write only**: unlike the ECS payload bucket + * there is no finalize-time delete on this backend (the bucket's lifecycle + * rule is the reaper), so `grantDelete` would be an unused permission. + */ + readonly payloadBucket: s3.IBucket; + }; } /** @@ -298,6 +374,26 @@ export class TaskOrchestrator extends Construct { // Bucket the orchestrator writes the ECS payload to (and deletes // from at finalize); the ECS strategy reads this to build the S3 URI. ...(props.ecsPayloadBucket && { ECS_PAYLOAD_BUCKET: props.ecsPayloadBucket.bucketName }), + // ADR-021: the MicroVM substrate's deployment-time configuration. Names + // are the contract `lambda-microvm-strategy.ts` reads verbatim — do not + // rename one side without the other. + ...(props.microvmConfig && { + MICROVM_IMAGE_IDENTIFIER: props.microvmConfig.imageIdentifier, + MICROVM_EXECUTION_ROLE_ARN: props.microvmConfig.executionRoleArn, + MICROVM_EGRESS_CONNECTOR_ARNS: props.microvmConfig.egressConnectorArns.join(','), + // ALWAYS injected, never conditional — part of the same all-or-nothing + // block as the four above. The value is a control (`NO_INGRESS`), not a + // configuration nicety: `RunMicrovm` attaches a PUBLIC HTTP_INGRESS + // connector when the field is absent from the request, so a deployment + // that omitted this var would hand every agent MicroVM a public + // endpoint. Making the prop required is what lets this be + // unconditional; there is no "no ingress configured" state to express. + MICROVM_INGRESS_CONNECTOR_ARNS: props.microvmConfig.ingressConnectorArns.join(','), + MICROVM_PAYLOAD_BUCKET: props.microvmConfig.payloadBucket.bucketName, + ...(props.microvmConfig.imageVersion && { + MICROVM_IMAGE_VERSION: props.microvmConfig.imageVersion, + }), + }), ...(props.attachmentsBucket && { ATTACHMENTS_BUCKET_NAME: props.attachmentsBucket.bucketName }), }, bundling: orchestratorBundling, @@ -325,6 +421,16 @@ export class TaskOrchestrator extends Construct { props.ecsPayloadBucket.grantDelete(this.fn); } + // ADR-021: MicroVM payload bucket. WRITE only — the strategy uploads an + // oversized /run payload and never reads it back (the MicroVM execution + // role is the reader, with its own read-only grant), and it never deletes + // (the bucket's lifecycle rule reaps). No grantDelete, deliberately: the ECS + // path has one because the orchestrator deletes at finalize; this one does + // not, so the grant would be dead permission. + if (props.microvmConfig) { + props.microvmConfig.payloadBucket.grantPut(this.fn); + } + // Durable execution managed policy this.fn.role!.addManagedPolicy( iam.ManagedPolicy.fromAwsManagedPolicyName('service-role/AWSLambdaBasicDurableExecutionRolePolicy'), @@ -386,6 +492,91 @@ export class TaskOrchestrator extends Construct { })); } + // Lambda MicroVMs compute strategy permissions (only when configured). + // + // EXACTLY the four control-plane actions the P1 strategy calls, per + // ADR-021's "only the MicroVM lifecycle actions it calls" requirement: + // RunMicrovm — startSession + // GetMicrovm — pollSession + // TerminateMicrovm — stopSession / finalize (the active cleanup path) + // PassNetworkConnector — required to attach egress connectors, even the + // AWS-managed ones + // + // NOT granted, deliberately: + // - lambda:SuspendMicrovm / lambda:ResumeMicrovm — the ADR's grant list + // names them, but P1 has no suspend/resume code path. They land with the + // P3 interface widening (mandatory suspendSession/resumeSession across + // all three strategies) together with the approve/deny Lambdas' + // conditional ResumeMicrovm + GetMicrovm. + // - lambda:CreateMicrovmAuthToken — granted to no role in any phase; no + // JWE consumer exists (ADR-021 sub-decision 3). + if (props.microvmConfig) { + // RESOURCE SCOPING — every MicroVM lifecycle action authorizes against the + // *image*, not the running instance: the Service Authorization Reference + // lists `microvmImage` + // (`arn::lambda:::microvm-image:`) as the + // required resource for RunMicrovm, GetMicrovm and TerminateMicrovm alike. + // That is what makes ADR-021's "scoped to platform-created images" + // achievable even though `microvmId` is minted per session — the + // per-session identifier never appears in IAM. + // + // There is NO account-wide fallback: `imageArn` is a required prop, and + // `LambdaMicrovmCompute` resolves a bare image name to its exact ARN, so + // this statement always names exactly one image. + // + // The `:*` sibling is a version-suffix hedge, not a widening. The SAR + // pattern ends at the image name, but the CLI and console surface + // version-qualified `…:microvm-image::` forms, and the docs + // are already inconsistent about MicroVM ARN separators. If the authorized + // resource turns out to carry the version, an exact-only policy would + // AccessDenied every task; if it does not, this entry is inert. Either way + // it stays pinned to this image's name (names cannot contain `:`), so it + // can never match a different image. + const { imageArn } = props.microvmConfig; + const microvmImageResources = [imageArn, `${imageArn}:*`]; + + this.fn.addToRolePolicy(new iam.PolicyStatement({ + sid: 'MicrovmLifecycle', + actions: [ + 'lambda:RunMicrovm', + 'lambda:GetMicrovm', + 'lambda:TerminateMicrovm', + ], + resources: microvmImageResources, + })); + + // `lambda:PassNetworkConnector` supports NO resource-level permissions + // (the Service Authorization Reference lists no resource type for it), so + // `Resource: '*'` is mandatory — a narrowed ARN would simply never match + // and RunMicrovm would fail with AccessDenied. It is also why the ADR + // notes the action is needed "even for the default connectors": the + // AWS-managed connectors live in the `aws` account, outside any ARN we + // could enumerate. The action only permits *passing* a connector to a + // service, not creating or reading one. + this.fn.addToRolePolicy(new iam.PolicyStatement({ + sid: 'MicrovmPassNetworkConnector', + actions: ['lambda:PassNetworkConnector'], + resources: ['*'], + })); + + // RunMicrovm hands the MicroVM its execution role, which is a PassRole in + // IAM's eyes (the Reference lists `iam:PassRole` as a documented dependent + // of RunMicrovm) — same shape as the ECS branch above passing the task and + // execution roles to ecs-tasks.amazonaws.com. ADR-021's grant list omits + // it because it enumerates MicroVM actions, not the IAM plumbing they + // imply; without it RunMicrovm fails on the role hand-off. + this.fn.addToRolePolicy(new iam.PolicyStatement({ + sid: 'MicrovmPassExecutionRole', + actions: ['iam:PassRole'], + resources: [props.microvmConfig.executionRoleArn], + conditions: { + StringEquals: { + 'iam:PassedToService': 'lambda.amazonaws.com', + }, + }, + })); + } + // Per-repo Secrets Manager grants (e.g. per-repo GitHub tokens from Blueprints) for (const [index, secretArn] of (props.additionalSecretArns ?? []).entries()) { const secret = secretsmanager.Secret.fromSecretCompleteArn( @@ -437,7 +628,7 @@ export class TaskOrchestrator extends Construct { }, { id: 'AwsSolutions-IAM5', - reason: 'DynamoDB index/* wildcards generated by CDK grantReadWriteData; AgentCore runtime/* required for sub-resource invocation; Secrets Manager wildcards generated by CDK grantRead; AgentCore Memory wildcards generated by CDK grantRead/grantWrite; ECS RunTask/DescribeTasks/StopTask conditioned on cluster ARN; iam:PassRole scoped to ECS task/execution roles and conditioned on ecs-tasks.amazonaws.com', + reason: 'DynamoDB index/* wildcards generated by CDK grantReadWriteData; AgentCore runtime/* required for sub-resource invocation; Secrets Manager wildcards generated by CDK grantRead; AgentCore Memory wildcards generated by CDK grantRead/grantWrite; ECS RunTask/DescribeTasks/StopTask conditioned on cluster ARN; iam:PassRole scoped to ECS task/execution roles and conditioned on ecs-tasks.amazonaws.com; S3 object/* wildcard from CDK grantPut on the dedicated MicroVM payload bucket; MicroVM lifecycle actions (RunMicrovm/GetMicrovm/TerminateMicrovm) are scoped to the single platform MicroVM image ARN plus a :* version-suffix sibling (every one of them authorizes against the image resource, not the per-session instance; no account-wide wildcard is used); lambda:PassNetworkConnector requires Resource:* because the action supports no resource-level permissions and the AWS-managed connectors live outside this account; iam:PassRole is scoped to the MicroVM execution role and conditioned on lambda.amazonaws.com', }, ], true); } diff --git a/cdk/src/handlers/cancel-task.ts b/cdk/src/handlers/cancel-task.ts index 0f90c3d75..9963f1f9f 100644 --- a/cdk/src/handlers/cancel-task.ts +++ b/cdk/src/handlers/cancel-task.ts @@ -19,6 +19,7 @@ import { BedrockAgentCoreClient, StopRuntimeSessionCommand } from '@aws-sdk/client-bedrock-agentcore'; import { ECSClient, StopTaskCommand } from '@aws-sdk/client-ecs'; +import { LambdaMicrovmsClient, TerminateMicrovmCommand } from '@aws-sdk/client-lambda-microvms'; import { GetCommand, PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; import type { APIGatewayProxyEvent, APIGatewayProxyResult } from 'aws-lambda'; import { ulid } from 'ulid'; @@ -33,6 +34,7 @@ import { computeTtlEpoch } from './shared/validation'; const ddb = makeDocClient(); const agentCoreClient = makeClient(BedrockAgentCoreClient); const ecsClient = makeClient(ECSClient); +const microvmClient = makeClient(LambdaMicrovmsClient); const TABLE_NAME = process.env.TASK_TABLE_NAME!; const EVENTS_TABLE_NAME = process.env.TASK_EVENTS_TABLE_NAME!; const TASK_RETENTION_DAYS = Number(process.env.TASK_RETENTION_DAYS ?? '90'); @@ -144,6 +146,42 @@ export async function handler(event: APIGatewayProxyEvent): Promise = asyn // Returns the full SessionHandle (serializable) so ECS polling can use it in step 5. const sessionHandle = await context.step('start-session', async () => { let autoRetried = false; + // Hoisted out of the `try` so the catch can reap a MicroVM that STARTED but + // whose handle never made it into DynamoDB — see the catch block. + let strategy: ReturnType | undefined; + let startedHandle: Awaited>['handle'] | undefined; try { - const strategy = resolveComputeStrategy(blueprintConfig); + strategy = resolveComputeStrategy(blueprintConfig); const startInput = { taskId, userId: task.user_id, @@ -190,11 +196,12 @@ const durableHandler: DurableExecutionHandler = asyn }, ); autoRetried = retried; + startedHandle = handle; - // Build compute metadata for the task record so cancel-task can stop the right backend - const computeMetadata: Record = handle.strategyType === 'ecs' - ? { clusterArn: handle.clusterArn, taskArn: handle.taskArn } - : { runtimeArn: handle.runtimeArn }; + // Build compute metadata for the task record so cancel-task can stop the + // right backend (and, for lambda-microvm, so the P3 approve/deny Lambdas + // can load the handle they resume from — ADR-021 sub-decision 2). + const computeMetadata = buildComputeMetadata(handle); await transitionTask(taskId, TaskStatus.HYDRATING, TaskStatus.RUNNING, { session_id: handle.sessionId, @@ -215,6 +222,42 @@ const durableHandler: DurableExecutionHandler = asyn return handle; } catch (err) { + // ORPHAN REAP (ADR-021). `RunMicrovm` may have already succeeded and left a + // MicroVM RUNNING — the throw could have come from `buildComputeMetadata`, + // the `transitionTask` write, or the `session_started` emit. Nothing + // self-terminates on this substrate (live-verified: a MicroVM with no + // working hook reached RUNNING in 12 s and stayed RUNNING with no + // stateReason), and the handle only ever existed in this Lambda's memory — + // once we throw, no poll and no finalize step will ever see it. So the VM + // would bill until `maximumDurationInSeconds` (8 h) expired while also + // holding account memory quota that gates admission for everyone else. + // + // Best-effort in the strongest sense: `stopSession` is internally + // non-throwing for this backend, and the extra try/catch guarantees that + // even a surprise (a non-microvm strategy, a synchronous throw) cannot + // replace the ORIGINAL failure — which is the one the user needs to see. + // + // Scoped to `lambda-microvm` deliberately. ECS and AgentCore have the same + // structural window, but generalising here would issue `StopTask` / + // `StopRuntimeSession` calls the ORCHESTRATOR role may not be granted (those + // grants live on the cancel Lambda), turning a clean failure into a clean + // failure plus a misleading AccessDenied. Widening it is a follow-up that + // needs the IAM change reviewed alongside. + if (startedHandle?.strategyType === 'lambda-microvm' && strategy) { + try { + log.warn('Session start failed after the MicroVM was created — terminating the orphan', { + microvm_id: startedHandle.microvmId, + original_error: err instanceof Error ? err.message : String(err), + }); + await strategy.stopSession(startedHandle); + } catch (reapErr) { + log.error('Failed to terminate the orphaned MicroVM — it may bill until its 8h cap', { + microvm_id: startedHandle.microvmId, + error: reapErr instanceof Error ? reapErr.message : String(reapErr), + }); + } + } + // Carry the auto-retry fact into error_message: the `[auto-retried]` suffix // is persisted verbatim (the error classifier ignores it — it does not // affect classification). It is a breadcrumb for a failure renderer to @@ -238,6 +281,14 @@ const durableHandler: DurableExecutionHandler = asyn ? resolveComputeStrategy(blueprintConfig) : undefined; + // Kept as a SEPARATE local rather than widening `computeStrategy`'s condition: + // the ECS cross-check below is gated on `computeStrategy` truthiness, so + // reusing that local for lambda-microvm would route MicroVM polls through the + // ECS exit-code/patience logic. Two locals keep the ECS path byte-identical. + const microvmStrategy = blueprintConfig.compute_type === 'lambda-microvm' + ? resolveComputeStrategy(blueprintConfig) + : undefined; + // Step 5: Wait for agent to finish // Polls DynamoDB on each interval. The agent writes terminal status when done. // While RUNNING, the runtime updates `agent_heartbeat_at`; if that timestamp @@ -251,6 +302,9 @@ const durableHandler: DurableExecutionHandler = asyn const ddbState = await pollTaskStatus(taskId, state, blueprintConfig.compute_type); let consecutiveEcsPollFailures = 0; let consecutiveEcsCompletedPolls = 0; + // Carried forward by default: an unrelated poll (or a MicroVM poll that + // threw) must not silently re-arm the once-per-episode anomaly event. + let microvmSuspendAnomalyReported = state.microvmSuspendAnomalyReported ?? false; // ECS compute-level crash detection: if DDB is not terminal, check ECS task status if ( @@ -300,7 +354,52 @@ const durableHandler: DurableExecutionHandler = asyn } } - return { ...ddbState, consecutiveEcsPollFailures, consecutiveEcsCompletedPolls }; + // Lambda MicroVMs substrate cross-check (ADR-021 sub-decision 1). Same + // division of labour as the ECS block above — the strategy reports raw + // substrate state and `reconcileMicrovmSubstrateState` interprets it + // against the DDB status — but the rules differ: a `suspended` VM is + // healthy during an approval wait, an anomaly (not a failure) otherwise, + // and a terminal VM with a non-terminal task row is a substrate failure. + if ( + ddbState.lastStatus + && !TERMINAL_STATUSES.includes(ddbState.lastStatus) + && microvmStrategy + && sessionHandle.strategyType === 'lambda-microvm' + ) { + try { + const substrateStatus = await microvmStrategy.pollSession(sessionHandle); + const { taskFailed, suspendAnomalyReported } = await reconcileMicrovmSubstrateState({ + taskId, + ddbStatus: ddbState.lastStatus, + substrate: substrateStatus, + microvmId: sessionHandle.microvmId, + userId: task.user_id, + correlation, + log, + repo: task.repo, + // Threaded so `microvm_suspend_anomaly` is emitted once per anomaly + // EPISODE rather than on every ~30 s poll; a non-anomalous observation + // re-arms it (see reconcileMicrovmSubstrateState). + suspendAnomalyReported: microvmSuspendAnomalyReported, + }); + microvmSuspendAnomalyReported = suspendAnomalyReported; + if (taskFailed) { + return { attempts: ddbState.attempts, lastStatus: TaskStatus.FAILED }; + } + } catch (err) { + // Non-fatal: a GetMicrovm hiccup must not abort the durable poll step. + // The task stays bounded by MAX_POLL_ATTEMPTS (~8.5 h) and, once the + // agent is RUNNING, by its own terminal write. A repeated-failure + // escalation counter (the ECS `MAX_CONSECUTIVE_ECS_POLL_FAILURES` + // analogue) is deliberately deferred — it would add PollState fields + // that P3's suspend policy will need to reshape anyway. + log.warn('MicroVM pollSession check failed (non-fatal)', { + error: err instanceof Error ? err.message : String(err), + }); + } + } + + return { ...ddbState, consecutiveEcsPollFailures, consecutiveEcsCompletedPolls, microvmSuspendAnomalyReported }; }, { initialState: { attempts: 0 }, @@ -339,6 +438,20 @@ const durableHandler: DurableExecutionHandler = asyn if (blueprintConfig.compute_type === 'ecs') { await deleteEcsPayload(taskId); } + // ADR-021: "When the orchestrator finalizes a `lambda-microvm` task, the + // orchestrator shall call terminate-microvm (termination shall not rely on + // any substrate timeout)." Without this the VM lingers until + // `maximumDurationInSeconds` (8 h) expires — with `idlePolicy` omitted there + // is no tighter substrate bound — so we would keep paying for a full 8-hour + // reservation after every task, and every SUSPENDED/RUNNING VM keeps counting + // against the account memory quota that gates admission. + // + // `stopSession` is internally best-effort (it swallows and level-differentiates + // every failure), so this cannot fail the finalize step or strand the task in + // a non-terminal state. + if (microvmStrategy && sessionHandle.strategyType === 'lambda-microvm') { + await microvmStrategy.stopSession(sessionHandle); + } }); }; diff --git a/cdk/src/handlers/shared/api-key.ts b/cdk/src/handlers/shared/api-key.ts index 777df8705..5c207119b 100644 --- a/cdk/src/handlers/shared/api-key.ts +++ b/cdk/src/handlers/shared/api-key.ts @@ -27,7 +27,7 @@ import { API_KEY_SCOPES, type ApiKeyScope } from './types'; export const API_KEY_PREFIX = 'bgak'; /** Secret entropy per key (bytes; 256-bit). */ -export const API_KEY_SECRET_BYTES = 32; +const API_KEY_SECRET_BYTES = 32; /** Number of underscore-delimited segments in a well-formed key. */ const KEY_PART_COUNT = 3; diff --git a/cdk/src/handlers/shared/comment-trigger.ts b/cdk/src/handlers/shared/comment-trigger.ts index 4182bed17..14a1589f2 100644 --- a/cdk/src/handlers/shared/comment-trigger.ts +++ b/cdk/src/handlers/shared/comment-trigger.ts @@ -17,9 +17,6 @@ * SOFTWARE. */ -/** The explicit mention that turns a channel comment into an instruction. */ -export const COMMENT_TRIGGER_MENTION = '@bgagent'; - export interface CommentTrigger { readonly triggered: boolean; readonly instruction: string; diff --git a/cdk/src/handlers/shared/compute-strategy.ts b/cdk/src/handlers/shared/compute-strategy.ts index 48feb838d..99bb48073 100644 --- a/cdk/src/handlers/shared/compute-strategy.ts +++ b/cdk/src/handlers/shared/compute-strategy.ts @@ -20,13 +20,47 @@ import type { BlueprintConfig, ComputeType } from './repo-config'; import { AgentCoreComputeStrategy } from './strategies/agentcore-strategy'; import { EcsComputeStrategy } from './strategies/ecs-strategy'; +import { LambdaMicrovmComputeStrategy } from './strategies/lambda-microvm-strategy'; +/** + * Per-session compute handle, discriminated on ``strategyType``. + * + * ``sessionId`` is the shared key across every variant: it is what the + * orchestrator persists as ``TaskRecord.session_id`` (and what + * ``cancel-task.ts`` / ``pollTaskStatus`` read back), so every variant must + * supply a non-empty, substrate-meaningful value. AgentCore uses a fresh UUID + * (its ``runtimeSessionId`` must be ≥ 33 chars); ECS uses the task ARN; the + * MicroVM backend uses the ``microvmId`` — see the note on the + * ``lambda-microvm`` variant below. + * + * ADR-021 sub-decision 1: the MicroVM variant carries ``microvmId`` (every + * lifecycle API — suspend/resume/terminate/get — takes only that identifier) + * and ``endpoint`` (minted per session by ``RunMicrovm``, required for any + * future orchestrator→agent HTTP interaction). The image ARN/version is + * deliberately NOT in the handle: like the ECS task-definition ARN it is + * deployment-time configuration consumed by ``startSession`` from the + * construct-injected environment and recorded in the session-start log entry + * for diagnostics, not per-session lifecycle state. + */ export type SessionHandle = | { readonly sessionId: string; readonly strategyType: 'agentcore'; readonly runtimeArn: string } - | { readonly sessionId: string; readonly strategyType: 'ecs'; readonly clusterArn: string; readonly taskArn: string }; + | { readonly sessionId: string; readonly strategyType: 'ecs'; readonly clusterArn: string; readonly taskArn: string } + | { readonly sessionId: string; readonly strategyType: 'lambda-microvm'; readonly microvmId: string; readonly endpoint: string }; +/** + * Substrate-observed session state. Deliberately mechanical: the strategy + * REPORTS, the orchestrator INTERPRETS (ADR-021 sub-decision 1 — "Poll + * semantics"). ``pollSession`` receives only the handle and cannot see the + * task's DynamoDB status, so no health rule may live in a strategy. + * + * ``suspended`` exists only for backends with an orchestrator-visible suspend + * API (today: ``lambda-microvm``). It is NOT a health verdict — a suspended + * MicroVM is healthy while the task is ``AWAITING_APPROVAL`` and an anomaly + * otherwise, and only the orchestrator can tell the two apart. + */ export type SessionStatus = | { readonly status: 'running' } + | { readonly status: 'suspended' } | { readonly status: 'completed' } | { readonly status: 'failed'; readonly error: string }; @@ -52,9 +86,10 @@ export interface ComputeStrategy { * #299 ECS_RIGHTSIZED_PLANNING: true for a read-only workflow (e.g. * coding/pr-review-v1) that clones + reads but never * builds. The ECS strategy uses it to pick the smaller planning task def - * instead of the larger build def. AgentCore ignores it (its microVM is a - * single fixed size). Optional so callers/tests that omit it default to the - * build def (never worse than today). + * instead of the larger build def. The fixed-size substrates ignore it — + * AgentCore and lambda-microvm each run one microVM shape, so there is no + * second tier to route to. Optional so callers/tests that omit it default to + * the build def (never worse than today). */ readOnly?: boolean; }): Promise; @@ -69,6 +104,8 @@ export function resolveComputeStrategy(blueprintConfig: BlueprintConfig): Comput return new AgentCoreComputeStrategy(); case 'ecs': return new EcsComputeStrategy(); + case 'lambda-microvm': + return new LambdaMicrovmComputeStrategy(); default: { const _exhaustive: never = computeType; throw new Error(`Unknown compute_type: '${_exhaustive}'`); diff --git a/cdk/src/handlers/shared/error-classifier.ts b/cdk/src/handlers/shared/error-classifier.ts index 8364a5b69..8d0323ebf 100644 --- a/cdk/src/handlers/shared/error-classifier.ts +++ b/cdk/src/handlers/shared/error-classifier.ts @@ -17,6 +17,8 @@ * SOFTWARE. */ +import { LAMBDA_MICROVM_SUPPORTED_REGIONS } from './microvm-regions'; + /** * Error categories for runtime task errors. */ @@ -61,7 +63,7 @@ export const ErrorClass = { USER: 'user', } as const; -export type ErrorClassType = (typeof ErrorClass)[keyof typeof ErrorClass]; +type ErrorClassType = (typeof ErrorClass)[keyof typeof ErrorClass]; /** * Structured classification of a task error. @@ -189,6 +191,132 @@ const PATTERNS: readonly ErrorPattern[] = [ errorClass: ErrorClass.TRANSIENT, }, }, + { + // ADR-021 sub-decision 4, "Regional availability enforcement" (defense in + // depth): Lambda MicroVMs launched in 5 Regions, so a stack deployed + // elsewhere gets an endpoint-resolution failure from the SDK rather than a + // typed service error. Classify it as CONFIG + non-retryable so the task + // reply names the cause and the supported Regions instead of "temporary + // infrastructure issue — reply to retry" (which would loop forever). + // + // ORDERING: this entry MUST stay ABOVE the generic `Session start failed` + // pattern below, which would otherwise win on the persisted (wrapped) + // error_message and hand the user a retry remedy. Both alternatives are + // anchored on a `lambda`/`microvm` marker precisely so an AgentCore or ECS + // endpoint failure cannot be hijacked into MicroVM copy — their endpoint + // hosts are `bedrock-agentcore.*` / `ecs.*`. + pattern: /(?:UnknownEndpoint|Inaccessible host|Could not resolve endpoint).{0,120}lambda|(?:lambda[- ]?microvms?|microvms?).{0,60}(?:not available|not supported|unavailable)|(?:not available|not supported|unavailable).{0,60}(?:lambda[- ]?microvms?)/i, + classification: { + category: ErrorCategory.CONFIG, + title: 'Lambda MicroVMs is not available in this Region', + description: + 'The task\'s Blueprint selects the `lambda-microvm` compute backend, but the AWS Lambda MicroVMs service could not be reached in this stack\'s Region — the regional endpoint does not resolve.', + remedy: + 'Lambda MicroVMs is available in: ' + + LAMBDA_MICROVM_SUPPORTED_REGIONS.join(', ') + + '. Either redeploy the platform in a supported Region, or move this repo to a backend that is available here ' + + '(bgagent repo onboard --compute-type agentcore, or --compute-type ecs for large repos). ' + + 'If AWS has since launched MicroVMs in this Region, update the supported-Region list in ' + + 'cdk/src/handlers/shared/microvm-regions.ts. Retrying as-is will not help.', + retryable: false, + errorClass: ErrorClass.SERVICE, + }, + }, + // --- Lambda MicroVMs (ADR-021) --- + // + // SCOPING: the three SDK-exception entries below are anchored on the + // ``MicroVM failed`` marker that + // ``lambda-microvm-strategy.wrapMicrovmError`` puts on every error it lets + // escape (see ``MICROVM_ERROR_MARKER``). The anchor is mandatory, not + // decorative: ``ThrottlingException``, ``ServiceQuotaExceededException`` and + // ``ResourceNotFoundException`` are generic AWS exception names that AgentCore, + // ECS, DynamoDB and Secrets Manager all throw too. Matching them unscoped + // would retroactively re-classify every other backend's throttle — changing + // its ``errorClass`` and therefore whether ``startSessionWithRetry`` + // auto-retries it. Unmarked occurrences must keep falling through to whatever + // classified them before this section existed (a precise earlier pattern, or + // UNKNOWN). + // + // Both orders are accepted in each pattern so a future wrapper that puts the + // exception name ahead of the marker still matches; ``[\s\S]`` rather than + // ``.`` because SDK messages can span lines. + // + // ORDERING: this section sits immediately ABOVE the generic + // `Session start failed` catch-all. That is only safe BECAUSE of the marker: + // an unmarked `Session start failed: ThrottlingException` from AgentCore or ECS + // cannot match here and still reaches the generic entry with its original + // copy, while a MicroVM failure gets the precise remedy (quota-increase path, + // the MICROVM_* env vars to check) instead of "Check AgentCore Runtime or ECS + // cluster health" — advice that names the wrong substrate entirely. If the + // marker anchor is ever dropped, these MUST move back below the catch-all. + { + pattern: /MicroVM substrate terminated before the agent wrote a terminal status/i, + classification: { + category: ErrorCategory.COMPUTE, + title: 'The MicroVM stopped before the agent reported a result', + description: + 'The Lambda MicroVM running this task reached a terminal state (session duration cap, host fault, or an external terminate) while the task was still mid-flight, so no result was ever written.', + remedy: + 'This is a compute-substrate fault, not a problem with your request — reply here to try again. ' + + 'If it repeats, check the MicroVM logs for the session and whether the task is exceeding the 8-hour session cap; ' + + 'a long-running repo may belong on --compute-type ecs.', + retryable: true, + errorClass: ErrorClass.TRANSIENT, + }, + }, + { + // Account-level MicroVM memory quota. Deliberately TRANSIENT rather than + // SERVICE: this quota is capacity-shaped, not configuration-shaped — it + // frees as running/suspended MicroVMs terminate (AWS counts SUSPENDED VMs + // toward the quota), which is the same "wait and retry" character as the + // existing per-user `concurrency limit` entry above. The remedy still names + // the quota-increase path for the case where the ceiling is genuinely too + // low, so a persistently failing deployment is not left guessing. + pattern: /MicroVM [\w ]+failed[\s\S]*ServiceQuotaExceededException|ServiceQuotaExceededException[\s\S]*MicroVM [\w ]+failed/i, + classification: { + category: ErrorCategory.COMPUTE, + title: 'Couldn\'t start — the MicroVM compute quota is currently exhausted', + description: 'Starting the MicroVM was rejected because the account\'s Lambda MicroVMs quota (memory across running and suspended MicroVMs) is fully consumed.', + remedy: + 'Wait for in-flight tasks to finish and retry — the quota frees as MicroVMs terminate. ' + + 'If the platform hits this routinely, request a Lambda MicroVMs quota increase in Service Quotas, ' + + 'or lower the per-user concurrency cap so admission stops accepting tasks the substrate cannot start.', + retryable: true, + errorClass: ErrorClass.TRANSIENT, + }, + }, + { + pattern: /MicroVM [\w ]+failed[\s\S]*(?:ThrottlingException|TooManyRequestsException)|(?:ThrottlingException|TooManyRequestsException)[\s\S]*MicroVM [\w ]+failed/i, + classification: { + category: ErrorCategory.COMPUTE, + title: 'Couldn\'t start — the MicroVM control plane throttled the request', + description: 'The Lambda MicroVMs control plane returned a throttling error for this session\'s lifecycle call.', + remedy: 'This is a rate limit, not a problem with your request — reply here to try again. If it is constant, an admin should check the account\'s Lambda MicroVMs request-rate quotas.', + retryable: true, + errorClass: ErrorClass.TRANSIENT, + }, + }, + { + // A named resource the strategy was handed does not exist: a MicroVM image + // identifier/version, an execution role, or a network connector ARN. Always + // a deploy/config fault — retrying with the same ARNs cannot succeed. + // + // NOTE: `pollSession` maps a ResourceNotFoundException from `GetMicrovm` to + // `completed` (a reaped MicroVM) and never lets it reach here, so this entry + // only ever sees start-time resource faults. + pattern: /MicroVM [\w ]+failed[\s\S]*ResourceNotFoundException|ResourceNotFoundException[\s\S]*MicroVM [\w ]+failed/i, + classification: { + category: ErrorCategory.CONFIG, + title: 'A MicroVM resource referenced by this deployment does not exist', + description: 'The Lambda MicroVMs control plane rejected the session because a resource it was pointed at — MicroVM image, execution role, or network connector — could not be found.', + remedy: + 'Retrying won\'t help: an admin needs to re-deploy the MicroVM substrate so the image, execution role, and network connector ARNs wired into the orchestrator actually exist. ' + + 'Check the MICROVM_IMAGE_IDENTIFIER / MICROVM_EXECUTION_ROLE_ARN / MICROVM_*_CONNECTOR_ARNS values on the orchestrator function against the deployed resources.', + retryable: false, + errorClass: ErrorClass.SERVICE, + }, + }, + { pattern: /Session start failed/i, classification: { @@ -521,6 +649,7 @@ const PATTERNS: readonly ErrorPattern[] = [ errorClass: ErrorClass.TRANSIENT, }, }, + ]; /** diff --git a/cdk/src/handlers/shared/jira-adf.ts b/cdk/src/handlers/shared/jira-adf.ts index 62322525b..77287e89f 100644 --- a/cdk/src/handlers/shared/jira-adf.ts +++ b/cdk/src/handlers/shared/jira-adf.ts @@ -36,7 +36,7 @@ */ /** Deepest markdown heading level (`######`) ADF heading nodes are clamped to. */ -export const MAX_MARKDOWN_HEADING_LEVEL = 6; +const MAX_MARKDOWN_HEADING_LEVEL = 6; export interface AdfNode { readonly type?: string; diff --git a/cdk/src/handlers/shared/jira-attachments.ts b/cdk/src/handlers/shared/jira-attachments.ts index 4e7e47bd2..51b10936f 100644 --- a/cdk/src/handlers/shared/jira-attachments.ts +++ b/cdk/src/handlers/shared/jira-attachments.ts @@ -77,7 +77,7 @@ const ATTACHMENT_FETCH_TIMEOUT_MS = 10_000; const COMMENT_FETCH_TIMEOUT_MS = 5_000; /** Default cap on recent comments folded into the task context. */ -export const DEFAULT_MAX_COMMENTS = 10; +const DEFAULT_MAX_COMMENTS = 10; /** * Tenant-scoped context for a Jira REST call, resolved once per task by the diff --git a/cdk/src/handlers/shared/microvm-regions.ts b/cdk/src/handlers/shared/microvm-regions.ts new file mode 100644 index 000000000..06e926281 --- /dev/null +++ b/cdk/src/handlers/shared/microvm-regions.ts @@ -0,0 +1,72 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Regional availability of the AWS Lambda MicroVMs backend (ADR-021, + * sub-decision 4 — "Regional availability enforcement"). + * + * ABCA is a single-region deployment, so the constraint is binary per stack: + * either the stack's Region offers Lambda MicroVMs or the `lambda-microvm` + * backend does not exist there at all. Enforcement is layered, and THIS module + * is the one *static* layer: + * + * - **synth/deploy** (static, this constant) — offline determinism is + * required, so the CDK gate compares the stack Region against + * {@link LAMBDA_MICROVM_SUPPORTED_REGIONS}. + * - **repo onboarding / `platform doctor` / orchestration** (live) — a + * `ListManagedMicrovmImages` probe, so those layers **self-heal** the day + * AWS adds a Region, with no code change and no dependency on this list. + * + * ## Documented update path (this list rots by design) + * + * When AWS launches Lambda MicroVMs in a new Region: + * + * 1. Add the Region id to {@link LAMBDA_MICROVM_SUPPORTED_REGIONS} below + * (this is the ONLY place the list is declared — do not copy it). + * 2. Ship it; nothing else needs changing. The live probes already accepted + * the Region before this edit, so the edit only unblocks synth. + * + * Until step 1 lands, operators in a newly launched Region use the CDK + * context-flag escape hatch rather than waiting on a release — which is why a + * stale list is a friction bug, never an availability outage. + * + * Source: ADR-021 capability table ("Regions (launch)"); 5 Regions at the + * 2026-06-22 launch. + */ +export const LAMBDA_MICROVM_SUPPORTED_REGIONS: readonly string[] = [ + 'us-east-1', + 'us-east-2', + 'us-west-2', + 'eu-west-1', + 'ap-northeast-1', +]; + +/** + * True when `region` is in the statically documented + * {@link LAMBDA_MICROVM_SUPPORTED_REGIONS} list. + * + * A `false` result means "not known to be supported *by this build*", NOT + * "unsupported by AWS" — the live probes are authoritative. Never use this to + * reject at runtime what a probe already accepted. + * + * @param region - an AWS Region id (e.g. `us-east-1`); compared verbatim. + */ +export function isLambdaMicrovmRegionSupported(region: string | undefined): boolean { + return !!region && LAMBDA_MICROVM_SUPPORTED_REGIONS.includes(region); +} diff --git a/cdk/src/handlers/shared/orchestrator.ts b/cdk/src/handlers/shared/orchestrator.ts index 22475f80a..547927b5b 100644 --- a/cdk/src/handlers/shared/orchestrator.ts +++ b/cdk/src/handlers/shared/orchestrator.ts @@ -20,6 +20,7 @@ import { S3Client } from '@aws-sdk/client-s3'; import { GetCommand, PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; import { ulid } from 'ulid'; +import type { SessionHandle, SessionStatus } from './compute-strategy'; import { AttachmentBudgetExceededError, AttachmentConfigurationError, AttachmentResolutionError, hydrateContext, resolveGitHubToken } from './context-hydration'; import { logger, type Logger } from './logger'; import { writeMinimalEpisode } from './memory'; @@ -58,6 +59,18 @@ export interface PollState { readonly consecutiveEcsPollFailures?: number; /** Consecutive polls where ECS reports completed but DDB is not terminal — escalated after 5. */ readonly consecutiveEcsCompletedPolls?: number; + /** + * True once `microvm_suspend_anomaly` has been emitted for the CURRENT anomaly + * episode, so the event fires once per episode instead of on every ~30 s poll + * (an 8-hour suspended task would otherwise write ~960 identical events). + * + * Re-armed (set back to false) by any non-anomalous observation — see + * {@link reconcileMicrovmSubstrateState}. Kept as a plain boolean rather than a + * counter/timestamp on purpose: P3's suspend policy will reshape this area + * anyway, and one flag is the smallest thing that fixes the duplication without + * pre-committing to a shape that work will have to undo. + */ + readonly microvmSuspendAnomalyReported?: boolean; } /** After RUNNING this long, we expect `agent_heartbeat_at` from the agent (if ever set). */ @@ -228,6 +241,202 @@ const MIN_POLL_INTERVAL_MS = 5_000; /** Maximum allowed poll interval (5 minutes). */ const MAX_POLL_INTERVAL_MS = 300_000; +/** + * Build the ``compute_metadata`` map persisted on the task row at session start, + * so a later handler can act on the right backend without re-deriving anything: + * ``cancel-task.ts`` reads ``clusterArn``/``taskArn`` from it today, and ADR-021 + * sub-decision 2 has the approve/deny Lambdas read ``microvmId`` from it in P3. + * + * Kept as an exhaustive switch (not a ternary + spread) so a fourth backend is a + * compile error here rather than a silently empty metadata map — the field is + * load-bearing for compute shutdown, and a missing handle is exactly what + * ``task_cancel_compute_orphan`` exists to alarm on. + * + * DynamoDB stores this as ``Record`` (see + * ``TaskRecord.compute_metadata``), so every value must already be a string. + */ +export function buildComputeMetadata(handle: SessionHandle): Record { + switch (handle.strategyType) { + case 'ecs': + return { clusterArn: handle.clusterArn, taskArn: handle.taskArn }; + case 'agentcore': + return { runtimeArn: handle.runtimeArn }; + case 'lambda-microvm': + // ADR-021: `microvmId` keys every lifecycle API; `endpoint` is per-session + // state that becomes load-bearing the day an orchestrator→agent HTTP + // consumer appears (none exists in P1–P3). + return { microvmId: handle.microvmId, endpoint: handle.endpoint }; + default: { + const _exhaustive: never = handle; + throw new Error(`Unknown strategyType on session handle: ${JSON.stringify(_exhaustive)}`); + } + } +} + +/** Outcome of a MicroVM substrate cross-check. */ +export interface MicrovmReconcileResult { + /** + * True when this call drove the task to FAILED — the caller must stop polling + * and report ``lastStatus: FAILED``. False for every healthy or + * anomalous-but-not-fatal observation. + */ + readonly taskFailed: boolean; + + /** + * Value the caller must carry into the next poll cycle's + * ``PollState.microvmSuspendAnomalyReported``. + * + * True while a suspend anomaly is being (or has been) reported; false whenever + * the anomaly condition is absent, which RE-ARMS the event for a genuinely new + * episode. Returned rather than mutated so the durable poll's state stays a + * plain serializable value. + */ + readonly suspendAnomalyReported: boolean; +} + +/** + * Cross-reference a MicroVM's substrate state against the task's DynamoDB status + * — the interpretation half of ADR-021's "the strategy reports, the orchestrator + * interprets" split. ``pollSession`` can only see the handle, so every rule that + * needs the task status lives here, exactly as ``finalPollState`` already does + * the substrate/DDB cross-check for ECS and ``pollTaskStatus`` does it for + * AgentCore heartbeats. + * + * Rules (all three from ADR-021 sub-decision 1's EARS requirements): + * - substrate ``suspended`` + task ``AWAITING_APPROVAL`` → HEALTHY. This is the + * orchestrator's own intended suspend (P3); say nothing. + * - substrate ``suspended`` + any other task status → ANOMALY, not a failure. + * Surface a task event and keep polling: a suspended VM preserves full + * memory/disk state and can be resumed, so failing the task would destroy + * recoverable work over a condition we cannot yet explain. + * - substrate terminal + non-terminal task status → FAIL the task with the + * substrate-failure reason (``error-classifier`` has the matching entry). + * + * ## The anomaly event fires ONCE PER EPISODE, not once per poll + * + * The anomaly branch is reached on every poll while the condition holds, so a + * task suspended out of band for hours would write one identical TaskEvent every + * ~30 s (~960 of them across the 8 h poll window) — noise that buries the first, + * informative one and inflates TaskEvents. ``suspendAnomalyReported`` (threaded + * through {@link PollState}) suppresses the repeats while leaving the + * do-not-fail-fast behaviour untouched: the function still returns + * ``taskFailed: false`` on every one of those polls. + * + * **Recovery re-arms it.** Any observation that is NOT an anomaly — the VM + * resumed (``running``), or the task moved into ``AWAITING_APPROVAL`` so the + * suspend is now intended — resets the flag to false. A second, later episode is + * new information (something suspended this VM twice), so it earns its own event. + * The alternative (latch forever) would silently hide a flapping suspend loop, + * which is exactly the pathology an operator most needs to see. + * + * The WARN log is emitted on every poll regardless. Logs are cheap, per-poll + * evidence is what a timeline investigation needs, and CloudWatch is not a + * user-facing surface the way TaskEvents is. + * + * The terminal branch RE-READS the task row before failing. The DynamoDB status + * handed in was read earlier in the same poll cycle, and the ordinary happy path + * is "agent writes terminal status, agent exits, VM terminates" — so a stale read + * plus a fast teardown would otherwise fail a task that actually succeeded. The + * re-read is the same "confirm before acting on a lost race" move ``finalizeTask`` + * makes after a failed transition. (ECS buys the same protection with a + * 5-consecutive-poll patience counter; one extra GetItem on a path that is about + * to end the task is cheaper and does not add state to ``PollState``.) + * + * @param taskId - the task being polled. + * @param ddbStatus - the task status observed by this poll cycle. + * @param substrate - what ``pollSession`` reported. + * @param microvmId - for event/log correlation. + * @param userId - owner, for ``failTask``. + * @param correlation - the #245 envelope for emitted events. + * @param log - the caller's child logger (already carries task/user/repo). + * @param repo - optional target repo for the correlation envelope. + * @param suspendAnomalyReported - the previous cycle's flag; see the section above. + */ +export async function reconcileMicrovmSubstrateState(args: { + taskId: string; + ddbStatus: TaskStatusType; + substrate: SessionStatus; + microvmId: string; + userId: string; + correlation: EventCorrelation; + log: Logger; + repo?: string; + suspendAnomalyReported?: boolean; +}): Promise { + const { + taskId, ddbStatus, substrate, microvmId, userId, correlation, log, repo, + suspendAnomalyReported = false, + } = args; + + if (substrate.status === 'running') { + // Healthy — and it also ENDS any anomaly episode, so the next one reports. + return { taskFailed: false, suspendAnomalyReported: false }; + } + + if (substrate.status === 'suspended') { + if (ddbStatus === TaskStatus.AWAITING_APPROVAL) { + // Orchestrator-intended suspend during an approval wait — the whole point + // of this backend. Nothing to report, and the anomaly is re-armed: if the + // task later leaves AWAITING_APPROVAL while still suspended, that is a new + // and genuinely reportable episode. + return { taskFailed: false, suspendAnomalyReported: false }; + } + // Suspended outside an approval wait. Nothing in ABCA suspends a MicroVM + // except the orchestrator's (P3) approval-wait policy, so this means either + // an out-of-band SuspendMicrovm call or a substrate-side suspend we did not + // ask for. Surface it — do NOT fail-fast (ADR-021: "an anomaly to surface, + // not fail-fast"); the VM's state is intact and resumable. + log.warn('MicroVM is suspended while the task is not awaiting approval', { + microvm_id: microvmId, + task_status: ddbStatus, + anomaly_already_reported: suspendAnomalyReported, + }); + if (!suspendAnomalyReported) { + await emitTaskEvent(taskId, 'microvm_suspend_anomaly', { + microvm_id: microvmId, + task_status: ddbStatus, + reason: 'suspended_outside_approval_wait', + }, correlation); + } + return { taskFailed: false, suspendAnomalyReported: true }; + } + + // Terminal substrate report (`completed` or `failed`). `pollSession` reports + // TERMINATING/TERMINATED/NotFound as `completed` because it cannot see an exit + // code; `failed` only reaches here if a future mapping adds one. + const detail = substrate.status === 'failed' ? substrate.error : `substrate state ${substrate.status}`; + + const reread = await loadTask(taskId); + if (TERMINAL_STATUSES.includes(reread.status)) { + // The agent wrote its terminal status between this cycle's status read and + // now — the normal shutdown ordering. Not a failure. + log.info('MicroVM terminated after the agent wrote a terminal status', { + microvm_id: microvmId, + task_status: reread.status, + }); + // Terminal either way, so the flag no longer matters; carried through + // unchanged rather than reset so the value never lies about what happened. + return { taskFailed: false, suspendAnomalyReported }; + } + + log.error('MicroVM reached a terminal state before the agent wrote a terminal status', { + microvm_id: microvmId, + task_status: reread.status, + detail, + }); + // `releaseConcurrency: false` — the finalize step sees the now-terminal task + // and decrements, matching the ECS substrate-failure branch in orchestrate-task. + await failTask( + taskId, + reread.status, + `MicroVM substrate terminated before the agent wrote a terminal status: ${detail}`, + userId, + false, + repo, + ); + return { taskFailed: true, suspendAnomalyReported }; +} + /** * Load blueprint configuration for a task's repository and merge with platform defaults. * @param task - the task record (needs task.repo). diff --git a/cdk/src/handlers/shared/repo-config.ts b/cdk/src/handlers/shared/repo-config.ts index 00823dd6d..4fe37aebf 100644 --- a/cdk/src/handlers/shared/repo-config.ts +++ b/cdk/src/handlers/shared/repo-config.ts @@ -25,7 +25,7 @@ import { makeDocClient } from './ua'; * Per-repository configuration written by the Blueprint CDK construct * and read at runtime by the task API gate and the orchestrator. */ -export type ComputeType = 'agentcore' | 'ecs'; +export type ComputeType = 'agentcore' | 'ecs' | 'lambda-microvm'; export interface RepoConfig { readonly repo: string; diff --git a/cdk/src/handlers/shared/strategies/lambda-microvm-strategy.ts b/cdk/src/handlers/shared/strategies/lambda-microvm-strategy.ts new file mode 100644 index 000000000..cb736e3af --- /dev/null +++ b/cdk/src/handlers/shared/strategies/lambda-microvm-strategy.ts @@ -0,0 +1,619 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { + GetMicrovmCommand, + LambdaMicrovmsClient, + MicrovmState, + RunMicrovmCommand, + TerminateMicrovmCommand, +} from '@aws-sdk/client-lambda-microvms'; +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import type { ComputeStrategy, SessionHandle, SessionStatus } from '../compute-strategy'; +import { logger } from '../logger'; +import type { BlueprintConfig } from '../repo-config'; +import { makeClient } from '../ua'; + +let sharedClient: LambdaMicrovmsClient | undefined; +function getClient(): LambdaMicrovmsClient { + if (!sharedClient) { + sharedClient = makeClient(LambdaMicrovmsClient); + } + return sharedClient; +} + +let sharedS3Client: S3Client | undefined; +function getS3Client(): S3Client { + if (!sharedS3Client) { + sharedS3Client = makeClient(S3Client); + } + return sharedS3Client; +} + +/** + * Fully-qualified MicroVM image **ARN** passed as `imageIdentifier` on every + * `RunMicrovm`. + * + * MUST be an ARN, never a bare image name: `RunMicrovm` rejects a name outright + * (live 2026-07-31 — `ValidationException: Malformed ARN - doesn't start with + * 'arn:'`), as does `list-microvm-image-builds`. `LambdaMicrovmCompute` already + * derives the exact `…:microvm-image:` ARN for the lifecycle IAM scope and + * injects THAT value here, so the two can never disagree; {@link assertImageArn} + * fails fast if a hand-edited deployment breaks the contract. + */ +const MICROVM_IMAGE_IDENTIFIER = process.env.MICROVM_IMAGE_IDENTIFIER; +const MICROVM_IMAGE_VERSION = process.env.MICROVM_IMAGE_VERSION; +const MICROVM_EXECUTION_ROLE_ARN = process.env.MICROVM_EXECUTION_ROLE_ARN; +const MICROVM_EGRESS_CONNECTOR_ARNS = process.env.MICROVM_EGRESS_CONNECTOR_ARNS; +/** + * Ingress connectors to pass on every `RunMicrovm`. Injected by + * `LambdaMicrovmCompute` as exactly the Lambda-managed `NO_INGRESS` connector in + * P1–P3; a deployment that genuinely needs ingress (#391 operator shell access) + * can widen it without a strategy change. + * + * **Always present in a CDK-deployed stack.** `TaskOrchestrator.microvmConfig` + * requires `ingressConnectorArns` and injects this var unconditionally alongside + * the other four, so the fallback below is DEAD CODE on any stack this repo + * deploys — kept only as defense in depth for a hand-edited Lambda environment, + * because the failure mode it guards (a PUBLIC endpoint on every agent MicroVM) + * is too severe to leave to the type system alone. + */ +const MICROVM_INGRESS_CONNECTOR_ARNS = process.env.MICROVM_INGRESS_CONNECTOR_ARNS; +const MICROVM_PAYLOAD_BUCKET = process.env.MICROVM_PAYLOAD_BUCKET; + +/** + * Session wall-clock ceiling passed on EVERY ``RunMicrovm`` call, pinned to the + * service maximum of 28 800 s / 8 h (ADR-021 sub-decision 1). + * + * Three reasons it is a constant and not a knob: it matches AgentCore's 8-hour + * session cap (backend parity), it sits inside the orchestrator's ~8.5 h + * safety-net poll window, and — because ``idlePolicy`` is omitted (see + * {@link RunMicrovmCommand} construction below) — it is the ONLY substrate-level + * bound on *suspended* time as well as running time. There is no wall-clock task + * budget in the platform today (budgets are ``max_turns`` / ``max_budget_usd``), + * so a Blueprint override would be policy without a driver; add one only if a + * real need appears. + */ +export const MICROVM_MAX_DURATION_SECONDS = 28_800; + +/** + * Hard service cap on ``runHookPayload`` (bytes), measured live rather than read + * off the SDK docs. + * + * The SDK's ``RunMicrovmRequest.runHookPayload`` documents "Maximum: 16,384 + * bytes"; the service enforces **4 096** (2026-07-31, us-east-1): + * + * ``` + * ValidationException: 1 validation error detected: Value at 'runHookPayload' + * failed to satisfy constraint: Member must have length less than or equal to 4096 + * ``` + * + * Probed exactly: 4 096 bytes passes length validation, 4 097 is rejected. The + * old 16 384 threshold would have inlined every envelope between 4 097 and + * 16 384 bytes and had the service reject all of them. + * + * This is the EXACT branch point for the inline/S3-pointer decision, with no + * safety margin — deliberately unlike ``ecs-strategy``, which keeps its inline + * warn line at 6 144 of ECS's 8 192-byte cap. That margin exists because ECS + * counts the *whole* ``containerOverrides`` blob (env vars, command, and payload + * share one budget), so the strategy cannot know how much of the 8 192 the + * payload actually gets. ``runHookPayload`` is a single standalone string, so + * the counted size is exactly what we measure and the boundary is computable. + * + * Consequence worth stating plainly: at 4 KB the **S3-pointer path is the + * dominant one**. A hydrated task payload (prompt + issue thread + repo context) + * essentially always exceeds 4 KB, so the inline branch is the exception (tiny + * repo-less prompts), not the common case. + */ +const RUN_HOOK_PAYLOAD_LIMIT_BYTES = 4_096; + +/** + * Stable marker prefixed onto every error this strategy lets escape, via + * {@link wrapMicrovmError}. Load-bearing, not cosmetic: ``error-classifier`` + * anchors its ``ThrottlingException`` / ``ServiceQuotaExceededException`` / + * ``ResourceNotFoundException`` entries on this marker so those bare AWS + * exception names classify as MicroVM faults ONLY when they came from this + * backend. Without the anchor an identically-named AgentCore or ECS throttle + * would silently inherit MicroVM copy and MicroVM retry semantics. + * + * Keep in lockstep with the MicroVM section of ``error-classifier.ts``. + */ +export const MICROVM_ERROR_MARKER = 'MicroVM'; + +/** + * Wrap an error escaping a MicroVM control-plane (or payload-upload) call so it + * carries {@link MICROVM_ERROR_MARKER} plus the originating operation. + * + * The AWS exception NAME is spliced into the message explicitly because + * ``err.message`` alone omits it (``String(err)`` would include it, but the + * classifier is handed the *wrapped* error) and the classifier keys on that + * name. ``cause`` retains the original for anyone who needs ``err.name``. + * + * The wrapper's own ``name`` is intentionally left as ``Error`` so + * ``String(wrapped)`` reads ``Error: MicroVM failed: : `` — + * marker first, which is the order the classifier patterns document. + */ +function wrapMicrovmError(operation: string, err: unknown): Error { + const name = err instanceof Error ? err.name : undefined; + const message = err instanceof Error ? err.message : String(err); + const detail = name && name !== 'Error' && !message.includes(name) + ? `${name}: ${message}` + : message; + return new Error(`${MICROVM_ERROR_MARKER} ${operation} failed: ${detail}`, { cause: err }); +} + +/** + * S3 object key for a task's MicroVM ``/run`` payload. Same key shape as the + * ECS payload bucket (``ecsPayloadKey``): one object per task under its own + * task-id prefix. The payload bucket's lifecycle-expiry rule (ADR-021 + * sub-decision 3) is the reaper — unlike the ECS path there is no orchestrator + * finalize-time delete on this backend yet. + */ +export function microvmPayloadKey(taskId: string): string { + return `${taskId}/payload.json`; +} + +/** Split a comma-separated env-var list into trimmed, non-empty entries. */ +function parseArnList(raw: string | undefined): string[] { + return (raw ?? '').split(',').map(s => s.trim()).filter(Boolean); +} + +/** + * Resource-name half of the Lambda-managed **`NO_INGRESS`** connector ARN. + * + * Kept in lockstep with `MICROVM_NO_INGRESS_CONNECTOR_RESOURCE` in + * `constructs/lambda-microvm-compute.ts` — the construct is the normal source of + * this ARN (via `MICROVM_INGRESS_CONNECTOR_ARNS`); this copy exists only for the + * fallback below, which must not depend on a construct the Lambda bundle does + * not include. + */ +const NO_INGRESS_CONNECTOR_RESOURCE = 'aws-network-connector:NO_INGRESS'; + +/** + * ARN of the Lambda-managed `NO_INGRESS` connector for the running Region. + * + * **Dead code in a CDK-deployed stack** — `TaskOrchestrator` requires + * `ingressConnectorArns` and always injects + * `MICROVM_INGRESS_CONNECTOR_ARNS`, so `configuredIngress` is never empty there. + * This exists for the one path the type system cannot reach: a Lambda + * environment edited outside CDK. Kept rather than deleted because the failure + * mode of *omitting* `ingressNetworkConnectors` is a PUBLIC endpoint on every + * agent MicroVM — the service attaches `HTTP_INGRESS` by default (live + * 2026-07-31) — and a silent public endpoint is worse than a few dead lines. + * Deriving the ARN needs only the Region: partition follows from the Region + * prefix, and the account segment is the literal `aws` because these connectors + * are service-owned. + */ +function noIngressConnectorArn(): string { + const region = process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION ?? ''; + const partition = region.startsWith('cn-') + ? 'aws-cn' + : region.startsWith('us-gov-') + ? 'aws-us-gov' + : 'aws'; + return `arn:${partition}:lambda:${region}:aws:network-connector:${NO_INGRESS_CONNECTOR_RESOURCE}`; +} + +/** + * Fail fast when `MICROVM_IMAGE_IDENTIFIER` is not an ARN. + * + * The service's own error (`ValidationException: Malformed ARN - doesn't start + * with 'arn:'`) names neither the env var nor the remedy, and it arrives after + * the payload has already been written to S3. Checking here keeps the diagnosis + * one hop from the cause. + */ +function assertImageArn(identifier: string): void { + if (identifier.startsWith('arn:')) { + return; + } + throw new Error( + `MICROVM_IMAGE_IDENTIFIER must be a full MicroVM image ARN, got ${JSON.stringify(identifier)}. ` + + 'RunMicrovm rejects bare image names ("Malformed ARN - doesn\'t start with \'arn:\'"). ' + + 'LambdaMicrovmCompute injects the exact arn::lambda:::' + + 'microvm-image: ARN it also scopes the lifecycle IAM grant to, so this indicates the ' + + 'orchestrator function\'s environment was edited outside CDK — redeploy the stack with ' + + '`--context compute_type=lambda-microvm` (plus the image context flags) to restore it.', + ); +} + +/** + * AWS Lambda MicroVMs compute backend (ADR-021). + * + * A serverless Firecracker sandbox per session: snapshot-based launch, native + * disk that survives suspend/resume, and — unlike AgentCore — a real + * control-plane state machine the orchestrator can observe through + * {@link LambdaMicrovmComputeStrategy.pollSession}. + * + * P1 scope is start / poll / stop only. ``suspendSession`` / ``resumeSession`` + * (the interface widening across all three strategies) land in P3 — do NOT add + * them here piecemeal, ADR-021 sub-decision 1 requires them in one commit so + * the exhaustive-``never`` switch culture forces every backend to make a + * compile-checked decision about its suspend semantics. + */ +export class LambdaMicrovmComputeStrategy implements ComputeStrategy { + readonly type = 'lambda-microvm'; + + async startSession(input: { + taskId: string; + /** Accepted to satisfy the ComputeStrategy interface. MicroVMs have no + * workload-token-injecting runtime (they inherit the ECS env-var identity + * posture until #249 / ADR-016 redesign the seam), so this is unused. */ + userId: string; + payload: Record; + blueprintConfig: BlueprintConfig; + }): Promise { + if (!MICROVM_IMAGE_IDENTIFIER || !MICROVM_EXECUTION_ROLE_ARN || !MICROVM_EGRESS_CONNECTOR_ARNS || !MICROVM_PAYLOAD_BUCKET) { + // Config/deploy mismatch: this repo is compute_type=lambda-microvm but the + // stack was deployed WITHOUT the MicroVM substrate, so the orchestrator has + // no MICROVM_* env vars. Name the root cause + remedy so an admin doesn't + // have to reverse-engineer it from a bare env-var list — same posture as + // the ECS branch. (The CLI `repo onboard --compute-type lambda-microvm` + // availability probe normally prevents this; a repo onboarded before that + // guard, or edited directly, can still reach here.) + throw new Error( + 'This repository is configured compute_type=lambda-microvm, but this stack was deployed without the ' + + 'Lambda MicroVMs substrate (missing MICROVM_IMAGE_IDENTIFIER/MICROVM_EXECUTION_ROLE_ARN/' + + 'MICROVM_EGRESS_CONNECTOR_ARNS/MICROVM_PAYLOAD_BUCKET). Redeploy the stack with ' + + '`--context compute_type=lambda-microvm` to provision the MicroVM substrate, or set this repo to ' + + 'compute_type=agentcore (bgagent repo onboard --compute-type agentcore).', + ); + } + + const { taskId, payload } = input; + + // An identifier that is not an ARN cannot launch anything — check before the + // payload upload so a misconfiguration never leaves an orphan S3 object. + assertImageArn(MICROVM_IMAGE_IDENTIFIER); + + // Payload delivery (ADR-021 sub-decision 3): the `/run` lifecycle hook + // receives `runHookPayload` as its request body, capped at 4 KB by the + // service. The hydrated_context essentially always blows that, so the + // S3-pointer path (mirroring ECS #502) is the DOMINANT one here and the + // inline branch is the exception. The MicroVM EXECUTION role holds the read + // grant, exactly as the ECS task role does today. + // + // Two keys, deliberately mirroring the ECS container env contract + // (AGENT_PAYLOAD / AGENT_PAYLOAD_S3_URI) so the agent's `/run` hook has one + // self-describing shape to branch on: + // { "agent_payload": {...} } — inline + // { "agent_payload_s3_uri": "s3://..." } — pointer + const inlineEnvelope = JSON.stringify({ agent_payload: payload }); + // Measure the SERIALIZED envelope, not the bare payload: the envelope is + // what the service counts against the 4 KB cap. Byte length (not + // String.length) because a multi-byte prompt/diff makes chars an undercount. + const inlineBytes = Buffer.byteLength(inlineEnvelope, 'utf8'); + + let runHookPayload: string; + let payloadS3Uri: string | undefined; + // EXACT boundary: `<= limit` inlines, `> limit` uploads. The service accepts + // 4 096 bytes and rejects 4 097 (measured), so 4 096 must still go inline. + if (inlineBytes <= RUN_HOOK_PAYLOAD_LIMIT_BYTES) { + runHookPayload = inlineEnvelope; + } else { + const key = microvmPayloadKey(taskId); + const payloadJson = JSON.stringify(payload); + try { + await getS3Client().send(new PutObjectCommand({ + Bucket: MICROVM_PAYLOAD_BUCKET, + Key: key, + Body: payloadJson, + ContentType: 'application/json', + })); + } catch (err) { + // Marked so the classifier attributes an upload failure to this backend + // rather than letting a bare S3 exception name fall through to UNKNOWN. + throw wrapMicrovmError('payload upload', err); + } + payloadS3Uri = `s3://${MICROVM_PAYLOAD_BUCKET}/${key}`; + runHookPayload = JSON.stringify({ agent_payload_s3_uri: payloadS3Uri }); + logger.info('Wrote MicroVM run-hook payload to S3', { + task_id: taskId, + bytes: Buffer.byteLength(payloadJson, 'utf8'), + inline_bytes: inlineBytes, + inline_limit_bytes: RUN_HOOK_PAYLOAD_LIMIT_BYTES, + uri: payloadS3Uri, + }); + } + + // Explicit ingress control (F7, live 2026-07-31): `RunMicrovm` does NOT + // default to "no ingress" — omitting the field attaches the AWS-managed + // PUBLIC `HTTP_INGRESS` connector and mints a public + // `*.lambda-microvm..on.aws` endpoint. So the field is ALWAYS sent. + // + // The env var is unconditional in every CDK-deployed stack (its prop is + // required), so in practice this always takes the `configuredIngress` branch + // and carries the construct's `NO_INGRESS` ARN — or real connectors once #391 + // widens it. The fallback is unreachable there by construction; see + // `noIngressConnectorArn`. + const configuredIngress = parseArnList(MICROVM_INGRESS_CONNECTOR_ARNS); + const ingressNetworkConnectors = configuredIngress.length > 0 + ? configuredIngress + : [noIngressConnectorArn()]; + + const command = new RunMicrovmCommand({ + imageIdentifier: MICROVM_IMAGE_IDENTIFIER, + ...(MICROVM_IMAGE_VERSION && { imageVersion: MICROVM_IMAGE_VERSION }), + executionRoleArn: MICROVM_EXECUTION_ROLE_ARN, + // Egress rides the platform VPC through an egress network connector so the + // DNS Firewall / security-group / flow-log stack applies unchanged + // (ADR-021 sub-decision 4). + egressNetworkConnectors: parseArnList(MICROVM_EGRESS_CONNECTOR_ARNS), + // Never omitted — see the comment above. `NO_INGRESS` is the suppression + // mechanism, not an empty list. + ingressNetworkConnectors, + runHookPayload, + maximumDurationInSeconds: MICROVM_MAX_DURATION_SECONDS, + // `idlePolicy` is OMITTED — never passed, in any phase (ADR-021 + // sub-decision 1, asserted by an invariant unit test). MicroVM idle + // policies measure idleness as *inbound traffic at the endpoint*, and the + // ABCA agent is outbound-only: "no inbound traffic" is its normal state, + // so a naive idle policy would suspend an agent mid-way through a + // 40-minute build. All three idlePolicy fields are required when the block + // is present, so omission is the unambiguous disabled state. Suspension is + // orchestrator-owned (P3) — do NOT reintroduce this field. + // + // `clientToken` is also deliberately omitted. It is an idempotency token, + // and the one place a MicroVM start is retried is `startSessionWithRetry`, + // which retries precisely BECAUSE the first attempt FAILED. Passing a + // task-derived token there would ask the service to dedupe against that + // failed attempt and could replay its outcome instead of genuinely + // retrying — turning the auto-retry into a no-op. Session start is already + // idempotent by construction at the ABCA level (no clone, commit, or PR has + // happened yet), so the token buys nothing and risks the retry. + }); + + let result; + try { + result = await getClient().send(command); + } catch (err) { + // Marker-scoped so `ThrottlingException` / `ServiceQuotaExceededException` + // / `ResourceNotFoundException` from THIS backend classify as MicroVM + // faults, while identically-named AgentCore/ECS errors keep their existing + // classification. See MICROVM_ERROR_MARKER. + throw wrapMicrovmError('RunMicrovm', err); + } + + const { microvmId, endpoint } = result; + if (!microvmId || !endpoint) { + // A malformed response means a MicroVM may ALREADY BE RUNNING (and billing) + // that no caller will ever receive a handle for — nothing self-terminates on + // this substrate. Reap it here, best-effort, before failing: this is the one + // orphan window the orchestrator's own catch cannot cover, because + // `startSession` never returned a handle to it. + if (microvmId) { + await this.terminateBestEffort(microvmId, 'incomplete RunMicrovm response'); + } + // Wrapped like every other escaping error so `error-classifier` can see the + // MicroVM marker: without it this lands in the generic `Session start + // failed` bucket with "Check AgentCore Runtime or ECS cluster health" — + // advice that names the wrong substrate entirely. `RunMicrovm` is the + // operation because that is the call whose response is malformed. + throw wrapMicrovmError( + 'RunMicrovm', + new Error( + `RunMicrovm returned an incomplete response (microvmId=${microvmId ?? 'missing'}, ` + + `endpoint=${endpoint ? 'present' : 'missing'}, state=${result.state ?? 'unknown'})`, + ), + ); + } + + // Image ARN/version is logged, NOT carried in the handle (ADR-021 + // sub-decision 1) — it is deployment-time config, and this line is the + // diagnostic record of which snapshot a given session actually booted. + logger.info('Lambda MicroVM session started', { + task_id: taskId, + microvm_id: microvmId, + state: result.state, + image_identifier: MICROVM_IMAGE_IDENTIFIER, + image_arn: result.imageArn, + image_version: result.imageVersion ?? MICROVM_IMAGE_VERSION, + maximum_duration_seconds: MICROVM_MAX_DURATION_SECONDS, + payload_delivery: payloadS3Uri ? 's3_pointer' : 'inline', + ...(payloadS3Uri && { payload_s3_uri: payloadS3Uri }), + }); + + return { + // sessionId = microvmId, mirroring the ECS variant's "sessionId = the + // substrate identifier" precedent (ECS uses the task ARN) rather than + // AgentCore's fresh UUID — AgentCore only needs a UUID because + // `runtimeSessionId` is a caller-minted value that must be ≥ 33 chars. + // Here the substrate mints the id, every lifecycle API keys on it, and it + // is what an operator needs to correlate `TaskRecord.session_id` with the + // MicroVM in logs/console. A second synthetic UUID would add a + // non-actionable identifier and leave `session_id` un-joinable. + sessionId: microvmId, + strategyType: 'lambda-microvm', + microvmId, + endpoint, + }; + } + + /** + * Report the substrate's view of the session — MECHANICALLY. No task-state + * interpretation happens here (ADR-021 sub-decision 1): this method sees only + * the handle, so the health rules that need the task's DynamoDB status live in + * the orchestrator (``reconcileMicrovmSubstrateState``). + * + * State mapping: + * - ``PENDING`` / ``RUNNING`` → ``running`` (PENDING is still booting, the + * same way ECS's PENDING/PROVISIONING map to ``running``). + * - ``SUSPENDING`` / ``SUSPENDED`` → ``suspended``. SUSPENDING is folded in + * because the VM is already on its way to frozen; reporting ``running`` + * would tell the orchestrator compute is still progressing when it is not. + * Both map to a state the orchestrator treats as benign-or-anomalous + * depending on the task status, never as a failure. (``SUSPENDING`` was + * never observable live — suspend reaches ``SUSPENDED`` in under a second + * — so nothing may WAIT for it; it is mapped for completeness only.) + * - ``TERMINATING`` / ``TERMINATED`` → ``completed``. Both are terminal or + * terminal-bound and carry no exit code, so "the substrate is gone" is all + * the strategy can honestly say; whether that is success or failure is the + * orchestrator's call (it cross-references the DynamoDB status). This is + * the load-bearing terminal signal: a terminated MicroVM stays observable + * as ``TERMINATED`` for at least ~10 minutes (live-measured), so a poller + * that waited for NotFound would spin on a finished VM. + * - anything else (an unrecognized future state) → ``running``, so a service + * enum addition can never fail a healthy task. + */ + async pollSession(handle: SessionHandle): Promise { + if (handle.strategyType !== 'lambda-microvm') { + throw new Error('pollSession called with non-lambda-microvm handle'); + } + const { microvmId } = handle; + + let state: string | undefined; + try { + const result = await getClient().send(new GetMicrovmCommand({ + microvmIdentifier: microvmId, + })); + state = result.state; + } catch (err) { + // A MicroVM that the control plane no longer knows about is gone, not + // broken — treat NotFound as terminal (``completed``) rather than + // ``failed``. This deliberately DIVERGES from ecs-strategy's + // "DescribeTasks returned no task ⇒ failed": ECS keeps stopped tasks + // describable for ~1 h, so a missing task there really is anomalous, + // whereas a MicroVM is eventually reaped from the control plane by design + // and would otherwise fail every task that finished cleanly. + // + // NOTE (live 2026-07-31): this is a LATE signal, not the near-term one. A + // terminated MicroVM reported ``TERMINATED`` at +3 s and was STILL + // ``TERMINATED`` ~10 minutes later; ``ResourceNotFoundException`` was never + // observed in that window. The mapping is still correct — and load-bearing + // for a VM reaped after a long gap — but the branch that actually fires in + // practice is ``TERMINATED → completed`` in the switch below. Neither may + // be removed in favour of the other. + // + // The orchestrator still fails the task when this terminal report lands + // while the DynamoDB status is non-terminal, so a genuine mid-run + // disappearance is not swallowed — it just gets the substrate-failure + // classification instead of a misleading poll error. + if (err instanceof Error && err.name === 'ResourceNotFoundException') { + logger.info('MicroVM not found on poll — treating as terminal', { + microvm_id: microvmId, + }); + return { status: 'completed' }; + } + throw wrapMicrovmError('GetMicrovm', err); + } + + switch (state) { + case MicrovmState.PENDING: + case MicrovmState.RUNNING: + return { status: 'running' }; + case MicrovmState.SUSPENDING: + case MicrovmState.SUSPENDED: + return { status: 'suspended' }; + case MicrovmState.TERMINATING: + case MicrovmState.TERMINATED: + return { status: 'completed' }; + default: + logger.warn('Unrecognized MicroVM state — reporting running', { + microvm_id: microvmId, + state, + }); + return { status: 'running' }; + } + } + + /** + * Terminate the MicroVM. Best-effort with differentiated error handling + * matching ``agentcore-strategy.stopSession``: a stop that cannot happen must + * never fail the caller, but the log LEVEL has to distinguish "already gone" + * (expected) from "we were throttled / denied" (an operator signal that + * MicroVMs may be leaking) from "something else" (worth a warning). + * + * ADR-021: termination is the active cleanup path — it must not rely on + * ``maximumDurationInSeconds`` expiring, which would keep paying for an + * 8-hour reservation after the task is done. Live verification made that + * mandatory rather than belt-and-braces: a hook-less MicroVM reached + * ``RUNNING`` in 12 s and stayed ``RUNNING`` indefinitely with no + * ``stateReason`` — nothing self-terminates, so nothing cleans up if the + * orchestrator does not. + */ + async stopSession(handle: SessionHandle): Promise { + if (handle.strategyType !== 'lambda-microvm') { + throw new Error('stopSession called with non-lambda-microvm handle'); + } + await this.terminateBestEffort(handle.microvmId, 'session stop'); + } + + /** + * `TerminateMicrovm` that never throws, with the log LEVEL carrying the + * diagnosis. + * + * The single implementation behind BOTH {@link stopSession} and the + * incomplete-response orphan reap in {@link startSession}, so every terminate + * ABCA issues has identical error semantics — a second, subtly-different + * best-effort copy is exactly how one of them ends up throwing and masking the + * failure it was cleaning up after. + * + * @param microvmId - the MicroVM to terminate. + * @param reason - why we are terminating, for the log line (the orphan-reap and + * the ordinary finalize path are worth telling apart in CloudWatch). + */ + private async terminateBestEffort(microvmId: string, reason: string): Promise { + try { + await getClient().send(new TerminateMicrovmCommand({ + microvmIdentifier: microvmId, + })); + logger.info('Lambda MicroVM terminated', { microvm_id: microvmId, reason }); + } catch (err) { + const errName = err instanceof Error ? err.name : undefined; + if (errName === 'ResourceNotFoundException' || errName === 'ConflictException') { + // Already terminated (reaped) or already TERMINATING — the desired end + // state either way. ConflictException joins the info branch because a + // concurrent terminate (orchestrator finalize racing a user cancel) is + // routine here, and warning on it would train operators to ignore warns. + logger.info('MicroVM already terminated or terminating', { + microvm_id: microvmId, + reason, + error_type: errName, + }); + } else if (errName === 'ThrottlingException' || errName === 'AccessDeniedException') { + // A throttle or a missing lambda:TerminateMicrovm grant means the VM is + // probably STILL RUNNING and billing — escalate. + logger.error('Failed to terminate MicroVM', { + microvm_id: microvmId, + reason, + error_type: errName, + error: err instanceof Error ? err.message : String(err), + }); + } else { + logger.warn('Failed to terminate MicroVM (best-effort)', { + microvm_id: microvmId, + reason, + error: err instanceof Error ? err.message : String(err), + }); + } + } + } +} + +/** + * Re-exported so tests and future callers can assert the documented cap without + * duplicating the literal. This is BOTH the service's limit and our exact + * inline/S3-pointer branch point — there is no separate threshold. + */ +export const MICROVM_RUN_HOOK_PAYLOAD_LIMIT_BYTES = RUN_HOOK_PAYLOAD_LIMIT_BYTES; + +/** + * Re-exported for tests: the `NO_INGRESS` fallback the strategy substitutes when + * `MICROVM_INGRESS_CONNECTOR_ARNS` is missing (see {@link noIngressConnectorArn}). + */ +export const microvmNoIngressConnectorArnForRegion = noIngressConnectorArn; diff --git a/cdk/src/main.ts b/cdk/src/main.ts index a43abc723..98266bc98 100644 --- a/cdk/src/main.ts +++ b/cdk/src/main.ts @@ -65,6 +65,15 @@ const excludeResourceTypes = [ 'AWS::Route53Resolver::ResolverQueryLoggingConfigAssociation', ]; +// TODO(#645): with three backends this single-valued tag is no longer an honest +// statement of what a stack runs — a `--context compute_type=lambda-microvm` +// deploy still provisions the AgentCore runtime, so every resource gets tagged +// `compute_type=lambda-microvm` including the AgentCore ones. ADR-021 +// sub-decision 4 flags revisiting the semantics (e.g. a `compute_types` list). +// Deliberately NOT changed here: retagging every resource in the stack is a +// replacement-risk change of its own, and MicroVM spend is already attributable +// through the per-resource `abca:compute-backend` tags the +// LambdaMicrovmCompute construct applies. Tags.of(stack).add('compute_type', computeType, { excludeResourceTypes }); const githubTagKeys = [ diff --git a/cdk/src/stacks/agent.ts b/cdk/src/stacks/agent.ts index 91ef0ea26..93b77d423 100644 --- a/cdk/src/stacks/agent.ts +++ b/cdk/src/stacks/agent.ts @@ -46,6 +46,11 @@ import { FanOutConsumer } from '../constructs/fanout-consumer'; import { GitHubScreenshotIntegration } from '../constructs/github-screenshot-integration'; import { IterationHeartbeat } from '../constructs/iteration-heartbeat'; import { JiraIntegration } from '../constructs/jira-integration'; +import { + LambdaMicrovmCompute, + isLambdaMicrovmImageConfigured, + type LambdaMicrovmImageInputs, +} from '../constructs/lambda-microvm-compute'; import { LinearIntegration } from '../constructs/linear-integration'; import { OrchestrationReconciler } from '../constructs/orchestration-reconciler'; import { OrchestrationTable } from '../constructs/orchestration-table'; @@ -209,6 +214,52 @@ export class AgentStack extends Stack { }, ]); + // --- Compute-backend deploy gate (read early) --- + // Which optional compute substrate this deploy provisions, from the + // ``compute_type`` deploy context (default 'agentcore' — the AgentCore + // runtime is always present, the other backends are additive). Read HERE, + // well above the constructs it gates, because TaskApi is instantiated + // before them and needs to know whether to wire the cancel Lambda's + // MicroVM termination grant (ADR-021 sub-decision 4). + const computeType = this.node.tryGetContext('compute_type') ?? 'agentcore'; + const lambdaMicrovmEnabled = computeType === 'lambda-microvm'; + + // The operator-supplied MicroVM image inputs, resolved HERE (pure context + // reads, no construct dependency) rather than at the construct's call site + // below, because TaskApi — created well before the MicroVM construct — needs + // to know whether an image will exist in order to decide whether the cancel + // Lambda gets a `lambda:TerminateMicrovm` grant at all. The same object is + // handed to the construct, and `isLambdaMicrovmImageConfigured` is the single + // shared predicate, so the two cannot drift. + // + // Base-image ARNs/versions are Region-scoped service data only discoverable + // through ``aws lambda-microvms list-managed-microvm-images``, and the + // artifact has to be uploaded to the bucket THIS stack creates — hence + // context values rather than defaults. See the construct's "three states" + // table and cdk/scripts/package-microvm-artifact.sh for the bootstrap + // sequence. + const microvmImageInputs: LambdaMicrovmImageInputs = { + baseImageArn: this.node.tryGetContext('microvm_base_image_arn'), + baseImageVersion: this.node.tryGetContext('microvm_base_image_version'), + externalImageIdentifier: this.node.tryGetContext('microvm_image_identifier'), + externalImageVersion: this.node.tryGetContext('microvm_image_version'), + }; + const microvmImageConfigured = lambdaMicrovmEnabled + && isLambdaMicrovmImageConfigured(microvmImageInputs); + + // MicroVM image ARN placeholder — the image is created AFTER TaskApi, but the + // cancel Lambda's grant must be scoped to it. Same Lazy.string cycle-break as + // the runtime / orchestrator / SessionRole ARNs below. + let microvmImageArnHolder: string | undefined; + const lazyMicrovmImageArn = Lazy.string({ + produce: () => { + if (!microvmImageArnHolder) { + throw new Error('MicroVM image ARN was accessed before LambdaMicrovmCompute was created'); + } + return microvmImageArnHolder; + }, + }); + // Network isolation — VPC with restricted egress const agentVpc = new AgentVpc(this, 'AgentVpc'); @@ -308,6 +359,11 @@ export class AgentStack extends Stack { traceArtifactsBucket: traceArtifactsBucket.bucket, attachmentsBucket: attachmentsBucket.bucket, userConcurrencyTable: userConcurrencyTable.table, + // ADR-021: gives the cancel Lambda `lambda:TerminateMicrovm`, scoped to the + // platform MicroVM image, so cancelling a MicroVM-backed task stops compute + // immediately. Omitted when no image is configured — there can be no + // MicroVM-backed task to cancel then. + ...(microvmImageConfigured && { lambdaMicrovmImageArn: lazyMicrovmImageArn }), }); // --- AgentCore Runtime (IAM-authed orchestrator path) --- @@ -640,7 +696,8 @@ export class AgentStack extends Stack { // ``--context compute_type=ecs``, so the default synth (and the // bootstrap-coverage test that synths with default context) stays // agentcore-only, matching how other optional constructs are context-gated. - const computeType = this.node.tryGetContext('compute_type') ?? 'agentcore'; + // (``computeType`` is read near the top of the constructor — TaskApi needs it + // for the conditional MicroVM cancel grant.) // Ephemeral bucket for ECS task payloads — the orchestrator writes the // payload here (it exceeds the 8 KB RunTask containerOverrides limit) and // passes only an S3 URI pointer; the container fetches it on boot, the @@ -701,17 +758,88 @@ export class AgentStack extends Stack { }) : undefined; + // --- AWS Lambda MicroVMs compute backend (CONTEXT-GATED) --- + // ADR-021 P1: a serverless Firecracker sandbox per session — VM-level + // isolation with no cluster to operate, and (from P3) suspend/resume so a + // task parked on a HITL approval gate stops billing compute while keeping + // its cloned repo and warm build caches in memory. + // + // Gated exactly like the ECS backend above: resources synthesize only under + // ``--context compute_type=lambda-microvm``, so the default synth — and the + // bootstrap-coverage test that synths with default context — stays + // agentcore-only. The construct itself enforces the ADR's Region gate, so a + // deploy into a Region without Lambda MicroVMs fails at synth rather than on + // the first task. + const lambdaMicrovm = lambdaMicrovmEnabled + ? new LambdaMicrovmCompute(this, 'LambdaMicrovmCompute', { + vpc: agentVpc.vpc, + // Per-session IAM scoping (#209): the MicroVM execution role is admitted + // to the same per-task SessionRole the AgentCore runtime and the Fargate + // task role use, so tenant-data access is tag-scoped on every substrate. + agentSessionRole, + // Resolved above TaskApi — see `microvmImageInputs`. + ...microvmImageInputs, + }) + : undefined; + + // Resolve the Lazy TaskApi's cancel grant is scoped by. The invariant the + // Lazy's `produce` guards: `microvmImageConfigured` (computed from the same + // inputs, via the same predicate) is true exactly when the construct sets + // `imageArn`, so a configured deployment always has an ARN to resolve and an + // unconfigured one never asks for it. + microvmImageArnHolder = lambdaMicrovm?.imageArn; + // Advertise which compute substrate this deploy actually provisioned, so the // CLI can refuse to onboard a repo as ``compute_type: ecs`` when the ECS gate // wasn't on (``--context compute_type=ecs``) — otherwise that mismatch only // surfaces per-task as "ECS compute strategy requires ECS_CLUSTER_ARN…" at // runtime. ``ecs`` implies the AgentCore runtime is ALSO available (the ECS - // gate is additive), so an agentcore repo works on either substrate. + // gate is additive), so an agentcore repo works on either substrate — and the + // same holds for ``lambda-microvm`` (ADR-021). new CfnOutput(this, 'ComputeSubstrate', { - value: ecsCluster ? 'ecs' : 'agentcore', - description: 'Compute substrate provisioned by this deploy: "agentcore" (default) or "ecs" ' - + '(deployed with --context compute_type=ecs; adds the Fargate substrate alongside AgentCore).', - }); + value: ecsCluster ? 'ecs' : (lambdaMicrovm ? 'lambda-microvm' : 'agentcore'), + description: 'Compute substrate provisioned by this deploy: "agentcore" (default), "ecs" ' + + '(deployed with --context compute_type=ecs; adds the Fargate substrate alongside AgentCore) ' + + 'or "lambda-microvm" (--context compute_type=lambda-microvm; adds the Lambda MicroVMs ' + + 'substrate alongside AgentCore).', + }); + + if (lambdaMicrovm) { + // Emitted so the packaging helper (cdk/scripts/package-microvm-artifact.sh) + // can discover where to upload the zip+Dockerfile and which log group / + // build role to hand `create-microvm-image` — none of which have + // predictable physical names. + new CfnOutput(this, 'MicrovmArtifactBucketName', { + value: lambdaMicrovm.artifactBucket.bucketName, + description: 'S3 bucket the Lambda MicroVMs zip+Dockerfile artifact is uploaded to (ADR-021)', + }); + new CfnOutput(this, 'MicrovmArtifactObjectKey', { + value: lambdaMicrovm.artifactObjectKey, + description: 'S3 key the Lambda MicroVMs artifact must be uploaded to (matches the build role\'s s3:GetObject scope)', + }); + new CfnOutput(this, 'MicrovmBuildRoleArn', { + value: lambdaMicrovm.buildRole.roleArn, + description: 'IAM role for `aws lambda-microvms create-microvm-image --build-role-arn`', + }); + new CfnOutput(this, 'MicrovmExecutionRoleArn', { + value: lambdaMicrovm.executionRole.roleArn, + description: 'IAM role the running MicroVM assumes (`run-microvm --execution-role-arn`)', + }); + new CfnOutput(this, 'MicrovmEgressConnectorArns', { + value: lambdaMicrovm.egressConnectorArns.join(','), + description: 'Lambda network connector ARNs routing MicroVM egress through the platform VPC', + }); + new CfnOutput(this, 'MicrovmBuildEgressConnectorArns', { + value: lambdaMicrovm.buildEgressConnectorArns.join(','), + description: 'Lambda network connector ARNs for the IMAGE BUILD path (TCP 443 + 80 — ' + + 'agent/Dockerfile runs apt-get over HTTP; pass these to ' + + '`create-microvm-image --egress-network-connectors`, NOT the runtime connectors)', + }); + new CfnOutput(this, 'MicrovmLogGroupName', { + value: lambdaMicrovm.logGroup.logGroupName, + description: 'CloudWatch log group for MicroVM build- and run-time logs', + }); + } // --- Task Orchestrator (durable Lambda function) --- // Per-user concurrency cap, shared by the orchestrator (admission control) @@ -750,6 +878,29 @@ export class AgentStack extends Stack { // Pass the payload bucket so the orchestrator writes/deletes the // out-of-band payload and the ECS strategy builds the S3 URI pointer. ...(ecsPayloadBucket && { ecsPayloadBucket: ecsPayloadBucket.bucket }), + // ADR-021: route ``compute_type: 'lambda-microvm'`` repos to the MicroVM + // substrate. Wired only when an image is actually configured — without one + // the strategy has nothing to run, and injecting a partial MICROVM_* env + // block would trade the strategy's precise "deployed without the MicroVM + // substrate" error for an opaque service-side failure. The construct sets + // ``imageIdentifier`` and ``imageArn`` together (a bare image name is + // resolved to its exact ARN), so testing both keeps the all-or-nothing + // contract compile-checked rather than assumed. + ...(lambdaMicrovm?.imageIdentifier && lambdaMicrovm.imageArn && { + microvmConfig: { + imageIdentifier: lambdaMicrovm.imageIdentifier, + imageArn: lambdaMicrovm.imageArn, + imageVersion: lambdaMicrovm.imageVersion, + executionRoleArn: lambdaMicrovm.executionRole.roleArn, + egressConnectorArns: lambdaMicrovm.egressConnectorArns, + // Explicit NO_INGRESS, not an omission: RunMicrovm attaches a PUBLIC + // HTTP_INGRESS connector (and mints a public endpoint) when the field is + // absent, so ADR-021's "no inbound exposure" posture is a control the + // construct has to pass on every launch. Still no JWE tokens minted. + ingressConnectorArns: lambdaMicrovm.ingressConnectorArns, + payloadBucket: lambdaMicrovm.payloadBucket, + }, + }), }); // Now that the orchestrator exists, resolve the Lazy used by TaskApi at synth. diff --git a/cdk/test/bootstrap/__snapshots__/version.test.ts.snap b/cdk/test/bootstrap/__snapshots__/version.test.ts.snap index 71788609a..f8b1a2e7e 100644 --- a/cdk/test/bootstrap/__snapshots__/version.test.ts.snap +++ b/cdk/test/bootstrap/__snapshots__/version.test.ts.snap @@ -1,3 +1,3 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing -exports[`bootstrap version module hash is stable 1`] = `"b0501c8a57f20e4b5bf50d6fe8bbb934310adc4b64480228565d8392a19d1503"`; +exports[`bootstrap version module hash is stable 1`] = `"40d0a8b2343663084f614423fc3e6210761377dedf0bc079956ba7d4cea84c5e"`; diff --git a/cdk/test/bootstrap/artifact-sync.test.ts b/cdk/test/bootstrap/artifact-sync.test.ts index 95fb9a9c5..b0aae2c95 100644 --- a/cdk/test/bootstrap/artifact-sync.test.ts +++ b/cdk/test/bootstrap/artifact-sync.test.ts @@ -19,7 +19,14 @@ import { readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { infrastructurePolicy, applicationPolicy, observabilityPolicy } from '../../src/bootstrap/policies'; +import { + infrastructurePolicy, + applicationPolicy, + observabilityPolicy, + computeAgentcorePolicy, + computeEcsPolicy, + computeLambdaMicrovmPolicy, +} from '../../src/bootstrap/policies'; import { BOOTSTRAP_VERSION, computeBootstrapHash } from '../../src/bootstrap/version'; const artifactsDir = join(__dirname, '..', '..', 'bootstrap'); @@ -40,6 +47,9 @@ describe('Bootstrap artifact sync', () => { { name: 'infrastructure', fn: infrastructurePolicy }, { name: 'application', fn: applicationPolicy }, { name: 'observability', fn: observabilityPolicy }, + { name: 'compute-agentcore', fn: computeAgentcorePolicy }, + { name: 'compute-ecs', fn: computeEcsPolicy }, + { name: 'compute-lambda-microvm', fn: computeLambdaMicrovmPolicy }, ] as const; for (const { name, fn } of cases) { diff --git a/cdk/test/bootstrap/bootstrap-template.test.ts b/cdk/test/bootstrap/bootstrap-template.test.ts index d4bae4965..f028501a1 100644 --- a/cdk/test/bootstrap/bootstrap-template.test.ts +++ b/cdk/test/bootstrap/bootstrap-template.test.ts @@ -48,6 +48,15 @@ describe('Bootstrap template', () => { expect(template.Conditions.IncludeComputeEcs).toBeDefined(); expect(template.Conditions.IncludeComputeEcs['Fn::Not']).toBeDefined(); }); + + it('has IncludeComputeLambdaMicrovms condition matching the full backend token', () => { + const condition = template.Conditions.IncludeComputeLambdaMicrovms; + expect(condition).toBeDefined(); + expect(condition['Fn::Not']).toBeDefined(); + // Split on the WHOLE `lambda-microvm` token: splitting on a `lambda` + // prefix would also fire for any future `lambda*` backend name. + expect(JSON.stringify(condition)).toContain('lambda-microvm'); + }); }); describe('Managed policy resources', () => { @@ -57,6 +66,7 @@ describe('Bootstrap template', () => { 'IaCRoleABCAObservability', 'IaCRoleABCAComputeAgentcore', 'IaCRoleABCAComputeEcs', + 'IaCRoleABCAComputeLambdaMicrovms', ]; for (const logicalId of expectedPolicies) { @@ -75,9 +85,16 @@ describe('Bootstrap template', () => { expect(template.Resources.IaCRoleABCAComputeEcs.Condition).toBe('IncludeComputeEcs'); }); - it('non-ECS policies do not have a condition', () => { - const nonEcs = expectedPolicies.filter((p) => p !== 'IaCRoleABCAComputeEcs'); - for (const logicalId of nonEcs) { + it('IaCRoleABCAComputeLambdaMicrovms has IncludeComputeLambdaMicrovms condition', () => { + expect(template.Resources.IaCRoleABCAComputeLambdaMicrovms.Condition) + .toBe('IncludeComputeLambdaMicrovms'); + }); + + it('non-optional-compute policies do not have a condition', () => { + const unconditional = expectedPolicies.filter( + (p) => p !== 'IaCRoleABCAComputeEcs' && p !== 'IaCRoleABCAComputeLambdaMicrovms', + ); + for (const logicalId of unconditional) { expect(template.Resources[logicalId].Condition).toBeUndefined(); } }); @@ -122,6 +139,15 @@ describe('Bootstrap template', () => { expect(ecsEntry).toBeDefined(); expect(ecsEntry['Fn::If'][1]).toEqual({ Ref: 'IaCRoleABCAComputeEcs' }); expect(ecsEntry['Fn::If'][2]).toEqual({ Ref: 'AWS::NoValue' }); + + // ...and so should Lambda MicroVMs (ADR-021). + const microvmEntry = fallback.find( + + (item: any) => item['Fn::If'] && item['Fn::If'][0] === 'IncludeComputeLambdaMicrovms', + ); + expect(microvmEntry).toBeDefined(); + expect(microvmEntry['Fn::If'][1]).toEqual({ Ref: 'IaCRoleABCAComputeLambdaMicrovms' }); + expect(microvmEntry['Fn::If'][2]).toEqual({ Ref: 'AWS::NoValue' }); }); it('does not reference AdministratorAccess', () => { @@ -157,10 +183,13 @@ describe('Bootstrap template', () => { // ECS should be conditional - const ecsItem = items.find((item: any) => item['Fn::If']); - expect(ecsItem).toBeDefined(); - expect(ecsItem['Fn::If'][0]).toBe('IncludeComputeEcs'); - expect(ecsItem['Fn::If'][1]).toBe('Compute-ECS'); + const conditionalItems = items.filter((item: any) => item['Fn::If']); + expect(conditionalItems.map((item: any) => item['Fn::If'][0])).toEqual([ + 'IncludeComputeEcs', + 'IncludeComputeLambdaMicrovms', + ]); + expect(conditionalItems[0]['Fn::If'][1]).toBe('Compute-ECS'); + expect(conditionalItems[1]['Fn::If'][1]).toBe('Compute-LambdaMicrovms'); }); }); diff --git a/cdk/test/bootstrap/golden-baseline.test.ts b/cdk/test/bootstrap/golden-baseline.test.ts index ead2b9734..2283618ca 100644 --- a/cdk/test/bootstrap/golden-baseline.test.ts +++ b/cdk/test/bootstrap/golden-baseline.test.ts @@ -26,6 +26,7 @@ import { applicationPolicy, computeAgentcorePolicy, computeEcsPolicy, + computeLambdaMicrovmPolicy, infrastructurePolicy, observabilityPolicy, } from '../../src/bootstrap/policies'; @@ -73,12 +74,13 @@ describe('Golden-file parity: TypeScript policies match DEPLOYMENT_ROLES.md', () const markdownPath = join(__dirname, '..', '..', '..', 'docs', 'design', 'DEPLOYMENT_ROLES.md'); const markdown = readFileSync(markdownPath, 'utf-8'); - // Extract JSON blocks: [0]=trust, [1]=infrastructure, [2]=application, [3]=observability, [4]=ECS + // Extract JSON blocks: [0]=trust, [1]=infrastructure, [2]=application, [3]=observability, [4]=ECS, [5]=Lambda MicroVMs const jsonBlocks = extractJsonBlocks(markdown); const goldenInfra = JSON.parse(jsonBlocks[1]); const goldenApp = JSON.parse(jsonBlocks[2]); const goldenObs = JSON.parse(jsonBlocks[3]); const goldenEcs = JSON.parse(jsonBlocks[4]); + const goldenLambdaMicrovm = JSON.parse(jsonBlocks[5]); // Resolve TypeScript policies via CDK Stack const tsInfra = stack.resolve(infrastructurePolicy()); @@ -86,6 +88,7 @@ describe('Golden-file parity: TypeScript policies match DEPLOYMENT_ROLES.md', () const tsObs = stack.resolve(observabilityPolicy()); const tsComputeAgentcore = stack.resolve(computeAgentcorePolicy()); const tsComputeEcs = stack.resolve(computeEcsPolicy()); + const tsComputeLambdaMicrovm = stack.resolve(computeLambdaMicrovmPolicy()); // --- Infrastructure and Application: direct parity --- const directTestCases: Array<{ @@ -212,4 +215,28 @@ describe('Golden-file parity: TypeScript policies match DEPLOYMENT_ROLES.md', () } }); }); + + // --- Compute-LambdaMicrovms: matches block 5 from markdown --- + describe('Compute-LambdaMicrovms policy', () => { + const goldenNorm = normalizeStatements(goldenLambdaMicrovm.Statement); + const tsNorm = normalizeStatements( + (tsComputeLambdaMicrovm as { Statement: Array<{ Sid?: string; Action?: string | string[]; Resource?: string | string[] }> }).Statement, + ); + + it('has the same SIDs', () => { + expect(tsNorm.map((s) => s.sid)).toEqual(goldenNorm.map((s) => s.sid)); + }); + + it('has identical actions (sorted)', () => { + for (let i = 0; i < goldenNorm.length; i++) { + expect(tsNorm[i].actions).toEqual(goldenNorm[i].actions); + } + }); + + it('has identical resources (sorted)', () => { + for (let i = 0; i < goldenNorm.length; i++) { + expect(tsNorm[i].resources).toEqual(goldenNorm[i].resources); + } + }); + }); }); diff --git a/cdk/test/bootstrap/policies.test.ts b/cdk/test/bootstrap/policies.test.ts index 2ee788c3a..e3cdae579 100644 --- a/cdk/test/bootstrap/policies.test.ts +++ b/cdk/test/bootstrap/policies.test.ts @@ -22,6 +22,7 @@ import { allPolicies } from '../../src/bootstrap/policies'; import { applicationPolicy } from '../../src/bootstrap/policies/application'; import { computeAgentcorePolicy } from '../../src/bootstrap/policies/compute-agentcore'; import { computeEcsPolicy } from '../../src/bootstrap/policies/compute-ecs'; +import { computeLambdaMicrovmPolicy } from '../../src/bootstrap/policies/compute-lambda-microvm'; import { infrastructurePolicy } from '../../src/bootstrap/policies/infrastructure'; import { observabilityPolicy } from '../../src/bootstrap/policies/observability'; @@ -399,6 +400,79 @@ describe('IaCRole-ABCA-Compute-ECS', () => { }); }); +describe('computeLambdaMicrovmPolicy', () => { + const stack = new Stack(); + const doc = computeLambdaMicrovmPolicy(); + const json = doc.toJSON(); + const rendered = JSON.stringify(json); + + it('produces valid JSON', () => { + expect(() => JSON.parse(rendered)).not.toThrow(); + }); + + it('is under 6144 characters when serialized', () => { + // AWS managed policy size limit + expect(rendered.length).toBeLessThan(6144); + }); + + it('contains the expected SIDs', () => { + const resolvedDoc = stack.resolve(doc); + const statements = resolvedDoc.Statement as Array<{ Sid: string }>; + + expect(statements.map((s) => s.Sid)).toEqual(['LambdaMicrovms']); + }); + + it('covers the expected service prefixes', () => { + const resolvedDoc = stack.resolve(doc); + const statements = resolvedDoc.Statement as Array<{ Action: string | string[] }>; + const allActions = statements.flatMap((s) => + Array.isArray(s.Action) ? s.Action : [s.Action], + ); + const prefixes = new Set(allActions.map((a) => a.split(':')[0])); + + expect(prefixes).toEqual(new Set(['lambda'])); + }); + + it('covers both CFN resource types the construct synthesizes', () => { + const resolvedDoc = stack.resolve(doc); + const statements = resolvedDoc.Statement as Array<{ Action: string | string[] }>; + const allActions = statements.flatMap((s) => + Array.isArray(s.Action) ? s.Action : [s.Action], + ); + + // AWS::Lambda::MicrovmImage + AWS::Lambda::NetworkConnector, plus the + // documented dependent of CreateMicrovmImage. + expect(allActions).toContain('lambda:CreateMicrovmImage'); + expect(allActions).toContain('lambda:CreateNetworkConnector'); + expect(allActions).toContain('lambda:PassNetworkConnector'); + }); + + it('grants NO runtime session lifecycle actions (those belong to the orchestrator role)', () => { + // A deploy role that can start, suspend, or mint tokens for MicroVMs is a + // privilege escalation — the orchestrator gets those, per-deployment. + // Compared as exact actions, not substrings: `lambda:GetMicrovmImage` (which + // the deploy role legitimately needs) contains `lambda:GetMicrovm`. + const resolvedDoc = stack.resolve(doc); + const statements = resolvedDoc.Statement as Array<{ Action: string | string[] }>; + const allActions = new Set(statements.flatMap((s) => + Array.isArray(s.Action) ? s.Action : [s.Action], + )); + + for (const action of [ + 'lambda:RunMicrovm', + 'lambda:GetMicrovm', + 'lambda:TerminateMicrovm', + 'lambda:SuspendMicrovm', + 'lambda:ResumeMicrovm', + 'lambda:CreateMicrovmAuthToken', + 'lambda:CreateMicrovmShellAuthToken', + 'lambda:ConnectMicrovm', + ]) { + expect(allActions.has(action)).toBe(false); + } + }); +}); + describe('Cross-policy validation', () => { const stack = new Stack(); const policies = allPolicies(); @@ -416,8 +490,10 @@ describe('Cross-policy validation', () => { expect(unique.size).toBe(allSids.length); }); - it('returns exactly 5 policies', () => { - expect(policies).toHaveLength(5); + it('returns exactly 6 policies', () => { + // infrastructure, application, observability, compute-agentcore, + // compute-ecs, compute-lambda-microvm (ADR-021). + expect(policies).toHaveLength(6); }); it('every policy is under 6144 character limit', () => { diff --git a/cdk/test/constructs/lambda-microvm-compute.test.ts b/cdk/test/constructs/lambda-microvm-compute.test.ts new file mode 100644 index 000000000..6e70e595a --- /dev/null +++ b/cdk/test/constructs/lambda-microvm-compute.test.ts @@ -0,0 +1,796 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { App, Stack } from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import { AgentSessionRole } from '../../src/constructs/agent-session-role'; +import { + DEFAULT_MINIMUM_MEMORY_MIB, + LambdaMicrovmCompute, + MICROVM_ARTIFACT_OBJECT_KEY, + MICROVM_BACKEND_TAG_KEY, + MICROVM_BACKEND_TAG_VALUE, + MICROVM_LOG_GROUP_PREFIX, + MICROVM_NO_INGRESS_CONNECTOR_RESOURCE, + MICROVM_PAYLOAD_TTL_DAYS, + MICROVM_REGION_OVERRIDE_CONTEXT, + MICROVM_SUPPORTED_MEMORY_MIB, + assertLambdaMicrovmRegionSupported, + isLambdaMicrovmImageConfigured, + microvmNoIngressConnectorArn, +} from '../../src/constructs/lambda-microvm-compute'; +import { LAMBDA_MICROVM_SUPPORTED_REGIONS } from '../../src/handlers/shared/microvm-regions'; + +const BASE_IMAGE_ARN = 'arn:aws:lambda:us-east-1:aws:microvm-image:al2023-1'; + +interface BuildOptions { + readonly region?: string; + readonly context?: Record; + readonly withImage?: boolean; + readonly externalImageIdentifier?: string; + readonly externalImageVersion?: string; + readonly withSessionRole?: boolean; + readonly regionAgnostic?: boolean; + readonly minimumMemoryInMiB?: number; +} + +interface Built { + readonly stack: Stack; + readonly construct: LambdaMicrovmCompute; + readonly template: Template; +} + +/** + * Construct a minimal stack around the construct WITHOUT synthesizing it. + * + * Deliberately NOT the full `AgentStack`: this file asserts the construct's own + * contract, so a bare `Stack` + VPC keeps each template small and makes the + * resource counts below unambiguous. Stack-level wiring is covered in + * `test/stacks/agent.test.ts`. + * + * Separate from {@link build} because the Region gate throws inside the + * construct's constructor — those cases must never reach a synth, and skipping + * it keeps the one describe that cannot cache a `Template` cheap. + */ +function instantiate(options: BuildOptions = {}): Omit { + const app = new App({ context: options.context }); + const stack = new Stack(app, 'TestStack', { + env: options.regionAgnostic + ? undefined + : { account: '123456789012', region: options.region ?? 'us-east-1' }, + }); + const vpc = new ec2.Vpc(stack, 'Vpc', { maxAzs: 2 }); + + let agentSessionRole: AgentSessionRole | undefined; + if (options.withSessionRole) { + // AgentSessionRole requires at least one assuming role; a stand-in for the + // AgentCore runtime role keeps this test focused on the MicroVM execution + // role being admitted *in addition* to it. + const runtimeRole = new iam.Role(stack, 'RuntimeRole', { + assumedBy: new iam.ServicePrincipal('bedrock-agentcore.amazonaws.com'), + }); + agentSessionRole = new AgentSessionRole(stack, 'AgentSessionRole', { + assumingRoles: [runtimeRole], + taskScopedTables: [ + new dynamodb.Table(stack, 'TaskTable', { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + }), + ], + traceArtifactsBucket: new s3.Bucket(stack, 'TraceBucket'), + attachmentsBucket: new s3.Bucket(stack, 'AttachmentsBucket'), + }); + } + + const construct = new LambdaMicrovmCompute(stack, 'LambdaMicrovmCompute', { + vpc, + agentSessionRole, + ...(options.withImage && { + baseImageArn: BASE_IMAGE_ARN, + baseImageVersion: '1', + }), + externalImageIdentifier: options.externalImageIdentifier, + externalImageVersion: options.externalImageVersion, + minimumMemoryInMiB: options.minimumMemoryInMiB, + }); + + return { stack, construct }; +} + +/** {@link instantiate} plus one synth, for assertions against the template. */ +function build(options: BuildOptions = {}): Built { + const { stack, construct } = instantiate(options); + return { stack, construct, template: Template.fromStack(stack) }; +} + +describe('LambdaMicrovmCompute — image provisioned from a managed base image', () => { + let built: Built; + let template: Template; + + beforeAll(() => { + built = build({ withImage: true, withSessionRole: true }); + template = built.template; + }); + + test('synthesizes exactly one MicroVM image from the artifact bucket object', () => { + template.resourceCountIs('AWS::Lambda::MicrovmImage', 1); + template.hasResourceProperties('AWS::Lambda::MicrovmImage', { + BaseImageArn: BASE_IMAGE_ARN, + BaseImageVersion: '1', + CodeArtifact: { + Uri: { + 'Fn::Join': ['', [ + 's3://', + { Ref: Match.stringLikeRegexp('LambdaMicrovmComputeArtifactBucket') }, + `/${MICROVM_ARTIFACT_OBJECT_KEY}`, + ]], + }, + }, + }); + }); + + test('builds an ARM64 image at the largest ACCEPTED BASELINE (8 GiB)', () => { + // 32768 was rejected live: "The requested memory size of 32768 MiB is not + // supported by base MicroVM image …al2023-1. Supported memory sizes in MiB + // are: [512, 1024, 2048, 4096, 8192]." Note this configures the BASELINE — + // the service scales vertically to a 32 GiB / 16 vCPU peak on its own, which + // is why nothing here asks for the peak. + template.hasResourceProperties('AWS::Lambda::MicrovmImage', { + CpuConfigurations: [{ Architecture: 'arm64' }], + Resources: [{ MinimumMemoryInMiB: 8192 }], + }); + expect(DEFAULT_MINIMUM_MEMORY_MIB).toBe(8192); + expect(MICROVM_SUPPORTED_MEMORY_MIB).toEqual([512, 1024, 2048, 4096, 8192]); + expect(Math.max(...MICROVM_SUPPORTED_MEMORY_MIB)).toBe(DEFAULT_MINIMUM_MEMORY_MIB); + }); + + test('configures /ready + /run and NOTHING else (the rest fail their transition)', () => { + const images = template.findResources('AWS::Lambda::MicrovmImage'); + const hooks = Object.values(images)[0]!.Properties.Hooks; + expect(hooks.Port).toBe(8080); + // Paths are the routes `agent/src/server.py` actually serves (the API model + // has no path field at all, so a made-up path would be a silent lie). + expect(hooks.MicrovmHooks).toEqual({ + Run: '/aws/lambda-microvms/runtime/v1/run', + RunTimeoutInSeconds: 60, + }); + // /ready is MANDATORY: create-microvm-image refuses ANY lifecycle hook + // without it, so "declare /run in P1, serve it in P2" was unreachable. + expect(hooks.MicrovmImageHooks).toEqual({ + Ready: '/aws/lambda-microvms/runtime/v1/ready', + ReadyTimeoutInSeconds: 60, + }); + // A hook the service calls but the agent does not serve fails the lifecycle + // transition (or every build), so nothing else may be advertised. + expect(hooks.MicrovmHooks.Suspend).toBeUndefined(); + expect(hooks.MicrovmHooks.Resume).toBeUndefined(); + expect(hooks.MicrovmHooks.Terminate).toBeUndefined(); + expect(hooks.MicrovmImageHooks.Validate).toBeUndefined(); + }); + + test('bakes NO environment variables into the snapshot (ADR-021: no secrets in the image)', () => { + template.hasResourceProperties('AWS::Lambda::MicrovmImage', { + EnvironmentVariables: [], + }); + }); + + test('routes image build-time egress through the BUILD connector, not the runtime one', () => { + // The runtime connector is 443-only, and `agent/Dockerfile` runs `apt-get` + // over HTTP/80 — pointing the build at it fails every snapshot build. + template.hasResourceProperties('AWS::Lambda::MicrovmImage', { + EgressNetworkConnectors: [ + { 'Fn::GetAtt': [Match.stringLikeRegexp('LambdaMicrovmComputeBuildEgressConnector'), 'Arn'] }, + ], + }); + expect(built.construct.buildEgressConnectorArns) + .not.toEqual(built.construct.egressConnectorArns); + }); + + test('exposes the image ARN as both the identifier and the IAM grant scope', () => { + expect(built.construct.imageIdentifier).toBeDefined(); + expect(built.construct.imageIdentifier).toBe(built.construct.imageArn); + // Version deliberately unpinned so the service resolves the latest ACTIVE + // version after a rebuild without a stack update. + expect(built.construct.imageVersion).toBeUndefined(); + }); + + test('creates TWO egress connectors on the private subnets, both with an operator role', () => { + template.resourceCountIs('AWS::Lambda::NetworkConnector', 2); + + // Runtime connector. + template.hasResourceProperties('AWS::Lambda::NetworkConnector', { + // REQUIRED for VPC_EGRESS despite the generated L1 typing it optional: + // "NetworkConnectorOperatorRole is required for VPC_EGRESS connector type". + OperatorRole: { + 'Fn::GetAtt': [Match.stringLikeRegexp('LambdaMicrovmComputeConnectorOperatorRole'), 'Arn'], + }, + Configuration: { + VpcEgressConfiguration: { + // The only value the service accepts today. + AssociatedComputeResourceTypes: ['MicroVm'], + NetworkProtocol: 'IPv4', + SubnetIds: Match.anyValue(), + SecurityGroupIds: [ + { 'Fn::GetAtt': [Match.stringLikeRegexp('LambdaMicrovmComputeMicrovmSG'), 'GroupId'] }, + ], + }, + }, + }); + + // Build connector — same shape, different security group. + template.hasResourceProperties('AWS::Lambda::NetworkConnector', { + OperatorRole: { + 'Fn::GetAtt': [Match.stringLikeRegexp('LambdaMicrovmComputeConnectorOperatorRole'), 'Arn'], + }, + Configuration: { + VpcEgressConfiguration: { + SecurityGroupIds: [ + { 'Fn::GetAtt': [Match.stringLikeRegexp('LambdaMicrovmComputeMicrovmBuildSG'), 'GroupId'] }, + ], + }, + }, + }); + + // ONE operator role shared by both — a second identical role would double + // the IAM surface with no isolation gain. + const operatorRoles = Object.keys(template.findResources('AWS::IAM::Role')) + .filter(id => id.includes('LambdaMicrovmComputeConnectorOperatorRole')); + expect(operatorRoles).toHaveLength(1); + }); + + test('operator role trusts lambda.amazonaws.com and can manage ENIs', () => { + const [, role] = Object.entries(template.findResources('AWS::IAM::Role')) + .find(([id]) => id.includes('LambdaMicrovmComputeConnectorOperatorRole'))!; + + // Same confused-deputy posture as the build/execution roles. + const statements = role.Properties.AssumeRolePolicyDocument.Statement; + expect(statements).toHaveLength(1); + expect(statements[0].Action).toBe('sts:AssumeRole'); + expect(statements[0].Principal).toEqual({ Service: 'lambda.amazonaws.com' }); + expect(statements[0].Condition).toEqual({ + StringEquals: { 'aws:SourceAccount': '123456789012' }, + }); + + // The AWS-managed policy for exactly this job... + expect(JSON.stringify(role.Properties.ManagedPolicyArns)) + .toContain('service-role/AWSLambdaVPCAccessExecutionRole'); + + // ...plus the ENI/tag/private-IP actions the probe run showed it needs. + const policies = Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('LambdaMicrovmComputeConnectorOperatorRole')); + const actions = policies + .flatMap(([, p]) => p.Properties.PolicyDocument.Statement as Array<{ Action: string | string[] }>) + .flatMap(s => Array.isArray(s.Action) ? s.Action : [s.Action]); + for (const action of [ + 'ec2:CreateNetworkInterface', + 'ec2:DeleteNetworkInterface', + 'ec2:DescribeNetworkInterfaces', + 'ec2:DescribeSubnets', + 'ec2:DescribeVpcs', + 'ec2:DescribeSecurityGroups', + 'ec2:AssignPrivateIpAddresses', + 'ec2:UnassignPrivateIpAddresses', + 'ec2:CreateTags', + ]) { + expect(actions).toContain(action); + } + // It manages ENIs, nothing else — no data-plane reach. + const rendered = JSON.stringify(policies); + expect(rendered).not.toContain('s3:'); + expect(rendered).not.toContain('dynamodb:'); + expect(rendered).not.toContain('lambda:'); + }); + + test('runtime security group allows TCP 443 egress only', () => { + template.hasResourceProperties('AWS::EC2::SecurityGroup', { + GroupDescription: 'Lambda MicroVMs agent sessions - egress TCP 443 only', + SecurityGroupEgress: [ + Match.objectLike({ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, CidrIp: '0.0.0.0/0' }), + ], + }); + }); + + test('BUILD security group additionally allows TCP 80 (apt-get), and only there', () => { + template.hasResourceProperties('AWS::EC2::SecurityGroup', { + GroupDescription: 'Lambda MicroVMs image BUILD - egress TCP 443 + 80 (apt-get)', + SecurityGroupEgress: [ + Match.objectLike({ IpProtocol: 'tcp', FromPort: 443, ToPort: 443, CidrIp: '0.0.0.0/0' }), + Match.objectLike({ IpProtocol: 'tcp', FromPort: 80, ToPort: 80, CidrIp: '0.0.0.0/0' }), + ], + }); + + // The point of two groups: port 80 must NOT leak into the runtime posture. + const [, runtimeSg] = Object.entries(template.findResources('AWS::EC2::SecurityGroup')) + .find(([id]) => id.includes('LambdaMicrovmComputeMicrovmSG'))!; + expect(runtimeSg.Properties.SecurityGroupEgress).toHaveLength(1); + expect(JSON.stringify(runtimeSg.Properties.SecurityGroupEgress)).not.toContain('"FromPort":80'); + }); + + test('exposes the explicit NO_INGRESS connector for RunMicrovm', () => { + // Load-bearing: RunMicrovm attaches a PUBLIC HTTP_INGRESS connector when the + // field is omitted, so "no inbound" must be requested, not assumed. + expect(built.construct.ingressConnectorArns).toHaveLength(1); + expect(built.construct.ingressConnectorArns[0]) + .toMatch(/^arn:.+:lambda:us-east-1:aws:network-connector:aws-network-connector:NO_INGRESS$/); + expect(MICROVM_NO_INGRESS_CONNECTOR_RESOURCE).toBe('aws-network-connector:NO_INGRESS'); + // Service-owned, not account-owned — the account segment is the literal `aws`. + expect(built.construct.ingressConnectorArns[0]).not.toContain('123456789012'); + }); + + test('creates a retention-managed log group under the service log namespace', () => { + template.hasResourceProperties('AWS::Logs::LogGroup', { + LogGroupName: `${MICROVM_LOG_GROUP_PREFIX}/TestStack-abca-agent`, + RetentionInDays: 90, + }); + }); + + test('creates the artifact and payload buckets, both private + TLS-only + encrypted', () => { + // artifact + payload, plus the trace/attachments buckets the test's + // AgentSessionRole fixture needs. + template.resourceCountIs('AWS::S3::Bucket', 4); + const buckets = template.findResources('AWS::S3::Bucket'); + const microvmBuckets = Object.entries(buckets).filter(([id]) => id.includes('LambdaMicrovmCompute')); + expect(microvmBuckets).toHaveLength(2); + for (const [, bucket] of microvmBuckets) { + expect(bucket.Properties.PublicAccessBlockConfiguration).toEqual({ + BlockPublicAcls: true, + BlockPublicPolicy: true, + IgnorePublicAcls: true, + RestrictPublicBuckets: true, + }); + expect(bucket.Properties.BucketEncryption).toEqual({ + ServerSideEncryptionConfiguration: [ + { ServerSideEncryptionByDefault: { SSEAlgorithm: 'AES256' } }, + ], + }); + } + }); + + test('payload bucket expires objects (the ONLY reaper on this backend)', () => { + template.hasResourceProperties('AWS::S3::Bucket', { + LifecycleConfiguration: { + Rules: Match.arrayWith([ + Match.objectLike({ + Id: 'microvm-payload-ttl', + Status: 'Enabled', + ExpirationInDays: MICROVM_PAYLOAD_TTL_DAYS, + AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 }, + }), + ]), + }, + }); + }); + + test('artifact bucket has NO object expiry (the snapshot may be rebuilt from it)', () => { + const buckets = template.findResources('AWS::S3::Bucket'); + const [, artifactBucket] = Object.entries(buckets) + .find(([id]) => id.includes('LambdaMicrovmComputeArtifactBucket'))!; + const rules = artifactBucket.Properties.LifecycleConfiguration.Rules; + expect(rules).toEqual([ + expect.objectContaining({ + Id: 'microvm-artifact-mpu-abort', + AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 }, + }), + ]); + expect(rules[0].ExpirationInDays).toBeUndefined(); + }); + + test('both roles are trusted by lambda.amazonaws.com for AssumeRole AND TagSession, pinned to this account', () => { + const roles = Object.entries(template.findResources('AWS::IAM::Role')) + .filter(([id]) => id.includes('LambdaMicrovmComputeBuildRole') || id.includes('LambdaMicrovmComputeExecutionRole')); + expect(roles).toHaveLength(2); + + for (const [, role] of roles) { + const statements = role.Properties.AssumeRolePolicyDocument.Statement; + expect(statements.map((s: { Action: string }) => s.Action).sort()) + .toEqual(['sts:AssumeRole', 'sts:TagSession']); + for (const statement of statements) { + // `microvms.lambda.amazonaws.com` does not exist — using it is rejected + // with MalformedPolicyDocument. + expect(statement.Principal).toEqual({ Service: 'lambda.amazonaws.com' }); + // Confused-deputy: lambda.amazonaws.com is shared with every other + // Lambda feature, so aws:SourceAccount must be on BOTH actions. + expect(statement.Condition).toEqual({ + StringEquals: { 'aws:SourceAccount': '123456789012' }, + }); + expect(JSON.stringify(statement.Condition)).not.toContain('aws:SourceArn'); + } + } + }); + + test('build role reads exactly the one artifact object and writes MicroVM logs', () => { + const policies = Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('LambdaMicrovmComputeBuildRole')); + expect(policies).toHaveLength(1); + const statements = policies[0]![1].Properties.PolicyDocument.Statement; + const actions = statements.flatMap((s: { Action: string | string[] }) => + Array.isArray(s.Action) ? s.Action : [s.Action]); + expect(actions.sort()).toEqual([ + 'logs:CreateLogGroup', + 'logs:CreateLogStream', + 'logs:PutLogEvents', + 's3:GetObject', + ]); + // Object-scoped, not bucket-scoped. + const s3Statement = statements.find((s: { Action: string }) => s.Action === 's3:GetObject'); + expect(JSON.stringify(s3Statement.Resource)).toContain(MICROVM_ARTIFACT_OBJECT_KEY); + }); + + test('execution role gets READ-ONLY on the payload bucket and no write/delete', () => { + const policies = Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('LambdaMicrovmComputeExecutionRole')); + const statements = policies.flatMap(([, p]) => p.Properties.PolicyDocument.Statement); + const actions: string[] = statements.flatMap((s: { Action: string | string[] }) => + Array.isArray(s.Action) ? s.Action : [s.Action]); + + // CDK's grantRead renders the Get*/List* read set. + expect(actions).toContain('s3:GetObject*'); + // The MicroVM runs untrusted repo code — it must not be able to clobber + // another task's payload, so nothing mutating may appear. + const s3Actions = actions.filter(a => a.startsWith('s3:')); + expect(s3Actions).toEqual(['s3:GetObject*', 's3:GetBucket*', 's3:List*']); + for (const action of s3Actions) { + expect(action).not.toMatch(/Put|Delete|Abort|Write|^s3:\*$/); + } + }); + + test('execution role has NO Bedrock / Secrets Manager / DynamoDB grants (P2 scope)', () => { + const policies = Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('LambdaMicrovmComputeExecutionRole')); + const rendered = JSON.stringify(policies); + expect(rendered).not.toContain('bedrock:'); + expect(rendered).not.toContain('secretsmanager:'); + expect(rendered).not.toContain('dynamodb:'); + expect(rendered).not.toContain('bedrock-agentcore:'); + }); + + test('execution role is admitted to the per-task SessionRole (tenant-data delegation)', () => { + const sessionRoles = Object.entries(template.findResources('AWS::IAM::Role')) + .filter(([id]) => id.includes('AgentSessionRole')); + expect(sessionRoles).toHaveLength(1); + const trust = JSON.stringify(sessionRoles[0]![1].Properties.AssumeRolePolicyDocument); + expect(trust).toContain('sts:TagSession'); + expect(trust).toContain('LambdaMicrovmComputeExecutionRole'); + + // ...and the other half of admitComputeRole: the execution role may assume it. + const execPolicies = JSON.stringify( + Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('LambdaMicrovmComputeExecutionRole')), + ); + expect(execPolicies).toContain('sts:AssumeRole'); + }); + + test('never grants lambda:CreateMicrovmAuthToken to anything (no JWE consumer in P1–P3)', () => { + expect(JSON.stringify(template.toJSON())).not.toContain('CreateMicrovmAuthToken'); + }); + + test('warns that a P1 image has no smoke-parity guarantee (hook phasing)', () => { + // ADR-021 sub-decision 3, as corrected by the live P1 run: P1 declares AND + // serves /ready + /run, so the image is creatable, launchable and + // payload-deliverable — which makes it look even more like a working backend + // than before, while nothing has exercised clone → change → PR on it. + const warnings = built.construct.node.metadata.filter(m => m.type === 'aws:cdk:warning'); + const message = warnings.map(w => String(w.data)).join('\n'); + expect(JSON.stringify(built.construct.node.metadata)) + .toContain('abca:microvm-image-p1-smoke-unverified'); + // The superseded id must be gone, not merely reworded — operators grep for it. + expect(JSON.stringify(built.construct.node.metadata)) + .not.toContain('abca:microvm-image-p1-not-runnable'); + expect(message).toContain('smoke'); + expect(message).toContain('P2'); + // It must state what IS true now, or it reads as the old (wrong) claim. + expect(message).toContain('/run'); + expect(message).toContain('/ready'); + }); + + test('declares no hook the agent does not serve yet', () => { + const images = template.findResources('AWS::Lambda::MicrovmImage'); + const rendered = JSON.stringify(Object.values(images)[0]!.Properties.Hooks); + for (const hook of ['Suspend', 'Resume', 'Terminate', 'Validate']) { + expect(rendered).not.toContain(hook); + } + expect(rendered).toContain('"Run":"/aws/lambda-microvms/runtime/v1/run"'); + expect(rendered).toContain('"Ready":"/aws/lambda-microvms/runtime/v1/ready"'); + }); + + test('tags every MicroVM resource with the backend cost-allocation tag', () => { + const json = template.toJSON(); + const taggedTypes = [ + 'AWS::Lambda::MicrovmImage', + 'AWS::Lambda::NetworkConnector', + 'AWS::S3::Bucket', + 'AWS::IAM::Role', + 'AWS::EC2::SecurityGroup', + ]; + for (const type of taggedTypes) { + const allResources = json.Resources as Record }; + }>; + const resources = Object.entries(allResources) + .filter(([id, r]) => r.Type === type && id.includes('LambdaMicrovmCompute')); + expect(resources.length).toBeGreaterThan(0); + for (const [id, resource] of resources) { + expect(resource.Properties?.Tags).toEqual( + expect.arrayContaining([ + { Key: MICROVM_BACKEND_TAG_KEY, Value: MICROVM_BACKEND_TAG_VALUE }, + ]), + ); + expect(id).toContain('LambdaMicrovmCompute'); + } + } + }); +}); + +describe('LambdaMicrovmCompute — image built out of band', () => { + const EXTERNAL_IMAGE_ARN = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:my-agent'; + + // Two distinct configurations (identifier given as an ARN vs as a bare name), + // so two cached fixtures — synthesized once each, per cdk/AGENTS.md. + let byArn: Built; + let byName: Built; + let byNameVersioned: Built; + + beforeAll(() => { + byArn = build({ + externalImageIdentifier: EXTERNAL_IMAGE_ARN, + externalImageVersion: '7', + }); + byName = build({ externalImageIdentifier: 'my-agent' }); + byNameVersioned = build({ externalImageIdentifier: 'my-agent', externalImageVersion: '3' }); + }); + + test('uses the supplied identifier and synthesizes no image resource', () => { + byArn.template.resourceCountIs('AWS::Lambda::MicrovmImage', 0); + // The buckets/roles/connectors still exist — the substrate is provisioned. + byArn.template.resourceCountIs('AWS::Lambda::NetworkConnector', 2); + expect(byArn.construct.imageIdentifier).toBe(EXTERNAL_IMAGE_ARN); + expect(byArn.construct.imageVersion).toBe('7'); + expect(byArn.construct.imageArn).toBe(EXTERNAL_IMAGE_ARN); + }); + + test('resolves a bare image NAME to its exact ARN and uses THAT as the identifier', () => { + // Load-bearing twice over: + // - IAM: a bare name is not an IAM resource, but the `microvmImage` ARN + // shape is fully derivable from it, so the grant stays pinned to THIS + // image instead of widening to `microvm-image:*`. + // - RunMicrovm: it rejects a bare name outright ("Malformed ARN - doesn't + // start with 'arn:'"), so the ORCHESTRATOR must receive the ARN too. The + // construct therefore publishes one value for both jobs. + // The partition stays the Aws.PARTITION pseudo-parameter (unresolved until + // deploy) so the same code is correct in aws-cn / aws-us-gov — hence the + // suffix assertion rather than a literal `arn:aws:` comparison. + expect(byName.construct.imageArn) + .toMatch(/^arn:.+:lambda:us-east-1:123456789012:microvm-image:my-agent$/); + expect(byName.construct.imageArn).not.toContain('microvm-image:*'); + expect(byName.construct.imageIdentifier).toBe(byName.construct.imageArn); + expect(byName.construct.imageIdentifier).not.toBe('my-agent'); + }); + + test('warns about the missing smoke guarantee, out-of-band path included', () => { + // The warning is keyed on "an image is configured", not on which of the two + // image states produced it — an out-of-band build is just as unverified. + for (const fixture of [byArn, byName]) { + expect(JSON.stringify(fixture.construct.node.metadata)) + .toContain('abca:microvm-image-p1-smoke-unverified'); + } + }); + + test('an image version never changes the IAM scope', () => { + // The SAR `microvmImage` pattern ends at the name; `imageVersion` travels as + // a separate RunMicrovm request field. + expect(byNameVersioned.construct.imageArn).toBe(byName.construct.imageArn); + expect(byNameVersioned.construct.imageVersion).toBe('3'); + }); +}); + +describe('LambdaMicrovmCompute — first deploy, no image configured', () => { + let built: Built; + + beforeAll(() => { + built = build(); + }); + + test('provisions the substrate but no image, and reports no identifier', () => { + built.template.resourceCountIs('AWS::Lambda::MicrovmImage', 0); + built.template.resourceCountIs('AWS::Lambda::NetworkConnector', 2); + expect(built.construct.imageIdentifier).toBeUndefined(); + }); + + test('warns (does not throw) so the artifact bucket can be created before the upload', () => { + const warnings = built.construct.node.metadata.filter(m => m.type === 'aws:cdk:warning'); + expect(warnings).toHaveLength(1); + const message = String(warnings[0]!.data); + // The warning has to be actionable: it names the bootstrap remedy and the + // context flags that move the deployment out of this state. + expect(message).toContain('package-microvm-artifact.sh'); + expect(message).toContain('microvm_base_image_arn'); + expect(message).toContain('microvm_image_identifier'); + expect(JSON.stringify(built.construct.node.metadata)) + .toContain('abca:microvm-image-not-provisioned'); + // ...and NOT the "configured but unverified" warning — there is no image to + // be unverified, and stacking both would blur two different remedies. + expect(JSON.stringify(built.construct.node.metadata)) + .not.toContain('abca:microvm-image-p1-smoke-unverified'); + }); +}); + +describe('LambdaMicrovmCompute — memory sizing', () => { + // TEST-CONVENTION EXEMPTION (cdk/AGENTS.md "synth once in beforeAll"): the + // rejection cases assert the CONSTRUCTOR throws, so there is no template to + // cache; they use `instantiate()`, which never calls Template.fromStack(). + let at512: Built; + + beforeAll(() => { + at512 = build({ withImage: true, minimumMemoryInMiB: 512 }); + }); + + test.each(MICROVM_SUPPORTED_MEMORY_MIB)('accepts the supported size %i MiB', (mib) => { + expect(() => instantiate({ withImage: true, minimumMemoryInMiB: mib })).not.toThrow(); + }); + + test('an explicitly supported size reaches the image resource verbatim', () => { + at512.template.hasResourceProperties('AWS::Lambda::MicrovmImage', { + Resources: [{ MinimumMemoryInMiB: 512 }], + }); + expect(at512.construct.minimumMemoryInMiB).toBe(512); + }); + + test('rejects the old 32768 default at SYNTH, naming the accepted baselines', () => { + // The service rejects it minutes into a build; failing at synth is the whole + // point of validating here. + let error: Error | undefined; + try { + instantiate({ withImage: true, minimumMemoryInMiB: 32768 }); + } catch (err) { + error = err as Error; + } + expect(error).toBeDefined(); + expect(error!.message).toContain('32768'); + expect(error!.message).toContain('512, 1024, 2048, 4096, 8192'); + // The message must say BASELINE, or an operator reads the rejection as "this + // backend caps at 8 GiB" and moves a repo to ECS it did not need to. + expect(error!.message).toContain('BASELINE'); + expect(error!.message).toContain('32 GiB'); + // ...and points at the backend that DOES have the SUSTAINED capacity. + expect(error!.message).toContain('compute_type=ecs'); + }); + + test.each([0, 256, 6144, 16384, 8193])('rejects the unsupported baseline %i MiB', (mib) => { + expect(() => instantiate({ withImage: true, minimumMemoryInMiB: mib })) + .toThrow(/is not a BASELINE memory size AWS Lambda MicroVMs accepts/); + }); +}); + +describe('microvmNoIngressConnectorArn — explicit no-inbound control', () => { + test('builds the service-owned NO_INGRESS ARN for the stack Region', () => { + // Shape taken verbatim from the HTTP_INGRESS ARN the service attached on a + // launch that passed NO ingress connectors, with the name swapped. The + // partition stays the Aws.PARTITION pseudo-parameter so the same code is + // correct in aws-cn / aws-us-gov. + const stack = new Stack(new App(), 'S', { env: { account: '123456789012', region: 'eu-west-1' } }); + expect(microvmNoIngressConnectorArn(stack)) + .toMatch(/^arn:.+:lambda:eu-west-1:aws:network-connector:aws-network-connector:NO_INGRESS$/); + }); + + test('never names the deployment account (these connectors are AWS-owned)', () => { + const stack = new Stack(new App(), 'S', { env: { account: '999999999999', region: 'us-west-2' } }); + expect(microvmNoIngressConnectorArn(stack)).not.toContain('999999999999'); + expect(microvmNoIngressConnectorArn(stack)).toContain(':aws:network-connector:'); + }); +}); + +describe('assertLambdaMicrovmRegionSupported — Region gate', () => { + // TEST-CONVENTION EXEMPTION (cdk/AGENTS.md "synth once in beforeAll"): + // these cases assert that the construct's CONSTRUCTOR throws, so there is no + // template to cache — a fixture built in `beforeAll` would fail the hook + // rather than the test. They stay per-case but use `instantiate()`, which + // skips `Template.fromStack()` entirely: the gate runs as the construct's + // first statement, so no synth is needed to observe it. The one case that + // DOES need a template (the escape hatch succeeding) is cached below. + let overridden: Built; + + beforeAll(() => { + overridden = build({ + region: 'eu-central-1', + withImage: true, + context: { [MICROVM_REGION_OVERRIDE_CONTEXT]: true }, + }); + }); + + test.each(LAMBDA_MICROVM_SUPPORTED_REGIONS)('accepts %s', (region) => { + expect(() => instantiate({ region, withImage: true })).not.toThrow(); + }); + + test('fails synth in an unsupported Region, naming the list and the escape hatch', () => { + let error: Error | undefined; + try { + instantiate({ region: 'eu-central-1', withImage: true }); + } catch (err) { + error = err as Error; + } + + expect(error).toBeDefined(); + // One instantiation, three assertions on its message — the remedy has to + // name the Region, the supported list, and the escape hatch, or an operator + // in a just-launched Region is stuck. + expect(error!.message).toMatch(/AWS Lambda MicroVMs are not available in eu-central-1/); + expect(error!.message).toContain(MICROVM_REGION_OVERRIDE_CONTEXT); + expect(error!.message).toContain('us-east-1, us-east-2, us-west-2, eu-west-1, ap-northeast-1'); + }); + + test('the context escape hatch unblocks an unsupported Region', () => { + overridden.template.resourceCountIs('AWS::Lambda::MicrovmImage', 1); + // Still surfaced as a warning — the operator is ahead of the constant. + expect(JSON.stringify(overridden.construct.node.metadata)) + .toContain('abca:microvm-region-override'); + }); + + test('skips the check for a region-agnostic (token) Region rather than rejecting it', () => { + // `cdk synth` with no CDK_DEFAULT_REGION resolves Stack.region to the + // AWS::Region pseudo-parameter; comparing that to a Region list would fail + // every region-agnostic synth. The live probes cover this case instead. + expect(() => instantiate({ regionAgnostic: true, withImage: true })).not.toThrow(); + }); + + test('is callable standalone on any construct scope', () => { + const supported = new Stack(new App(), 'S', { env: { account: '1', region: 'us-west-2' } }); + expect(() => assertLambdaMicrovmRegionSupported(supported)).not.toThrow(); + + const unsupported = new Stack(new App(), 'S', { env: { account: '1', region: 'sa-east-1' } }); + expect(() => assertLambdaMicrovmRegionSupported(unsupported)).toThrow(/not available in sa-east-1/); + }); +}); + +describe('isLambdaMicrovmImageConfigured — the shared three-state predicate', () => { + // Public because `stacks/agent.ts` must answer "will an image exist?" BEFORE + // TaskApi is constructed (the cancel grant's Lazy ARN depends on it), while the + // construct answers the same question later. One predicate, so they cannot + // drift — a drift means either a missing cancel grant or a grant scoped to an + // image that was never created. + test('true for a complete managed-base-image build', () => { + expect(isLambdaMicrovmImageConfigured({ + baseImageArn: BASE_IMAGE_ARN, + baseImageVersion: '1', + })).toBe(true); + }); + + test('true for an out-of-band image, by ARN or by bare name', () => { + expect(isLambdaMicrovmImageConfigured({ externalImageIdentifier: 'abca-agent' })).toBe(true); + expect(isLambdaMicrovmImageConfigured({ + externalImageIdentifier: 'arn:aws:lambda:us-east-1:123456789012:microvm-image:abca-agent', + })).toBe(true); + }); + + test('false for a PARTIAL base-image config (CFN requires both fields)', () => { + // The construct falls through to its no-image branch here, so the predicate + // must agree — otherwise the stack would wire a cancel grant whose Lazy ARN + // has nothing to resolve to. + expect(isLambdaMicrovmImageConfigured({ baseImageArn: BASE_IMAGE_ARN })).toBe(false); + expect(isLambdaMicrovmImageConfigured({ baseImageVersion: '1' })).toBe(false); + }); + + test('false when nothing is configured, and a version alone never counts', () => { + expect(isLambdaMicrovmImageConfigured({})).toBe(false); + expect(isLambdaMicrovmImageConfigured({ externalImageVersion: '7' })).toBe(false); + }); +}); diff --git a/cdk/test/constructs/task-api.test.ts b/cdk/test/constructs/task-api.test.ts index c0a748e6a..12c742fe7 100644 --- a/cdk/test/constructs/task-api.test.ts +++ b/cdk/test/constructs/task-api.test.ts @@ -448,6 +448,60 @@ describe('TaskApi construct', () => { expect(vars).not.toHaveProperty('ECS_CLUSTER_ARN'); } }); + + describe('Lambda MicroVMs cancel wiring (ADR-021)', () => { + const MICROVM_IMAGE_ARN = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:abca-agent'; + + // One synth per configuration, cached (cdk/AGENTS.md): the backend enabled, + // and — via the outer describe's `baseTemplate` — the backend disabled. + let microvmTemplate: Template; + + beforeAll(() => { + microvmTemplate = createStack({ lambdaMicrovmImageArn: MICROVM_IMAGE_ARN }).template; + }); + + /** Every MicroVM-related IAM action granted anywhere in the template. */ + function microvmActions(template: Template): string[] { + return Object.values(template.findResources('AWS::IAM::Policy')) + .flatMap((p) => (p as any).Properties.PolicyDocument.Statement as Array<{ Action: string | string[] }>) + .flatMap((stmt) => (Array.isArray(stmt.Action) ? stmt.Action : [stmt.Action])) + .filter((action) => action.includes('Microvm')); + } + + /** MICROVM_* env keys across every Lambda in the template. */ + function microvmEnvKeys(template: Template): string[] { + return Object.values(template.findResources('AWS::Lambda::Function')) + .flatMap((fn) => Object.keys(((fn as any).Properties?.Environment?.Variables ?? {}) as Record)) + .filter((key) => key.startsWith('MICROVM_')); + } + + test('grants ONLY lambda:TerminateMicrovm when the backend is enabled', () => { + // cancel-task.ts sends TerminateMicrovmCommand and nothing else — it never + // reads MicroVM state — so lambda:GetMicrovm would be an unused grant. + expect(microvmActions(microvmTemplate)).toEqual(['lambda:TerminateMicrovm']); + }); + + test('scopes the grant to the one platform image, never an account wildcard', () => { + const statement = Object.values(microvmTemplate.findResources('AWS::IAM::Policy')) + .flatMap((p) => (p as any).Properties.PolicyDocument.Statement as Array<{ Action: string | string[]; Resource: unknown }>) + .find((stmt) => stmt.Action === 'lambda:TerminateMicrovm')!; + + // Every MicroVM lifecycle action authorizes against the image resource, so + // the per-session microvmId never appears in IAM and this can be exact. + expect(statement.Resource).toEqual([MICROVM_IMAGE_ARN, `${MICROVM_IMAGE_ARN}:*`]); + expect(statement.Resource).not.toBe('*'); + expect(JSON.stringify(statement.Resource)).not.toContain('microvm-image:*'); + }); + + test('adds no MicroVM grant when no image is configured', () => { + expect(microvmActions(baseTemplate)).toEqual([]); + expect(microvmEnvKeys(baseTemplate)).toEqual([]); + }); + + test('needs no env var — the handler reads microvmId from the task row', () => { + expect(microvmEnvKeys(microvmTemplate)).toEqual([]); + }); + }); }); describe('TaskApi construct with webhooks', () => { diff --git a/cdk/test/constructs/task-orchestrator.test.ts b/cdk/test/constructs/task-orchestrator.test.ts index 7027a135f..418617928 100644 --- a/cdk/test/constructs/task-orchestrator.test.ts +++ b/cdk/test/constructs/task-orchestrator.test.ts @@ -20,6 +20,7 @@ import { App, Stack } from 'aws-cdk-lib'; import { Template, Match } from 'aws-cdk-lib/assertions'; import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +import * as s3 from 'aws-cdk-lib/aws-s3'; import { TaskOrchestrator } from '../../src/constructs/task-orchestrator'; interface StackOverrides { @@ -537,3 +538,227 @@ describe('TaskOrchestrator construct', () => { }); }); }); + +describe('TaskOrchestrator with the Lambda MicroVMs backend (ADR-021)', () => { + const IMAGE_ARN = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:abca-agent'; + /** What the construct derives from the bare image name `abca-agent`. */ + const NAME_DERIVED_IMAGE_ARN = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:abca-agent'; + const EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/MicrovmExecutionRole'; + const CONNECTOR_ARN = 'arn:aws:lambda:us-east-1:123456789012:network-connector:nc-123'; + // What LambdaMicrovmCompute always supplies: the Lambda-managed NO_INGRESS + // connector. Service-owned, hence the literal `aws` account segment. + const NO_INGRESS_ARN = 'arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:NO_INGRESS'; + + /** + * Build a stack with `microvmConfig` wired. Separate from `createStack` above + * because the payload bucket has to be a real construct (the prop takes an + * `s3.IBucket` so the grant can be rendered), which the shared helper's + * override shape does not model. + */ + function createMicrovmStack(config?: { + imageIdentifier?: string; + imageArn?: string; + imageVersion?: string; + ingressConnectorArns?: string[]; + }): { template: Template } { + const app = new App(); + const stack = new Stack(app, 'TestStack', { + env: { account: '123456789012', region: 'us-east-1' }, + }); + const mkTable = (id: string, sortKey?: string) => new dynamodb.Table(stack, id, { + partitionKey: { name: 'task_id', type: dynamodb.AttributeType.STRING }, + ...(sortKey ? { sortKey: { name: sortKey, type: dynamodb.AttributeType.STRING } } : {}), + }); + + new TaskOrchestrator(stack, 'TaskOrchestrator', { + taskTable: mkTable('TaskTable'), + taskEventsTable: mkTable('TaskEventsTable', 'event_id'), + userConcurrencyTable: new dynamodb.Table(stack, 'UserConcurrencyTable', { + partitionKey: { name: 'user_id', type: dynamodb.AttributeType.STRING }, + }), + runtimeArn: 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test-runtime', + microvmConfig: { + imageIdentifier: config?.imageIdentifier ?? IMAGE_ARN, + imageArn: config?.imageArn ?? IMAGE_ARN, + imageVersion: config?.imageVersion, + executionRoleArn: EXECUTION_ROLE_ARN, + egressConnectorArns: [CONNECTOR_ARN], + // REQUIRED prop — mirrors what LambdaMicrovmCompute passes on every + // deploy. There is no "unset" case to fixture, by design. + ingressConnectorArns: config?.ingressConnectorArns ?? [NO_INGRESS_ARN], + payloadBucket: new s3.Bucket(stack, 'MicrovmPayloadBucket'), + }, + }); + + return { template: Template.fromStack(stack) }; + } + + function orchestratorEnv(template: Template): Record { + const [, fn] = Object.entries(template.findResources('AWS::Lambda::Function')) + .find(([id]) => id.includes('OrchestratorFn'))!; + return fn.Properties.Environment.Variables as Record; + } + + function microvmStatements(template: Template): Array<{ + Sid?: string; + Action: string | string[]; + Resource: unknown; + Condition?: unknown; + }> { + return Object.values(template.findResources('AWS::IAM::Policy')) + .flatMap(p => p.Properties.PolicyDocument.Statement) + .filter((s: { Sid?: string }) => (s.Sid ?? '').startsWith('Microvm')); + } + + // Three distinct configurations, synthesized once each in beforeAll per + // cdk/AGENTS.md: the default wiring, the same wiring with an image version + + // ingress connectors pinned, and a name-derived (not operator-supplied) image + // ARN. `noMicrovmTemplate` is the negative control. + let template: Template; + let pinnedTemplate: Template; + let nameDerivedTemplate: Template; + let noMicrovmTemplate: Template; + + beforeAll(() => { + template = createMicrovmStack().template; + pinnedTemplate = createMicrovmStack({ + imageVersion: '4', + ingressConnectorArns: ['arn:aws:lambda:us-east-1:aws:network-connector:x', 'arn:y'], + }).template; + // What LambdaMicrovmCompute passes when the operator gave a bare image NAME: + // the exact ARN it derived, never a wildcard. + nameDerivedTemplate = createMicrovmStack({ + imageIdentifier: 'abca-agent', + imageArn: NAME_DERIVED_IMAGE_ARN, + }).template; + noMicrovmTemplate = createStack().template; + }); + + test('injects the required MICROVM_* env vars verbatim (the strategy contract)', () => { + const env = orchestratorEnv(template); + expect(env.MICROVM_IMAGE_IDENTIFIER).toBe(IMAGE_ARN); + expect(env.MICROVM_EXECUTION_ROLE_ARN).toBe(EXECUTION_ROLE_ARN); + expect(env.MICROVM_EGRESS_CONNECTOR_ARNS).toBe(CONNECTOR_ARN); + expect(env.MICROVM_INGRESS_CONNECTOR_ARNS).toBe(NO_INGRESS_ARN); + expect(env.MICROVM_PAYLOAD_BUCKET).toBeDefined(); + }); + + test('ALWAYS injects the ingress var, carrying the NO_INGRESS control', () => { + // OUTCOME assertion, not an omission assertion. This var used to be injected + // only when non-empty, and the test asserted it was `undefined` — which is + // precisely the shape of test that passes while the service silently picks + // something wider: `RunMicrovm` attaches a PUBLIC HTTP_INGRESS connector (and + // a public *.lambda-microvm..on.aws endpoint) when the request omits + // the field. So "no inbound" has to be a value we can SEE in the template. + const env = orchestratorEnv(template); + expect(Object.keys(env)).toContain('MICROVM_INGRESS_CONNECTOR_ARNS'); + expect(env.MICROVM_INGRESS_CONNECTOR_ARNS).toBe(NO_INGRESS_ARN); + expect(env.MICROVM_INGRESS_CONNECTOR_ARNS).toContain('NO_INGRESS'); + expect(env.MICROVM_INGRESS_CONNECTOR_ARNS).not.toContain('HTTP_INGRESS'); + // Not an empty string either — the strategy would then fall back rather than + // pass what the deployment configured. + expect(env.MICROVM_INGRESS_CONNECTOR_ARNS).not.toBe(''); + }); + + test('the ingress var belongs to the all-or-nothing block, not the optional one', () => { + // Exactly one MICROVM_* var is genuinely optional (`imageVersion`, whose + // absent state means "resolve the latest ACTIVE version"). Everything else, + // ingress included, is present whenever microvmConfig is. + const env = orchestratorEnv(template); + expect(Object.keys(env).filter((k) => k.startsWith('MICROVM_')).sort()).toEqual([ + 'MICROVM_EGRESS_CONNECTOR_ARNS', + 'MICROVM_EXECUTION_ROLE_ARN', + 'MICROVM_IMAGE_IDENTIFIER', + 'MICROVM_INGRESS_CONNECTOR_ARNS', + 'MICROVM_PAYLOAD_BUCKET', + ]); + expect(env.MICROVM_IMAGE_VERSION).toBeUndefined(); + }); + + test('pins the image version, and real ingress connectors WIN over NO_INGRESS', () => { + // How #391 (operator shell access) lands without a strategy change: the + // construct supplies different connectors and they are joined verbatim. + const env = orchestratorEnv(pinnedTemplate); + expect(env.MICROVM_IMAGE_VERSION).toBe('4'); + expect(env.MICROVM_INGRESS_CONNECTOR_ARNS).toBe('arn:aws:lambda:us-east-1:aws:network-connector:x,arn:y'); + expect(env.MICROVM_INGRESS_CONNECTOR_ARNS).not.toContain('NO_INGRESS'); + }); + + test('grants exactly the four P1 lifecycle actions and nothing more', () => { + const actions = microvmStatements(template) + .flatMap(s => Array.isArray(s.Action) ? s.Action : [s.Action]) + .filter(a => a.startsWith('lambda:')); + expect(actions.sort()).toEqual([ + 'lambda:GetMicrovm', + 'lambda:PassNetworkConnector', + 'lambda:RunMicrovm', + 'lambda:TerminateMicrovm', + ]); + }); + + test('scopes the lifecycle statement to the one platform image ARN', () => { + const lifecycle = microvmStatements(template).find(s => s.Sid === 'MicrovmLifecycle')!; + // Exact ARN plus the `:*` version-suffix hedge — both pinned to this + // image's name, so neither can match another image. + expect(lifecycle.Resource).toEqual([IMAGE_ARN, `${IMAGE_ARN}:*`]); + }); + + test('a name-derived image ARN is scoped to that exact name, never a wildcard', () => { + // An out-of-band image referenced by bare NAME is a valid RunMicrovm + // identifier but not an IAM resource. LambdaMicrovmCompute resolves it to the + // exact `microvmImage` ARN, so ADR-021's "scoped to platform-created images" + // holds here too — the account/Region-wide `microvm-image:*` widening this + // test previously accepted is a compliance violation, not a fallback. + const lifecycle = microvmStatements(nameDerivedTemplate).find(s => s.Sid === 'MicrovmLifecycle')!; + expect(lifecycle.Resource).toEqual([ + NAME_DERIVED_IMAGE_ARN, + `${NAME_DERIVED_IMAGE_ARN}:*`, + ]); + expect(JSON.stringify(lifecycle.Resource)).not.toContain('microvm-image:*'); + expect(JSON.stringify(lifecycle.Resource)).toContain('microvm-image:abca-agent'); + }); + + test('PassNetworkConnector must be Resource:* (the action has no resource type)', () => { + const pass = microvmStatements(template).find(s => s.Sid === 'MicrovmPassNetworkConnector')!; + expect(pass.Resource).toBe('*'); + }); + + test('passes the execution role to lambda.amazonaws.com only', () => { + const passRole = microvmStatements(template).find(s => s.Sid === 'MicrovmPassExecutionRole')!; + expect(passRole.Action).toBe('iam:PassRole'); + expect(passRole.Resource).toBe(EXECUTION_ROLE_ARN); + expect(passRole.Condition).toEqual({ + StringEquals: { 'iam:PassedToService': 'lambda.amazonaws.com' }, + }); + }); + + test('grants NO suspend/resume (P3) and NO auth-token minting (never)', () => { + const actions = new Set( + Object.values(template.findResources('AWS::IAM::Policy')) + .flatMap(p => p.Properties.PolicyDocument.Statement as Array<{ Action: string | string[] }>) + .flatMap(s => Array.isArray(s.Action) ? s.Action : [s.Action]), + ); + expect(actions.has('lambda:SuspendMicrovm')).toBe(false); + expect(actions.has('lambda:ResumeMicrovm')).toBe(false); + expect(actions.has('lambda:CreateMicrovmAuthToken')).toBe(false); + expect(actions.has('lambda:CreateMicrovmShellAuthToken')).toBe(false); + }); + + test('gets write on the payload bucket but NOT delete (lifecycle rule is the reaper)', () => { + const payloadStatements = Object.values(template.findResources('AWS::IAM::Policy')) + .flatMap(p => p.Properties.PolicyDocument.Statement as Array<{ + Action: string | string[]; + Resource: unknown; + }>) + .filter(s => JSON.stringify(s.Resource).includes('MicrovmPayloadBucket')); + + const actions = payloadStatements.flatMap(s => Array.isArray(s.Action) ? s.Action : [s.Action]); + expect(actions).toContain('s3:PutObject'); + expect(actions).not.toContain('s3:DeleteObject'); + }); + + test('adds no MicroVM statements when microvmConfig is omitted', () => { + expect(microvmStatements(noMicrovmTemplate)).toEqual([]); + expect(orchestratorEnv(noMicrovmTemplate).MICROVM_IMAGE_IDENTIFIER).toBeUndefined(); + }); +}); diff --git a/cdk/test/handlers/cancel-task.test.ts b/cdk/test/handlers/cancel-task.test.ts index 68e7b1de0..b64465766 100644 --- a/cdk/test/handlers/cancel-task.test.ts +++ b/cdk/test/handlers/cancel-task.test.ts @@ -31,6 +31,11 @@ jest.mock('@aws-sdk/client-ecs', () => ({ ECSClient: jest.fn(() => ({ send: mockEcsSend })), StopTaskCommand: jest.fn((input: unknown) => ({ _type: 'StopTask', input })), })); +const mockMicrovmSend = jest.fn(); +jest.mock('@aws-sdk/client-lambda-microvms', () => ({ + LambdaMicrovmsClient: jest.fn(() => ({ send: mockMicrovmSend })), + TerminateMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'TerminateMicrovm', input })), +})); jest.mock('@aws-sdk/client-dynamodb', () => ({ DynamoDBClient: jest.fn(() => ({})) })); jest.mock('@aws-sdk/lib-dynamodb', () => ({ DynamoDBDocumentClient: { from: jest.fn(() => ({ send: mockSend })) }, @@ -327,6 +332,70 @@ describe('cancel-task handler', () => { expect(mockAgentCoreSend).not.toHaveBeenCalled(); }); + test('cancels a lambda-microvm task via TerminateMicrovm, not AgentCore or ECS', async () => { + mockSend.mockReset(); + // NOTE: RUNNING_TASK carries `agent_runtime_arn`, and RUNTIME_ARN is also set + // in this suite's env — exactly the mixed-deployment shape that would send a + // MicroVM task down the AgentCore branch if the microvm check did not come + // first. This test is the regression guard for that branch ordering. + const microvmTask = { + ...RUNNING_TASK, + compute_type: 'lambda-microvm', + session_id: 'mvm-0123456789abcdef', + compute_metadata: { + microvmId: 'mvm-0123456789abcdef', + endpoint: 'https://mvm-0123456789abcdef.microvm.lambda.us-east-1.amazonaws.com', + }, + }; + mockSend + .mockResolvedValueOnce({ Item: microvmTask }) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({}); + + const result = await handler(makeEvent()); + expect(result.statusCode).toBe(200); + expect(mockMicrovmSend).toHaveBeenCalledTimes(1); + const cmd = mockMicrovmSend.mock.calls[0][0]; + expect(cmd._type).toBe('TerminateMicrovm'); + // The SDK request key is `microvmIdentifier`, not `microvmId`. + expect(cmd.input).toEqual({ microvmIdentifier: 'mvm-0123456789abcdef' }); + expect(mockAgentCoreSend).not.toHaveBeenCalled(); + expect(mockEcsSend).not.toHaveBeenCalled(); + }); + + test('skips TerminateMicrovm when compute_metadata.microvmId is missing', async () => { + mockSend.mockReset(); + const microvmTask = { ...RUNNING_TASK, compute_type: 'lambda-microvm', compute_metadata: {} }; + mockSend + .mockResolvedValueOnce({ Item: microvmTask }) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({}); + + const result = await handler(makeEvent()); + expect(result.statusCode).toBe(200); + expect(mockMicrovmSend).not.toHaveBeenCalled(); + // Must NOT silently fall through to stopping an unrelated AgentCore runtime. + expect(mockAgentCoreSend).not.toHaveBeenCalled(); + }); + + test('a TerminateMicrovm failure still returns 200 (the CANCELLED write stands)', async () => { + mockSend.mockReset(); + const microvmTask = { + ...RUNNING_TASK, + compute_type: 'lambda-microvm', + compute_metadata: { microvmId: 'mvm-0123456789abcdef', endpoint: 'https://x' }, + }; + mockSend + .mockResolvedValueOnce({ Item: microvmTask }) + .mockResolvedValueOnce({}) + .mockResolvedValueOnce({}); + mockMicrovmSend.mockRejectedValueOnce(new Error('ResourceNotFoundException')); + + const result = await handler(makeEvent()); + expect(result.statusCode).toBe(200); + expect(mockMicrovmSend).toHaveBeenCalledTimes(1); + }); + test('falls back to AgentCore stop when compute_type is unrecognized but RUNTIME_ARN is available', async () => { mockSend.mockReset(); const unknownTask = { diff --git a/cdk/test/handlers/orchestrate-task-microvm.test.ts b/cdk/test/handlers/orchestrate-task-microvm.test.ts new file mode 100644 index 000000000..75c5fc881 --- /dev/null +++ b/cdk/test/handlers/orchestrate-task-microvm.test.ts @@ -0,0 +1,420 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/** + * Drives the FULL orchestrate-task durable handler for a `lambda-microvm` task + * with a fake durable-execution context, so the wiring the unit tests can't see + * is covered end to end: strategy resolution → RunMicrovm → compute_metadata + * persistence → substrate poll cross-check → **TerminateMicrovm on finalize**. + * + * The finalize terminate is an ADR-021 normative requirement ("When the + * orchestrator finalizes a `lambda-microvm` task, the orchestrator shall call + * terminate-microvm") and is invisible to strategy-level unit tests, which is + * exactly how it was missed the first time. + */ + +// `withDurableExecution` wraps the handler at import time; unwrap it so the raw +// (event, context) function is testable (same trick as orchestrate-task-feedback). +jest.mock('@aws/durable-execution-sdk-js', () => ({ + withDurableExecution: (fn: unknown) => fn, +})); + +const MICROVM_ID = 'mvm-0123456789abcdef'; +const ENDPOINT = 'https://mvm-0123456789abcdef.microvm.lambda.us-east-1.amazonaws.com'; + +const mockMicrovmSend = jest.fn(); +jest.mock('@aws-sdk/client-lambda-microvms', () => ({ + LambdaMicrovmsClient: jest.fn(() => ({ send: mockMicrovmSend })), + RunMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'RunMicrovm', input })), + GetMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'GetMicrovm', input })), + TerminateMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'TerminateMicrovm', input })), + MicrovmState: { + PENDING: 'PENDING', + RUNNING: 'RUNNING', + SUSPENDED: 'SUSPENDED', + SUSPENDING: 'SUSPENDING', + TERMINATED: 'TERMINATED', + TERMINATING: 'TERMINATING', + }, +})); + +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(() => ({ send: jest.fn().mockResolvedValue({}) })), + PutObjectCommand: jest.fn((input: unknown) => ({ _type: 'PutObject', input })), + DeleteObjectCommand: jest.fn((input: unknown) => ({ _type: 'DeleteObject', input })), +})); + +// Real orchestrator helpers would talk to DynamoDB; stub them and assert on the +// calls. `buildComputeMetadata` is kept REAL (re-exported from the actual module) +// so the persisted metadata shape is genuinely exercised, not mirrored. +const realOrchestrator = jest.requireActual('../../src/handlers/shared/orchestrator'); +const mockTransitionTask = jest.fn(); +const mockEmitTaskEvent = jest.fn(); +const mockFinalizeTask = jest.fn(); +const mockPollTaskStatus = jest.fn(); +const mockReconcile = jest.fn(); +const mockFailTask = jest.fn(); +jest.mock('../../src/handlers/shared/orchestrator', () => ({ + admissionControl: jest.fn().mockResolvedValue(true), + emitTaskEvent: (...a: unknown[]) => mockEmitTaskEvent(...a), + envelopeFor: () => ({ + log: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, + correlation: { user_id: 'user-1', repo: 'org/repo' }, + }), + failTask: (...a: unknown[]) => mockFailTask(...a), + finalizeTask: (...a: unknown[]) => mockFinalizeTask(...a), + hydrateAndTransition: jest.fn().mockResolvedValue({ repo_url: 'org/repo', task_id: 'TASK001' }), + loadBlueprintConfig: jest.fn().mockResolvedValue({ compute_type: 'lambda-microvm', runtime_arn: '' }), + loadTask: jest.fn().mockResolvedValue({ + task_id: 'TASK001', user_id: 'user-1', status: 'SUBMITTED', repo: 'org/repo', + }), + pollTaskStatus: (...a: unknown[]) => mockPollTaskStatus(...a), + reconcileMicrovmSubstrateState: (...a: unknown[]) => mockReconcile(...a), + transitionTask: (...a: unknown[]) => mockTransitionTask(...a), + buildComputeMetadata: realOrchestrator.buildComputeMetadata, +})); + +jest.mock('../../src/handlers/shared/preflight', () => ({ + runPreflightChecks: jest.fn().mockResolvedValue({ passed: true, checks: {} }), +})); + +const mockDeleteEcsPayload = jest.fn(); +jest.mock('../../src/handlers/shared/strategies/ecs-strategy', () => ({ + deleteEcsPayload: (...a: unknown[]) => mockDeleteEcsPayload(...a), + EcsComputeStrategy: jest.fn(), +})); + +// MICROVM_* env must be set before the strategy module is imported. +process.env.MICROVM_IMAGE_IDENTIFIER = 'arn:aws:lambda:us-east-1:123456789012:microvm-image/abca-agent'; +process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/AbcaMicrovmExecution'; +process.env.MICROVM_EGRESS_CONNECTOR_ARNS = 'arn:aws:lambda:us-east-1:123456789012:network-connector/egress-1'; +process.env.MICROVM_PAYLOAD_BUCKET = 'test-microvm-payload-bucket'; +process.env.TASK_TABLE_NAME = 'Tasks'; +process.env.TASK_EVENTS_TABLE_NAME = 'TaskEvents'; +process.env.USER_CONCURRENCY_TABLE_NAME = 'UserConcurrency'; +process.env.TASK_RETENTION_DAYS = '90'; + +import { TaskStatus } from '../../src/constructs/task-status'; +import { handler } from '../../src/handlers/orchestrate-task'; +import { LambdaMicrovmComputeStrategy } from '../../src/handlers/shared/strategies/lambda-microvm-strategy'; + +/** + * Minimal stand-in for the durable-execution context: `step` runs its body + * inline, `waitForCondition` runs the poll body ONCE and returns the resulting + * state (enough to exercise the substrate cross-check without looping). + */ +function fakeContext(opts: { pollOnce?: boolean } = {}) { + const steps: string[] = []; + return { + steps, + ctx: { + step: async (name: string, fn: () => Promise) => { + steps.push(name); + return fn(); + }, + waitForCondition: async ( + name: string, + fn: (state: { attempts: number }) => Promise, + cfg: { initialState: { attempts: number } }, + ) => { + steps.push(name); + if (opts.pollOnce === false) return cfg.initialState; + return fn(cfg.initialState); + }, + }, + }; +} + +function runMicrovmOk() { + mockMicrovmSend.mockResolvedValueOnce({ + microvmId: MICROVM_ID, + endpoint: ENDPOINT, + state: 'RUNNING', + imageArn: 'arn:image', + imageVersion: '7', + }); +} + +function commandsOfType(type: string) { + return mockMicrovmSend.mock.calls.map(c => c[0]).filter(c => c._type === type); +} + +beforeEach(() => { + jest.clearAllMocks(); + mockMicrovmSend.mockReset(); + mockPollTaskStatus.mockResolvedValue({ attempts: 1, lastStatus: TaskStatus.COMPLETED }); + mockReconcile.mockResolvedValue({ taskFailed: false }); +}); + +describe('orchestrate-task for a lambda-microvm task', () => { + test('persists microvmId and endpoint in compute_metadata on the RUNNING transition', async () => { + runMicrovmOk(); + const { ctx } = fakeContext(); + + await handler({ task_id: 'TASK001' }, ctx as never); + + // RunMicrovm actually went out. + expect(commandsOfType('RunMicrovm')).toHaveLength(1); + + expect(mockTransitionTask).toHaveBeenCalledTimes(1); + const [, from, to, attrs] = mockTransitionTask.mock.calls[0]; + expect(from).toBe(TaskStatus.HYDRATING); + expect(to).toBe(TaskStatus.RUNNING); + expect(attrs.compute_type).toBe('lambda-microvm'); + // ADR-021: the P3 approve/deny Lambdas resume from exactly these two keys. + expect(attrs.compute_metadata).toEqual({ microvmId: MICROVM_ID, endpoint: ENDPOINT }); + // sessionId is the microvmId (substrate identifier, mirroring ECS). + expect(attrs.session_id).toBe(MICROVM_ID); + // agent_runtime_arn is an AgentCore-only attribute and must not appear. + expect(attrs.agent_runtime_arn).toBeUndefined(); + }); + + test('sends TerminateMicrovm on finalize (ADR-021 active-cleanup requirement)', async () => { + runMicrovmOk(); + const { ctx, steps } = fakeContext(); + + await handler({ task_id: 'TASK001' }, ctx as never); + + expect(steps).toContain('finalize'); + expect(mockFinalizeTask).toHaveBeenCalledTimes(1); + + const terminates = commandsOfType('TerminateMicrovm'); + expect(terminates).toHaveLength(1); + expect(terminates[0].input).toEqual({ microvmIdentifier: MICROVM_ID }); + }); + + test('terminates AFTER finalizeTask, so the task row is already terminal', async () => { + const order: string[] = []; + mockFinalizeTask.mockImplementation(async () => { order.push('finalizeTask'); }); + mockMicrovmSend.mockImplementation(async (cmd: { _type: string }) => { + if (cmd._type === 'RunMicrovm') { + return { microvmId: MICROVM_ID, endpoint: ENDPOINT, state: 'RUNNING' }; + } + if (cmd._type === 'TerminateMicrovm') order.push('terminate'); + return {}; + }); + + await handler({ task_id: 'TASK001' }, fakeContext().ctx as never); + + expect(order).toEqual(['finalizeTask', 'terminate']); + }); + + test('a TerminateMicrovm failure does not fail the finalize step (best-effort)', async () => { + runMicrovmOk(); + const err = new Error('boom'); + err.name = 'InternalServerException'; + mockMicrovmSend.mockRejectedValueOnce(err); + + // stopSession swallows every failure internally, so the handler resolves. + await expect(handler({ task_id: 'TASK001' }, fakeContext().ctx as never)).resolves.toBeUndefined(); + expect(mockFinalizeTask).toHaveBeenCalledTimes(1); + }); + + test('does not call deleteEcsPayload for a microvm task', async () => { + runMicrovmOk(); + + await handler({ task_id: 'TASK001' }, fakeContext().ctx as never); + + expect(mockDeleteEcsPayload).not.toHaveBeenCalled(); + }); + + test('cross-checks the substrate through reconcileMicrovmSubstrateState while non-terminal', async () => { + runMicrovmOk(); + mockPollTaskStatus.mockResolvedValue({ attempts: 1, lastStatus: TaskStatus.RUNNING }); + // GetMicrovm during the poll, then TerminateMicrovm on finalize. + mockMicrovmSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'SUSPENDED' }); + + await handler({ task_id: 'TASK001' }, fakeContext().ctx as never); + + expect(commandsOfType('GetMicrovm')).toHaveLength(1); + expect(mockReconcile).toHaveBeenCalledTimes(1); + const args = mockReconcile.mock.calls[0][0]; + expect(args.microvmId).toBe(MICROVM_ID); + expect(args.ddbStatus).toBe(TaskStatus.RUNNING); + // The strategy's mechanical mapping is what the orchestrator interprets. + expect(args.substrate).toEqual({ status: 'suspended' }); + }); + + test('returns a failed poll state when reconciliation fails the task', async () => { + runMicrovmOk(); + mockPollTaskStatus.mockResolvedValue({ attempts: 3, lastStatus: TaskStatus.RUNNING }); + mockMicrovmSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'TERMINATED' }); + mockReconcile.mockResolvedValue({ taskFailed: true }); + + await handler({ task_id: 'TASK001' }, fakeContext().ctx as never); + + expect(mockFinalizeTask).toHaveBeenCalledWith( + 'TASK001', + { attempts: 3, lastStatus: TaskStatus.FAILED }, + 'user-1', + ); + }); + + test('skips the substrate cross-check once the DDB status is terminal', async () => { + runMicrovmOk(); + mockPollTaskStatus.mockResolvedValue({ attempts: 1, lastStatus: TaskStatus.COMPLETED }); + + await handler({ task_id: 'TASK001' }, fakeContext().ctx as never); + + expect(commandsOfType('GetMicrovm')).toHaveLength(0); + expect(mockReconcile).not.toHaveBeenCalled(); + // Finalize still terminates. + expect(commandsOfType('TerminateMicrovm')).toHaveLength(1); + }); + + test('a GetMicrovm poll failure is non-fatal and finalize still terminates', async () => { + runMicrovmOk(); + mockPollTaskStatus.mockResolvedValue({ attempts: 1, lastStatus: TaskStatus.RUNNING }); + mockMicrovmSend.mockRejectedValueOnce(new Error('transient')); + + await expect(handler({ task_id: 'TASK001' }, fakeContext().ctx as never)).resolves.toBeUndefined(); + + expect(mockReconcile).not.toHaveBeenCalled(); + expect(commandsOfType('TerminateMicrovm')).toHaveLength(1); + }); + + test('threads suspendAnomalyReported into the reconcile call so the event fires once', async () => { + runMicrovmOk(); + mockPollTaskStatus.mockResolvedValue({ attempts: 1, lastStatus: TaskStatus.RUNNING }); + mockMicrovmSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'SUSPENDED' }); + mockReconcile.mockResolvedValue({ taskFailed: false, suspendAnomalyReported: true }); + + await handler({ task_id: 'TASK001' }, fakeContext().ctx as never); + + // First poll of the task: nothing reported yet. + expect(mockReconcile.mock.calls[0][0].suspendAnomalyReported).toBe(false); + // ...and the reconciler's answer is carried into the state the next poll reads. + expect(mockFinalizeTask).toHaveBeenCalledWith( + 'TASK001', + expect.objectContaining({ microvmSuspendAnomalyReported: true }), + 'user-1', + ); + }); + + test('a MicroVM poll failure carries the anomaly flag forward rather than re-arming it', async () => { + // A GetMicrovm hiccup is not evidence that the anomaly ended, so it must not + // silently re-arm the event and produce a duplicate on the next cycle. + runMicrovmOk(); + mockPollTaskStatus.mockResolvedValue({ attempts: 1, lastStatus: TaskStatus.RUNNING }); + mockMicrovmSend.mockRejectedValueOnce(new Error('transient')); + + const { ctx } = fakeContext(); + // Seed the poll state as if a previous cycle had already reported. + const seededCtx = { + ...ctx, + waitForCondition: async ( + _name: string, + fn: (state: Record) => Promise, + ) => fn({ attempts: 1, microvmSuspendAnomalyReported: true }), + }; + + await handler({ task_id: 'TASK001' }, seededCtx as never); + + expect(mockFinalizeTask).toHaveBeenCalledWith( + 'TASK001', + expect.objectContaining({ microvmSuspendAnomalyReported: true }), + 'user-1', + ); + }); + + describe('orphan reap when session start fails AFTER RunMicrovm succeeded', () => { + test('terminates the MicroVM from the in-memory handle when the persist write fails', async () => { + // The MicroVM is already RUNNING and billing, and its id exists ONLY in this + // Lambda's memory — no poll or finalize step will ever see it. Nothing + // self-terminates on this substrate, so without the reap it bills for the + // full 8 h cap while holding admission-gating memory quota. + runMicrovmOk(); + mockTransitionTask.mockRejectedValueOnce(new Error('ConditionalCheckFailedException')); + + await expect(handler({ task_id: 'TASK001' }, fakeContext().ctx as never)) + .rejects.toThrow('ConditionalCheckFailedException'); + + const terminates = commandsOfType('TerminateMicrovm'); + expect(terminates).toHaveLength(1); + expect(terminates[0].input).toEqual({ microvmIdentifier: MICROVM_ID }); + }); + + test('still fails the task with the ORIGINAL error — the reap never masks it', async () => { + runMicrovmOk(); + mockTransitionTask.mockRejectedValueOnce(new Error('persist exploded')); + + await expect(handler({ task_id: 'TASK001' }, fakeContext().ctx as never)) + .rejects.toThrow('persist exploded'); + + expect(mockFailTask).toHaveBeenCalledTimes(1); + const [, fromStatus, reason] = mockFailTask.mock.calls[0]; + expect(fromStatus).toBe(TaskStatus.HYDRATING); + expect(reason).toContain('Session start failed'); + expect(reason).toContain('persist exploded'); + }); + + test('a failing TerminateMicrovm does not replace the original error', async () => { + runMicrovmOk(); + mockTransitionTask.mockRejectedValueOnce(new Error('persist exploded')); + const reapErr = new Error('terminate denied'); + reapErr.name = 'AccessDeniedException'; + mockMicrovmSend.mockRejectedValueOnce(reapErr); + + // stopSession is internally best-effort (it logs AccessDenied at error level + // and returns), so the reap is a no-op here — the user must still see why + // session start actually failed. + await expect(handler({ task_id: 'TASK001' }, fakeContext().ctx as never)) + .rejects.toThrow('persist exploded'); + }); + + test('even a stopSession that BREAKS its no-throw contract cannot mask the original error', async () => { + // Defense in depth for the handler's inner try/catch: `stopSession` is + // contractually non-throwing for this backend, but "contractually" is not + // "structurally". If it ever regresses, the reap must still not become the + // error the user is shown — the session-start failure is the actionable one. + runMicrovmOk(); + mockTransitionTask.mockRejectedValueOnce(new Error('persist exploded')); + const stopSpy = jest.spyOn(LambdaMicrovmComputeStrategy.prototype, 'stopSession') + .mockRejectedValueOnce(new Error('stopSession itself threw')); + + try { + await expect(handler({ task_id: 'TASK001' }, fakeContext().ctx as never)) + .rejects.toThrow('persist exploded'); + } finally { + stopSpy.mockRestore(); + } + }); + + test('does NOT terminate when RunMicrovm itself failed (there is nothing to reap)', async () => { + const err = new Error('Rate exceeded'); + err.name = 'ThrottlingException'; + mockMicrovmSend.mockRejectedValueOnce(err); + + await expect(handler({ task_id: 'TASK001' }, fakeContext().ctx as never)).rejects.toThrow(); + + expect(commandsOfType('TerminateMicrovm')).toHaveLength(0); + }); + + test('does not reap on a healthy start (no spurious terminate before the task runs)', async () => { + runMicrovmOk(); + mockPollTaskStatus.mockResolvedValue({ attempts: 1, lastStatus: TaskStatus.RUNNING }); + mockMicrovmSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'RUNNING' }); + + await handler({ task_id: 'TASK001' }, fakeContext().ctx as never); + + // Exactly ONE terminate — the finalize one, not an extra reap. + expect(commandsOfType('TerminateMicrovm')).toHaveLength(1); + }); + }); +}); diff --git a/cdk/test/handlers/shared/compute-strategy.test.ts b/cdk/test/handlers/shared/compute-strategy.test.ts index 1fac73d7b..d00dea2ea 100644 --- a/cdk/test/handlers/shared/compute-strategy.test.ts +++ b/cdk/test/handlers/shared/compute-strategy.test.ts @@ -30,9 +30,26 @@ jest.mock('@aws-sdk/client-ecs', () => ({ StopTaskCommand: jest.fn(), })); +jest.mock('@aws-sdk/client-lambda-microvms', () => ({ + LambdaMicrovmsClient: jest.fn(() => ({ send: jest.fn() })), + RunMicrovmCommand: jest.fn(), + GetMicrovmCommand: jest.fn(), + TerminateMicrovmCommand: jest.fn(), + MicrovmState: { + PENDING: 'PENDING', + RUNNING: 'RUNNING', + SUSPENDED: 'SUSPENDED', + SUSPENDING: 'SUSPENDING', + TERMINATED: 'TERMINATED', + TERMINATING: 'TERMINATING', + }, +})); + import { resolveComputeStrategy } from '../../../src/handlers/shared/compute-strategy'; +import type { ComputeType } from '../../../src/handlers/shared/repo-config'; import { AgentCoreComputeStrategy } from '../../../src/handlers/shared/strategies/agentcore-strategy'; import { EcsComputeStrategy } from '../../../src/handlers/shared/strategies/ecs-strategy'; +import { LambdaMicrovmComputeStrategy } from '../../../src/handlers/shared/strategies/lambda-microvm-strategy'; describe('resolveComputeStrategy', () => { test('returns AgentCoreComputeStrategy for compute_type agentcore', () => { @@ -52,4 +69,34 @@ describe('resolveComputeStrategy', () => { expect(strategy).toBeInstanceOf(EcsComputeStrategy); expect(strategy.type).toBe('ecs'); }); + + test('returns LambdaMicrovmComputeStrategy for compute_type lambda-microvm', () => { + const strategy = resolveComputeStrategy({ + compute_type: 'lambda-microvm', + runtime_arn: 'arn:test', + }); + expect(strategy).toBeInstanceOf(LambdaMicrovmComputeStrategy); + expect(strategy.type).toBe('lambda-microvm'); + }); + + test('throws a named error for an unknown compute_type (exhaustive-never default)', () => { + expect(() => + resolveComputeStrategy({ + // Cast past the exhaustive union to exercise the runtime default arm — + // a hand-edited RepoTable row can carry a value the type system forbids. + compute_type: 'quantum-annealer' as ComputeType, + runtime_arn: 'arn:test', + }), + ).toThrow("Unknown compute_type: 'quantum-annealer'"); + }); + + test('every ComputeType member resolves to a strategy whose type field round-trips', () => { + // Guards the pairing itself: a new backend added to ComputeType but wired to + // the wrong class would still be instanceof-correct in the tests above. + const all: ComputeType[] = ['agentcore', 'ecs', 'lambda-microvm']; + for (const computeType of all) { + const strategy = resolveComputeStrategy({ compute_type: computeType, runtime_arn: 'arn:test' }); + expect(strategy.type).toBe(computeType); + } + }); }); diff --git a/cdk/test/handlers/shared/error-classifier.test.ts b/cdk/test/handlers/shared/error-classifier.test.ts index 9649b3502..9a2e4ae40 100644 --- a/cdk/test/handlers/shared/error-classifier.test.ts +++ b/cdk/test/handlers/shared/error-classifier.test.ts @@ -18,6 +18,7 @@ */ import { classifyError, ErrorCategory, ErrorClass, isTransientError, retryGuidance, type ErrorClassification } from '../../../src/handlers/shared/error-classifier'; +import { LAMBDA_MICROVM_SUPPORTED_REGIONS } from '../../../src/handlers/shared/microvm-regions'; import { toTaskDetail, type TaskRecord } from '../../../src/handlers/shared/types'; describe('classifyError', () => { @@ -476,6 +477,212 @@ describe('classifyError', () => { }); }); + // --- Lambda MicroVMs (ADR-021) --- + + describe('Lambda MicroVMs errors', () => { + test('classifies regional unavailability as a non-retryable CONFIG fault with the supported-Region list', () => { + // ADR-021: "If startSession fails because the MicroVM service is unavailable + // in the stack region, then the orchestrator shall classify the failure with + // a configuration remedy and shall not retry." + const result = classifyError( + 'Session start failed: UnknownEndpoint: Inaccessible host: `lambda.eu-central-1.amazonaws.com\'. ' + + 'This service may not be available in the `eu-central-1\' region.', + )!; + expect(result.category).toBe(ErrorCategory.CONFIG); + expect(result.title).toBe('Lambda MicroVMs is not available in this Region'); + expect(result.retryable).toBe(false); + expect(result.errorClass).toBe(ErrorClass.SERVICE); + expect(isTransientError(result)).toBe(false); + // The remedy must name the supported Regions AND the alternative backends. + for (const region of LAMBDA_MICROVM_SUPPORTED_REGIONS) { + expect(result.remedy).toContain(region); + } + expect(result.remedy).toContain('--compute-type agentcore'); + expect(result.remedy).toContain('microvm-regions.ts'); + }); + + test('classifies regional unavailability from the raw (unwrapped) SDK error too', () => { + // startSessionWithRetry classifies String(err) — no "Session start failed" + // wrapper — so the no-retry verdict must hold on the raw string as well. + const result = classifyError('Could not resolve endpoint for lambda in region ap-south-1')!; + expect(result.title).toBe('Lambda MicroVMs is not available in this Region'); + expect(isTransientError(result)).toBe(false); + }); + + test('regional unavailability wins over the generic "Session start failed" copy (ordering)', () => { + // The generic compute entry would tell the user "reply to retry", which + // loops forever against a Region that will never have the service. + const generic = classifyError('Session start failed: boom')!; + expect(generic.title).toBe('Agent session failed to start'); + const regional = classifyError('Session start failed: MicroVMs not available in this region')!; + expect(regional.title).toBe('Lambda MicroVMs is not available in this Region'); + }); + + test('does NOT hijack an AgentCore or ECS endpoint failure (marker-anchored pattern)', () => { + // Their endpoint hosts are bedrock-agentcore.* / ecs.*, so the MicroVM + // entry must not match — otherwise a transient AgentCore blip would be + // reported as a permanent regional misconfiguration. + const agentcore = classifyError( + 'Session start failed: UnknownEndpoint: Inaccessible host: `bedrock-agentcore.us-east-1.amazonaws.com\'.', + )!; + expect(agentcore.title).toBe('Agent session failed to start'); + const ecs = classifyError( + 'Session start failed: UnknownEndpoint: Inaccessible host: `ecs.us-east-1.amazonaws.com\'.', + )!; + expect(ecs.title).toBe('Agent session failed to start'); + }); + + test('classifies ServiceQuotaExceededException as a retryable capacity fault', () => { + const result = classifyError( + 'Session start failed: Error: MicroVM RunMicrovm failed: ServiceQuotaExceededException: MicroVM memory quota exceeded', + )!; + expect(result.category).toBe(ErrorCategory.COMPUTE); + expect(result.retryable).toBe(true); + // Capacity-shaped, not configuration-shaped: it frees as MicroVMs + // terminate, so the session-start auto-retry is allowed to try once. + expect(result.errorClass).toBe(ErrorClass.TRANSIENT); + expect(isTransientError(result)).toBe(true); + expect(result.remedy).toMatch(/quota increase/i); + }); + + test('classifies ThrottlingException as retryable/transient', () => { + const result = classifyError('MicroVM RunMicrovm failed: ThrottlingException: Rate exceeded')!; + expect(result.category).toBe(ErrorCategory.COMPUTE); + expect(result.retryable).toBe(true); + expect(result.errorClass).toBe(ErrorClass.TRANSIENT); + expect(isTransientError(result)).toBe(true); + }); + + test('classifies TooManyRequestsException alongside ThrottlingException', () => { + const result = classifyError('MicroVM GetMicrovm failed: TooManyRequestsException: slow down')!; + expect(result.errorClass).toBe(ErrorClass.TRANSIENT); + }); + + test('classifies a marked ResourceNotFoundException as a non-retryable deploy fault', () => { + const result = classifyError('MicroVM RunMicrovm failed: ResourceNotFoundException: MicroVM image not found')!; + expect(result.category).toBe(ErrorCategory.CONFIG); + expect(result.retryable).toBe(false); + expect(result.errorClass).toBe(ErrorClass.SERVICE); + // A retry with the same ARNs cannot succeed — no auto-retry. + expect(isTransientError(result)).toBe(false); + expect(result.remedy).toContain('MICROVM_IMAGE_IDENTIFIER'); + }); + + test('classifies a marked payload-upload failure', () => { + const result = classifyError('MicroVM payload upload failed: ThrottlingException: SlowDown')!; + // The marker admits multi-word operation names ("payload upload"). + expect(result.errorClass).toBe(ErrorClass.TRANSIENT); + }); + + test('classifies a MicroVM substrate-failure reason written by the orchestrator', () => { + // Must stay in lockstep with the reason string + // `reconcileMicrovmSubstrateState` persists. + const result = classifyError( + 'MicroVM substrate terminated before the agent wrote a terminal status: substrate state completed', + )!; + expect(result.category).toBe(ErrorCategory.COMPUTE); + expect(result.retryable).toBe(true); + expect(result.errorClass).toBe(ErrorClass.TRANSIENT); + expect(result.title).not.toMatch(/Unexpected error/i); + }); + + test('every new MicroVM classification carries a full, non-empty guidance shape', () => { + const messages = [ + 'Session start failed: UnknownEndpoint: Inaccessible host: `lambda.eu-central-1.amazonaws.com\'', + 'MicroVM RunMicrovm failed: ServiceQuotaExceededException: quota exceeded', + 'MicroVM RunMicrovm failed: ThrottlingException: Rate exceeded', + 'MicroVM RunMicrovm failed: ResourceNotFoundException: image not found', + 'MicroVM substrate terminated before the agent wrote a terminal status: substrate state completed', + ]; + for (const msg of messages) { + const result = classifyError(msg) as ErrorClassification; + expect(result.title.length).toBeGreaterThan(0); + expect(result.description.length).toBeGreaterThan(0); + expect(result.remedy.length).toBeGreaterThan(0); + expect(typeof result.retryable).toBe('boolean'); + expect([ErrorClass.TRANSIENT, ErrorClass.SERVICE, ErrorClass.USER]).toContain(result.errorClass); + expect(result.category).not.toBe(ErrorCategory.UNKNOWN); + } + }); + }); + + // --- Cross-backend scoping regression (ADR-021) --- + + describe('MicroVM exception patterns must NOT change other backends', () => { + /** + * `ThrottlingException`, `ServiceQuotaExceededException` and + * `ResourceNotFoundException` are generic AWS exception names thrown by + * AgentCore, ECS, DynamoDB and Secrets Manager too. The MicroVM entries are + * therefore anchored on the `MicroVM failed` marker that + * `lambda-microvm-strategy.wrapMicrovmError` adds. + * + * These cases pin the PRE-ADR-021 behaviour for UNMARKED occurrences: on + * `main` the classifier contained no pattern for any of these names, so a + * bare occurrence fell through to UNKNOWN — category `unknown`, title + * "Unexpected error", `retryable: false`, `errorClass: USER`, and therefore + * NOT auto-retried by `startSessionWithRetry`. If a future edit drops the + * marker anchor, these fail instead of silently flipping every other + * backend's retry semantics. + */ + const PRE_CHANGE_UNKNOWN = { + category: ErrorCategory.UNKNOWN, + title: 'Unexpected error', + retryable: false, + errorClass: ErrorClass.USER, + }; + + test.each([ + ['ThrottlingException: Rate exceeded'], + ['ServiceQuotaExceededException: quota exceeded'], + ['ResourceNotFoundException: Requested resource not found'], + ['TooManyRequestsException: slow down'], + ])('an unmarked "%s" still classifies as UNKNOWN, exactly as before', (message) => { + const result = classifyError(message)!; + expect(result.category).toBe(PRE_CHANGE_UNKNOWN.category); + expect(result.title).toBe(PRE_CHANGE_UNKNOWN.title); + expect(result.retryable).toBe(PRE_CHANGE_UNKNOWN.retryable); + expect(result.errorClass).toBe(PRE_CHANGE_UNKNOWN.errorClass); + // The retry gate must be unchanged for other backends. + expect(isTransientError(result)).toBe(false); + }); + + test('an AgentCore StopRuntimeSession throttle is not re-classified as a MicroVM fault', () => { + const result = classifyError( + 'ThrottlingException: Too many requests for agentRuntimeArn arn:aws:bedrock-agentcore:us-east-1:1:runtime/r', + )!; + expect(result.title).toBe('Unexpected error'); + expect(result.title).not.toMatch(/MicroVM/i); + }); + + test('an ECS RunTask quota error is not re-classified as a MicroVM fault', () => { + const result = classifyError( + 'ECS RunTask returned no task: arn:test: ServiceQuotaExceededException', + )!; + expect(result.title).not.toMatch(/MicroVM/i); + expect(result.category).toBe(ErrorCategory.UNKNOWN); + }); + + test('the generic "Session start failed" wrapper still wins for agentcore/ECS quota + throttle', () => { + // Pinned by the pre-existing compute-errors test too; restated here so the + // scoping intent is explicit at the regression site. + expect(classifyError('Session start failed: ServiceQuotaExceededException')!.title) + .toBe('Agent session failed to start'); + expect(classifyError('Session start failed: ThrottlingException: Rate exceeded')!.title) + .toBe('Agent session failed to start'); + expect(classifyError('Session start failed: ResourceNotFoundException')!.title) + .toBe('Agent session failed to start'); + }); + + test('Blueprint / hydration ResourceNotFoundException keep their precise CONFIG copy', () => { + expect( + classifyError('Blueprint config load failed: ResourceNotFoundException: Requested resource not found')!.title, + ).toBe('Blueprint configuration error'); + expect( + classifyError('Hydration failed: ResourceNotFoundException: secret missing')!.title, + ).toBe('Context hydration failed'); + }); + }); + // --- Environmental blockers --- describe('blocker errors (canonical BLOCKED[] prefix)', () => { diff --git a/cdk/test/handlers/shared/microvm-regions.test.ts b/cdk/test/handlers/shared/microvm-regions.test.ts new file mode 100644 index 000000000..020858cba --- /dev/null +++ b/cdk/test/handlers/shared/microvm-regions.test.ts @@ -0,0 +1,63 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { + isLambdaMicrovmRegionSupported, + LAMBDA_MICROVM_SUPPORTED_REGIONS, +} from '../../../src/handlers/shared/microvm-regions'; + +describe('LAMBDA_MICROVM_SUPPORTED_REGIONS', () => { + test('lists exactly the 5 launch Regions from ADR-021', () => { + expect([...LAMBDA_MICROVM_SUPPORTED_REGIONS]).toEqual([ + 'us-east-1', + 'us-east-2', + 'us-west-2', + 'eu-west-1', + 'ap-northeast-1', + ]); + }); + + test('has no duplicate entries', () => { + expect(new Set(LAMBDA_MICROVM_SUPPORTED_REGIONS).size).toBe(LAMBDA_MICROVM_SUPPORTED_REGIONS.length); + }); +}); + +describe('isLambdaMicrovmRegionSupported', () => { + test.each([...LAMBDA_MICROVM_SUPPORTED_REGIONS])('accepts the supported Region %s', (region) => { + expect(isLambdaMicrovmRegionSupported(region)).toBe(true); + }); + + test('rejects a Region that has not launched the backend', () => { + expect(isLambdaMicrovmRegionSupported('eu-central-1')).toBe(false); + expect(isLambdaMicrovmRegionSupported('ap-southeast-2')).toBe(false); + }); + + test('rejects undefined and empty input rather than defaulting to supported', () => { + // A missing AWS_REGION must never read as "supported" — the static gate is a + // deny-by-default check, and the live probes are what self-heal false denials. + expect(isLambdaMicrovmRegionSupported(undefined)).toBe(false); + expect(isLambdaMicrovmRegionSupported('')).toBe(false); + }); + + test('compares Region ids verbatim (no case folding, no prefix matching)', () => { + expect(isLambdaMicrovmRegionSupported('US-EAST-1')).toBe(false); + expect(isLambdaMicrovmRegionSupported('us-east-1a')).toBe(false); + expect(isLambdaMicrovmRegionSupported('us-east')).toBe(false); + }); +}); diff --git a/cdk/test/handlers/shared/orchestrator.test.ts b/cdk/test/handlers/shared/orchestrator.test.ts new file mode 100644 index 000000000..615adda22 --- /dev/null +++ b/cdk/test/handlers/shared/orchestrator.test.ts @@ -0,0 +1,403 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// --- Mocks --- +const mockDdbSend = jest.fn(); +jest.mock('@aws-sdk/client-dynamodb', () => ({ + DynamoDBClient: jest.fn(() => ({})), +})); +jest.mock('@aws-sdk/lib-dynamodb', () => ({ + DynamoDBDocumentClient: { from: jest.fn(() => ({ send: mockDdbSend })) }, + GetCommand: jest.fn((input: unknown) => ({ _type: 'Get', input })), + PutCommand: jest.fn((input: unknown) => ({ _type: 'Put', input })), + UpdateCommand: jest.fn((input: unknown) => ({ _type: 'Update', input })), +})); +jest.mock('@aws-sdk/client-s3', () => ({ S3Client: jest.fn(() => ({ send: jest.fn() })) })); + +const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn(), child: jest.fn() }; +jest.mock('../../../src/handlers/shared/logger', () => ({ logger: mockLogger })); + +process.env.TASK_TABLE_NAME = 'Tasks'; +process.env.TASK_EVENTS_TABLE_NAME = 'TaskEvents'; +process.env.USER_CONCURRENCY_TABLE_NAME = 'Concurrency'; +process.env.TASK_RETENTION_DAYS = '90'; + +import { TaskStatus } from '../../../src/constructs/task-status'; +import type { SessionHandle, SessionStatus } from '../../../src/handlers/shared/compute-strategy'; +import { buildComputeMetadata, reconcileMicrovmSubstrateState } from '../../../src/handlers/shared/orchestrator'; + +const MICROVM_ID = 'mvm-0123456789abcdef'; +const ENDPOINT = 'https://mvm-0123456789abcdef.microvm.lambda.us-east-1.amazonaws.com'; + +/** Commands the mocked document client received, in order. */ +function sentCommands(): Array<{ _type: string; input: Record }> { + return mockDdbSend.mock.calls.map(c => c[0]); +} + +function commandsOfType(type: string): Array<{ _type: string; input: Record }> { + return sentCommands().filter(c => c._type === type); +} + +/** + * Prime the mocked doc client: the FIRST Get returns a task row with + * ``rereadStatus``; every Put/Update resolves empty. Mirrors the single re-read + * `reconcileMicrovmSubstrateState` performs before failing a task. + */ +function primeReread(rereadStatus: string): void { + mockDdbSend.mockImplementation((cmd: { _type: string }) => { + if (cmd._type === 'Get') { + return Promise.resolve({ + Item: { task_id: 'TASK001', user_id: 'user-1', repo: 'org/repo', status: rereadStatus }, + }); + } + return Promise.resolve({}); + }); +} + +const CORRELATION = { user_id: 'user-1', repo: 'org/repo' }; + +function reconcile(substrate: SessionStatus, ddbStatus: string, suspendAnomalyReported?: boolean) { + return reconcileMicrovmSubstrateState({ + taskId: 'TASK001', + ddbStatus: ddbStatus as never, + substrate, + microvmId: MICROVM_ID, + userId: 'user-1', + correlation: CORRELATION, + log: mockLogger, + repo: 'org/repo', + ...(suspendAnomalyReported !== undefined && { suspendAnomalyReported }), + }); +} + +beforeEach(() => { + jest.clearAllMocks(); + mockDdbSend.mockReset(); + mockDdbSend.mockResolvedValue({}); +}); + +describe('buildComputeMetadata', () => { + test('persists clusterArn and taskArn for an ECS handle (unchanged behaviour)', () => { + const handle: SessionHandle = { + sessionId: 'arn:aws:ecs:us-east-1:123456789012:task/c/abc', + strategyType: 'ecs', + clusterArn: 'arn:aws:ecs:us-east-1:123456789012:cluster/c', + taskArn: 'arn:aws:ecs:us-east-1:123456789012:task/c/abc', + }; + // cancel-task.ts reads exactly these two keys — do not rename them. + expect(buildComputeMetadata(handle)).toEqual({ + clusterArn: 'arn:aws:ecs:us-east-1:123456789012:cluster/c', + taskArn: 'arn:aws:ecs:us-east-1:123456789012:task/c/abc', + }); + }); + + test('persists runtimeArn for an AgentCore handle (unchanged behaviour)', () => { + const handle: SessionHandle = { + sessionId: 'a-uuid', + strategyType: 'agentcore', + runtimeArn: 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/r', + }; + expect(buildComputeMetadata(handle)).toEqual({ + runtimeArn: 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/r', + }); + }); + + test('persists microvmId and endpoint for a lambda-microvm handle', () => { + const handle: SessionHandle = { + sessionId: MICROVM_ID, + strategyType: 'lambda-microvm', + microvmId: MICROVM_ID, + endpoint: ENDPOINT, + }; + // ADR-021: the P3 approve/deny Lambdas load the resume handle from these keys. + expect(buildComputeMetadata(handle)).toEqual({ microvmId: MICROVM_ID, endpoint: ENDPOINT }); + }); + + test('never carries the MicroVM image ARN (deployment config, not session state)', () => { + const metadata = buildComputeMetadata({ + sessionId: MICROVM_ID, + strategyType: 'lambda-microvm', + microvmId: MICROVM_ID, + endpoint: ENDPOINT, + }); + expect(Object.keys(metadata).sort()).toEqual(['endpoint', 'microvmId']); + }); + + test('produces only string values (compute_metadata is Record in DDB)', () => { + for (const handle of [ + { sessionId: 's', strategyType: 'agentcore', runtimeArn: 'a' }, + { sessionId: 's', strategyType: 'ecs', clusterArn: 'c', taskArn: 't' }, + { sessionId: 's', strategyType: 'lambda-microvm', microvmId: 'm', endpoint: 'e' }, + ] as SessionHandle[]) { + for (const value of Object.values(buildComputeMetadata(handle))) { + expect(typeof value).toBe('string'); + } + } + }); + + test('throws for an unrecognized strategyType (exhaustive-never guard)', () => { + expect(() => + buildComputeMetadata({ sessionId: 's', strategyType: 'firecracker-v2' } as unknown as SessionHandle), + ).toThrow(/Unknown strategyType on session handle/); + }); +}); + +describe('reconcileMicrovmSubstrateState', () => { + describe('running substrate', () => { + test('is a no-op: no DDB reads, no events, task not failed', async () => { + const result = await reconcile({ status: 'running' }, TaskStatus.RUNNING); + + // `suspendAnomalyReported: false` RE-ARMS the once-per-episode event: a VM + // that resumed and is later suspended again earns a fresh anomaly event. + expect(result).toEqual({ taskFailed: false, suspendAnomalyReported: false }); + expect(mockDdbSend).not.toHaveBeenCalled(); + }); + }); + + describe('suspended substrate', () => { + test('is healthy while the task is AWAITING_APPROVAL — no event, no failure', async () => { + const result = await reconcile({ status: 'suspended' }, TaskStatus.AWAITING_APPROVAL); + + // The orchestrator-intended suspend during an approval wait is the whole + // economic point of the backend: it must be silent — and it re-arms the + // anomaly event, because leaving AWAITING_APPROVAL while still suspended + // would be a new, genuinely reportable episode. + expect(result).toEqual({ taskFailed: false, suspendAnomalyReported: false }); + expect(mockDdbSend).not.toHaveBeenCalled(); + expect(mockLogger.warn).not.toHaveBeenCalled(); + }); + + test('writes an anomaly event and does NOT fail the task when the status is RUNNING', async () => { + const result = await reconcile({ status: 'suspended' }, TaskStatus.RUNNING); + + expect(result).toEqual({ taskFailed: false, suspendAnomalyReported: true }); + + const puts = commandsOfType('Put'); + expect(puts).toHaveLength(1); + expect(puts[0].input.TableName).toBe('TaskEvents'); + const item = puts[0].input.Item as Record; + expect(item.event_type).toBe('microvm_suspend_anomaly'); + expect(item.task_id).toBe('TASK001'); + // Correlation envelope (#245) stamped as top-level fields. + expect(item.user_id).toBe('user-1'); + expect(item.repo).toBe('org/repo'); + expect(item.metadata).toEqual({ + microvm_id: MICROVM_ID, + task_status: TaskStatus.RUNNING, + reason: 'suspended_outside_approval_wait', + }); + + // Crucially: no status transition — a suspended VM is resumable, so + // failing the task would destroy recoverable work. + expect(commandsOfType('Update')).toHaveLength(0); + expect(mockLogger.warn).toHaveBeenCalled(); + }); + + test.each([ + TaskStatus.HYDRATING, + TaskStatus.RUNNING, + TaskStatus.FINALIZING, + ])('treats suspended + %s as an anomaly rather than a failure', async (status) => { + const result = await reconcile({ status: 'suspended' }, status); + + expect(result).toEqual({ taskFailed: false, suspendAnomalyReported: true }); + expect(commandsOfType('Put')[0].input.Item).toMatchObject({ + event_type: 'microvm_suspend_anomaly', + metadata: { task_status: status }, + }); + }); + + test('emits the anomaly event ONCE across repeated polls of the same episode', async () => { + // The poll runs every ~30 s for up to 8.5 h; without the flag an + // out-of-band suspend would write ~960 identical TaskEvents, burying the + // first informative one. The caller threads the returned flag back in. + let reported: boolean | undefined; + for (let poll = 0; poll < 5; poll += 1) { + const result = await reconcile({ status: 'suspended' }, TaskStatus.RUNNING, reported); + reported = result.suspendAnomalyReported; + // The no-fail-fast behaviour is unchanged on EVERY poll — that is the + // property the suppression must not break. + expect(result.taskFailed).toBe(false); + expect(result.suspendAnomalyReported).toBe(true); + } + + expect(commandsOfType('Put')).toHaveLength(1); + expect((commandsOfType('Put')[0].input.Item as Record).event_type) + .toBe('microvm_suspend_anomaly'); + // The WARN log is deliberately NOT suppressed: per-poll evidence is what a + // timeline investigation needs, and CloudWatch is not a user-facing surface. + expect(mockLogger.warn).toHaveBeenCalledTimes(5); + }); + + test('the repeat-suppressed polls record that the event was already reported', async () => { + await reconcile({ status: 'suspended' }, TaskStatus.RUNNING, true); + + expect(commandsOfType('Put')).toHaveLength(0); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('suspended while the task is not awaiting approval'), + expect.objectContaining({ anomaly_already_reported: true }), + ); + }); + + test('RE-ARMS after the VM resumes, so a second episode emits again', async () => { + // Recovery genuinely re-arms (documented decision): a flapping suspend loop + // is the pathology an operator most needs to see, and latching forever + // would hide it after the first occurrence. + const first = await reconcile({ status: 'suspended' }, TaskStatus.RUNNING, false); + expect(first.suspendAnomalyReported).toBe(true); + + const recovered = await reconcile({ status: 'running' }, TaskStatus.RUNNING, first.suspendAnomalyReported); + expect(recovered.suspendAnomalyReported).toBe(false); + + const second = await reconcile({ status: 'suspended' }, TaskStatus.RUNNING, recovered.suspendAnomalyReported); + expect(second.suspendAnomalyReported).toBe(true); + + // Two episodes → two events. + expect(commandsOfType('Put')).toHaveLength(2); + }); + + test('RE-ARMS when the task enters AWAITING_APPROVAL, so a later out-of-band suspend reports', async () => { + const first = await reconcile({ status: 'suspended' }, TaskStatus.RUNNING, false); + expect(first.suspendAnomalyReported).toBe(true); + + // The gate opened: this suspend is now the intended one. + const intended = await reconcile( + { status: 'suspended' }, TaskStatus.AWAITING_APPROVAL, first.suspendAnomalyReported); + expect(intended.suspendAnomalyReported).toBe(false); + expect(commandsOfType('Put')).toHaveLength(1); + + // The gate closed but the VM is still suspended — a new anomaly. + const third = await reconcile( + { status: 'suspended' }, TaskStatus.RUNNING, intended.suspendAnomalyReported); + expect(third.suspendAnomalyReported).toBe(true); + expect(commandsOfType('Put')).toHaveLength(2); + }); + + test('defaults to NOT-yet-reported when the caller omits the flag', async () => { + // Back-compat for any caller (and the first poll of every task) that has no + // prior state: the event must fire, not be suppressed by an undefined flag. + const result = await reconcile({ status: 'suspended' }, TaskStatus.RUNNING); + expect(result.suspendAnomalyReported).toBe(true); + expect(commandsOfType('Put')).toHaveLength(1); + }); + }); + + describe('terminal substrate', () => { + test('fails the task when the re-read status is still non-terminal', async () => { + primeReread(TaskStatus.RUNNING); + + const result = await reconcile({ status: 'completed' }, TaskStatus.RUNNING); + + expect(result).toEqual({ taskFailed: true, suspendAnomalyReported: false }); + + // Re-read before acting (guards the "agent wrote terminal, VM torn down" + // race), then the FAILED transition. + expect(commandsOfType('Get')).toHaveLength(1); + const updates = commandsOfType('Update'); + expect(updates).toHaveLength(1); + expect(updates[0].input.TableName).toBe('Tasks'); + const values = updates[0].input.ExpressionAttributeValues as Record; + expect(values[':toStatus']).toBe(TaskStatus.FAILED); + expect(values[':fromStatus']).toBe(TaskStatus.RUNNING); + // The reason string is what error-classifier keys the substrate-failure + // classification on — keep the two in lockstep. + expect(values[':attr_error_message']).toBe( + 'MicroVM substrate terminated before the agent wrote a terminal status: substrate state completed', + ); + + // Plus the task_failed audit event. + const puts = commandsOfType('Put'); + expect(puts).toHaveLength(1); + expect((puts[0].input.Item as Record).event_type).toBe('task_failed'); + }); + + test('does NOT fail the task when the re-read shows the agent already wrote a terminal status', async () => { + primeReread(TaskStatus.COMPLETED); + + const result = await reconcile({ status: 'completed' }, TaskStatus.RUNNING); + + // Normal shutdown ordering: agent writes COMPLETED, exits, VM terminates. + // Without the re-read this would have failed a successful task. + expect(result).toEqual({ taskFailed: false, suspendAnomalyReported: false }); + expect(commandsOfType('Update')).toHaveLength(0); + expect(commandsOfType('Put')).toHaveLength(0); + }); + + test.each([ + TaskStatus.COMPLETED, + TaskStatus.FAILED, + TaskStatus.CANCELLED, + TaskStatus.TIMED_OUT, + ])('accepts a re-read terminal status of %s without failing the task', async (status) => { + primeReread(status); + + const result = await reconcile({ status: 'completed' }, TaskStatus.RUNNING); + + expect(result).toEqual({ taskFailed: false, suspendAnomalyReported: false }); + expect(commandsOfType('Update')).toHaveLength(0); + }); + + test('carries the substrate error detail into the failure reason', async () => { + primeReread(TaskStatus.RUNNING); + + const result = await reconcile({ status: 'failed', error: 'host fault' }, TaskStatus.RUNNING); + + expect(result).toEqual({ taskFailed: true, suspendAnomalyReported: false }); + const values = commandsOfType('Update')[0].input.ExpressionAttributeValues as Record; + expect(values[':attr_error_message']).toBe( + 'MicroVM substrate terminated before the agent wrote a terminal status: host fault', + ); + }); + + test('fails from AWAITING_APPROVAL too — a terminated VM cannot resume the gate', async () => { + primeReread(TaskStatus.AWAITING_APPROVAL); + + const result = await reconcile({ status: 'completed' }, TaskStatus.AWAITING_APPROVAL); + + expect(result).toEqual({ taskFailed: true, suspendAnomalyReported: false }); + const values = commandsOfType('Update')[0].input.ExpressionAttributeValues as Record; + expect(values[':fromStatus']).toBe(TaskStatus.AWAITING_APPROVAL); + expect(values[':toStatus']).toBe(TaskStatus.FAILED); + }); + + test('transitions from the RE-READ status, not the stale polled status', async () => { + // Task moved HYDRATING → RUNNING between the poll read and the re-read; the + // conditional transition must use the fresh value or it fails its own + // ConditionExpression and the task is left stuck. + primeReread(TaskStatus.RUNNING); + + await reconcile({ status: 'completed' }, TaskStatus.HYDRATING); + + const values = commandsOfType('Update')[0].input.ExpressionAttributeValues as Record; + expect(values[':fromStatus']).toBe(TaskStatus.RUNNING); + }); + + test('does not decrement concurrency — the finalize step owns the release', async () => { + primeReread(TaskStatus.RUNNING); + + await reconcile({ status: 'completed' }, TaskStatus.RUNNING); + + // Matches the ECS substrate-failure branch: failTask(..., releaseConcurrency=false). + const concurrencyWrites = commandsOfType('Update').filter( + c => c.input.TableName === 'Concurrency', + ); + expect(concurrencyWrites).toHaveLength(0); + }); + }); +}); diff --git a/cdk/test/handlers/shared/session-start-retry.test.ts b/cdk/test/handlers/shared/session-start-retry.test.ts index f6227278e..ee27474c1 100644 --- a/cdk/test/handlers/shared/session-start-retry.test.ts +++ b/cdk/test/handlers/shared/session-start-retry.test.ts @@ -128,3 +128,129 @@ describe('startSessionWithRetry — retry-event emit is best-effort (#599 B1)', expect(warns.some((w) => w.message.includes('event emit failed'))).toBe(true); }); }); + +describe('startSessionWithRetry — Lambda MicroVM start failures (ADR-021)', () => { + /** + * The retry gate is strategy-AGNOSTIC by construction: it takes + * `Pick` and asks `classifyError` about the + * RAW error string, so a new backend needs no change here — only correct + * classifier entries. These cases pin that contract for the MicroVM errors + * ADR-021 calls out, so a future classifier edit can't silently turn a + * non-retryable regional misconfiguration into a wasted retry (or drop the + * retry on a throttle). + */ + const MICROVM_HANDLE: SessionHandle = { + sessionId: 'mvm-0123456789abcdef', + strategyType: 'lambda-microvm', + microvmId: 'mvm-0123456789abcdef', + endpoint: 'https://mvm-0123456789abcdef.microvm.lambda.us-east-1.amazonaws.com', + }; + + it('auto-retries a ThrottlingException once and returns the MicroVM handle', async () => { + // The realistic shape: the strategy wraps escaping errors with the + // `MicroVM failed` marker (wrapMicrovmError), and the retry gate + // classifies String(err) — so the marker is present when it is classified. + const throttle = new Error('MicroVM RunMicrovm failed: ThrottlingException: Rate exceeded'); + const startSession = jest + .fn() + .mockRejectedValueOnce(throttle) + .mockResolvedValueOnce(MICROVM_HANDLE); + const { d, emitReasons } = deps(); + + const res = await startSessionWithRetry({ startSession }, {} as never, d); + + expect(res).toEqual({ handle: MICROVM_HANDLE, autoRetried: true }); + expect(startSession).toHaveBeenCalledTimes(2); + expect(emitReasons).toHaveLength(1); + }); + + it('auto-retries a ServiceQuotaExceededException (capacity frees as MicroVMs terminate)', async () => { + const quota = new Error( + 'MicroVM RunMicrovm failed: ServiceQuotaExceededException: MicroVM memory quota exceeded', + ); + const startSession = jest + .fn() + .mockRejectedValueOnce(quota) + .mockResolvedValueOnce(MICROVM_HANDLE); + const { d } = deps(); + + const res = await startSessionWithRetry({ startSession }, {} as never, d); + + expect(res).toEqual({ handle: MICROVM_HANDLE, autoRetried: true }); + expect(startSession).toHaveBeenCalledTimes(2); + }); + + it('does NOT retry a regional-unavailability failure — the Region will never gain the service', async () => { + const regional = new Error( + "UnknownEndpoint: Inaccessible host: `lambda.eu-central-1.amazonaws.com'. " + + "This service may not be available in the `eu-central-1' region.", + ); + const startSession = jest.fn().mockRejectedValueOnce(regional); + const { d, emitReasons } = deps(); + + await expect(startSessionWithRetry({ startSession }, {} as never, d)).rejects.toBe(regional); + + expect(startSession).toHaveBeenCalledTimes(1); + expect(emitReasons).toEqual([]); + }); + + it('does NOT retry a missing-substrate config error (no MICROVM_* env vars)', async () => { + const configErr = new Error( + 'This repository is configured compute_type=lambda-microvm, but this stack was deployed without the ' + + 'Lambda MicroVMs substrate (missing MICROVM_IMAGE_IDENTIFIER/...).', + ); + const startSession = jest.fn().mockRejectedValueOnce(configErr); + const { d } = deps(); + + await expect(startSessionWithRetry({ startSession }, {} as never, d)).rejects.toBe(configErr); + expect(startSession).toHaveBeenCalledTimes(1); + }); + + it('does NOT retry a ResourceNotFoundException (bad image/role/connector ARN)', async () => { + const notFound = new Error('MicroVM RunMicrovm failed: ResourceNotFoundException: MicroVM image not found'); + const startSession = jest.fn().mockRejectedValueOnce(notFound); + const { d } = deps(); + + await expect(startSessionWithRetry({ startSession }, {} as never, d)).rejects.toBe(notFound); + expect(startSession).toHaveBeenCalledTimes(1); + }); +}); + +describe('startSessionWithRetry — other backends keep their pre-ADR-021 retry behaviour', () => { + /** + * Companion to the classifier scoping regression: the marker anchor means an + * AgentCore or ECS error carrying the SAME AWS exception name is still + * classified as before (UNKNOWN ⇒ not transient ⇒ no auto-retry). If the + * anchor is dropped, these flip to two attempts and this fails. + */ + it('does NOT auto-retry an unmarked ThrottlingException (agentcore/ECS path)', async () => { + const throttle = new Error('ThrottlingException: Rate exceeded'); + const startSession = jest.fn().mockRejectedValueOnce(throttle); + const { d, emitReasons } = deps(); + + await expect(startSessionWithRetry({ startSession }, {} as never, d)).rejects.toBe(throttle); + expect(startSession).toHaveBeenCalledTimes(1); + expect(emitReasons).toEqual([]); + }); + + it('does NOT auto-retry an unmarked ServiceQuotaExceededException (agentcore/ECS path)', async () => { + const quota = new Error('ServiceQuotaExceededException: quota exceeded'); + const startSession = jest.fn().mockRejectedValueOnce(quota); + const { d } = deps(); + + await expect(startSessionWithRetry({ startSession }, {} as never, d)).rejects.toBe(quota); + expect(startSession).toHaveBeenCalledTimes(1); + }); + + it('still auto-retries the ECS deploy-race, proving the gate itself is unchanged', async () => { + const startSession = jest + .fn() + .mockRejectedValueOnce(new Error('TaskDefinition is inactive')) + .mockResolvedValueOnce(HANDLE); + const { d } = deps(); + + const res = await startSessionWithRetry({ startSession }, {} as never, d); + expect(res).toEqual({ handle: HANDLE, autoRetried: true }); + expect(startSession).toHaveBeenCalledTimes(2); + }); +}); diff --git a/cdk/test/handlers/shared/strategies/lambda-microvm-strategy.test.ts b/cdk/test/handlers/shared/strategies/lambda-microvm-strategy.test.ts new file mode 100644 index 000000000..bf9ad7792 --- /dev/null +++ b/cdk/test/handlers/shared/strategies/lambda-microvm-strategy.test.ts @@ -0,0 +1,914 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// The image identifier MUST be a full ARN — RunMicrovm rejects a bare name +// ("Malformed ARN - doesn't start with 'arn:'"), and the live run observed the +// colon form (`microvm-image:`), which is what the construct derives. +const IMAGE_IDENTIFIER = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:abca-agent'; +const IMAGE_VERSION = '7'; +const EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/AbcaMicrovmExecution'; +const EGRESS_CONNECTOR_ARN = 'arn:aws:lambda:us-east-1:123456789012:network-connector/egress-1'; +const INGRESS_CONNECTOR_ARN = 'arn:aws:lambda:us-east-1:123456789012:network-connector/ingress-1'; +const NO_INGRESS_CONNECTOR_ARN = + 'arn:aws:lambda:us-east-1:aws:network-connector:aws-network-connector:NO_INGRESS'; +const PAYLOAD_BUCKET = 'test-microvm-payload-bucket'; +const MICROVM_ID = 'mvm-0123456789abcdef'; +const ENDPOINT = 'https://mvm-0123456789abcdef.microvm.lambda.us-east-1.amazonaws.com'; + +// Set env vars BEFORE import — LambdaMicrovmComputeStrategy reads them as +// module-level constants (same pattern as ecs-strategy). The top-of-file import +// is the FULLY-CONFIGURED substrate; the missing-config describe block below +// re-imports under jest.isolateModules with vars deleted, and the ingress block +// re-imports with a REAL ingress connector configured. Ingress is deliberately +// absent here so the default (explicit NO_INGRESS fallback) assertions are +// hermetic; AWS_REGION is set because that fallback derives the ARN from it. +process.env.MICROVM_IMAGE_IDENTIFIER = IMAGE_IDENTIFIER; +process.env.MICROVM_IMAGE_VERSION = IMAGE_VERSION; +process.env.MICROVM_EXECUTION_ROLE_ARN = EXECUTION_ROLE_ARN; +process.env.MICROVM_EGRESS_CONNECTOR_ARNS = EGRESS_CONNECTOR_ARN; +process.env.MICROVM_PAYLOAD_BUCKET = PAYLOAD_BUCKET; +process.env.AWS_REGION = 'us-east-1'; +delete process.env.MICROVM_INGRESS_CONNECTOR_ARNS; + +const mockSend = jest.fn(); +jest.mock('@aws-sdk/client-lambda-microvms', () => ({ + LambdaMicrovmsClient: jest.fn(() => ({ send: mockSend })), + RunMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'RunMicrovm', input })), + GetMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'GetMicrovm', input })), + TerminateMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'TerminateMicrovm', input })), + // Mirrors the real SDK's const-object enum so the strategy's switch keys on + // the same literals the service returns. + MicrovmState: { + PENDING: 'PENDING', + RUNNING: 'RUNNING', + SUSPENDED: 'SUSPENDED', + SUSPENDING: 'SUSPENDING', + TERMINATED: 'TERMINATED', + TERMINATING: 'TERMINATING', + }, +})); + +const mockS3Send = jest.fn(); +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(() => ({ send: mockS3Send })), + PutObjectCommand: jest.fn((input: unknown) => ({ _type: 'PutObject', input })), + DeleteObjectCommand: jest.fn((input: unknown) => ({ _type: 'DeleteObject', input })), +})); + +// The real logger writes JSON to process.stdout/stderr, so level assertions need +// the module mocked rather than console spied (see repo-config.test.ts). +const mockLogger = { info: jest.fn(), warn: jest.fn(), error: jest.fn(), child: jest.fn() }; +jest.mock('../../../../src/handlers/shared/logger', () => ({ logger: mockLogger })); + +import type { BlueprintConfig } from '../../../../src/handlers/shared/repo-config'; +import { + LambdaMicrovmComputeStrategy, + MICROVM_ERROR_MARKER, + MICROVM_MAX_DURATION_SECONDS, + MICROVM_RUN_HOOK_PAYLOAD_LIMIT_BYTES, + microvmNoIngressConnectorArnForRegion, + microvmPayloadKey, +} from '../../../../src/handlers/shared/strategies/lambda-microvm-strategy'; + +const BLUEPRINT: BlueprintConfig = { compute_type: 'lambda-microvm', runtime_arn: '' }; + +/** + * Build a payload whose serialized `{"agent_payload": …}` envelope is EXACTLY + * `targetBytes` long, so the 4 KB boundary can be probed on both sides. Asserts + * its own arithmetic — if the envelope shape ever changes, this fails loudly + * rather than silently testing the wrong boundary. + */ +function payloadWithEnvelopeBytes(targetBytes: number): Record { + const overhead = Buffer.byteLength(JSON.stringify({ agent_payload: { p: '' } }), 'utf8'); + const payload = { p: 'x'.repeat(targetBytes - overhead) }; + expect(Buffer.byteLength(JSON.stringify({ agent_payload: payload }), 'utf8')).toBe(targetBytes); + return payload; +} + +function runMicrovmOk() { + mockSend.mockResolvedValueOnce({ + microvmId: MICROVM_ID, + endpoint: ENDPOINT, + state: 'RUNNING', + imageArn: IMAGE_IDENTIFIER, + imageVersion: IMAGE_VERSION, + }); +} + +/** The handle shape pollSession/stopSession expect. */ +const makeHandle = () => ({ + sessionId: MICROVM_ID, + strategyType: 'lambda-microvm' as const, + microvmId: MICROVM_ID, + endpoint: ENDPOINT, +}); + +/** + * Run `body` with only the given Region env vars set, restoring both afterwards. + * + * `noIngressConnectorArn()` reads the Region at CALL time (not import time), so + * the NO_INGRESS fallback tests need no module reload — just a scoped env. + */ +function withRegion(env: { AWS_REGION?: string; AWS_DEFAULT_REGION?: string }, body: () => void): void { + const saved = { + AWS_REGION: process.env.AWS_REGION, + AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION, + }; + try { + for (const key of ['AWS_REGION', 'AWS_DEFAULT_REGION'] as const) { + if (env[key] === undefined) delete process.env[key]; + else process.env[key] = env[key]; + } + body(); + } finally { + for (const key of ['AWS_REGION', 'AWS_DEFAULT_REGION'] as const) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + } +} + +/** {@link withRegion} for an async body. */ +async function withRegionAsync( + env: { AWS_REGION?: string; AWS_DEFAULT_REGION?: string }, + body: () => Promise, +): Promise { + const saved = { + AWS_REGION: process.env.AWS_REGION, + AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION, + }; + try { + for (const key of ['AWS_REGION', 'AWS_DEFAULT_REGION'] as const) { + if (env[key] === undefined) delete process.env[key]; + else process.env[key] = env[key]; + } + await body(); + } finally { + for (const key of ['AWS_REGION', 'AWS_DEFAULT_REGION'] as const) { + if (saved[key] === undefined) delete process.env[key]; + else process.env[key] = saved[key]; + } + } +} + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe('LambdaMicrovmComputeStrategy', () => { + test('type is lambda-microvm', () => { + expect(new LambdaMicrovmComputeStrategy().type).toBe('lambda-microvm'); + }); + + describe('startSession', () => { + test('sends RunMicrovm with the configured image, execution role and egress connector', async () => { + runMicrovmOk(); + + const handle = await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo', prompt: 'Fix the bug' }, + blueprintConfig: BLUEPRINT, + }); + + expect(mockSend).toHaveBeenCalledTimes(1); + const call = mockSend.mock.calls[0][0]; + expect(call._type).toBe('RunMicrovm'); + expect(call.input.imageIdentifier).toBe(IMAGE_IDENTIFIER); + expect(call.input.imageVersion).toBe(IMAGE_VERSION); + expect(call.input.executionRoleArn).toBe(EXECUTION_ROLE_ARN); + expect(call.input.egressNetworkConnectors).toEqual([EGRESS_CONNECTOR_ARN]); + + expect(handle).toEqual({ + sessionId: MICROVM_ID, + strategyType: 'lambda-microvm', + microvmId: MICROVM_ID, + endpoint: ENDPOINT, + }); + }); + + test('sets maximumDurationInSeconds to the 28800s service maximum', async () => { + runMicrovmOk(); + + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + expect(MICROVM_MAX_DURATION_SECONDS).toBe(28_800); + expect(mockSend.mock.calls[0][0].input.maximumDurationInSeconds).toBe(28_800); + }); + + test('OMITS idlePolicy entirely — the field must be absent, not disabled (ADR-021 invariant)', async () => { + runMicrovmOk(); + + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + const input = mockSend.mock.calls[0][0].input; + // Absence, not `undefined`/falsy: all three idlePolicy fields are required + // when the block is present, so omission is the unambiguous disabled state. + // Traffic-based auto-suspend would freeze the outbound-only agent mid-build. + expect(Object.keys(input)).not.toContain('idlePolicy'); + expect('idlePolicy' in input).toBe(false); + }); + + test('passes the explicit NO_INGRESS connector when no ingress is configured', async () => { + runMicrovmOk(); + + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + const input = mockSend.mock.calls[0][0].input; + // NOT omitted: a RunMicrovm call with no ingressNetworkConnectors comes + // back with the AWS-managed PUBLIC HTTP_INGRESS connector attached and a + // public *.lambda-microvm..on.aws endpoint (live-observed). "No + // inbound" is a control we have to request. + expect('ingressNetworkConnectors' in input).toBe(true); + expect(input.ingressNetworkConnectors).toEqual([NO_INGRESS_CONNECTOR_ARN]); + expect(JSON.stringify(input)).not.toContain('HTTP_INGRESS'); + expect(JSON.stringify(input)).not.toContain('SHELL_INGRESS'); + }); + + test('derives the NO_INGRESS fallback ARN from the running Region', () => { + // Region-derived rather than hardcoded so the fallback is right in all five + // supported Regions; partition follows the Region prefix for aws-cn/-gov. + expect(microvmNoIngressConnectorArnForRegion()).toBe(NO_INGRESS_CONNECTOR_ARN); + }); + + test.each([ + ['us-east-1', 'aws'], + ['ap-northeast-1', 'aws'], + ['cn-north-1', 'aws-cn'], + ['us-gov-west-1', 'aws-us-gov'], + ])('the NO_INGRESS fallback uses the right partition in %s', (region, partition) => { + // A wrong partition would make the connector ARN unresolvable and the + // launch would fail — which is safer than a public endpoint, but still a + // hard outage in aws-cn / aws-us-gov if we ever ship there. The Region is + // read at CALL time (not import time), so no module reload is needed. + withRegion({ AWS_REGION: region }, () => { + expect(microvmNoIngressConnectorArnForRegion()).toBe( + `arn:${partition}:lambda:${region}:aws:network-connector:aws-network-connector:NO_INGRESS`, + ); + }); + }); + + test('the NO_INGRESS fallback reads AWS_DEFAULT_REGION when AWS_REGION is unset', () => { + withRegion({ AWS_DEFAULT_REGION: 'eu-west-1' }, () => { + expect(microvmNoIngressConnectorArnForRegion()) + .toContain(':lambda:eu-west-1:aws:network-connector:'); + }); + }); + + test('the NO_INGRESS fallback never splices `undefined` into the ARN', () => { + // Neither var set is impossible in Lambda (the runtime always injects + // AWS_REGION), but a Region-less ARN must still be a well-formed string the + // service can reject cleanly rather than `arn:aws:lambda:undefined:...`. + withRegion({}, () => { + expect(microvmNoIngressConnectorArnForRegion()).not.toContain('undefined'); + }); + }); + + test('the fallback reaches the RunMicrovm INPUT, per-Region, not just the helper', async () => { + // Outcome-level assertion for the fallback path: what matters is the ARN the + // service actually receives. Asserting only the helper would let a wiring + // regression (field omitted, wrong variable) pass while every agent MicroVM + // silently got the service-default PUBLIC endpoint. + runMicrovmOk(); + + await withRegionAsync({ AWS_REGION: 'eu-west-1' }, async () => { + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + }); + + expect(mockSend.mock.calls[0][0].input.ingressNetworkConnectors).toEqual([ + 'arn:aws:lambda:eu-west-1:aws:network-connector:aws-network-connector:NO_INGRESS', + ]); + }); + + test('inlines a small payload in runHookPayload and never touches S3', async () => { + runMicrovmOk(); + + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo', prompt: 'Fix the bug', max_turns: 50 }, + blueprintConfig: BLUEPRINT, + }); + + expect(mockS3Send).not.toHaveBeenCalled(); + const envelope = JSON.parse(mockSend.mock.calls[0][0].input.runHookPayload); + expect(envelope.agent_payload).toEqual({ repo_url: 'org/repo', prompt: 'Fix the bug', max_turns: 50 }); + expect(envelope.agent_payload_s3_uri).toBeUndefined(); + }); + + test('uploads an oversized payload to S3 and inlines only the pointer', async () => { + mockS3Send.mockResolvedValueOnce({}); + runMicrovmOk(); + + const big = { repo_url: 'org/repo', hydrated_context: { blob: 'x'.repeat(20_000) } }; + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: big, + blueprintConfig: BLUEPRINT, + }); + + // Same key shape as the ECS payload bucket: /payload.json + expect(mockS3Send).toHaveBeenCalledTimes(1); + const put = mockS3Send.mock.calls[0][0]; + expect(put._type).toBe('PutObject'); + expect(put.input.Bucket).toBe(PAYLOAD_BUCKET); + expect(put.input.Key).toBe('TASK001/payload.json'); + expect(put.input.ContentType).toBe('application/json'); + expect(JSON.parse(put.input.Body)).toEqual(big); + + const runHookPayload = mockSend.mock.calls[0][0].input.runHookPayload; + const envelope = JSON.parse(runHookPayload); + expect(envelope.agent_payload_s3_uri).toBe(`s3://${PAYLOAD_BUCKET}/TASK001/payload.json`); + expect(envelope.agent_payload).toBeUndefined(); + // The whole point: the hook body must sit far under the 4 KB cap. + expect(Buffer.byteLength(runHookPayload, 'utf8')).toBeLessThan(MICROVM_RUN_HOOK_PAYLOAD_LIMIT_BYTES); + }); + + test('the inline/S3 branch point IS the 4096-byte service cap, with no headroom', () => { + // Live-measured, NOT read off the SDK docs (which say 16,384): the service + // rejects 4097 with "Member must have length less than or equal to 4096". + // Unlike ECS (whose 8192-byte cap is shared with env vars + command, so it + // needs a margin), runHookPayload is the entire counted string. + expect(MICROVM_RUN_HOOK_PAYLOAD_LIMIT_BYTES).toBe(4_096); + }); + + test('a payload whose envelope is EXACTLY 4096 bytes stays inline', async () => { + runMicrovmOk(); + + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: payloadWithEnvelopeBytes(4_096), + blueprintConfig: BLUEPRINT, + }); + + // Boundary is `<=`: 4096 passed the service's length validation live. + expect(mockS3Send).not.toHaveBeenCalled(); + const runHookPayload = mockSend.mock.calls[0][0].input.runHookPayload; + expect(Buffer.byteLength(runHookPayload, 'utf8')).toBe(4_096); + expect(JSON.parse(runHookPayload).agent_payload).toBeDefined(); + }); + + test('a payload whose envelope is 4097 bytes — one over — goes to S3', async () => { + mockS3Send.mockResolvedValueOnce({}); + runMicrovmOk(); + + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: payloadWithEnvelopeBytes(4_097), + blueprintConfig: BLUEPRINT, + }); + + expect(mockS3Send).toHaveBeenCalledTimes(1); + const envelope = JSON.parse(mockSend.mock.calls[0][0].input.runHookPayload); + expect(envelope.agent_payload_s3_uri).toBe(`s3://${PAYLOAD_BUCKET}/TASK001/payload.json`); + expect(envelope.agent_payload).toBeUndefined(); + }); + + test('a mid-sized envelope the SDK-documented 16KB cap would have inlined goes to S3', async () => { + mockS3Send.mockResolvedValueOnce({}); + runMicrovmOk(); + + // Regression guard for the live-verification fix: anything from 4,097 to + // 16,384 bytes used to be inlined and would be REJECTED by the service. + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: payloadWithEnvelopeBytes(13_000), + blueprintConfig: BLUEPRINT, + }); + + expect(mockS3Send).toHaveBeenCalledTimes(1); + const envelope = JSON.parse(mockSend.mock.calls[0][0].input.runHookPayload); + expect(envelope.agent_payload_s3_uri).toBeDefined(); + expect(envelope.agent_payload).toBeUndefined(); + }); + + test('measures the serialized envelope in BYTES, so a multi-byte payload still goes to S3', async () => { + mockS3Send.mockResolvedValueOnce({}); + runMicrovmOk(); + + // 3-byte UTF-8 characters: 2000 chars is ~6 KB of bytes but only 2 KB of + // chars, so measuring String.length would have wrongly inlined this. + const payload = { prompt: '\u4f60'.repeat(2_000) }; + expect(JSON.stringify({ agent_payload: payload }).length) + .toBeLessThan(MICROVM_RUN_HOOK_PAYLOAD_LIMIT_BYTES); + await new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload, + blueprintConfig: BLUEPRINT, + }); + + expect(mockS3Send).toHaveBeenCalledTimes(1); + expect(JSON.parse(mockSend.mock.calls[0][0].input.runHookPayload).agent_payload_s3_uri).toBeDefined(); + }); + + test('throws when RunMicrovm returns no microvmId', async () => { + mockSend.mockResolvedValueOnce({ endpoint: ENDPOINT, state: 'PENDING' }); + + await expect( + new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }), + ).rejects.toThrow('RunMicrovm returned an incomplete response'); + }); + + test('throws when RunMicrovm returns no endpoint', async () => { + mockSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'PENDING' }); + + await expect( + new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }), + ).rejects.toThrow('RunMicrovm returned an incomplete response'); + }); + + test('MARKS the incomplete-response throw so the classifier can see the backend', async () => { + // Unmarked, this landed in error-classifier's generic `Session start failed` + // bucket, whose remedy is "Check AgentCore Runtime or ECS cluster health" — + // the wrong substrate entirely. + mockSend.mockResolvedValueOnce({ endpoint: ENDPOINT, state: 'PENDING' }); + + await expect( + new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }), + ).rejects.toThrow( + `${MICROVM_ERROR_MARKER} RunMicrovm failed: RunMicrovm returned an incomplete response`, + ); + }); + + test('REAPS the MicroVM when the response carries an id but no endpoint', async () => { + // The one orphan window the orchestrator cannot cover: startSession never + // returns a handle, so nothing downstream knows the id. A MicroVM is already + // running and nothing self-terminates on this substrate. + mockSend + .mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'RUNNING' }) + .mockResolvedValueOnce({}); + + await expect( + new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }), + ).rejects.toThrow(`${MICROVM_ERROR_MARKER} RunMicrovm failed`); + + const terminate = mockSend.mock.calls.find(c => c[0]._type === 'TerminateMicrovm'); + expect(terminate).toBeDefined(); + expect(terminate![0].input).toEqual({ microvmIdentifier: MICROVM_ID }); + }); + + test('a failing reap does not mask the incomplete-response error', async () => { + mockSend + .mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'RUNNING' }) + .mockRejectedValueOnce(new Error('terminate blew up')); + + await expect( + new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }), + ).rejects.toThrow('RunMicrovm returned an incomplete response'); + }); + + test('does NOT attempt a reap when there is no id to reap', async () => { + mockSend.mockResolvedValueOnce({ endpoint: ENDPOINT, state: 'PENDING' }); + + await expect( + new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }), + ).rejects.toThrow(); + + expect(mockSend.mock.calls.filter(c => c[0]._type === 'TerminateMicrovm')).toHaveLength(0); + }); + + test('microvmPayloadKey matches the ECS payload key shape', () => { + expect(microvmPayloadKey('TASK001')).toBe('TASK001/payload.json'); + }); + }); + + describe('pollSession — mechanical state mapping, no task-state interpretation', () => { + test.each([ + ['PENDING', 'running'], + ['RUNNING', 'running'], + ['SUSPENDING', 'suspended'], + ['SUSPENDED', 'suspended'], + ['TERMINATING', 'completed'], + ['TERMINATED', 'completed'], + ])('maps MicroVM state %s to session status %s', async (state, expected) => { + mockSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state }); + + const result = await new LambdaMicrovmComputeStrategy().pollSession(makeHandle()); + expect(result).toEqual({ status: expected }); + }); + + test('sends GetMicrovm keyed on microvmIdentifier', async () => { + mockSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'RUNNING' }); + + await new LambdaMicrovmComputeStrategy().pollSession(makeHandle()); + + const call = mockSend.mock.calls[0][0]; + expect(call._type).toBe('GetMicrovm'); + expect(call.input).toEqual({ microvmIdentifier: MICROVM_ID }); + }); + + test('reports SUSPENDED as suspended without inspecting task state', async () => { + mockSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'SUSPENDED' }); + + // No task id, no DDB read — the strategy cannot see the task row at all, + // which is exactly why the health rules live in the orchestrator. + const result = await new LambdaMicrovmComputeStrategy().pollSession(makeHandle()); + expect(result).toEqual({ status: 'suspended' }); + }); + + test('treats ResourceNotFoundException as completed (a reaped MicroVM is gone, not broken)', async () => { + const err = new Error('MicroVM mvm-0123456789abcdef not found'); + err.name = 'ResourceNotFoundException'; + mockSend.mockRejectedValueOnce(err); + + const result = await new LambdaMicrovmComputeStrategy().pollSession(makeHandle()); + expect(result).toEqual({ status: 'completed' }); + }); + + test('rethrows non-NotFound errors so the caller can count poll failures', async () => { + const err = new Error('Rate exceeded'); + err.name = 'ThrottlingException'; + mockSend.mockRejectedValueOnce(err); + + await expect(new LambdaMicrovmComputeStrategy().pollSession(makeHandle())).rejects.toThrow('Rate exceeded'); + }); + test('reports running for an unrecognized future state rather than failing the task', async () => { + mockSend.mockResolvedValueOnce({ microvmId: MICROVM_ID, state: 'HIBERNATING_SOMEDAY' }); + + const result = await new LambdaMicrovmComputeStrategy().pollSession(makeHandle()); + expect(result).toEqual({ status: 'running' }); + }); + + test('throws when the handle is not a lambda-microvm handle', async () => { + await expect( + new LambdaMicrovmComputeStrategy().pollSession({ + sessionId: 'test', + strategyType: 'agentcore', + runtimeArn: 'arn:test', + }), + ).rejects.toThrow('pollSession called with non-lambda-microvm handle'); + }); + }); + + describe('stopSession — best-effort with differentiated error handling', () => { + test('sends TerminateMicrovm keyed on microvmIdentifier', async () => { + mockSend.mockResolvedValueOnce({}); + + await new LambdaMicrovmComputeStrategy().stopSession(makeHandle()); + + expect(mockSend).toHaveBeenCalledTimes(1); + const call = mockSend.mock.calls[0][0]; + expect(call._type).toBe('TerminateMicrovm'); + expect(call.input).toEqual({ microvmIdentifier: MICROVM_ID }); + }); + + test.each([ + ['ResourceNotFoundException', 'info'], + ['ConflictException', 'info'], + ['ThrottlingException', 'error'], + ['AccessDeniedException', 'error'], + ['InternalServerException', 'warn'], + ])('logs %s at %s level and never throws', async (errName, expectedLevel) => { + const err = new Error('boom'); + err.name = errName; + mockSend.mockRejectedValueOnce(err); + + await expect(new LambdaMicrovmComputeStrategy().stopSession(makeHandle())).resolves.toBeUndefined(); + + const byLevel: Record = { + info: mockLogger.info, + warn: mockLogger.warn, + error: mockLogger.error, + }; + expect(byLevel[expectedLevel]).toHaveBeenCalledTimes(1); + for (const [level, spy] of Object.entries(byLevel)) { + if (level !== expectedLevel) expect(spy).not.toHaveBeenCalled(); + } + }); + + test('throws when the handle is not a lambda-microvm handle', async () => { + await expect( + new LambdaMicrovmComputeStrategy().stopSession({ + sessionId: 'test', + strategyType: 'ecs', + clusterArn: 'arn:cluster', + taskArn: 'arn:task', + }), + ).rejects.toThrow('stopSession called with non-lambda-microvm handle'); + }); + }); + + describe('error marking — every escaping error carries the MicroVM marker', () => { + // The marker is what lets error-classifier scope its bare AWS exception-name + // patterns to THIS backend, so an unmarked escape is a silent classification + // hole: the error would fall through to UNKNOWN ("Unexpected error"). + const markerRe = new RegExp(`${MICROVM_ERROR_MARKER} [\\w ]+failed`); + + test('the marker constant matches the pattern shape the classifier anchors on', () => { + expect(MICROVM_ERROR_MARKER).toBe('MicroVM'); + expect(`${MICROVM_ERROR_MARKER} RunMicrovm failed: x`).toMatch(markerRe); + }); + + test('marks a RunMicrovm failure and splices in the SDK exception name', async () => { + const err = new Error('Rate exceeded'); + err.name = 'ThrottlingException'; + mockSend.mockRejectedValueOnce(err); + + const start = new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + await expect(start).rejects.toThrow(markerRe); + // The exception NAME must appear: err.message alone omits it, and the + // classifier keys on ` … `. + await expect(start).rejects.toThrow('MicroVM RunMicrovm failed: ThrottlingException: Rate exceeded'); + }); + + test('preserves the original error as `cause` so err.name stays inspectable', async () => { + const err = new Error('quota'); + err.name = 'ServiceQuotaExceededException'; + mockSend.mockRejectedValueOnce(err); + + await new LambdaMicrovmComputeStrategy() + .startSession({ taskId: 'TASK001', userId: 'u', payload: {}, blueprintConfig: BLUEPRINT }) + .catch((thrown: Error) => { + expect(thrown.cause).toBe(err); + expect((thrown.cause as Error).name).toBe('ServiceQuotaExceededException'); + }); + expect.assertions(2); + }); + + test('marks a GetMicrovm failure', async () => { + const err = new Error('Rate exceeded'); + err.name = 'ThrottlingException'; + mockSend.mockRejectedValueOnce(err); + + await expect(new LambdaMicrovmComputeStrategy().pollSession(makeHandle())) + .rejects.toThrow('MicroVM GetMicrovm failed: ThrottlingException: Rate exceeded'); + }); + + test('marks a payload-upload failure so an S3 fault is attributed to this backend', async () => { + const err = new Error('Access Denied'); + err.name = 'AccessDenied'; + mockS3Send.mockRejectedValueOnce(err); + + const start = new LambdaMicrovmComputeStrategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: payloadWithEnvelopeBytes(20_000), + blueprintConfig: BLUEPRINT, + }); + + await expect(start).rejects.toThrow('MicroVM payload upload failed: AccessDenied: Access Denied'); + // Never reaches RunMicrovm — no half-started MicroVM on an upload fault. + expect(mockSend).not.toHaveBeenCalled(); + }); + + test('does not double-prefix when the message already contains the exception name', async () => { + const err = new Error('ThrottlingException: Rate exceeded'); + err.name = 'ThrottlingException'; + mockSend.mockRejectedValueOnce(err); + + await expect(new LambdaMicrovmComputeStrategy().pollSession(makeHandle())) + .rejects.toThrow('MicroVM GetMicrovm failed: ThrottlingException: Rate exceeded'); + }); + + test('marks a non-Error rejection too', async () => { + mockSend.mockRejectedValueOnce('a bare string failure'); + + await expect(new LambdaMicrovmComputeStrategy().pollSession(makeHandle())) + .rejects.toThrow('MicroVM GetMicrovm failed: a bare string failure'); + }); + }); +}); + +// The env-var guard reads module-level constants, so the missing-config case +// needs a fresh module graph with the vars deleted (same isolateModules pattern +// ecs-strategy's #502 tests use). +describe('LambdaMicrovmComputeStrategy without the MicroVM substrate deployed', () => { + function loadStrategyWithout(missing: string[]): typeof LambdaMicrovmComputeStrategy { + let Strategy!: typeof LambdaMicrovmComputeStrategy; + const saved: Record = {}; + jest.isolateModules(() => { + for (const key of missing) { + saved[key] = process.env[key]; + delete process.env[key]; + } + // eslint-disable-next-line @typescript-eslint/no-require-imports + Strategy = require('../../../../src/handlers/shared/strategies/lambda-microvm-strategy').LambdaMicrovmComputeStrategy; + }); + for (const [key, value] of Object.entries(saved)) { + if (value !== undefined) process.env[key] = value; + } + return Strategy; + } + + test.each([ + ['MICROVM_IMAGE_IDENTIFIER'], + ['MICROVM_EXECUTION_ROLE_ARN'], + ['MICROVM_EGRESS_CONNECTOR_ARNS'], + ['MICROVM_PAYLOAD_BUCKET'], + ])('throws a descriptive config/deploy-mismatch error when %s is missing', async (envVar) => { + const Strategy = loadStrategyWithout([envVar]); + + const start = new Strategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + // Names the root cause AND both remedies — an admin must not have to + // reverse-engineer this from a bare env-var list. + await expect(start).rejects.toThrow(/compute_type=lambda-microvm/); + await expect(start).rejects.toThrow(/deployed without the Lambda MicroVMs substrate/); + await expect(start).rejects.toThrow(/--context compute_type=lambda-microvm/); + await expect(start).rejects.toThrow(/--compute-type agentcore/); + await expect(start).rejects.toThrow(new RegExp(envVar)); + // Fails BEFORE any AWS call — no half-started MicroVM, no orphan S3 object. + expect(mockSend).not.toHaveBeenCalled(); + expect(mockS3Send).not.toHaveBeenCalled(); + }); + + test('MICROVM_IMAGE_VERSION is optional — the field is omitted so the service picks the default', async () => { + const Strategy = loadStrategyWithout(['MICROVM_IMAGE_VERSION']); + runMicrovmOk(); + + await new Strategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + expect('imageVersion' in mockSend.mock.calls[0][0].input).toBe(false); + }); +}); + +describe('LambdaMicrovmComputeStrategy with ingress connectors configured', () => { + test('passes the configured ingress connectors when the env var is set', async () => { + let Strategy!: typeof LambdaMicrovmComputeStrategy; + jest.isolateModules(() => { + process.env.MICROVM_INGRESS_CONNECTOR_ARNS = `${INGRESS_CONNECTOR_ARN}, ${INGRESS_CONNECTOR_ARN}-b`; + // eslint-disable-next-line @typescript-eslint/no-require-imports + Strategy = require('../../../../src/handlers/shared/strategies/lambda-microvm-strategy').LambdaMicrovmComputeStrategy; + }); + delete process.env.MICROVM_INGRESS_CONNECTOR_ARNS; + + runMicrovmOk(); + await new Strategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + // Comma-separated, whitespace-trimmed — same parsing as ECS_SUBNETS. A + // configured value WINS over the NO_INGRESS default (that is how #391 + // operator shell access lands without a strategy change). + expect(mockSend.mock.calls[0][0].input.ingressNetworkConnectors).toEqual([ + INGRESS_CONNECTOR_ARN, + `${INGRESS_CONNECTOR_ARN}-b`, + ]); + }); + + test('a BLANK env var still yields NO_INGRESS, never an omitted field', async () => { + let Strategy!: typeof LambdaMicrovmComputeStrategy; + jest.isolateModules(() => { + process.env.MICROVM_INGRESS_CONNECTOR_ARNS = ' , '; + // eslint-disable-next-line @typescript-eslint/no-require-imports + Strategy = require('../../../../src/handlers/shared/strategies/lambda-microvm-strategy').LambdaMicrovmComputeStrategy; + }); + delete process.env.MICROVM_INGRESS_CONNECTOR_ARNS; + + runMicrovmOk(); + await new Strategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + // A blank/misconfigured value must not fall back to the service default, + // which is a PUBLIC endpoint on every agent MicroVM. + expect(mockSend.mock.calls[0][0].input.ingressNetworkConnectors) + .toEqual([NO_INGRESS_CONNECTOR_ARN]); + }); +}); + +describe('LambdaMicrovmComputeStrategy image-identifier validation', () => { + function loadStrategyWithIdentifier(identifier: string): typeof LambdaMicrovmComputeStrategy { + let Strategy!: typeof LambdaMicrovmComputeStrategy; + const saved = process.env.MICROVM_IMAGE_IDENTIFIER; + jest.isolateModules(() => { + process.env.MICROVM_IMAGE_IDENTIFIER = identifier; + // eslint-disable-next-line @typescript-eslint/no-require-imports + Strategy = require('../../../../src/handlers/shared/strategies/lambda-microvm-strategy').LambdaMicrovmComputeStrategy; + }); + if (saved !== undefined) process.env.MICROVM_IMAGE_IDENTIFIER = saved; + return Strategy; + } + + test.each([ + ['abca-agent'], + ['backgroundagent-dev-abca-agent'], + ['microvm-image:abca-agent'], + ])('rejects the bare identifier %s BEFORE any AWS call', async (identifier) => { + const Strategy = loadStrategyWithIdentifier(identifier); + + const start = new Strategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + // Oversized on purpose: the guard must fire before the payload upload, or a + // misconfiguration leaves orphan objects in the payload bucket. + payload: payloadWithEnvelopeBytes(20_000), + blueprintConfig: BLUEPRINT, + }); + + await expect(start).rejects.toThrow(/must be a full MicroVM image ARN/); + // Names the service's own error text so an operator can match the two up. + await expect(start).rejects.toThrow(/Malformed ARN/); + // ...and the remedy. + await expect(start).rejects.toThrow(/--context compute_type=lambda-microvm/); + expect(mockSend).not.toHaveBeenCalled(); + expect(mockS3Send).not.toHaveBeenCalled(); + }); + + test('accepts a full image ARN', async () => { + const Strategy = loadStrategyWithIdentifier(IMAGE_IDENTIFIER); + runMicrovmOk(); + + await new Strategy().startSession({ + taskId: 'TASK001', + userId: 'cognito-test', + payload: { repo_url: 'org/repo' }, + blueprintConfig: BLUEPRINT, + }); + + expect(mockSend.mock.calls[0][0].input.imageIdentifier).toBe(IMAGE_IDENTIFIER); + }); +}); diff --git a/cdk/test/handlers/start-session-composition.test.ts b/cdk/test/handlers/start-session-composition.test.ts index 1fe29fbd6..43f7e146a 100644 --- a/cdk/test/handlers/start-session-composition.test.ts +++ b/cdk/test/handlers/start-session-composition.test.ts @@ -40,6 +40,28 @@ jest.mock('@aws-sdk/client-bedrock-agentcore', () => ({ StopRuntimeSessionCommand: jest.fn((input: unknown) => ({ _type: 'StopRuntimeSession', input })), })); +const mockMicrovmSend = jest.fn(); +jest.mock('@aws-sdk/client-lambda-microvms', () => ({ + LambdaMicrovmsClient: jest.fn(() => ({ send: mockMicrovmSend })), + RunMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'RunMicrovm', input })), + GetMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'GetMicrovm', input })), + TerminateMicrovmCommand: jest.fn((input: unknown) => ({ _type: 'TerminateMicrovm', input })), + MicrovmState: { + PENDING: 'PENDING', + RUNNING: 'RUNNING', + SUSPENDED: 'SUSPENDED', + SUSPENDING: 'SUSPENDING', + TERMINATED: 'TERMINATED', + TERMINATING: 'TERMINATING', + }, +})); + +jest.mock('@aws-sdk/client-s3', () => ({ + S3Client: jest.fn(() => ({ send: jest.fn().mockResolvedValue({}) })), + PutObjectCommand: jest.fn((input: unknown) => ({ _type: 'PutObject', input })), + DeleteObjectCommand: jest.fn((input: unknown) => ({ _type: 'DeleteObject', input })), +})); + jest.mock('../../src/handlers/shared/repo-config', () => ({ loadRepoConfig: jest.fn(), checkRepoOnboarded: jest.fn(), @@ -66,10 +88,14 @@ process.env.USER_CONCURRENCY_TABLE_NAME = 'UserConcurrency'; process.env.RUNTIME_ARN = 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test'; process.env.MAX_CONCURRENT_TASKS_PER_USER = '3'; process.env.TASK_RETENTION_DAYS = '90'; +process.env.MICROVM_IMAGE_IDENTIFIER = 'arn:aws:lambda:us-east-1:123456789012:microvm-image/abca-agent'; +process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/AbcaMicrovmExecution'; +process.env.MICROVM_EGRESS_CONNECTOR_ARNS = 'arn:aws:lambda:us-east-1:123456789012:network-connector/egress-1'; +process.env.MICROVM_PAYLOAD_BUCKET = 'test-microvm-payload-bucket'; import { TaskStatus } from '../../src/constructs/task-status'; import { resolveComputeStrategy } from '../../src/handlers/shared/compute-strategy'; -import { transitionTask, emitTaskEvent, failTask } from '../../src/handlers/shared/orchestrator'; +import { transitionTask, emitTaskEvent, failTask, buildComputeMetadata } from '../../src/handlers/shared/orchestrator'; import type { BlueprintConfig } from '../../src/handlers/shared/repo-config'; beforeEach(() => { @@ -156,3 +182,79 @@ describe('start-session step composition', () => { expect(mockDdbSend).toHaveBeenCalledTimes(4); }); }); + +describe('start-session step composition — lambda-microvm (ADR-021)', () => { + const taskId = 'TASK001'; + const MICROVM_ID = 'mvm-0123456789abcdef'; + const ENDPOINT = 'https://mvm-0123456789abcdef.microvm.lambda.us-east-1.amazonaws.com'; + const blueprintConfig: BlueprintConfig = { compute_type: 'lambda-microvm', runtime_arn: '' }; + const payload = { repo_url: 'org/repo', task_id: taskId }; + + test('startSession → buildComputeMetadata → transitionTask persists microvmId and endpoint', async () => { + mockMicrovmSend.mockResolvedValueOnce({ + microvmId: MICROVM_ID, + endpoint: ENDPOINT, + state: 'RUNNING', + imageArn: 'arn:image', + imageVersion: '7', + }); + mockDdbSend.mockResolvedValue({}); + + const strategy = resolveComputeStrategy(blueprintConfig); + const handle = await strategy.startSession({ taskId, userId: 'cognito-test', payload, blueprintConfig }); + + await transitionTask(taskId, TaskStatus.HYDRATING, TaskStatus.RUNNING, { + session_id: handle.sessionId, + started_at: new Date().toISOString(), + compute_type: handle.strategyType, + compute_metadata: buildComputeMetadata(handle), + }); + + // RunMicrovm actually went out. + expect(mockMicrovmSend).toHaveBeenCalledTimes(1); + expect(mockMicrovmSend.mock.calls[0][0]._type).toBe('RunMicrovm'); + + // The persisted attributes are what cancel-task (and P3's approve/deny + // resume) read back, so assert them on the real UpdateCommand input. + const update = mockDdbSend.mock.calls.find(c => c[0]._type === 'Update')![0]; + const values = update.input.ExpressionAttributeValues as Record; + expect(values[':attr_compute_type']).toBe('lambda-microvm'); + expect(values[':attr_compute_metadata']).toEqual({ microvmId: MICROVM_ID, endpoint: ENDPOINT }); + expect(values[':attr_session_id']).toBe(MICROVM_ID); + expect(values[':toStatus']).toBe(TaskStatus.RUNNING); + }); + + test('compute_metadata carries ONLY the two lifecycle keys (no image ARN)', async () => { + mockMicrovmSend.mockResolvedValueOnce({ + microvmId: MICROVM_ID, + endpoint: ENDPOINT, + state: 'RUNNING', + imageArn: 'arn:aws:lambda:us-east-1:123456789012:microvm-image/abca-agent', + imageVersion: '7', + }); + + const strategy = resolveComputeStrategy(blueprintConfig); + const handle = await strategy.startSession({ taskId, userId: 'cognito-test', payload, blueprintConfig }); + const metadata = buildComputeMetadata(handle); + + // ADR-021: the image ARN is deployment-time config, logged not persisted. + expect(Object.keys(metadata).sort()).toEqual(['endpoint', 'microvmId']); + expect(JSON.stringify(metadata)).not.toContain('microvm-image'); + }); + + test('error path: a marked RunMicrovm failure flows into failTask', async () => { + const err = new Error('Rate exceeded'); + err.name = 'ThrottlingException'; + mockMicrovmSend.mockRejectedValueOnce(err); + mockDdbSend.mockResolvedValue({}); + + const strategy = resolveComputeStrategy(blueprintConfig); + + await expect( + strategy.startSession({ taskId, userId: 'cognito-test', payload, blueprintConfig }), + ).rejects.toThrow('MicroVM RunMicrovm failed: ThrottlingException: Rate exceeded'); + + await failTask(taskId, TaskStatus.HYDRATING, 'Session start failed: boom', 'user-123', true); + expect(mockDdbSend).toHaveBeenCalled(); + }); +}); diff --git a/cdk/test/stacks/agent.test.ts b/cdk/test/stacks/agent.test.ts index 82cb22dfc..2d2b45f5e 100644 --- a/cdk/test/stacks/agent.test.ts +++ b/cdk/test/stacks/agent.test.ts @@ -21,6 +21,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { App, AspectPriority, Aspects } from 'aws-cdk-lib'; import { Match, Template } from 'aws-cdk-lib/assertions'; +import * as lambdaMicrovmCompute from '../../src/constructs/lambda-microvm-compute'; import { buildAppId, SolutionUaAspect } from '../../src/constructs/solution-ua-aspect'; import { AgentStack } from '../../src/stacks/agent'; @@ -751,6 +752,329 @@ describe('AgentStack with the ECS substrate gate (--context compute_type=ecs)', }); }); +describe('AgentStack with the Lambda MicroVMs substrate gate (--context compute_type=lambda-microvm)', () => { + const BASE_IMAGE_ARN = 'arn:aws:lambda:us-east-1:aws:microvm-image:al2023-1'; + + let template: Template; + + beforeAll(() => { + // Gate ON *and* an image configured — the steady state. The intermediate + // "gate on, no image yet" state is covered in the construct test; here the + // point is the stack-level wiring (env vars + IAM + outputs) that only + // exists once an image identifier is available. + const app = new App({ + context: { + compute_type: 'lambda-microvm', + microvm_base_image_arn: BASE_IMAGE_ARN, + microvm_base_image_version: '1', + }, + }); + const stack = new AgentStack(app, 'TestAgentStackMicrovm', { + env: { account: '123456789012', region: 'us-east-1' }, + }); + template = Template.fromStack(stack); + }); + + test('provisions the MicroVM image + BOTH egress network connectors', () => { + template.resourceCountIs('AWS::Lambda::MicrovmImage', 1); + // Runtime (443) + build-time (443 + 80, for apt-get) — see ADR-021's + // build-time-egress security-table row. + template.resourceCountIs('AWS::Lambda::NetworkConnector', 2); + }); + + test('does NOT provision the ECS substrate (the gates are mutually exclusive)', () => { + template.resourceCountIs('AWS::ECS::Cluster', 0); + template.resourceCountIs('AWS::ECS::TaskDefinition', 0); + }); + + test('outputs ComputeSubstrate=lambda-microvm so the CLI allows that onboarding', () => { + template.hasOutput('ComputeSubstrate', { Value: 'lambda-microvm' }); + }); + + test('outputs everything the packaging script needs to find (no predictable physical names)', () => { + for (const output of [ + 'MicrovmArtifactBucketName', + 'MicrovmArtifactObjectKey', + 'MicrovmBuildRoleArn', + 'MicrovmExecutionRoleArn', + 'MicrovmEgressConnectorArns', + // The script passes THIS one to create-microvm-image: the runtime + // connector is 443-only and the Dockerfile's apt-get needs port 80. + 'MicrovmBuildEgressConnectorArns', + 'MicrovmLogGroupName', + ]) { + template.hasOutput(output, {}); + } + }); + + test('the build and runtime egress connector outputs are DIFFERENT connectors', () => { + const outputs = template.toJSON().Outputs as Record; + expect(JSON.stringify(outputs.MicrovmEgressConnectorArns.Value)) + .not.toEqual(JSON.stringify(outputs.MicrovmBuildEgressConnectorArns.Value)); + }); + + test('injects the MICROVM_* env vars the strategy reads, including explicit NO_INGRESS', () => { + const fns = template.findResources('AWS::Lambda::Function'); + const [, orchestrator] = Object.entries(fns) + .find(([id]) => id.includes('TaskOrchestratorOrchestratorFn'))!; + const env = orchestrator.Properties.Environment.Variables as Record; + + expect(Object.keys(env).filter(k => k.startsWith('MICROVM_')).sort()).toEqual([ + 'MICROVM_EGRESS_CONNECTOR_ARNS', + 'MICROVM_EXECUTION_ROLE_ARN', + 'MICROVM_IMAGE_IDENTIFIER', + 'MICROVM_INGRESS_CONNECTOR_ARNS', + 'MICROVM_PAYLOAD_BUCKET', + ]); + // Image version is deliberately unpinned. + expect(env.MICROVM_IMAGE_VERSION).toBeUndefined(); + // Ingress is NOT empty and NOT omitted: RunMicrovm attaches a PUBLIC + // HTTP_INGRESS connector (with a public endpoint) when the field is absent, + // so "no inbound" is an explicit control on every launch. + expect(JSON.stringify(env.MICROVM_INGRESS_CONNECTOR_ARNS)).toContain('NO_INGRESS'); + expect(JSON.stringify(env.MICROVM_INGRESS_CONNECTOR_ARNS)).not.toContain('HTTP_INGRESS'); + }); + + test('MICROVM_IMAGE_IDENTIFIER is the image ARN, not a bare name', () => { + // RunMicrovm rejects a bare name ("Malformed ARN - doesn't start with + // 'arn:'"), so the identifier the orchestrator receives must be the same ARN + // the lifecycle IAM grant is scoped to. + const fns = template.findResources('AWS::Lambda::Function'); + const [, orchestrator] = Object.entries(fns) + .find(([id]) => id.includes('TaskOrchestratorOrchestratorFn'))!; + const env = orchestrator.Properties.Environment.Variables as Record; + expect(JSON.stringify(env.MICROVM_IMAGE_IDENTIFIER)) + .toMatch(/"Fn::GetAtt":\["LambdaMicrovmComputeImage[^"]*","ImageArn"\]/); + }); + + test('grants the orchestrator exactly the P1 lifecycle actions, image-scoped', () => { + const policies = Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('TaskOrchestrator')); + const statements = policies.flatMap(([, p]) => p.Properties.PolicyDocument.Statement as Array<{ + Sid?: string; + Action: string | string[]; + Resource: unknown; + }>); + + const lifecycle = statements.find(s => s.Sid === 'MicrovmLifecycle')!; + expect(lifecycle.Action).toEqual([ + 'lambda:RunMicrovm', + 'lambda:GetMicrovm', + 'lambda:TerminateMicrovm', + ]); + // Every MicroVM lifecycle action authorizes against the *image* resource, + // which is why "scoped to platform-created images" is achievable at all. + expect(JSON.stringify(lifecycle.Resource)).toMatch( + /"Fn::GetAtt":\["LambdaMicrovmComputeImage[^"]*","ImageArn"\]/, + ); + + // PassNetworkConnector supports no resource-level permissions. + const pass = statements.find(s => s.Sid === 'MicrovmPassNetworkConnector')!; + expect(pass.Action).toBe('lambda:PassNetworkConnector'); + expect(pass.Resource).toBe('*'); + + // iam:PassRole for the execution role hand-off, service-conditioned. + const passRole = statements.find(s => s.Sid === 'MicrovmPassExecutionRole')!; + expect(passRole.Action).toBe('iam:PassRole'); + }); + + test('does NOT grant suspend/resume (P3) or auth-token minting (never)', () => { + const rendered = JSON.stringify(template.toJSON()); + expect(rendered).not.toContain('lambda:SuspendMicrovm'); + expect(rendered).not.toContain('lambda:ResumeMicrovm'); + expect(rendered).not.toContain('lambda:CreateMicrovmAuthToken'); + expect(rendered).not.toContain('lambda:CreateMicrovmShellAuthToken'); + expect(rendered).not.toContain('lambda:ConnectMicrovm'); + }); + + test('orchestrator may WRITE the payload bucket; nothing grants it delete', () => { + const policies = Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('TaskOrchestrator')); + const statements = policies.flatMap(([, p]) => p.Properties.PolicyDocument.Statement as Array<{ + Action: string | string[]; + Resource: unknown; + }>); + const payloadStatements = statements.filter(s => + JSON.stringify(s.Resource).includes('LambdaMicrovmComputePayloadBucket')); + + const actions = payloadStatements.flatMap(s => Array.isArray(s.Action) ? s.Action : [s.Action]); + expect(actions).toContain('s3:PutObject'); + // The bucket's lifecycle rule is the reaper on this backend — unlike the ECS + // path the orchestrator never deletes, so the grant must not exist. + expect(actions).not.toContain('s3:DeleteObject'); + }); + + test('cancel Lambda may terminate a MicroVM (and only terminate), image-scoped', () => { + const policies = Object.entries(template.findResources('AWS::IAM::Policy')) + .filter(([id]) => id.includes('CancelTaskFn')); + const statements = policies.flatMap(([, p]) => p.Properties.PolicyDocument.Statement as Array<{ + Action: string | string[]; + Resource: unknown; + }>); + const microvmStatements = statements.filter(s => + JSON.stringify(s.Action).includes('Microvm')); + + expect(microvmStatements).toHaveLength(1); + expect(microvmStatements[0]!.Action).toBe('lambda:TerminateMicrovm'); + // Resolved through the stack's Lazy.string to the actual image resource + // (TaskApi is built before the MicroVM construct), so the grant names ONE + // image instead of an account/Region-wide `microvm-image:*`. + const rendered = JSON.stringify(microvmStatements[0]!.Resource); + expect(rendered).toMatch(/LambdaMicrovmComputeImage[^"]*","ImageArn"/); + expect(rendered).not.toContain('microvm-image:*'); + }); + + test('MicroVM resources carry the backend cost-allocation tag', () => { + template.hasResourceProperties('AWS::Lambda::MicrovmImage', { + Tags: Match.arrayWith([{ Key: 'abca:compute-backend', Value: 'lambda-microvm' }]), + }); + template.hasResourceProperties('AWS::Lambda::NetworkConnector', { + Tags: Match.arrayWith([{ Key: 'abca:compute-backend', Value: 'lambda-microvm' }]), + }); + }); + + describe('Region gate', () => { + // TEST-CONVENTION EXEMPTION (cdk/AGENTS.md "synth once in beforeAll"): the + // failure case asserts the STACK CONSTRUCTOR throws, so there is no template + // to cache. It is also cheap — the gate runs inside the MicroVM construct + // before any resource is created, and no `Template.fromStack()` is called. + // The success case (escape hatch) does need a template, so it is cached here. + let overriddenTemplate: Template; + + beforeAll(() => { + const app = new App({ + context: { + compute_type: 'lambda-microvm', + microvm_region_override: true, + microvm_base_image_arn: BASE_IMAGE_ARN, + microvm_base_image_version: '1', + }, + }); + overriddenTemplate = Template.fromStack(new AgentStack(app, 'TestAgentStackMicrovmOverride', { + env: { account: '123456789012', region: 'eu-central-1' }, + })); + }); + + test('fails synth when the stack Region has no Lambda MicroVMs', () => { + const app = new App({ context: { compute_type: 'lambda-microvm' } }); + expect(() => new AgentStack(app, 'TestAgentStackMicrovmBadRegion', { + env: { account: '123456789012', region: 'eu-central-1' }, + })).toThrow(/AWS Lambda MicroVMs are not available in eu-central-1/); + }); + + test('the microvm_region_override context flag unblocks an unsupported Region', () => { + overriddenTemplate.resourceCountIs('AWS::Lambda::MicrovmImage', 1); + }); + }); +}); + +describe('AgentStack default (agentcore) deploy — MicroVM substrate absent', () => { + let template: Template; + + beforeAll(() => { + const app = new App(); + const stack = new AgentStack(app, 'TestAgentStackNoMicrovm', { + env: { account: '123456789012', region: 'us-east-1' }, + }); + template = Template.fromStack(stack); + }); + + test('synthesizes no MicroVM resources', () => { + template.resourceCountIs('AWS::Lambda::MicrovmImage', 0); + template.resourceCountIs('AWS::Lambda::NetworkConnector', 0); + }); + + test('injects no MICROVM_* env vars', () => { + const fns = Object.values(template.findResources('AWS::Lambda::Function')); + for (const fn of fns) { + const env = (fn.Properties.Environment?.Variables ?? {}) as Record; + expect(Object.keys(env).filter(k => k.startsWith('MICROVM_'))).toEqual([]); + } + }); + + test('grants no MicroVM IAM actions anywhere', () => { + // Scoped to policy documents rather than the whole template: cdk-nag + // suppression *reasons* legitimately mention the actions in prose. + const statements = Object.values(template.findResources('AWS::IAM::Policy')) + .flatMap(p => p.Properties.PolicyDocument.Statement as Array<{ Action: string | string[] }>); + const actions = statements.flatMap(s => Array.isArray(s.Action) ? s.Action : [s.Action]); + expect(actions.filter(a => a.includes('Microvm'))).toEqual([]); + }); +}); + +describe('AgentStack with the MicroVM gate on but no image configured (first deploy)', () => { + let template: Template; + + beforeAll(() => { + // The bootstrap state: substrate provisioned so the artifact bucket exists, + // but no image yet. Exercises the false branch of the shared + // `isLambdaMicrovmImageConfigured` predicate that gates BOTH the + // orchestrator's MICROVM_* wiring and the cancel Lambda's grant. + const app = new App({ context: { compute_type: 'lambda-microvm' } }); + const stack = new AgentStack(app, 'TestAgentStackMicrovmNoImage', { + env: { account: '123456789012', region: 'us-east-1' }, + }); + template = Template.fromStack(stack); + }); + + test('provisions the substrate (buckets, roles, both connectors) but no image', () => { + template.resourceCountIs('AWS::Lambda::NetworkConnector', 2); + template.resourceCountIs('AWS::Lambda::MicrovmImage', 0); + template.hasOutput('MicrovmArtifactBucketName', {}); + // The build-time connector output is what the packaging script reads next, so + // it must exist in exactly this pre-image state. + template.hasOutput('MicrovmBuildEgressConnectorArns', {}); + }); + + test('grants no MicroVM IAM actions at all — nothing to run or cancel yet', () => { + // Notably this also proves the stack never resolves the image-ARN Lazy in + // this state: doing so would throw "accessed before LambdaMicrovmCompute was + // created"-class errors rather than synthesize. + const actions = Object.values(template.findResources('AWS::IAM::Policy')) + .flatMap(p => p.Properties.PolicyDocument.Statement as Array<{ Action: string | string[] }>) + .flatMap(s => Array.isArray(s.Action) ? s.Action : [s.Action]); + expect(actions.filter(a => a.includes('Microvm'))).toEqual([]); + }); + + test('injects no MICROVM_* env vars (the strategy fails fast with its own remedy)', () => { + const fns = Object.values(template.findResources('AWS::Lambda::Function')); + const keys = fns.flatMap(fn => + Object.keys((fn.Properties.Environment?.Variables ?? {}) as Record)); + expect(keys.filter(k => k.startsWith('MICROVM_'))).toEqual([]); + }); +}); + +describe('AgentStack MicroVM image ARN invariant', () => { + let synthError: unknown; + + beforeAll(() => { + // Force the stack-side gate true while the real construct remains in its + // no-image bootstrap state. This is the only way to exercise the Lazy's + // defensive invariant without changing production behavior. + const configuredSpy = jest.spyOn(lambdaMicrovmCompute, 'isLambdaMicrovmImageConfigured') + .mockReturnValue(true); + try { + const app = new App({ context: { compute_type: 'lambda-microvm' } }); + const stack = new AgentStack(app, 'TestAgentStackMicrovmInvariant', { + env: { account: '123456789012', region: 'us-east-1' }, + }); + Template.fromStack(stack); + } catch (err) { + synthError = err; + } finally { + configuredSpy.mockRestore(); + } + }); + + test('fails synth if a configured deployment has no image ARN', () => { + expect(synthError).toEqual(expect.objectContaining({ + message: expect.stringContaining( + 'MicroVM image ARN was accessed before LambdaMicrovmCompute was created', + ), + })); + }); +}); + describe('AgentStack solution attribution (#319): AWS_SDK_UA_APP_ID via stack-level aspect', () => { let template: Template; diff --git a/cli/package.json b/cli/package.json index 204d749e4..fa076d20a 100644 --- a/cli/package.json +++ b/cli/package.json @@ -41,6 +41,7 @@ "@aws-sdk/client-cloudformation": "3.1078.0", "@aws-sdk/client-cognito-identity-provider": "3.1078.0", "@aws-sdk/client-dynamodb": "3.1078.0", + "@aws-sdk/client-lambda-microvms": "^3.1098.0", "@aws-sdk/client-secrets-manager": "3.1078.0", "@aws-sdk/lib-dynamodb": "3.1078.0", "commander": "^14.0.3" diff --git a/cli/src/commands/repo.ts b/cli/src/commands/repo.ts index 94a58f3d3..626c4cbdb 100644 --- a/cli/src/commands/repo.ts +++ b/cli/src/commands/repo.ts @@ -18,6 +18,7 @@ */ import { Command } from 'commander'; +import { assertComputeSubstrateDeployed } from '../compute-substrate'; import { CliError } from '../errors'; import { DEFAULT_STACK_NAME, redactSecretArn, resolveOperatorContext } from '../operator-context'; import { @@ -152,7 +153,7 @@ export function makeRepoCommand(): Command { .argument('', 'Repository identifier') .option('--region ', 'AWS region (defaults to configured region or AWS_REGION)') .option('--stack-name ', 'CloudFormation stack name', DEFAULT_STACK_NAME) - .option('--compute-type ', 'Compute substrate: agentcore or ecs') + .option('--compute-type ', 'Compute substrate: agentcore, ecs, or lambda-microvm') .option('--runtime-arn ', 'Override AgentCore runtime ARN (agentcore only)') .option('--model ', 'Foundation model ID override') .option('--token-secret-arn ', 'Per-repo GitHub token Secrets Manager ARN') @@ -161,8 +162,11 @@ export function makeRepoCommand(): Command { .option('--output ', 'Output format: text or json', 'text') .action(async (repoId: string, opts) => { assertRepoFormat(repoId); - if (opts.computeType && opts.computeType !== 'agentcore' && opts.computeType !== 'ecs') { - throw new CliError("--compute-type must be 'agentcore' or 'ecs'."); + if (opts.computeType + && opts.computeType !== 'agentcore' + && opts.computeType !== 'ecs' + && opts.computeType !== 'lambda-microvm') { + throw new CliError("--compute-type must be 'agentcore', 'ecs', or 'lambda-microvm'."); } const { region, stackName } = resolveOperatorContext(opts); @@ -177,21 +181,27 @@ export function makeRepoCommand(): Command { `Stack '${stackName}' is missing output 'RepoTableName'. Re-deploy the CDK stack.`, ); } - // Refuse to onboard a repo as compute_type=ecs when the deployed stack did - // NOT provision the ECS substrate — otherwise every task on this repo fails - // at session start with "ECS compute strategy requires ECS_CLUSTER_ARN…". - // Catch it here, at config time, with a fixable message. ComputeSubstrate is - // null on stacks predating this output; treat that as "unknown" and only - // hard-block on an explicit non-ecs value, so onboarding still works against - // an older deploy (the runtime error remains the backstop there). - if (opts.computeType === 'ecs' && computeSubstrate && computeSubstrate !== 'ecs') { - throw new CliError( - `Stack '${stackName}' was deployed without the ECS substrate (ComputeSubstrate=${computeSubstrate}), ` - + 'so a repo onboarded as --compute-type ecs would fail at task start. Redeploy the stack with ' - + '`--context compute_type=ecs` first (adds the Fargate substrate alongside AgentCore), then re-run this — ' - + 'or onboard with --compute-type agentcore.', - ); - } + // Refuse to onboard a repo onto a compute backend the deployed stack did + // NOT provision — otherwise every task on this repo fails at session + // start ("ECS compute strategy requires ECS_CLUSTER_ARN…" for ecs, or the + // MicroVM strategy's "deployed without the Lambda MicroVMs substrate" for + // lambda-microvm). Catch it here, at config time, with a fixable message. + // + // ORDERING — this runs BEFORE `onboardRepo`, and that is deliberate: + // `onboardRepo` performs the live `ListManagedMicrovmImages` regional + // availability probe for lambda-microvm. The substrate gate is both + // CHEAPER (it reuses the `ComputeSubstrate` output already fetched in the + // Promise.all above — zero extra API calls, no extra IAM) and MORE + // SPECIFIC (a stack with no MicroVM substrate cannot run the backend even + // in a Region that supports it, whereas the reverse cannot happen: the + // synth-time Region gate means a stack carrying the MicroVM substrate is + // already in a supported Region). Reporting "this stack has no MicroVM + // substrate" beats reporting "MicroVMs are unavailable in this Region" + // when both are true — the first names the actual fix. + // + // See `assertComputeSubstrateDeployed` for the ComputeSubstrate output's + // exact semantics (single-valued today, list-tolerant by construction). + assertComputeSubstrateDeployed({ stackName, computeType: opts.computeType, computeSubstrate }); const config = await onboardRepo(region, tableName, repoId, { computeType: opts.computeType, diff --git a/cli/src/commands/runtime.ts b/cli/src/commands/runtime.ts index 61795a26f..78088cb85 100644 --- a/cli/src/commands/runtime.ts +++ b/cli/src/commands/runtime.ts @@ -82,7 +82,9 @@ export function makeRuntimeCommand(): Command { ? `${b.runtime_arn} (${b.runtime_arn_source})` : b.compute_type === 'ecs' ? '(n/a — ECS uses platform cluster)' - : '(missing)'; + : b.compute_type === 'lambda-microvm' + ? '(n/a — Lambda MicroVMs are platform-managed)' + : '(missing)'; console.log( `${b.repo.padEnd(REPO_WIDTH)} ${b.status.padEnd(10)} ` + `${b.compute_type.padEnd(COMPUTE_WIDTH)} ${runtimeLabel}`, @@ -112,6 +114,14 @@ export function makeRuntimeCommand(): Command { console.log(` note: ${ecs.note}`); } } + + if ((report.lambda_microvm_substrates ?? []).length > 0) { + console.log('\nLambda MicroVM compute substrates:'); + for (const microvm of report.lambda_microvm_substrates ?? []) { + console.log(` repos: ${microvm.used_by_repos.join(', ')}`); + console.log(` note: ${microvm.note}`); + } + } }), ); diff --git a/cli/src/compute-substrate.ts b/cli/src/compute-substrate.ts new file mode 100644 index 000000000..7572d0989 --- /dev/null +++ b/cli/src/compute-substrate.ts @@ -0,0 +1,143 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { CliError } from './errors'; + +/** Per-repo compute backend, mirrored from `cdk/src/handlers/shared/repo-config.ts`. */ +export type OnboardComputeType = 'agentcore' | 'ecs' | 'lambda-microvm'; + +/** + * Parse the stack's `ComputeSubstrate` output into the set of OPTIONAL substrates + * the deploy provisioned. + * + * ## What the output actually contains today: ONE value + * + * `cdk/src/stacks/agent.ts` emits + * `ecsCluster ? 'ecs' : (lambdaMicrovm ? 'lambda-microvm' : 'agentcore')`, and both + * constructs are gated on the SAME single-valued `compute_type` deploy context + * (`--context compute_type=…`). So the two optional backends are **mutually + * exclusive today** — a mixed `ecs` + `lambda-microvm` deploy is not + * expressible, which is why that ternary can never have to arbitrate, and why + * the CDK test asserts "does NOT provision the ECS substrate (the gates are + * mutually exclusive)" on a MicroVM stack. + * + * The three reachable values are therefore: + * + * | Output | Substrates available to tasks | + * |---|---| + * | `agentcore` | AgentCore only | + * | `ecs` | AgentCore **and** ECS (the optional backends are additive) | + * | `lambda-microvm` | AgentCore **and** Lambda MicroVMs | + * + * ## Why this parses a LIST anyway + * + * ADR-021 sub-decision 4 explicitly flags the single-value tag as "already + * imprecise with two backends, wrong with three" and names a `compute_types` list + * as the intended follow-up. If that lands, a stack would emit + * `ecs,lambda-microvm` — and an `!== 'ecs'` equality check would then start + * REFUSING valid onboardings, silently, in the safe-looking direction. Splitting + * on commas makes that future value work correctly with no change here, while + * being byte-identical in behaviour for the single values above. + * + * @param raw - the raw `ComputeSubstrate` output value, or null when absent. + * @returns the provisioned substrate names, or `undefined` when the output is + * missing/blank (an older stack predating the output — "unknown", not "none"). + */ +export function parseComputeSubstrateOutput( + raw: string | null | undefined, +): readonly string[] | undefined { + if (raw === null || raw === undefined) { + return undefined; + } + const values = raw.split(',').map((value) => value.trim()).filter(Boolean); + return values.length > 0 ? values : undefined; +} + +/** Backend-specific remedy copy for {@link assertComputeSubstrateDeployed}. */ +const SUBSTRATE_REMEDIES: Record, { + readonly label: string; + readonly context: string; + readonly adds: string; + readonly runtimeFailure: string; +}> = { + 'ecs': { + label: 'ECS', + context: '`--context compute_type=ecs`', + adds: 'adds the Fargate substrate alongside AgentCore', + runtimeFailure: 'fail at task start', + }, + 'lambda-microvm': { + label: 'Lambda MicroVMs', + context: '`--context compute_type=lambda-microvm`', + adds: 'adds the Lambda MicroVMs substrate alongside AgentCore', + // More specific than the ECS wording because the MicroVM failure surfaces + // later and less legibly: the strategy's own env-var guard fires first if no + // image is configured, and otherwise RunMicrovm rejects the call. + runtimeFailure: 'fail at session start (no MICROVM_* configuration on the orchestrator)', + }, +}; + +/** + * Refuse to onboard a repo onto a compute backend the deployed stack never + * provisioned. + * + * Without this the row is written happily and every task on that repo dies at + * session start — for `ecs` with "ECS compute strategy requires ECS_CLUSTER_ARN…", + * for `lambda-microvm` with the strategy's "deployed without the Lambda MicroVMs + * substrate" error (or, if an image somehow IS configured, a `RunMicrovm` + * rejection). Catching it here turns a per-task runtime failure into one + * config-time message with a fixable remedy. + * + * Two deliberate non-strictnesses, both carried over from the original ECS check: + * + * - **`agentcore` is never gated.** The AgentCore runtime is unconditional; the + * other two backends are additive on top of it. + * - **An absent output means "unknown", not "none".** Stacks deployed before + * `ComputeSubstrate` existed return null, and hard-blocking there would break + * onboarding against a perfectly good older deploy. The runtime error remains + * the backstop in that case. + * + * @param args.stackName - stack the outputs were read from, for the message. + * @param args.computeType - the backend the operator asked for. + * @param args.computeSubstrate - raw `ComputeSubstrate` stack output (or null). + * @throws CliError when the requested backend is definitely not deployed. + */ +export function assertComputeSubstrateDeployed(args: { + stackName: string; + computeType: OnboardComputeType | undefined; + computeSubstrate: string | null | undefined; +}): void { + const { stackName, computeType, computeSubstrate } = args; + if (!computeType || computeType === 'agentcore') { + return; + } + + const provisioned = parseComputeSubstrateOutput(computeSubstrate); + if (!provisioned || provisioned.includes(computeType)) { + return; + } + + const remedy = SUBSTRATE_REMEDIES[computeType]; + throw new CliError( + `Stack '${stackName}' was deployed without the ${remedy.label} substrate ` + + `(ComputeSubstrate=${computeSubstrate}), so a repo onboarded as --compute-type ${computeType} ` + + `would ${remedy.runtimeFailure}. Redeploy the stack with ${remedy.context} first ` + + `(${remedy.adds}), then re-run this — or onboard with --compute-type agentcore.`, + ); +} diff --git a/cli/src/lambda-microvm-availability.ts b/cli/src/lambda-microvm-availability.ts new file mode 100644 index 000000000..602c54de8 --- /dev/null +++ b/cli/src/lambda-microvm-availability.ts @@ -0,0 +1,69 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { + LambdaMicrovmsClient, + ListManagedMicrovmImagesCommand, +} from '@aws-sdk/client-lambda-microvms'; +import { CliError } from './errors'; +import { makeClient } from './ua'; + +/** Informational copy; keep in sync with cdk/src/handlers/shared/microvm-regions.ts. */ +const LAMBDA_MICROVM_LAUNCH_REGIONS: readonly string[] = [ + 'us-east-1', + 'us-east-2', + 'us-west-2', + 'eu-west-1', + 'ap-northeast-1', +]; + +interface LambdaMicrovmProbeClient { + send(command: ListManagedMicrovmImagesCommand): Promise; +} + +export type LambdaMicrovmProbeClientFactory = (region: string) => LambdaMicrovmProbeClient; + +export const LAMBDA_MICROVM_REMEDY = + `Launch regions: ${LAMBDA_MICROVM_LAUNCH_REGIONS.join(', ')}. ` + + 'Use --compute-type agentcore or --compute-type ecs in other regions.'; + +/** Probe the read-only image catalog used to detect regional service availability. */ +export async function probeLambdaMicrovmAvailability( + region: string, + clientFactory: LambdaMicrovmProbeClientFactory = + (clientRegion) => makeClient(LambdaMicrovmsClient, { region: clientRegion }), +): Promise { + const client = clientFactory(region); + await client.send(new ListManagedMicrovmImagesCommand({})); +} + +/** Probe and convert any SDK/endpoint failure into an operator-actionable error. */ +export async function requireLambdaMicrovmAvailability( + region: string, + clientFactory?: LambdaMicrovmProbeClientFactory, +): Promise { + try { + await probeLambdaMicrovmAvailability(region, clientFactory); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new CliError( + `Lambda MicroVMs availability probe failed in ${region}: ${message}. ${LAMBDA_MICROVM_REMEDY}`, + ); + } +} diff --git a/cli/src/platform-doctor.ts b/cli/src/platform-doctor.ts index b68b816cf..1fad9f1e8 100644 --- a/cli/src/platform-doctor.ts +++ b/cli/src/platform-doctor.ts @@ -24,9 +24,14 @@ import { DescribeUserPoolCommand, } from '@aws-sdk/client-cognito-identity-provider'; import { isGithubTokenConfigured } from './github-token'; +import { + LAMBDA_MICROVM_REMEDY, + LambdaMicrovmProbeClientFactory, + probeLambdaMicrovmAvailability, +} from './lambda-microvm-availability'; import { checkLinearWorkspaceAuth, type LinearProbe, type LinearRefreshVerifier } from './linear-auth-health'; import { PLATFORM_REPO_DEFAULTS } from './repo-display'; -import { countActiveRepos } from './repo-lookup'; +import { listRepoConfigs, RepoConfigRow } from './repo-lookup'; import { getStackOutput } from './stack-outputs'; import { makeClient } from './ua'; @@ -38,7 +43,7 @@ import { makeClient } from './ua'; * (`us.anthropic.…`) used at invoke time, while `GetFoundationModel` requires * the bare foundation-model id, so we strip the regional inference prefix. */ -export const DEFAULT_BEDROCK_MODEL_ID = +const DEFAULT_BEDROCK_MODEL_ID = PLATFORM_REPO_DEFAULTS.model_id.replace(/^(us|eu|apac)\./, ''); export type DoctorCheckStatus = 'pass' | 'fail' | 'warn'; @@ -53,6 +58,8 @@ export interface DoctorCheckResult { export interface RunPlatformDoctorOptions { readonly region: string; readonly stackName: string; + /** Override for deterministic/offline tests. */ + readonly lambdaMicrovmClientFactory?: LambdaMicrovmProbeClientFactory; /** Injectable Linear auth probe (tests supply a fake; production uses the default). */ readonly linearProbe?: LinearProbe; /** @@ -89,8 +96,15 @@ export async function runPlatformDoctor( checks.push(await checkApiReachable(apiUrl)); checks.push(await checkCognitoConfig(region, userPoolId, appClientId)); checks.push(await checkGithubToken(region, githubTokenSecretArn)); - checks.push(await checkActiveRepos(region, repoTableName)); + const activeRepoResult = await loadActiveRepos(region, repoTableName); + checks.push(checkActiveRepos(repoTableName, activeRepoResult)); checks.push(await checkBedrockModel(region, DEFAULT_BEDROCK_MODEL_ID)); + if (activeRepoResult.repos.some((repo) => repo.compute_type === 'lambda-microvm')) { + checks.push(await checkLambdaMicrovmAvailability( + region, + options.lambdaMicrovmClientFactory, + )); + } checks.push(await checkLinearAuth( region, linearRegistryTableName, options.linearProbe, options.linearVerifyRefresh, )); @@ -192,33 +206,73 @@ async function checkGithubToken( }; } -async function checkActiveRepos( +async function loadActiveRepos( region: string, repoTableName: string | null, -): Promise { +): Promise<{ readonly repos: RepoConfigRow[]; readonly error?: string }> { + if (!repoTableName) return { repos: [] }; + try { + return { + repos: (await listRepoConfigs(region, repoTableName)) + .filter((repo) => repo.status === 'active'), + }; + } catch (err) { + return { repos: [], error: err instanceof Error ? err.message : String(err) }; + } +} + +function checkActiveRepos( + repoTableName: string | null, + result: { readonly repos: readonly RepoConfigRow[]; readonly error?: string }, +): DoctorCheckResult { const id = 'active_repos'; const label = 'At least one active onboarded repo'; if (!repoTableName) { return { id, label, status: 'fail', detail: 'Stack output RepoTableName is missing.' }; } + if (result.error) { + return { id, label, status: 'fail', detail: result.error }; + } + + const count = result.repos.length; + if (count >= 1) { + return { id, label, status: 'pass', detail: `${count} active repo(s) in ${repoTableName}.` }; + } + return { + id, + label, + status: 'fail', + detail: 'No active repos in RepoTable. Register a Blueprint and redeploy.', + }; +} + +async function checkLambdaMicrovmAvailability( + region: string, + clientFactory?: LambdaMicrovmProbeClientFactory, +): Promise { + const id = 'lambda_microvm_availability'; + const label = `Lambda MicroVMs service (${region})`; try { - const count = await countActiveRepos(region, repoTableName); - if (count >= 1) { - return { id, label, status: 'pass', detail: `${count} active repo(s) in ${repoTableName}.` }; - } + await probeLambdaMicrovmAvailability(region, clientFactory); return { id, label, - status: 'fail', - detail: 'No active repos in RepoTable. Register a Blueprint and redeploy.', + status: 'pass', + detail: `Managed MicroVM images are available in ${region}.`, }; } catch (err) { + const message = err instanceof Error ? err.message : String(err); + const errorName = err instanceof Error ? err.name : ''; + const accessDenied = /AccessDenied|Unauthorized|not authorized/i.test(`${errorName} ${message}`); return { id, label, - status: 'fail', - detail: err instanceof Error ? err.message : String(err), + status: accessDenied ? 'warn' : 'fail', + detail: accessDenied + ? `${message}. Cannot verify Lambda MicroVM availability in ${region}; check IAM permissions for ` + + 'lambda-microvms List* actions.' + : `${message}. ${LAMBDA_MICROVM_REMEDY}`, }; } } diff --git a/cli/src/repo-lookup.ts b/cli/src/repo-lookup.ts index 529997c30..7301d5fdc 100644 --- a/cli/src/repo-lookup.ts +++ b/cli/src/repo-lookup.ts @@ -149,9 +149,3 @@ export async function listRepoConfigs( return items.sort((a, b) => a.repo.localeCompare(b.repo)); } - -/** Count active repos in RepoTable. */ -export async function countActiveRepos(region: string, tableName: string): Promise { - const repos = await listRepoConfigs(region, tableName); - return repos.filter((r) => r.status === 'active').length; -} diff --git a/cli/src/repo-onboard-notes.ts b/cli/src/repo-onboard-notes.ts index bcaa48207..f6c1e9c3c 100644 --- a/cli/src/repo-onboard-notes.ts +++ b/cli/src/repo-onboard-notes.ts @@ -59,5 +59,25 @@ export function buildRepoOnboardNotes(input: RepoOnboardNotesInput): readonly st ); } + if (input.config.compute_type === 'lambda-microvm') { + // Mirrors the ECS note, plus the one thing that has no ECS analogue: the + // substrate can be fully deployed and still carry no IMAGE (ADR-021's + // three-state table — the artifact bucket must exist before the artifact can + // be uploaded), in which case the orchestrator gets no MICROVM_* env block at + // all and tasks fail with the strategy's own remedy. The `bgagent repo + // onboard` substrate gate cannot see that: `ComputeSubstrate` says the + // backend was provisioned, not that an image was configured. + notes.push( + 'NOTE: compute_type=lambda-microvm requires the MicroVM substrate wired into the stack ' + + '(TaskOrchestrator microvmConfig) AND a MicroVM image configured. A stack deployed with ' + + '--context compute_type=lambda-microvm but no image yet (the expected FIRST deploy) still ' + + 'rejects tasks: package the artifact with cdk/scripts/package-microvm-artifact.sh, then ' + + 'redeploy with --context microvm_base_image_arn= --context microvm_base_image_version= ' + + '(or --context microvm_image_identifier=).', + 'NOTE: the lambda-microvm backend has no smoke-parity guarantee yet (ADR-021 P1 — synth emits ' + + 'abca:microvm-image-p1-smoke-unverified). Keep production repos on agentcore or ecs until P2.', + ); + } + return notes; } diff --git a/cli/src/repo-onboard.ts b/cli/src/repo-onboard.ts index 40825fd4c..4e30009fd 100644 --- a/cli/src/repo-onboard.ts +++ b/cli/src/repo-onboard.ts @@ -19,6 +19,10 @@ import { PutCommand, UpdateCommand } from '@aws-sdk/lib-dynamodb'; import { documentClient } from './dynamo-clients'; +import { + LambdaMicrovmProbeClientFactory, + requireLambdaMicrovmAvailability, +} from './lambda-microvm-availability'; import { assertRepoFormat, loadRepoConfig, @@ -31,7 +35,7 @@ import { export const REMOVED_REPO_TTL_DAYS = 30; export interface OnboardRepoOptions { - readonly computeType?: 'agentcore' | 'ecs'; + readonly computeType?: 'agentcore' | 'ecs' | 'lambda-microvm'; readonly runtimeArn?: string; readonly modelId?: string; readonly maxTurns?: number; @@ -39,12 +43,18 @@ export interface OnboardRepoOptions { readonly pollIntervalMs?: number; } +export interface OnboardRepoDependencies { + /** Override for deterministic/offline tests; production uses the regional AWS client. */ + readonly lambdaMicrovmClientFactory?: LambdaMicrovmProbeClientFactory; +} + /** Register or re-activate a repository in RepoTable (operator path). */ export async function onboardRepo( region: string, tableName: string, repo: string, options: OnboardRepoOptions = {}, + dependencies: OnboardRepoDependencies = {}, ): Promise { assertRepoFormat(repo); const now = new Date().toISOString(); @@ -63,6 +73,11 @@ export async function onboardRepo( existing = undefined; } + const effectiveComputeType = options.computeType ?? existing?.compute_type ?? 'agentcore'; + if (effectiveComputeType === 'lambda-microvm') { + await requireLambdaMicrovmAvailability(region, dependencies.lambdaMicrovmClientFactory); + } + const item: Record = { repo, status: 'active', diff --git a/cli/src/runtime-status.ts b/cli/src/runtime-status.ts index f1bfccc88..53e2776de 100644 --- a/cli/src/runtime-status.ts +++ b/cli/src/runtime-status.ts @@ -25,7 +25,7 @@ import { PLATFORM_REPO_DEFAULTS } from './repo-display'; import { listRepoConfigs, RepoConfigRow } from './repo-lookup'; import { makeClient } from './ua'; -export interface BlueprintRuntimeBinding { +interface BlueprintRuntimeBinding { readonly repo: string; readonly status: RepoConfigRow['status']; readonly compute_type: string; @@ -45,7 +45,7 @@ interface RuntimeProbeBase { * carries the GetAgentRuntime response, an `'error'` probe carries only the * failure message. */ -export type RuntimeProbeResult = +type RuntimeProbeResult = | (RuntimeProbeBase & { readonly probe_status: 'ok'; readonly agent_runtime_id?: string; @@ -59,17 +59,24 @@ export type RuntimeProbeResult = readonly error: string; }); -export interface EcsSubstrateSummary { +interface EcsSubstrateSummary { readonly compute_type: 'ecs'; readonly used_by_repos: readonly string[]; readonly note: string; } +interface LambdaMicrovmSubstrateSummary { + readonly compute_type: 'lambda-microvm'; + readonly used_by_repos: readonly string[]; + readonly note: string; +} + export interface RuntimeStatusReport { readonly platform_default_runtime_arn: string | null; readonly blueprints: readonly BlueprintRuntimeBinding[]; readonly agentcore_runtimes: readonly RuntimeProbeResult[]; readonly ecs_substrates: readonly EcsSubstrateSummary[]; + readonly lambda_microvm_substrates: readonly LambdaMicrovmSubstrateSummary[]; } /** Parse ``agentRuntimeId`` (and optional version) from an AgentCore runtime ARN. */ @@ -91,7 +98,9 @@ function bindingForRepo( ): BlueprintRuntimeBinding { const computeType = config.compute_type ?? PLATFORM_REPO_DEFAULTS.compute_type; const hasBlueprintRuntime = config.runtime_arn !== undefined; - const runtimeArn = hasBlueprintRuntime ? config.runtime_arn : platformRuntimeArn ?? undefined; + const runtimeArn = computeType === 'agentcore' + ? hasBlueprintRuntime ? config.runtime_arn : platformRuntimeArn ?? undefined + : undefined; return { repo: config.repo, @@ -155,6 +164,7 @@ export async function buildRuntimeStatusReport( const agentcoreMap = new Map(); const ecsRepos: string[] = []; + const lambdaMicrovmRepos: string[] = []; for (const binding of blueprints) { if (binding.status !== 'active') continue; @@ -162,6 +172,10 @@ export async function buildRuntimeStatusReport( ecsRepos.push(binding.repo); continue; } + if (binding.compute_type === 'lambda-microvm') { + lambdaMicrovmRepos.push(binding.repo); + continue; + } if (binding.runtime_arn) { const list = agentcoreMap.get(binding.runtime_arn) ?? []; list.push(binding.repo); @@ -183,10 +197,20 @@ export async function buildRuntimeStatusReport( }] : []; + const lambda_microvm_substrates: LambdaMicrovmSubstrateSummary[] = lambdaMicrovmRepos.length > 0 + ? [{ + compute_type: 'lambda-microvm', + used_by_repos: lambdaMicrovmRepos, + note: 'Lambda MicroVM sessions use platform-managed images and per-session MicroVMs. ' + + 'Per-blueprint runtime_arn overrides do not apply to Lambda MicroVM compute.', + }] + : []; + return { platform_default_runtime_arn: platformRuntimeArn, blueprints, agentcore_runtimes, ecs_substrates, + lambda_microvm_substrates, }; } diff --git a/cli/test/commands/platform.test.ts b/cli/test/commands/platform.test.ts index f362e07e2..abee5db29 100644 --- a/cli/test/commands/platform.test.ts +++ b/cli/test/commands/platform.test.ts @@ -25,6 +25,7 @@ import * as stackOutputs from '../../src/stack-outputs'; const cognitoSend = jest.fn(); const bedrockSend = jest.fn(); +const microvmSend = jest.fn(); jest.mock('@aws-sdk/client-cognito-identity-provider', () => { const actual = jest.requireActual('@aws-sdk/client-cognito-identity-provider'); @@ -42,6 +43,14 @@ jest.mock('@aws-sdk/client-bedrock', () => { }; }); +jest.mock('@aws-sdk/client-lambda-microvms', () => { + const actual = jest.requireActual('@aws-sdk/client-lambda-microvms'); + return { + ...actual, + LambdaMicrovmsClient: jest.fn(() => ({ send: microvmSend })), + }; +}); + jest.mock('../../src/github-token', () => { const actual = jest.requireActual('../../src/github-token'); return { @@ -54,13 +63,13 @@ jest.mock('../../src/repo-lookup', () => { const actual = jest.requireActual('../../src/repo-lookup'); return { ...actual, - countActiveRepos: jest.fn(), + listRepoConfigs: jest.fn(), }; }); const getStackOutputSpy = jest.spyOn(stackOutputs, 'getStackOutput'); const isGithubTokenConfiguredMock = githubToken.isGithubTokenConfigured as jest.Mock; -const countActiveReposMock = repoLookup.countActiveRepos as jest.Mock; +const listRepoConfigsMock = repoLookup.listRepoConfigs as jest.Mock; const originalFetch = global.fetch; function mockStackOutputs(): void { @@ -80,9 +89,10 @@ describe('runPlatformDoctor', () => { beforeEach(() => { getStackOutputSpy.mockReset(); isGithubTokenConfiguredMock.mockReset(); - countActiveReposMock.mockReset(); + listRepoConfigsMock.mockReset(); cognitoSend.mockReset().mockResolvedValue({}); bedrockSend.mockReset().mockResolvedValue({}); + microvmSend.mockReset().mockResolvedValue({ images: [] }); global.fetch = jest.fn().mockResolvedValue({ status: 401, ok: false }); }); @@ -93,7 +103,10 @@ describe('runPlatformDoctor', () => { test('returns pass when all checks succeed', async () => { mockStackOutputs(); isGithubTokenConfiguredMock.mockResolvedValue(true); - countActiveReposMock.mockResolvedValue(2); + listRepoConfigsMock.mockResolvedValue([ + { repo: 'acme/a', status: 'active' }, + { repo: 'acme/b', status: 'active' }, + ]); const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); expect(doctorChecksPassed(results)).toBe(true); @@ -107,7 +120,7 @@ describe('runPlatformDoctor', () => { test('fails when github token is not configured', async () => { mockStackOutputs(); isGithubTokenConfiguredMock.mockResolvedValue(false); - countActiveReposMock.mockResolvedValue(1); + listRepoConfigsMock.mockResolvedValue([{ repo: 'acme/a', status: 'active' }]); const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); expect(doctorChecksPassed(results)).toBe(false); @@ -117,13 +130,138 @@ describe('runPlatformDoctor', () => { test('warns when API returns an unexpected status code', async () => { mockStackOutputs(); isGithubTokenConfiguredMock.mockResolvedValue(true); - countActiveReposMock.mockResolvedValue(1); + listRepoConfigsMock.mockResolvedValue([{ repo: 'acme/a', status: 'active' }]); (global.fetch as jest.Mock).mockResolvedValue({ status: 500, ok: false }); const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); expect(results.find((r) => r.id === 'api_reachable')?.status).toBe('warn'); expect(doctorChecksPassed(results)).toBe(true); }); + + test('fails the active-repo check when the repo table output is missing', async () => { + mockStackOutputs(); + getStackOutputSpy.mockImplementation(async (_region, _stack, key) => + key === 'RepoTableName' ? null : ({ + ApiUrl: 'https://api.example/v1/', + UserPoolId: 'us-east-1_pool', + AppClientId: 'client123', + GitHubTokenSecretArn: 'arn:token', + }[key] ?? null)); + isGithubTokenConfiguredMock.mockResolvedValue(true); + + const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); + + expect(results.find((r) => r.id === 'active_repos')).toEqual(expect.objectContaining({ + status: 'fail', + detail: 'Stack output RepoTableName is missing.', + })); + expect(listRepoConfigsMock).not.toHaveBeenCalled(); + }); + + test('fails the active-repo check when the repo table has no active rows', async () => { + mockStackOutputs(); + isGithubTokenConfiguredMock.mockResolvedValue(true); + listRepoConfigsMock.mockResolvedValue([{ repo: 'acme/removed', status: 'removed' }]); + + const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); + + expect(results.find((r) => r.id === 'active_repos')).toEqual(expect.objectContaining({ + status: 'fail', + detail: 'No active repos in RepoTable. Register a Blueprint and redeploy.', + })); + }); + + test('reports a non-Error repo lookup failure', async () => { + mockStackOutputs(); + isGithubTokenConfiguredMock.mockResolvedValue(true); + listRepoConfigsMock.mockRejectedValue('DynamoDB unavailable'); + + const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); + + expect(results.find((r) => r.id === 'active_repos')).toEqual(expect.objectContaining({ + status: 'fail', + detail: 'DynamoDB unavailable', + })); + }); + + test('probes Lambda MicroVMs only when an active blueprint uses it', async () => { + mockStackOutputs(); + isGithubTokenConfiguredMock.mockResolvedValue(true); + listRepoConfigsMock.mockResolvedValue([ + { repo: 'acme/a', status: 'active', compute_type: 'lambda-microvm' }, + { repo: 'acme/removed', status: 'removed', compute_type: 'lambda-microvm' }, + ]); + + const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); + + expect(results.find((r) => r.id === 'lambda_microvm_availability')?.status).toBe('pass'); + expect(microvmSend).toHaveBeenCalledTimes(1); + }); + + test('omits Lambda MicroVM check when no active blueprint uses it', async () => { + mockStackOutputs(); + isGithubTokenConfiguredMock.mockResolvedValue(true); + listRepoConfigsMock.mockResolvedValue([ + { repo: 'acme/a', status: 'active', compute_type: 'agentcore' }, + { repo: 'acme/removed', status: 'removed', compute_type: 'lambda-microvm' }, + ]); + + const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); + + expect(results.some((r) => r.id === 'lambda_microvm_availability')).toBe(false); + expect(microvmSend).not.toHaveBeenCalled(); + }); + + test('reports Lambda MicroVM probe failure with remedy', async () => { + mockStackOutputs(); + isGithubTokenConfiguredMock.mockResolvedValue(true); + listRepoConfigsMock.mockResolvedValue([ + { repo: 'acme/a', status: 'active', compute_type: 'lambda-microvm' }, + ]); + microvmSend.mockRejectedValue(new Error('endpoint not found')); + + const results = await runPlatformDoctor({ region: 'eu-central-1', stackName: 'dev' }); + const check = results.find((r) => r.id === 'lambda_microvm_availability'); + + expect(check?.status).toBe('fail'); + expect(check?.detail).toContain('Launch regions: us-east-1'); + expect(check?.detail).toContain('--compute-type agentcore'); + }); + + test('warns when IAM prevents the Lambda MicroVM availability check', async () => { + mockStackOutputs(); + isGithubTokenConfiguredMock.mockResolvedValue(true); + listRepoConfigsMock.mockResolvedValue([ + { repo: 'acme/a', status: 'active', compute_type: 'lambda-microvm' }, + ]); + microvmSend.mockRejectedValue(Object.assign( + new Error('User is not authorized to perform lambda-microvms:ListManagedMicrovmImages'), + { name: 'AccessDeniedException' }, + )); + + const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); + const check = results.find((r) => r.id === 'lambda_microvm_availability'); + + expect(check?.status).toBe('warn'); + expect(check?.detail).toContain('Cannot verify Lambda MicroVM availability'); + expect(check?.detail).toContain('lambda-microvms List* actions'); + expect(doctorChecksPassed(results)).toBe(true); + }); + + test('formats a non-Error Lambda MicroVM probe failure', async () => { + mockStackOutputs(); + isGithubTokenConfiguredMock.mockResolvedValue(true); + listRepoConfigsMock.mockResolvedValue([ + { repo: 'acme/a', status: 'active', compute_type: 'lambda-microvm' }, + ]); + microvmSend.mockRejectedValue('service unavailable'); + + const results = await runPlatformDoctor({ region: 'us-east-1', stackName: 'dev' }); + const check = results.find((r) => r.id === 'lambda_microvm_availability'); + + expect(check?.status).toBe('fail'); + expect(check?.detail).toContain('service unavailable'); + }); }); describe('doctorChecksPassed', () => { diff --git a/cli/test/commands/repo-onboard-command.test.ts b/cli/test/commands/repo-onboard-command.test.ts index db0a9ac0b..56524c510 100644 --- a/cli/test/commands/repo-onboard-command.test.ts +++ b/cli/test/commands/repo-onboard-command.test.ts @@ -104,7 +104,20 @@ describe('repo onboard/offboard commands', () => { 'node', 'test', 'onboard', 'acme/a', '--region', 'us-east-1', '--compute-type', 'lambda', - ])).rejects.toThrow("compute-type must be 'agentcore' or 'ecs'"); + ])).rejects.toThrow("compute-type must be 'agentcore', 'ecs', or 'lambda-microvm'"); + }); + + test('repo onboard accepts lambda-microvm compute type', async () => { + const cmd = makeRepoCommand(); + await cmd.parseAsync([ + 'node', 'test', 'onboard', 'acme/a', + '--region', 'us-east-1', + '--compute-type', 'lambda-microvm', + ]); + + expect((onboardRepo as jest.Mock).mock.calls[0][3]).toEqual( + expect.objectContaining({ computeType: 'lambda-microvm' }), + ); }); test('repo offboard invokes offboardRepo', async () => { diff --git a/cli/test/commands/repo-onboard-notes.test.ts b/cli/test/commands/repo-onboard-notes.test.ts index fc3396091..386d5089f 100644 --- a/cli/test/commands/repo-onboard-notes.test.ts +++ b/cli/test/commands/repo-onboard-notes.test.ts @@ -72,4 +72,32 @@ describe('buildRepoOnboardNotes', () => { expect(notes.some((n) => n.includes('ecsConfig'))).toBe(true); }); + + test('notes lambda-microvm compute requirement, including the image step', () => { + const notes = buildRepoOnboardNotes({ + config: { repo: 'acme/a', status: 'active', compute_type: 'lambda-microvm' }, + platformRuntimeArn: PLATFORM_RUNTIME, + platformGithubTokenSecretArn: PLATFORM_SECRET, + }); + + expect(notes.some((n) => n.includes('microvmConfig'))).toBe(true); + // The substrate gate in `repo onboard` can prove the substrate exists but NOT + // that an image is configured — a deployed-but-imageless stack still rejects + // every task, so the note has to carry the packaging remedy. + expect(notes.some((n) => n.includes('package-microvm-artifact.sh'))).toBe(true); + expect(notes.some((n) => n.includes('microvm_base_image_arn'))).toBe(true); + // ...and the P1 honesty warning, matching the synth-time annotation id. + expect(notes.some((n) => n.includes('abca:microvm-image-p1-smoke-unverified'))).toBe(true); + }); + + test('emits no compute note for agentcore', () => { + const notes = buildRepoOnboardNotes({ + config: { repo: 'acme/a', status: 'active', compute_type: 'agentcore' }, + platformRuntimeArn: PLATFORM_RUNTIME, + platformGithubTokenSecretArn: PLATFORM_SECRET, + }); + + expect(notes.some((n) => n.includes('ecsConfig'))).toBe(false); + expect(notes.some((n) => n.includes('microvmConfig'))).toBe(false); + }); }); diff --git a/cli/test/commands/repo-onboard.test.ts b/cli/test/commands/repo-onboard.test.ts index d91dc85da..d6114e3a7 100644 --- a/cli/test/commands/repo-onboard.test.ts +++ b/cli/test/commands/repo-onboard.test.ts @@ -65,6 +65,73 @@ describe('repo onboard/offboard', () => { expect(put.input.Item?.compute_type).toBe('agentcore'); }); + test('onboardRepo probes when an existing repo is switched to lambda-microvm', async () => { + const { loadRepoConfig } = jest.requireMock('../../src/repo-lookup') as { + loadRepoConfig: jest.Mock; + }; + loadRepoConfig.mockResolvedValueOnce({ + repo: 'acme/a', + status: 'active', + compute_type: 'agentcore', + }); + const microvmSend = jest.fn().mockResolvedValue({ images: [] }); + + await onboardRepo( + 'us-east-1', + 'RepoTable', + 'acme/a', + { computeType: 'lambda-microvm' }, + { lambdaMicrovmClientFactory: () => ({ send: microvmSend }) }, + ); + + expect(microvmSend).toHaveBeenCalledTimes(1); + const put = ddbSend.mock.calls[0][0] as PutCommand; + expect(put.input.Item?.compute_type).toBe('lambda-microvm'); + }); + + test('onboardRepo probes when reactivating an existing lambda-microvm repo without a flag', async () => { + const { loadRepoConfig } = jest.requireMock('../../src/repo-lookup') as { + loadRepoConfig: jest.Mock; + }; + loadRepoConfig.mockResolvedValueOnce({ + repo: 'acme/a', + status: 'removed', + compute_type: 'lambda-microvm', + }); + const microvmSend = jest.fn().mockResolvedValue({ images: [] }); + + await onboardRepo( + 'us-east-1', + 'RepoTable', + 'acme/a', + {}, + { lambdaMicrovmClientFactory: () => ({ send: microvmSend }) }, + ); + + expect(microvmSend).toHaveBeenCalledTimes(1); + const put = ddbSend.mock.calls[0][0] as PutCommand; + expect(put.input.Item?.compute_type).toBe('lambda-microvm'); + }); + + test('onboardRepo rejects lambda-microvm when its availability probe fails', async () => { + const microvmSend = jest.fn().mockRejectedValue( + Object.assign(new Error('unknown service'), { name: 'UnrecognizedClientException' }), + ); + + await expect(onboardRepo( + 'eu-central-1', + 'RepoTable', + 'acme/a', + { computeType: 'lambda-microvm' }, + { lambdaMicrovmClientFactory: () => ({ send: microvmSend }) }, + )).rejects.toThrow( + 'Launch regions: us-east-1, us-east-2, us-west-2, eu-west-1, ap-northeast-1. ' + + 'Use --compute-type agentcore or --compute-type ecs', + ); + + expect(ddbSend).not.toHaveBeenCalled(); + }); + test('offboardRepo sets removed status and TTL', async () => { await offboardRepo('us-east-1', 'RepoTable', 'acme/a'); diff --git a/cli/test/commands/repo.test.ts b/cli/test/commands/repo.test.ts index cc115feb9..0dfcd2c61 100644 --- a/cli/test/commands/repo.test.ts +++ b/cli/test/commands/repo.test.ts @@ -219,6 +219,98 @@ describe('repo command JSON output', () => { expect(onboardRepoMock).toHaveBeenCalled(); }); + // --- lambda-microvm substrate gate (parity with the ECS gate above) --------- + // + // Before this gate existed, `--compute-type lambda-microvm` only ran the live + // regional availability probe, so a row could be written against an + // agentcore-only stack and every task on it would die at session start. + + test('onboard --compute-type lambda-microvm is REFUSED when the stack has no MicroVM substrate', async () => { + getStackOutputMock.mockReset().mockImplementation((_r: string, _s: string, key: string) => + Promise.resolve(key === 'ComputeSubstrate' ? 'agentcore' : 'RepoTable-dev')); + + const cmd = makeRepoCommand(); + await expect(cmd.parseAsync([ + 'node', 'test', 'onboard', 'acme/a', '--region', 'us-east-1', '--compute-type', 'lambda-microvm', + ])).rejects.toThrow(/without the Lambda MicroVMs substrate/i); + // Must NOT write the repo row when it would be dead-on-arrival... + expect(onboardRepoMock).not.toHaveBeenCalled(); + }); + + test('the substrate gate refuses BEFORE the live availability probe runs', async () => { + // ORDERING assertion (documented in repo.ts): the substrate gate is cheaper + // (reuses an already-fetched stack output) and more specific, so it must fire + // first. `onboardRepo` owns the ListManagedMicrovmImages probe, so "the probe + // did not run" is exactly "onboardRepo was never called". + getStackOutputMock.mockReset().mockImplementation((_r: string, _s: string, key: string) => + Promise.resolve(key === 'ComputeSubstrate' ? 'agentcore' : 'RepoTable-dev')); + onboardRepoMock.mockReset(); + + const cmd = makeRepoCommand(); + await expect(cmd.parseAsync([ + 'node', 'test', 'onboard', 'acme/a', '--region', 'us-east-1', '--compute-type', 'lambda-microvm', + ])).rejects.toThrow(CliError); + expect(onboardRepoMock).not.toHaveBeenCalled(); + }); + + test('onboard --compute-type lambda-microvm is ALLOWED when the stack provisioned it', async () => { + getStackOutputMock.mockReset().mockImplementation((_r: string, _s: string, key: string) => + Promise.resolve(key === 'ComputeSubstrate' ? 'lambda-microvm' : 'RepoTable-dev')); + onboardRepoMock.mockResolvedValue({ + repo: 'acme/a', status: 'active', compute_type: 'lambda-microvm', + }); + + const cmd = makeRepoCommand(); + await cmd.parseAsync([ + 'node', 'test', 'onboard', 'acme/a', '--region', 'us-east-1', '--compute-type', 'lambda-microvm', + ]); + // ...and the gate must then hand off to onboardRepo, which runs the live + // regional availability probe as the SECOND layer. + expect(onboardRepoMock).toHaveBeenCalledWith( + 'us-east-1', 'RepoTable-dev', 'acme/a', + expect.objectContaining({ computeType: 'lambda-microvm' })); + }); + + test('onboard --compute-type lambda-microvm is REFUSED on an ECS-only stack', async () => { + // The two optional backends are mutually exclusive today, so an ecs stack is + // a real (not hypothetical) way to get this wrong. + getStackOutputMock.mockReset().mockImplementation((_r: string, _s: string, key: string) => + Promise.resolve(key === 'ComputeSubstrate' ? 'ecs' : 'RepoTable-dev')); + + const cmd = makeRepoCommand(); + await expect(cmd.parseAsync([ + 'node', 'test', 'onboard', 'acme/a', '--region', 'us-east-1', '--compute-type', 'lambda-microvm', + ])).rejects.toThrow(/ComputeSubstrate=ecs/); + expect(onboardRepoMock).not.toHaveBeenCalled(); + }); + + test('onboard --compute-type lambda-microvm proceeds against an OLDER stack lacking ComputeSubstrate', async () => { + // Same back-compat posture as the ECS gate: null → "unknown", not "none". + getStackOutputMock.mockReset().mockImplementation((_r: string, _s: string, key: string) => + Promise.resolve(key === 'ComputeSubstrate' ? null : 'RepoTable-dev')); + onboardRepoMock.mockResolvedValue({ + repo: 'acme/a', status: 'active', compute_type: 'lambda-microvm', + }); + + const cmd = makeRepoCommand(); + await cmd.parseAsync([ + 'node', 'test', 'onboard', 'acme/a', '--region', 'us-east-1', '--compute-type', 'lambda-microvm', + ]); + expect(onboardRepoMock).toHaveBeenCalled(); + }); + + test('onboard with NO --compute-type is never gated (inherits/defaults elsewhere)', async () => { + // The effective compute type may come from the existing row, which the gate + // cannot see — `onboardRepo` resolves that and runs its own probe. + getStackOutputMock.mockReset().mockImplementation((_r: string, _s: string, key: string) => + Promise.resolve(key === 'ComputeSubstrate' ? 'agentcore' : 'RepoTable-dev')); + onboardRepoMock.mockResolvedValue({ repo: 'acme/a', status: 'active' }); + + const cmd = makeRepoCommand(); + await cmd.parseAsync(['node', 'test', 'onboard', 'acme/a', '--region', 'us-east-1']); + expect(onboardRepoMock).toHaveBeenCalled(); + }); + test('repo offboard --output json redacts the per-repo secret ARN', async () => { offboardRepoMock.mockResolvedValue({ repo: 'acme/a', diff --git a/cli/test/commands/runtime-status.test.ts b/cli/test/commands/runtime-status.test.ts index c3e925946..bae2c7d55 100644 --- a/cli/test/commands/runtime-status.test.ts +++ b/cli/test/commands/runtime-status.test.ts @@ -57,6 +57,11 @@ describe('buildRuntimeStatusReport', () => { status: 'active', compute_type: 'ecs', }, + { + repo: 'acme/microvm', + status: 'active', + compute_type: 'lambda-microvm', + }, ]); const report = await buildRuntimeStatusReport( @@ -67,6 +72,12 @@ describe('buildRuntimeStatusReport', () => { expect(report.agentcore_runtimes).toHaveLength(2); expect(report.ecs_substrates).toHaveLength(1); + expect(report.lambda_microvm_substrates).toEqual([expect.objectContaining({ + compute_type: 'lambda-microvm', + used_by_repos: ['acme/microvm'], + })]); + expect(report.blueprints.find((blueprint) => blueprint.repo === 'acme/microvm')?.runtime_arn) + .toBeUndefined(); expect(report.blueprints[0].runtime_arn_source).toBe('platform'); expect(report.blueprints[1].runtime_arn_source).toBe('blueprint'); expect(controlPlaneSend).toHaveBeenCalledTimes(2); diff --git a/cli/test/commands/runtime.test.ts b/cli/test/commands/runtime.test.ts index ffd19ef2f..11cb69627 100644 --- a/cli/test/commands/runtime.test.ts +++ b/cli/test/commands/runtime.test.ts @@ -51,6 +51,7 @@ describe('runtime status command', () => { control_plane_status: 'READY', }], ecs_substrates: [], + lambda_microvm_substrates: [], }); }); @@ -84,6 +85,7 @@ describe('runtime status command', () => { used_by_repos: ['acme/ecs'], note: 'ECS note', }], + lambda_microvm_substrates: [], }); const cmd = makeRuntimeCommand(); @@ -109,6 +111,7 @@ describe('runtime status command', () => { used_by_repos: ['acme/ecs'], note: 'ECS note', }], + lambda_microvm_substrates: [], }); const cmd = makeRuntimeCommand(); @@ -119,6 +122,34 @@ describe('runtime status command', () => { expect(output).toContain('ECS note'); }); + test('prints Lambda MicroVM substrate note without runtime probing', async () => { + (buildRuntimeStatusReport as jest.Mock).mockResolvedValue({ + platform_default_runtime_arn: 'arn:aws:bedrock-agentcore:us-east-1:123:runtime/platform', + blueprints: [{ + repo: 'acme/microvm', + status: 'active', + compute_type: 'lambda-microvm', + runtime_arn: undefined, + runtime_arn_source: 'platform', + }], + agentcore_runtimes: [], + ecs_substrates: [], + lambda_microvm_substrates: [{ + compute_type: 'lambda-microvm', + used_by_repos: ['acme/microvm'], + note: 'Lambda MicroVM sessions are platform-managed.', + }], + }); + + const cmd = makeRuntimeCommand(); + await cmd.parseAsync(['node', 'test', 'status', '--region', 'us-east-1']); + + const output = consoleSpy.mock.calls.map((c) => c[0]).join('\n'); + expect(output).toContain('n/a — Lambda MicroVMs are platform-managed'); + expect(output).toContain('Lambda MicroVM compute substrates'); + expect(output).toContain('Lambda MicroVM sessions are platform-managed.'); + }); + test('fails when RepoTable output is missing', async () => { (getStackOutput as jest.Mock).mockImplementation(async (_r: string, _s: string, key: string) => { if (key === 'RepoTableName') return null; @@ -136,6 +167,7 @@ describe('runtime status command', () => { blueprints: [], agentcore_runtimes: [], ecs_substrates: [], + lambda_microvm_substrates: [], }); const cmd = makeRuntimeCommand(); diff --git a/cli/test/compute-substrate.test.ts b/cli/test/compute-substrate.test.ts new file mode 100644 index 000000000..2260e925d --- /dev/null +++ b/cli/test/compute-substrate.test.ts @@ -0,0 +1,123 @@ +/** + * MIT No Attribution + * + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { + assertComputeSubstrateDeployed, + parseComputeSubstrateOutput, +} from '../src/compute-substrate'; +import { CliError } from '../src/errors'; + +const STACK = 'backgroundagent-dev'; + +function assertFor(computeType: 'agentcore' | 'ecs' | 'lambda-microvm' | undefined, substrate: string | null) { + return () => assertComputeSubstrateDeployed({ + stackName: STACK, + computeType, + computeSubstrate: substrate, + }); +} + +describe('parseComputeSubstrateOutput', () => { + test.each([ + ['agentcore', ['agentcore']], + ['ecs', ['ecs']], + ['lambda-microvm', ['lambda-microvm']], + ])('parses the single value %s that the stack emits today', (raw, expected) => { + expect(parseComputeSubstrateOutput(raw)).toEqual(expected); + }); + + test('tolerates a comma list, so a future compute_types output cannot silently over-refuse', () => { + // ADR-021 sub-decision 4 names a `compute_types` list as the intended + // follow-up to the single-valued tag. An `!== 'ecs'` equality check would + // start rejecting valid onboardings the day that lands. + expect(parseComputeSubstrateOutput('ecs,lambda-microvm')).toEqual(['ecs', 'lambda-microvm']); + expect(parseComputeSubstrateOutput(' ecs , lambda-microvm ')).toEqual(['ecs', 'lambda-microvm']); + }); + + test.each([[null], [undefined], [''], [' '], [',']])( + 'reports %p as UNKNOWN (undefined), not as an empty substrate set', + (raw) => { + // Load-bearing distinction: "unknown" must not hard-block onboarding + // against a stack deployed before the output existed. + expect(parseComputeSubstrateOutput(raw)).toBeUndefined(); + }, + ); +}); + +describe('assertComputeSubstrateDeployed', () => { + test('never gates agentcore — the runtime is unconditional', () => { + expect(assertFor('agentcore', 'agentcore')).not.toThrow(); + expect(assertFor('agentcore', 'ecs')).not.toThrow(); + expect(assertFor('agentcore', 'lambda-microvm')).not.toThrow(); + }); + + test('never gates an unspecified compute type', () => { + // The effective type may be inherited from the existing repo row, which this + // function cannot see; `onboardRepo` resolves and probes that case. + expect(assertFor(undefined, 'agentcore')).not.toThrow(); + }); + + test.each([['ecs'], ['lambda-microvm']] as const)( + 'allows %s when the stack provisioned it', + (computeType) => { + expect(assertFor(computeType, computeType)).not.toThrow(); + }, + ); + + test.each([['ecs'], ['lambda-microvm']] as const)( + 'allows %s when the output is absent (older stack → unknown)', + (computeType) => { + expect(assertFor(computeType, null)).not.toThrow(); + }, + ); + + test('refuses ecs on an agentcore-only stack, naming the substrate and the remedy', () => { + expect(assertFor('ecs', 'agentcore')).toThrow(CliError); + expect(assertFor('ecs', 'agentcore')).toThrow(/without the ECS substrate/); + expect(assertFor('ecs', 'agentcore')).toThrow(/ComputeSubstrate=agentcore/); + expect(assertFor('ecs', 'agentcore')).toThrow(/--context compute_type=ecs/); + expect(assertFor('ecs', 'agentcore')).toThrow(/--compute-type agentcore/); + }); + + test('refuses lambda-microvm on an agentcore-only stack, naming the substrate and the remedy', () => { + expect(assertFor('lambda-microvm', 'agentcore')).toThrow(CliError); + expect(assertFor('lambda-microvm', 'agentcore')).toThrow(/without the Lambda MicroVMs substrate/); + expect(assertFor('lambda-microvm', 'agentcore')).toThrow(/ComputeSubstrate=agentcore/); + expect(assertFor('lambda-microvm', 'agentcore')).toThrow(/--context compute_type=lambda-microvm/); + // The MicroVM remedy is more specific than ECS's about WHERE it fails, + // because the strategy's env-var guard fires before any AWS call. + expect(assertFor('lambda-microvm', 'agentcore')).toThrow(/MICROVM_\*/); + }); + + test('refuses each optional backend on the OTHER one (they are mutually exclusive today)', () => { + expect(assertFor('lambda-microvm', 'ecs')).toThrow(/without the Lambda MicroVMs substrate/); + expect(assertFor('ecs', 'lambda-microvm')).toThrow(/without the ECS substrate/); + }); + + test('names the stack so an operator pointed at the wrong stack sees it', () => { + expect(assertFor('lambda-microvm', 'agentcore')).toThrow(new RegExp(`'${STACK}'`)); + }); + + test('allows both optional backends against a hypothetical multi-substrate output', () => { + // Behavioural proof of the list tolerance above: this must NOT throw, or the + // `compute_types` follow-up would break onboarding for both backends. + expect(assertFor('ecs', 'ecs,lambda-microvm')).not.toThrow(); + expect(assertFor('lambda-microvm', 'ecs,lambda-microvm')).not.toThrow(); + }); +}); diff --git a/docs/decisions/ADR-021-lambda-microvms-compute-backend.md b/docs/decisions/ADR-021-lambda-microvms-compute-backend.md index 8e41bbe1f..9da6bc927 100644 --- a/docs/decisions/ADR-021-lambda-microvms-compute-backend.md +++ b/docs/decisions/ADR-021-lambda-microvms-compute-backend.md @@ -21,25 +21,38 @@ ABCA selects a per-repo compute backend through the Blueprint's `compute_type` f | | AgentCore Runtime | ECS on Fargate | **Lambda MicroVMs** | |---|---|---|---| | Isolation | MicroVM (managed) | Task-level (Firecracker) | MicroVM (Firecracker) | -| Max duration | 8 h | No cap | 8 h (running + suspended) | -| Suspend/resume | No orchestrator-visible API | No | **Yes** — explicit API + idle policy, state preserved, no compute charge while suspended | -| Resources | AgentCore-managed | 16 vCPU / 120 GB / 20–200 GB disk | **16 vCPU / 32 GB RAM / 32 GB disk (hard ceiling)** | +| Max duration | 8 h | No cap | 8 h (running + suspended) — **verified** (`L-B430C318 = 8 hours`) | +| Suspend/resume | No orchestrator-visible API | No | **Yes** — explicit API + idle policy, state preserved, no compute charge while suspended. **Verified**: suspend reaches `SUSPENDED` in ~1 s with no `idlePolicy`; resume restores `RUNNING` in ~1 s with `microvmId` **and** `endpoint` byte-identical | +| Resources | AgentCore-managed | 16 vCPU / 120 GB / 20–200 GB disk | **Baseline 8 GiB RAM / 4 vCPU, auto-scaling to a 32 GiB / 16 vCPU peak**; 32 GB disk. `minimumMemoryInMiB` configures the BASELINE (max 8,192 MiB); the service scales vertically on demand — capacity is baseline-priced with 4× burst headroom | | Packaging | ECR image ≤ 2 GB | ECR image, no hard cap | **Zip + Dockerfile in S3 → service-built snapshot image** (versioned, storage billed) | -| Invocation | `InvokeAgentRuntime` (SigV4) | `RunTask` + container overrides | `RunMicrovm` → dedicated HTTPS endpoint + JWE token (`CreateMicrovmAuthToken`, ≤ 60 min TTL) | +| Invocation | `InvokeAgentRuntime` (SigV4) | `RunTask` + container overrides | `RunMicrovm` (image **ARN** required — a bare name is rejected) → dedicated HTTPS endpoint + JWE token (`CreateMicrovmAuthToken`, ≤ 60 min TTL) | | Liveness | Agent heartbeat + `/ping` | `DescribeTasks` | MicroVM state (RUNNING / SUSPENDED / TERMINATED) via control-plane API | | Session storage | `/mnt/workspace` FUSE (no `flock()`) | Ephemeral disk | Native disk in snapshot — **survives suspend/resume, `flock()` works** | | Architecture | ARM64 | ARM64 | ARM64 (Graviton) | | Regions (launch) | Broad | Broad | 5 (us-east-1/2, us-west-2, eu-west-1, ap-northeast-1) | -> Service facts in the Lambda MicroVMs column (APIs, limits, quota semantics, regions) are externally sourced from the AWS developer guide and the Agent Toolkit skill (see References) and cannot be corroborated from this repository; P1 re-verifies the load-bearing ones empirically (see Testing). +> Rows marked **verified** were discharged empirically on 2026-07-31 (us-east-1); see `docs/verification/645-p1-lambda-microvm-runbook.md`. Two originally documented constants were **refuted** by that run and are corrected throughout this ADR: the `runHookPayload` cap (16 KB → 4 KB) and the claim that `RunMicrovm` accepts a bare image name (it does not). +> +> **Source hierarchy for service facts.** Where sources disagree, the higher tier wins, and the tier is recorded next to the claim: +> +> 1. **Live boundary probe** — a request the service actually accepted or rejected in this account/Region. Strongest, and the only tier that can refute the others. (Both corrections above came from here.) +> 2. **Modeled constraints** — the CLI/SDK request schema and enumerated allowed values (`--generate-cli-skeleton`, `ARM_64`, `ENABLED|DISABLED`). Authoritative about request *shape*; silent about runtime behaviour. +> 3. **Service developer guide** — including its sizing/scaling tables. Authoritative about *semantics* a probe cannot see, which is exactly how the memory row was fixed: a probe can only show that 32,768 MiB is rejected as a `minimumMemoryInMiB` value; only the guide explains that the field is a BASELINE and that the service scales to a 32 GiB peak on its own. A boundary probe is the strongest evidence about a boundary and says nothing about what the boundary *means*. +> 4. **SDK docstrings** — generated, and demonstrably stale here: `runHookPayload` documents "Maximum: 16,384 bytes" against an enforced 4,096. +> 5. **Launch blogs / skills / toolkit material** — orientation only; never load-bearing on its own. +> +> **Omitted API fields mean "service default", never "none".** Two live findings drove this rule: leaving `ingressNetworkConnectors` unset attaches a PUBLIC `HTTP_INGRESS` connector, and leaving `/ready` out of `hooks` makes the image un-creatable once any lifecycle hook is enabled. So for any security-relevant field, the desired posture must be **requested explicitly** and the test must assert the **outcome** (the ARN present in the request, the hook enabled) rather than the omission (`expect(field).toBeUndefined()`) — an omission assertion passes just as happily when the service is silently choosing something wider. +> +> On the **memory** row specifically: `CreateMicrovmImage` enumerates the baseline sizes a base image supports — `[512, 1024, 2048, 4096, 8192]` MiB for `al2023-1` — and rejects anything else, which is why the construct validates against that list at synth. The 32 GiB / 16 vCPU peak is reached by the service's own vertical scaling, not by asking for it. The account memory quota (`L-CD1C0CC4`, 1024 GB, "burst up to 4×") is an aggregate across MicroVMs, not a per-VM limit; note that concurrency arithmetic should be done against the PEAK, not the baseline, since that is what a busy fleet can actually consume. ### Design tensions the strategy must resolve 1. **Idle detection is inbound-traffic-based; the ABCA agent is outbound-only.** MicroVM idle policies suspend when no traffic arrives at the *endpoint*. A busy agent running a 40-minute build receives no inbound traffic and would be suspended mid-work by a naive idle policy. Conversely, "no inbound traffic" is the agent's *normal* state. 2. **No self-suspend.** The agent cannot suspend its own MicroVM from inside; only an external `SuspendMicrovm` call can. Suspend decisions must be owned by the orchestrator — which aligns with the unified liveness model proposed in [#491](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/491). -3. **Snapshots bake state.** The image snapshot is captured once at build time; every MicroVM resumes from it. Secrets, tokens, and per-task identity must arrive at run time (`runHookPayload`, ≤ 16 KB, or fetched in the `/run` hook), never at image build. CSPRNGs must be reseeded on `/run` and `/resume`. +3. **Snapshots bake state.** The image snapshot is captured once at build time; every MicroVM resumes from it. Secrets, tokens, and per-task identity must arrive at run time (`runHookPayload`, ≤ 4 KB, or fetched in the `/run` hook), never at image build. CSPRNGs must be reseeded on `/run` and `/resume`. 4. **Auth tokens are short-lived.** JWE tokens max out at 60 minutes; any orchestrator→agent HTTP interaction over the endpoint needs token refresh, unlike AgentCore's SigV4 invoke or ECS's no-endpoint model. 5. **Identity delta — narrower than it looks.** Most AgentCore services ABCA uses are standalone and substrate-portable: Memory is already consumed from ECS via an IAM grant plus `MEMORY_ID` (`EcsAgentCluster`), and Gateway (ADR-019) is portable by design (SigV4 inbound). The genuinely Runtime-coupled piece is the workload-access-token **delivery mechanism** (`runtimeUserId` → `WorkloadAccessToken` request header → `BedrockAgentCoreContext`, used by `resolve_linear_api_token()`), which has no MicroVM equivalent. The ECS backend already lives with this delta (env-var token delivery); MicroVMs inherit the same posture until the pluggable identity work ([#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249), ADR-016) redesigns the seam. +6. **The service's defaults are not our posture.** Two of them, both discovered live: `RunMicrovm` attaches a **public** `HTTP_INGRESS` connector (and mints a public `*.lambda-microvm..on.aws` endpoint) when `ingressNetworkConnectors` is omitted, and `CreateMicrovmImage` **requires** the `/ready` hook whenever any lifecycle hook is enabled. Neither posture can be reached by leaving a field out — each needs an explicit control (see sub-decisions 3 and 4). ## Decision @@ -47,25 +60,52 @@ Adopt **AWS Lambda MicroVMs as a third, opt-in `ComputeStrategy` backend** named ### 1. Strategy shape: extend the interface with mandatory suspend/resume -`ComputeType` widens to `'agentcore' | 'ecs' | 'lambda-microvm'` (mirrored in `cli/src/types.ts` and the CLI's inline unions). `SessionHandle` gains a `{ strategyType: 'lambda-microvm', microvmId, endpoint }` variant — `microvmId` because every lifecycle API (`suspend-microvm`, `resume-microvm`, `terminate-microvm`, and `create-microvm-auth-token` — the latter not called in P1–P3, see sub-decision 3) takes only the MicroVM identifier, and `endpoint` because it is per-session (minted by `RunMicrovm`) and required for any orchestrator→agent HTTP interaction. The image ARN is deliberately **not** in the handle: like the ECS task definition ARN, it is deployment-time configuration consumed by `startSession` (from construct-injected environment) and recorded in the session-start log entry for diagnostics, not per-session lifecycle state. +`ComputeType` widens to `'agentcore' | 'ecs' | 'lambda-microvm'` (mirrored in `cli/src/types.ts` and the CLI's inline unions). `SessionHandle` gains a `{ strategyType: 'lambda-microvm', microvmId, endpoint }` variant — `microvmId` because every lifecycle API (`suspend-microvm`, `resume-microvm`, `terminate-microvm`, `get-microvm`, and `create-microvm-auth-token` — the latter not called in P1–P3, see sub-decision 3) takes only the MicroVM identifier, and `endpoint` because it is per-session (minted by `RunMicrovm`) and required for any orchestrator→agent HTTP interaction. Note the naming seam: the handle field is `microvmId` (matching `RunMicrovmResponse.microvmId`), while the request key on every lifecycle command is `microvmIdentifier` — the strategy is the only place that translates between the two. The image ARN is deliberately **not** in the handle: like the ECS task definition ARN, it is deployment-time configuration consumed by `startSession` (from construct-injected environment) and recorded in the session-start log entry for diagnostics, not per-session lifecycle state. + +**A second, sharper naming seam: `imageIdentifier` must be an ARN.** The name suggests a bare image name is acceptable — `create-microvm-image --name` takes one, and this ADR originally assumed `run-microvm --image-identifier` would too. It does not: a bare name is rejected with `ValidationException: Malformed ARN - doesn't start with 'arn:'`, and so is `list-microvm-image-builds --image-identifier ` (`Invalid ARN format`). The construct therefore resolves an operator-supplied name to its exact `arn:${Partition}:lambda:${Region}:${Account}:microvm-image:${Name}` ARN **once** — the same value it scopes the lifecycle IAM grant to — and injects THAT as `MICROVM_IMAGE_IDENTIFIER`. One derivation, two consumers, so a request field and an IAM resource can never disagree. The strategy validates the invariant and fails fast with the remedy, because the service's own error names neither the env var nor the fix. The `ComputeStrategy` interface gains **mandatory** `suspendSession(handle)` / `resumeSession(handle)` methods returning a typed result (`{ supported: false } | { supported: true }`-shaped, exact type at implementation time); the widening lands in P3 (see sub-decision 5), in one commit across all three strategies. Mandatory-with-explicit-stub is the codebase idiom, not optional methods: no behavioral interface in the codebase has an optional method, `AgentCoreComputeStrategy.pollSession` is already a mandatory explicit stub rather than an optional member, and the exhaustive-`never` switch culture means a fourth backend must make a compile-checked decision about its suspend semantics instead of silently falling through a `strategy.suspendSession?.()` feature-detection. The agentcore and ecs strategies return `unsupported` (not a silent success — a suspend that silently no-ops would let the orchestrator believe compute billing stopped when it did not); the orchestrator gates its suspend policy on the typed response, consistent with how `pollTaskStatus` already branches explicitly on `computeType`. -**Poll semantics — the strategy reports, the orchestrator interprets.** `pollSession(handle)` receives only the session handle and cannot see task state, so the health rules must live where the DynamoDB status lives. `SessionStatus` gains a `'suspended'` variant; the strategy maps `GetMicrovm` state mechanically (`RUNNING` → `running`, `SUSPENDED` → `suspended`, `TERMINATED` → `completed`) and the **orchestrator** cross-references against the task row — the same division of labor `finalPollState` already uses for ECS (substrate stopped + non-terminal DynamoDB status → failed) and `pollTaskStatus` uses for agentcore heartbeats: substrate `suspended` + task `AWAITING_APPROVAL` is healthy (orchestrator-intended suspend); `suspended` with any other task status is an anomaly to surface, not fail-fast; substrate terminal + non-terminal task status → classify failed. +**Poll semantics — the strategy reports, the orchestrator interprets.** `pollSession(handle)` receives only the session handle and cannot see task state, so the health rules must live where the DynamoDB status lives. `SessionStatus` gains a `'suspended'` variant; the strategy maps `GetMicrovm` state mechanically and the **orchestrator** cross-references against the task row — the same division of labor `finalPollState` already uses for ECS (substrate stopped + non-terminal DynamoDB status → failed) and `pollTaskStatus` uses for agentcore heartbeats: substrate `suspended` + task `AWAITING_APPROVAL` is healthy (orchestrator-intended suspend); `suspended` with any other task status is an anomaly to surface, not fail-fast; substrate terminal + non-terminal task status → classify failed. + +The service's `MicrovmState` enum has **six** members, not three, so the mapping is stated exhaustively (one line of rationale each, mirrored in the strategy's doc comment): + +| `MicrovmState` | `SessionStatus` | Why | +|---|---|---| +| `PENDING` | `running` | Still booting; the same way ECS's `PENDING`/`PROVISIONING` map to `running`. | +| `RUNNING` | `running` | — | +| `SUSPENDING` | `suspended` | Already on its way to frozen; reporting `running` would tell the orchestrator compute is still progressing when it is not. Both suspend states land on a report the orchestrator treats as benign-or-anomalous depending on task status, never as failure. **Never observable in practice** — suspend reached `SUSPENDED` in under 1 s live — so it is mapped for completeness and nothing may *wait* for it. | +| `SUSPENDED` | `suspended` | — | +| `TERMINATING` | `completed` | Terminal-bound and carries no exit code, so "the substrate is gone" is all the strategy can honestly say. | +| `TERMINATED` | `completed` | Success vs failure is the orchestrator's call — it cross-references the DynamoDB status. **This is the load-bearing terminal signal**, not `NotFound` (see below). | +| *unrecognized* | `running` | A future service enum addition must never fail a healthy task; the strategy warns and keeps polling. | + +**`GetMicrovm` `ResourceNotFoundException` → `completed`, but as a LATE fallback.** This deliberately diverges from `ecs-strategy`, where `DescribeTasks` returning no task maps to `failed`. ECS keeps stopped tasks describable for roughly an hour, so a missing task there really is anomalous; a MicroVM is eventually reaped from the control plane **by design**, so `failed` would fail tasks that finished cleanly. + +What the live run corrected is the *timing*, not the mapping. `NotFound` is **not the near-term terminal signal**: a terminated MicroVM reported `TERMINATING` at +1 s, `TERMINATED` at +3 s, and was **still `TERMINATED` ~10 minutes later** and at every subsequent checkpoint — `ResourceNotFoundException` was never observed in that window. So the branch that actually fires in practice is `TERMINATED → completed` in the table above; the `NotFound` rule covers only a VM reaped after a long gap (a poll resumed after a crash, a stranded-task reconciler sweep). Both are required and neither substitutes for the other: without the `TERMINATED` row the orchestrator would poll a finished VM until its safety net fired, and without the `NotFound` rule a late sweep would classify a cleanly-finished task as a poll error. + +The divergence is safe because it does not weaken detection: the orchestrator still fails the task when a terminal report lands while the DynamoDB status is non-terminal, so a genuine mid-run disappearance is caught — it simply receives the substrate-failure classification instead of a misleading poll error. Because that cross-check acts on a status read earlier in the same poll cycle, the orchestrator **re-reads the task row before failing** (the normal shutdown order is "agent writes terminal status → agent exits → VM terminates", which a stale read would otherwise turn into a spurious failure); ECS buys the same protection with a five-consecutive-poll patience counter instead. + +Neither the mapping nor the NotFound rule is a health decision: both are mechanical restatements of substrate state, which is what keeps the "strategy reports, orchestrator interprets" split intact. Normative requirements (EARS, per [ADR-020](./ADR-020-ears-requirements-syntax.md)): - When a task's Blueprint sets `compute_type: 'lambda-microvm'`, the orchestrator shall resolve the `LambdaMicrovmComputeStrategy` via `resolveComputeStrategy`. - When `startSession` is invoked, the strategy shall call `RunMicrovm` with `maximumDurationInSeconds` set to 28 800 (the service maximum, matching AgentCore's 8-hour session cap and sitting inside the orchestrator's ~8.5 h safety-net poll window). +- When `startSession` is invoked, the strategy shall pass a fully-qualified MicroVM image ARN as `imageIdentifier`. +- If the configured image identifier is not an ARN, then the strategy shall fail the session start with an error naming the environment variable and the redeploy remedy, before performing any AWS call. - When `startSession` returns, the orchestrator shall persist the MicroVM handle (`microvmId`, `endpoint`) in the task row's `compute_metadata` (the field `cancel-task.ts` already reads ECS handles from). - The strategy shall omit `idlePolicy` on every `RunMicrovm` call, in every phase. - The orchestrator shall be the sole initiator of suspension, via `suspendSession`. -- When `pollSession` observes MicroVM state `SUSPENDED`, the strategy shall report `suspended` without interpreting task state. -- If the strategy reports a terminal substrate state while the task's DynamoDB status is non-terminal, then the orchestrator shall classify the task as failed with a substrate-failure remedy. +- When `pollSession` observes MicroVM state `SUSPENDED` or `SUSPENDING`, the strategy shall report `suspended` without interpreting task state. +- When `pollSession` observes MicroVM state `TERMINATED` or `TERMINATING`, the strategy shall report `completed` (the observable terminal state persists for at least ~10 minutes, so `completed` shall not depend on the MicroVM being reaped). +- When `pollSession` observes a MicroVM state it does not recognize, the strategy shall report `running`. +- If `GetMicrovm` reports that the MicroVM does not exist, then the strategy shall report `completed`. +- If the strategy reports a terminal substrate state while the task's DynamoDB status is non-terminal, then the orchestrator shall re-read the task row and, if it is still non-terminal, classify the task as failed with a substrate-failure remedy. - If the strategy reports `suspended` while the task's DynamoDB status is not `AWAITING_APPROVAL`, then the orchestrator shall surface an anomaly event and shall not fail-fast the task. - If `suspendSession` or `resumeSession` is invoked on a strategy that does not support suspension, then the strategy shall return an explicit unsupported result. - When the agent process reaches a terminal state, the agent shall exit. -- When the orchestrator finalizes a `lambda-microvm` task, the orchestrator shall call `terminate-microvm` (termination shall not rely on any substrate timeout). +- When the orchestrator finalizes a `lambda-microvm` task, the orchestrator shall call `terminate-microvm` (termination shall not rely on any substrate timeout, and shall not rely on the MicroVM self-terminating — it does not). *On the omitted `idlePolicy`:* if the block is present all three fields are required, so omission is the unambiguous disabled state the invariant test asserts. This deliberately forgoes `suspendedDurationSeconds` — it lives *inside* `idlePolicy` and cannot be set without re-enabling the traffic-idle machinery — so the suspended-state bound is `maximumDurationInSeconds` plus orchestrator termination and the stranded-approval reconciler (see sub-decision 2). A tighter substrate-level suspended-TTL remains available later as an additive `idlePolicy` change if operators want it. *On the fixed `maximumDurationInSeconds`:* no wall-clock task budget exists in the platform (budgets are `max_turns` / `max_budget_usd`), so the value is parity with AgentCore's 8 h cap rather than derived policy; a Blueprint override can be added later if a real need appears. @@ -80,8 +120,8 @@ The handshake must respect the existing approval mechanics: the agent **discover *Why inline rather than poll-only — codebase precedent:* resume-on-approve is structurally identical to task cancellation — a user-initiated, latency-sensitive action whose purpose is an immediate compute-lifecycle side effect. `cancel-task.ts` already resolves this exact tension: the API-plane handler invokes ECS `StopTask` / AgentCore `StopRuntimeSession` inline, best-effort (a failed stop logs a warning and the state transition stands; a `task_cancel_compute_orphan` event is written when no stoppable compute handle exists, `reason: missing_runtime_handle`) — with the conditional IAM wired in `task-api.ts`. The resume path goes one step further than the precedent by also writing the orphan event on *failed* resume calls, because a failed resume strands a suspended VM awaiting a decision — a stronger liveness consequence than a failed stop of an already-cancelled task. The alternative (orchestrator-poll-only resume) preserves single-owner lifecycle purity but pays up to a full poll interval (~30 s) of latency on every approval, and the purity argument was already litigated and declined for cancel. `approve-task.ts` is deliberately minimal today (security-critical ownership comparison, Cedar finding #6); the resume call is therefore added *after* the transaction commits, cannot alter the decision outcome, and carries one conditional `lambda:ResumeMicrovm` grant — the same blast-radius trade the cancel handler accepted in review. - **Timeout under freeze — the agent re-bases on the wall clock it already owns.** The agent's monotonic gate timer freezes while suspended, so resuming near the deadline is not enough: the frozen timer would still hold its remaining budget and fire the deny minutes *after* the user-visible window — colliding with the approval row's TTL (`created_at + timeout_s + 120s`) and triggering the "row reaped → stranded" fallback on a healthy gate. Instead, the gate expires at **`min(monotonic budget, created_at + timeout_s)`**, evaluated on each poll iteration and on `/resume`. This is not a new principle: Cedar decision #6 is already "min wins" for timeouts, the wall-clock deadline is already durable in the approval row the agent itself writes (`created_at` is in the agent's own clock domain — no skew), and §13.12's late-approval race fix already establishes that the durable row is authoritative over the agent's local timer. Deny authority stays agent-side (the conditional `TIMED_OUT` write + ConsistentRead re-read race protection is untouched); the orchestrator's resume at `deadline − margin` is purely the wake-up mechanism, with no correctness role. -- **Backstops, not mechanisms.** `maximumDurationInSeconds` (mandatory on every `RunMicrovm`, pinned at 28 800 s — see sub-decision 1) is the substrate kill switch bounding running **and** suspended time; the orchestrator's finalization `terminate-microvm` is the active cleanup path; the stranded-approval reconciler retains its role for orphaned waits. No `idlePolicy`-based bound is used in any phase — see sub-decision 1's omit-`idlePolicy` invariant. -- **Concurrency slot stays held** during suspend. Cedar decision #7's rationale ("container alive, consuming memory") weakens under suspend, but the conclusion survives for a harder reason: AWS counts `SUSPENDED` MicroVMs toward the account memory quota, so releasing ABCA's slot would not free real capacity — it would only invite admission of tasks the substrate cannot start. +- **Backstops, not mechanisms.** `maximumDurationInSeconds` (mandatory on every `RunMicrovm`, pinned at 28 800 s — see sub-decision 1) is the substrate kill switch bounding running **and** suspended time; the orchestrator's finalization `terminate-microvm` is the active cleanup path; the stranded-approval reconciler retains its role for orphaned waits. No `idlePolicy`-based bound is used in any phase — see sub-decision 1's omit-`idlePolicy` invariant. **The active terminate is mandatory, not belt-and-braces**: a MicroVM whose hook never ran reached `RUNNING` in 12 s and stayed `RUNNING` with no `stateReason` through every checkpoint (live). Nothing self-terminates on this substrate, so nothing cleans up — a leaked VM bills until the 8 h cap. +- **Concurrency slot stays held** during suspend. Cedar decision #7's rationale ("container alive, consuming memory") weakens under suspend, and the harder replacement rationale — "AWS counts `SUSPENDED` MicroVMs toward the account memory quota, so releasing ABCA's slot would not free real capacity" — is **undischarged**: the suspended VM stayed in `list-microvms` at every checkpoint, but that only proves *listed*. `L-CD1C0CC4` (1024 GB, account-scoped) exposes no `UsageMetric`, `AWS/Usage` carries only `CallCount` per API, and no MicroVM memory metric exists in any namespace, so consumption is **not observable safely** — proving it would need a large concurrent fleet. The conclusion (hold the slot) stands as the conservative choice, not as a verified fact. Size the arithmetic against the 32 GiB **peak** rather than the 8 GiB baseline: a busy fleet scales up, so peak is what actually competes for the account quota. The agent's `/suspend` hook flushes progress events (durable writes before returning 200, within the 60 s hook budget); `/resume` reseeds CSPRNGs and refreshes cached credentials. @@ -97,22 +137,55 @@ Normative requirements (EARS): ### 3. Packaging: same agent image source, new build path -The existing agent container (`agent/` Dockerfile, already ARM64) is repackaged as a zip + Dockerfile artifact in S3 and built into a versioned `MicrovmImage` via `CreateMicrovmImage` (with `/ready` and `/validate` build hooks for snapshot quality). The agent runs its existing FastAPI server (`agent/src/server.py`) — the MicroVM path uses the HTTP entrypoint like AgentCore, not ECS's batch bypass — plus the four runtime lifecycle hooks on a sidecar port. Runtime hooks are fast-notification only (1–60 s): `/run` validates the payload and starts the pipeline **asynchronously**, mirroring how the agent loop already runs in a background thread behind `/ping` on AgentCore. +The existing agent container (`agent/` Dockerfile, already ARM64) is repackaged as a zip + Dockerfile artifact in S3 and built into a versioned `MicrovmImage` via `CreateMicrovmImage`. The agent runs its existing FastAPI server (`agent/src/server.py`) — the MicroVM path uses the HTTP entrypoint like AgentCore, not ECS's batch bypass — plus the runtime lifecycle hooks (`/run`, `/suspend`, `/resume`, `/terminate`) and the `/ready` + `/validate` build hooks, all on the same port the server already listens on (8080, declared as the image's `hooks.port`). Runtime hooks are fast-notification only (1–60 s): `/run` validates the payload and starts the pipeline **asynchronously**, mirroring how the agent loop already runs in a background thread behind `/ping` on AgentCore. -**Payload delivery** reuses the ECS strategy's S3-pointer pattern, adapted to `runHookPayload` (≤ 16 KB): small payloads inline; larger ones uploaded by the strategy to a platform payload bucket (the ECS payload bucket pattern in `ecs-agent-cluster.ts`: orchestrator write access, compute-role read-only scoped to the bucket, lifecycle expiry on objects) with only the S3 URI in `runHookPayload` — the MicroVM **execution role** holds the read grant, exactly as the ECS task role does today. +**Hook phasing — corrected: `/ready` + `/run` are both P1.** The original plan split *declaring* a hook from *serving* it, putting `/run`'s declaration in P1 and its implementation in P2. Live verification proved that split is **not a reachable service state**, on two independent counts: -**No orchestrator→agent HTTP path exists in P1–P3**: payload arrives through the `/run` hook, all agent work is outbound, and therefore **no JWE auth tokens are minted at all** — token minting (and its ≤ 60 min TTL refresh problem) is deferred until a real consumer exists (e.g. operator shell access, [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391)). The `endpoint` stays in the `SessionHandle` because it is genuinely per-session state that becomes load-bearing the day such a consumer appears. **Constraint accepted:** 32 GB RAM / 32 GB disk (disk quota to be confirmed against current service quotas when COMPUTE.md is updated) means repos that motivated the 120 GB ECS sizing stay on `ecs`; MicroVMs target the default-sized workload with suspend economics, not the heavy-build niche. +- `CreateMicrovmImage` rejects an image that enables any lifecycle hook without `/ready`: *"The ready (/ready) MicroVM image hook must be enabled when any MicroVM lifecycle hook (run, resume, suspend, or terminate) is enabled."* So a P1 image declaring only `/run` is **not creatable**. +- With `/ready` added but unserved, both chipset builds fail: *"Ready hook check failed: the application returned a client error (HTTP 4xx) response."* So a declared hook must be served in the same phase. +- And an image with **no** hooks at all — the only other creatable shape — cannot receive a payload: *"The run hook must be enabled in the MicroVM image to pass the run hook payload."* So deferring hooks entirely also defers the whole payload-delivery channel. -- The image build shall not embed secrets, tokens, or per-task identity in the snapshot. -- The agent shall resolve credentials at `/run` time. -- When the `/run` hook receives the task payload, the agent shall validate it, start the pipeline asynchronously, and return HTTP 200 within the hook budget. -- The agent shall not execute the clone→verify→PR pipeline on the hook path. -- If the task payload exceeds the 16 KB `runHookPayload` limit, then the strategy shall upload the payload to the platform payload bucket and pass only its S3 URI in `runHookPayload`. -- The MicroVM execution role shall hold read-only access to the payload bucket, scoped to that bucket. +The phasing is therefore: + +| Hook | Declared by | Served by the agent | Notes | +|---|---|---|---| +| `/ready` | **P1** (construct sets `hooks.microvmImageHooks.ready`) | **P1** | MANDATORY, not a quality nicety — see above. A 200 once uvicorn is bound is the whole P1 contract: it also proves `server` imported cleanly (pulling in `pipeline` → `runner` → the policy engine), so a missing policy file fails the BUILD instead of the first task. | +| `/run` | **P1** (construct sets `hooks.microvmHooks.run`) | **P1** | The payload-delivery channel. Must be served in P1 because `/ready` forces hooks to exist at all, and a hook-less image cannot accept `runHookPayload`. | +| `/validate` | **P2** | **P2** | Build-time snapshot-quality hook. Still deliberately NOT declared: a `/validate` that 404s or reports failure fails every image build. Deeper warm-up assertions (Bedrock reachability, Memory access, tool availability) belong here. | +| `/suspend`, `/resume`, `/terminate` | **P3** (suspend/resume), P2 (`/terminate`) | P3 / P2 | Declaring a runtime hook the agent does not serve fails the corresponding lifecycle transition, so each is declared only in the phase that implements it. P1 termination is the orchestrator's `TerminateMicrovm`, which needs no in-guest cooperation. | + +Consequence to state plainly, replacing the original "a P1-built MicroVM image is not runnable end to end": **a P1 image is creatable, launchable and payload-deliverable, but carries no smoke-parity guarantee.** P1 delivers the strategy, the construct, the roles/buckets/connectors, the image resource, the packaging script, and the `/ready` + `/run` endpoints — so a `lambda-microvm` task can start a MicroVM and hand it a payload. What P1 has **not** established is anything P2 owns: AgentCore Memory grants and `MEMORY_ID` delivery, the agent's non-secret env parity inside the snapshot, egress specifics from a running MicroVM, and heartbeat/progress behaviour end to end. No clone → change → PR run has happened on this substrate. P2 ("smoke parity") is the phase that closes that gap. The construct and the packaging script both surface exactly this at synth/run time (`abca:microvm-image-p1-smoke-unverified`) so an operator cannot mistake a launchable substrate for a verified one. + +**Payload delivery** reuses the ECS strategy's S3-pointer pattern, adapted to `runHookPayload` (**≤ 4 KB** — measured, see below): payloads that fit ride inline; the rest are uploaded by the strategy to a platform payload bucket (the ECS payload bucket pattern in `ecs-agent-cluster.ts`: orchestrator write access, compute-role read-only scoped to the bucket, lifecycle expiry on objects) with only the S3 URI in `runHookPayload` — the MicroVM **execution role** holds the read grant, exactly as the ECS task role does today. + +The cap is **4 096 bytes**, not the 16 384 the SDK documents. Measured exactly: 4 096 passes, 4 097 is rejected with *"Value at 'runHookPayload' failed to satisfy constraint: Member must have length less than or equal to 4096"*. Two consequences follow. First, the original threshold would have inlined every envelope between 4 097 and 16 384 bytes and had the service reject all of them. Second, and more structurally: **the S3-pointer path is now the dominant one, and inline is the exception.** A hydrated task payload (prompt + issue thread + repo context) essentially always exceeds 4 KB, so "small payloads ride inline" describes tiny repo-less prompts rather than the common case. The payload bucket is therefore not a rarely-exercised overflow valve but a required part of every normal task, which raises its lifecycle rule (`MICROVM_PAYLOAD_TTL_DAYS`) and the execution role's read grant from edge-case plumbing to load-bearing. + +**No orchestrator→agent HTTP path exists in P1–P3**: payload arrives through the `/run` hook, all agent work is outbound, and therefore **no JWE auth tokens are minted at all** — token minting (and its ≤ 60 min TTL refresh problem) is deferred until a real consumer exists (e.g. operator shell access, [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391)). The `endpoint` stays in the `SessionHandle` because it is genuinely per-session state that becomes load-bearing the day such a consumer appears. But note the service does not agree by default: omitting `ingressNetworkConnectors` on `RunMicrovm` attaches a **public** `HTTP_INGRESS` connector, so the strategy passes the Lambda-managed `NO_INGRESS` connector explicitly on every launch (see sub-decision 4's security table). + +**Constraint accepted:** the configured **baseline is 8 GiB RAM / 4 vCPU** and the service scales vertically to a **32 GiB / 16 vCPU peak** on its own, with 32 GB of disk. So capacity is baseline-priced with 4× burst headroom — good for the bursty compile-and-test shape of an agent task — but the SUSTAINED ceiling is still 32 GiB, so repos that motivated the 120 GB ECS sizing stay on `ecs`. What the construct configures (and validates) is the baseline; the peak is not something a deployment asks for. + +Normative requirements (EARS). All of these are P1 now — the earlier P1/P2 split of this list existed only because the phasing table split declaring a hook from serving it, which the service does not permit: + +- (P1) The image build shall not embed secrets, tokens, or per-task identity in the snapshot. +- (P1) If the task payload exceeds the 4 KB `runHookPayload` limit, then the strategy shall upload the payload to the platform payload bucket and pass only its S3 URI in `runHookPayload`. +- (P1) The MicroVM execution role shall hold read-only access to the payload bucket, scoped to that bucket. +- (P1) Where no ingress is configured for a deployment, the strategy shall pass the Lambda-managed `NO_INGRESS` network connector on every `RunMicrovm` call (the field shall not be omitted). +- (P1) Where the image enables any MicroVM lifecycle hook, the image shall also enable the `/ready` hook and the agent shall serve it. +- (P1) When the `/run` hook receives the task payload, the agent shall validate it, start the pipeline asynchronously, and return HTTP 200 within the hook budget. +- (P1) If the `/run` hook payload cannot be resolved to a task payload, then the agent shall reject the hook with a client error and shall not start a pipeline. +- (P1) The agent shall not execute the clone→verify→PR pipeline on the hook path. +- (P1) The agent shall resolve credentials at `/run` time. +- (P1) Where a deployment configures a MicroVM image before smoke parity is verified, the platform shall warn that the backend has no smoke-parity guarantee. +- (P2) Where the image declares the `/validate` build hook, the agent shall serve it. ### 4. Infra and IAM: conditional resources behind bootstrap `ComputeTypes` -Mirroring the ECS pattern: a `compute-lambda-microvm` bootstrap policy (`cdk/src/bootstrap/policies/`) gated on the `ComputeTypes` CFN parameter; a CDK construct provisioning the build role, execution role (admitted to the per-session role via `AgentSessionRole.admitComputeRole`, which was designed for exactly this), the S3 artifact bucket wiring, and image build automation. Egress uses the platform VPC via an egress network connector so the DNS Firewall / security-group / flow-log stack in [COMPUTE.md](../design/COMPUTE.md) applies unchanged; ingress is restricted to the JWE-authenticated endpoint (no `SHELL_INGRESS` by default — it is noted as a candidate for [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391), operator session access, as a separate decision). +Mirroring the ECS pattern: a `compute-lambda-microvm` bootstrap policy (`cdk/src/bootstrap/policies/`) gated on the `ComputeTypes` CFN parameter; a CDK construct provisioning the build role, execution role (admitted to the per-session role via `AgentSessionRole.admitComputeRole`, which was designed for exactly this), the S3 artifact bucket wiring, and image build automation. Egress uses the platform VPC via egress network connectors so the DNS Firewall / security-group / flow-log stack in [COMPUTE.md](../design/COMPUTE.md) applies unchanged; ingress is suppressed with the Lambda-managed `NO_INGRESS` connector (no `SHELL_INGRESS` — it is noted as a candidate for [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391), operator session access, as a separate decision). + +Two networking facts the construct has to encode, both established live: + +- **A `VPC_EGRESS` connector requires an operator role.** CloudFormation's generated L1 types `operatorRole` as optional and this ADR originally assumed Lambda would manage the ENIs with its own service-linked role. It does not: the connector fails to create with *"NetworkConnectorOperatorRole is required for VPC_EGRESS connector type"*. The construct creates one role — trusting `lambda.amazonaws.com` with `aws:SourceAccount` pinned, carrying `AWSLambdaVPCAccessExecutionRole` plus the ENI / tag / private-IP actions that policy omits — and shares it across both connectors, since it manages interfaces rather than traffic. +- **Build-time egress needs port 80; runtime does not.** `agent/Dockerfile` installs Debian packages and `apt-get` fetches over plain HTTP, so a 443-only egress path fails every snapshot build (`Could not connect to deb.debian.org:80 … exit code: 100`). Rather than widen the runtime posture, the construct provisions a **second, build-only** connector on the same private subnets with a 443 + 80 security group, referenced solely by the image resource and the packaging script. The agent at run time still has 443-only egress. - Where the bootstrap `ComputeTypes` parameter includes `lambda-microvm`, the generated template shall attach the `IaCRole-ABCA-Compute-LambdaMicrovms` policy to the CloudFormation execution role. - The orchestrator role shall receive only the MicroVM lifecycle actions it calls (`lambda:RunMicrovm`, `lambda:SuspendMicrovm`, `lambda:ResumeMicrovm`, `lambda:TerminateMicrovm`, `lambda:GetMicrovm` for `pollSession`, and `lambda:PassNetworkConnector`, which is required even for the default connectors), scoped to platform-created images. @@ -128,14 +201,16 @@ Mirroring the ECS pattern: a `compute-lambda-microvm` bootstrap policy (`cdk/src | Control | AgentCore | ECS | Lambda MicroVMs | Delta | |---|---|---|---|---| -| Egress (DNS Firewall, TCP 443 SG, flow logs) | Platform VPC | Platform VPC | Platform VPC via egress network connector | None | +| Egress, runtime (DNS Firewall, TCP 443 SG, flow logs) | Platform VPC | Platform VPC | Platform VPC via egress network connector | None | +| Egress, image build | ECR build outside the platform VPC | ECR build outside the platform VPC | Platform VPC via a **separate build-only connector, TCP 443 + 80** (`apt-get` is plain HTTP) | New surface: build-time egress is wider than runtime egress by one port, on a connector no running MicroVM can use | | Tenant-data scoping | Per-session role (`admitComputeRole`) | Per-session role | Per-session role, execution role admitted identically | None | | Secrets delivery | Runtime env + Identity injection | Task env vars | Fetched at `/run`; never in snapshot | New surface: snapshot must stay secret-free (EARS req., sub-decision 3) | -| Inbound exposure | None (SigV4 invoke only) | None (no endpoint) | **Public HTTPS endpoint, JWE-gated, no unauthenticated access; no tokens minted in P1–P3** | New surface: endpoint exists but is unreachable without a minted token | +| Inbound exposure | None (SigV4 invoke only) | None (no endpoint) | **None — but only because the strategy passes `NO_INGRESS` explicitly.** The service default is a PUBLIC `HTTP_INGRESS` connector plus a public `*.lambda-microvm..on.aws` endpoint; no tokens are minted in P1–P3 either way | New surface **and** a new failure mode: "no inbound" is an active control, not an absence. Drop the `NO_INGRESS` argument and every agent MicroVM gets a public endpoint (EARS req., sub-decision 3) | | Session isolation | MicroVM | Task-level | MicroVM (Firecracker) | None (≥ ECS) | | State reuse | None | None | Snapshot shared across MicroVMs | New surface: CSPRNG reseed + credential refresh on `/run`/`/resume` (EARS req.) | | Workload-token injection | Yes (Runtime-coupled) | No (env-var posture) | No (env-var posture) | Shared with ECS; deferred to [#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249)/ADR-016 | | Operator shell access | No | No | Not enabled (`SHELL_INGRESS` omitted; candidate for [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391)) | None by default | +| Auth-token minting | n/a | n/a | `CreateMicrovmAuthToken` granted to no role in any phase | Verified static-only: any principal holding the action *can* mint a working JWE, including against a `SUSPENDED` MicroVM, so the posture rests entirely on the grant being absent | #### Regional availability enforcement @@ -154,21 +229,25 @@ Lambda MicroVMs launched in 5 regions (us-east-1/2, us-west-2, eu-west-1, ap-nor ### 5. Rollout: phased, default unchanged -- **P1 — strategy + infra:** `LambdaMicrovmComputeStrategy` (start/poll/stop), CDK construct, bootstrap policy, types sync, unit + CDK assertion tests. No suspend yet. -- **P2 — smoke parity:** agent completes clone → change → PR on the backend with progress visible to `bgagent watch`; failure classification entries in `error-classifier.ts`; **AgentCore Memory parity** (IAM grant + `MEMORY_ID` delivery, following the `EcsAgentCluster` pattern — Memory is a standalone service already consumed cross-substrate, and omitting the grant silently no-ops cross-session learning). +- **P1 — strategy + infra + minimal hook serving:** `LambdaMicrovmComputeStrategy` (start/poll/stop), CDK construct, bootstrap policy, types sync, unit + CDK assertion tests, and the agent's `/ready` + `/run` endpoints. No suspend yet. The image IS creatable and launchable and the payload DOES reach the agent — but there is **no smoke-parity guarantee** (sub-decision 3's phasing table). +- **P2 — smoke parity:** the agent serves the remaining hooks (`/terminate`, `/validate`); agent completes clone → change → PR on the backend with progress visible to `bgagent watch`; failure classification entries in `error-classifier.ts`; **AgentCore Memory parity** (IAM grant + `MEMORY_ID` delivery, following the `EcsAgentCluster` pattern — Memory is a standalone service already consumed cross-substrate, and omitting the grant silently no-ops cross-session learning); the agent's remaining non-secret env parity inside the snapshot. - **P3 — suspend/resume:** the interface widening from sub-decision 1 (mandatory methods, all three strategies in one commit), HITL-wait suspend policy, inline resume in the approve/deny Lambda with orchestrator-poll reconciliation (sub-decision 2), timeout-under-freeze wall-clock handling; coordinate with [#491](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/491)'s unified liveness model and update Cedar decision #7's rationale note. - **Out of scope:** replacing AgentCore as default; classic Lambda functions as a runtime; GPU; the Runtime-coupled workload-access-token injection path (delivery mechanism exists only on AgentCore Runtime; MicroVMs adopt the ECS env-var posture until [#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249)/ADR-016 redesign the seam). Gateway integration is orthogonal: ADR-019/[#641](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/641) is substrate-portable by design and applies to this backend when it lands. ## Consequences -- (+) **Suspend/resume economics.** Tasks idling on approval waits stop billing compute while preserving full state — bounded at ~1 h per gate under the current Cedar ceiling (decision #6), and the enabler for cheap off-hours gate-ceiling extensions later (§14.8). +- (+) **Suspend/resume economics.** Tasks idling on approval waits stop billing compute while preserving full state — bounded at ~1 h per gate under the current Cedar ceiling (decision #6), and the enabler for cheap off-hours gate-ceiling extensions later (§14.8). Verified end to end at the substrate level: suspend and resume each complete in ~1 s, and `microvmId` **and** `endpoint` survive the cycle byte-identical, so a stored `SessionHandle` remains valid. - (+) **VM-level isolation without cluster ops.** Firecracker isolation with no ECS cluster, task definition, or capacity management; one-session-per-MicroVM maps 1:1 onto ABCA's task model. -- (+) **Escapes AgentCore's 2 GB image limit and FUSE `flock()` workaround** — native disk in the snapshot supports `uv`/`mise` without the split-storage scheme. +- (+) **Escapes AgentCore's 2 GB image limit and FUSE `flock()` workaround** — native disk in the snapshot supports `uv`/`mise` without the split-storage scheme. Note the size comparison must say WHICH measure it means: the same agent tree is 1.799 GB as an OCI image (629.7 MB compressed, i.e. under AgentCore's limit) but reports `codeInstallSizeInBytes` of 2.17 GiB as a MicroVM snapshot (i.e. over it). The two straddle the limit and are not interchangeable; memory/disk snapshot sizes are a third thing again and must not be summed into the comparison. - (+) **Liveness becomes explicit.** Unlike AgentCore's stub `pollSession`, the strategy can report real substrate state, strengthening the [#491](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/491) unification. -- (−) **32 GB RAM / 32 GB disk hard ceiling** — not a successor to the ECS backend for heavy CI-parity builds; the platform now maintains three backends. -- (−) **New packaging pipeline.** Zip + Dockerfile + service-side image builds with versioned snapshots (storage billed per version) alongside the existing ECR flow; image versions need lifecycle cleanup. -- (−) **8-hour hard cap includes suspended time**, and with `idlePolicy` omitted there is no tighter substrate-level suspended-TTL — the suspended-state bound is `maximumDurationInSeconds` plus orchestrator termination and the stranded reconciler. Under today's 1 h gate ceiling this is comfortably sufficient, but any future extension of gate ceilings must revisit the bound (an additive `idlePolicy` change) and give the orchestrator a checkpoint-and-restart path (push branch, new session) beyond the cap. +- (−) **32 GiB sustained ceiling (8 GiB baseline + automatic 4× vertical scaling), 32 GB disk.** Not a successor to the ECS backend for heavy CI-parity builds; the platform now maintains three backends. +- (−) **Capacity is baseline-priced with burst headroom, which is a narrower promise than "32 GiB".** The deployment configures an 8 GiB / 4 vCPU baseline and the service scales to 32 GiB / 16 vCPU on demand — well matched to an agent task, which is idle-ish while waiting on the model and spiky during builds, and cheaper than reserving the peak. But it is *burst*, not a reservation: a workload that needs 32 GiB **sustained** is relying on scaling behaviour this ADR has not measured, and against ECS's 120 GB the gap for sustained-memory workloads is unchanged. So the value proposition remains the suspend economics, the observable control-plane state machine, and the absence of cluster ops — with capacity now a fair-to-good fit rather than a hard blocker. Repos with genuinely sustained heavy builds still belong on `ecs`. +- (−) **New packaging pipeline.** Zip + Dockerfile + service-side image builds with versioned snapshots (storage billed per version) alongside the existing ECR flow; image versions need lifecycle cleanup — including versions left behind by FAILED builds, and noting the last version of an image cannot be deleted individually (delete the image, which reaps it). +- (−) **The payload bucket is on the hot path, not the overflow path.** With a 4 KB `runHookPayload` cap, virtually every real task delivers its payload via S3, so the bucket, its TTL rule and the execution role's read grant are load-bearing for normal operation rather than an edge case (sub-decision 3). +- (−) **8-hour hard cap includes suspended time**, and with `idlePolicy` omitted there is no tighter substrate-level suspended-TTL — the suspended-state bound is `maximumDurationInSeconds` plus orchestrator termination and the stranded reconciler. A manually suspended VM was observed alive at 1 h with no TTL in sight (observation truncated there), so nothing contradicts this bound, but nothing narrows it either. Under today's 1 h gate ceiling it is comfortably sufficient; any future extension of gate ceilings must revisit the bound (an additive `idlePolicy` change) and give the orchestrator a checkpoint-and-restart path (push branch, new session) beyond the cap. - (!) **Idle-policy foot-gun.** Traffic-based auto-suspend would freeze a busy outbound-only agent; the decision to disable auto-suspend must be enforced in code and covered by tests, not left to configuration discipline. +- (!) **Service defaults are not the desired posture.** Two live-caught cases (public `HTTP_INGRESS` by default; `/ready` mandatory) mean an omitted field on this backend does not mean "off" — it can mean "the service picks, and it picks wider than we want". Every new `RunMicrovm` / `CreateMicrovmImage` field should be assumed to have an opinionated default until checked. +- (!) **Nothing self-terminates.** A MicroVM whose hook never ran still reaches `RUNNING` and stays there, billing, until the 8 h cap. The orchestrator's `TerminateMicrovm` on finalize is the only cleanup, so a leaked handle is a cost incident, not just an untidy state. - (!) **Snapshot uniqueness.** Shared memory snapshots require CSPRNG reseeding and credential refresh in `/run` / `/resume` hooks; missing this is a silent security defect. - (!) **Regional availability (5 regions at launch, expanding)** — enforced in layers (synth-time static check, onboarding + doctor live probes, orchestration-time classification; see sub-decision 4). The static CDK constant is the one piece that rots as AWS expands; its update path and context-flag escape hatch are deliberate. - (!) **Workload-token injection delta persists** (shared with the ECS backend) until [#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249)/ADR-016 land; document it in the security bar comparison rather than blocking on it. Memory and Gateway are explicitly *not* deltas — both are standalone services consumed via IAM from any substrate. @@ -177,11 +256,12 @@ Lambda MicroVMs launched in 5 regions (us-east-1/2, us-west-2, eu-west-1, ap-nor **P1 (start/poll/stop — no suspend):** -- Unit tests for the strategy: start/poll/stop mapping (including `SessionStatus` `'suspended'` reported mechanically, without task-state interpretation), payload-size branching (inline vs S3 pointer), error classification (`ServiceQuotaExceededException`, `ThrottlingException`, `ResourceNotFoundException`, regional-unavailability), the omit-`idlePolicy` invariant, and `maximumDurationInSeconds` fixed at 28 800. +- Unit tests for the strategy: start/poll/stop mapping (including `SessionStatus` `'suspended'` reported mechanically, without task-state interpretation), payload-size branching (inline vs S3 pointer, at the exact 4 096/4 097-byte boundary), the image-identifier-must-be-an-ARN guard, the explicit `NO_INGRESS` argument (including the blank-env-var fallback, which must never omit the field), error classification (`ServiceQuotaExceededException`, `ThrottlingException`, `ResourceNotFoundException`, regional-unavailability), the omit-`idlePolicy` invariant, and `maximumDurationInSeconds` fixed at 28 800. +- Agent tests: `/ready` returns 200 once the server is up and starts nothing; `/run` accepts both envelope shapes (inline and S3 pointer), starts the pipeline asynchronously through the same mapper `/invocations` uses, returns before the pipeline finishes, and rejects every unusable envelope with a named code before spawning; `/validate`, `/suspend`, `/resume` and `/terminate` are NOT served. - Orchestrator tests: substrate-terminal + non-terminal task status → failed classification; `suspended` + non-`AWAITING_APPROVAL` status → anomaly event, no fail-fast; `compute_metadata` persisted with `microvmId`/`endpoint` after `startSession`. -- CDK assertions: MicroVM resources present only when `ComputeTypes` includes the backend; synth failure for unsupported regions (plus the context-flag escape hatch); IAM actions scoped as specified (orchestrator lifecycle set; no `CreateMicrovmAuthToken` anywhere); payload-bucket grants (execution role read-only); backend cost-allocation tags; types-sync check covers the widened `ComputeType`. +- CDK assertions: MicroVM resources present only when `ComputeTypes` includes the backend; synth failure for unsupported regions (plus the context-flag escape hatch); memory size validated against the accepted list at synth; the connector operator role and its trust; two connectors with the build-only one carrying port 80 and the runtime one not; the `/ready` + `/run` hook declaration and the absence of the others; IAM actions scoped as specified (orchestrator lifecycle set; no `CreateMicrovmAuthToken` anywhere); payload-bucket grants (execution role read-only); backend cost-allocation tags; types-sync check covers the widened `ComputeType`. - CLI tests: onboarding rejection with remedy when the availability probe fails; doctor check present when a blueprint selects the backend. -- P1 verification items (external service facts): manual-suspend default TTL without `idlePolicy`; disk quota; `runHookPayload` 16 KB limit; IAM action names; region probe behavior; account-quota treatment of `SUSPENDED` MicroVMs — record answers in COMPUTE.md (`maximumDurationInSeconds` bounds the worst case either way). +- P1 verification items (external service facts) — **executed 2026-07-31, us-east-1**; see `docs/verification/645-p1-lambda-microvm-runbook.md` for the full evidence. Discharged: `runHookPayload` limit (**4 096**, not 16 KB), the accepted baseline memory sizes (`[512…8192]` MiB — note the developer guide, not the probe, is what establishes that this is a BASELINE with a 32 GiB peak), image-identifier ARN requirement, IAM action names and the observed image-ARN shape, region probe behaviour, manual suspend/resume without `idlePolicy`, terminate timing and the `TERMINATED`-persists-≥10-min finding, the default public `HTTP_INGRESS`, and the `/ready` requirement. **Not** discharged: account-quota treatment of `SUSPENDED` MicroVMs (not observable safely), suspended TTL beyond 1 h (truncated), the vertical-scaling behaviour itself (no workload here approached the baseline, so the 4× peak is documented rather than observed), and the `AWS::Lambda::MicrovmImage` CloudFormation value shapes (never exercised — the run used the out-of-band script path). Record the closed answers in COMPUTE.md. **P2 (smoke parity):** diff --git a/docs/design/DEPLOYMENT_ROLES.md b/docs/design/DEPLOYMENT_ROLES.md index 9f2f0baca..8a279edc8 100644 --- a/docs/design/DEPLOYMENT_ROLES.md +++ b/docs/design/DEPLOYMENT_ROLES.md @@ -2,7 +2,7 @@ This document defines least-privilege IAM policies for the CloudFormation execution role used during `cdk deploy`. The default CDK bootstrap grants `AdministratorAccess` to this role; the policies below scope it to only what ABCA needs. -> **Origin**: These IAM policies were derived from a thorough review of the repository's CDK constructs, stacks, and handler code, then **validated against a live deployment** in `us-east-1` (create, update, task execution, and destroy). CloudTrail analysis identified 36 additional actions beyond the initial code review, and 7 deployment iterations refined the policies to their current form. The policies are split into five managed policies — three always-applied (Infrastructure, Application, Observability) plus two compute-variant policies (Compute-AgentCore, Compute-ECS) — to stay under the IAM 6,144-character limit and to support per-compute-backend bootstrap selection. The policy sources live in `cdk/src/bootstrap/policies/*.ts` and are compiled to `cdk/bootstrap/policies/*.json`. +> **Origin**: These IAM policies were derived from a thorough review of the repository's CDK constructs, stacks, and handler code, then **validated against a live deployment** in `us-east-1` (create, update, task execution, and destroy). CloudTrail analysis identified 36 additional actions beyond the initial code review, and 7 deployment iterations refined the policies to their current form. The policies are split into six managed policies — three always-applied (Infrastructure, Application, Observability) plus three compute-variant policies (Compute-AgentCore, Compute-ECS, Compute-LambdaMicrovms) — to stay under the IAM 6,144-character limit and to support per-compute-backend bootstrap selection. The policy sources live in `cdk/src/bootstrap/policies/*.ts` and are compiled to `cdk/bootstrap/policies/*.json`. ## How CDK deployment roles work @@ -17,7 +17,7 @@ The policy below is a **CloudFormation Execution Role** replacement. The other t ## Using these policies -The policies are split into five IAM managed policies (each under the 6,144-character limit): +The policies are split into six IAM managed policies (each under the 6,144-character limit): | Policy Name | Scope | When applied | |-------------|-------|--------------| @@ -26,10 +26,11 @@ The policies are split into five IAM managed policies (each under the 6,144-char | `IaCRole-ABCA-Observability` | Bedrock Guardrails, CloudWatch, X-Ray, S3, ECR, KMS, SSM, STS | Always | | `IaCRole-ABCA-Compute-Agentcore` | Bedrock AgentCore (`bedrock-agentcore:*`) | Always (default compute backend) | | `IaCRole-ABCA-Compute-ECS` | ECS cluster + task-definition operations | Only when `ecs` is in `ComputeTypes` | +| `IaCRole-ABCA-Compute-LambdaMicrovms` | Lambda MicroVM image + network-connector operations | Only when `lambda-microvm` is in `ComputeTypes` | > **Placeholder substitution**: Replace `ACCOUNT_ID` with your 12-digit AWS account ID and `REGION` with your deployment region (e.g., `us-east-1`) throughout this document. -These policies are not created or attached manually. The repository generates them — and a custom bootstrap template that wires all five into the CloudFormation execution role — from the TypeScript sources, then bootstraps with that template: +These policies are not created or attached manually. The repository generates them — and a custom bootstrap template that wires all six into the CloudFormation execution role — from the TypeScript sources, then bootstraps with that template: ```bash # Regenerate artifacts (policies JSON + template YAML) and bootstrap. @@ -47,7 +48,7 @@ aws cloudformation update-stack --stack-name CDKToolkit --use-previous-template aws cloudformation describe-stacks --stack-name CDKToolkit --query 'Stacks[0].Parameters' ``` -Under the hood, `mise //cdk:bootstrap` runs `npx cdk bootstrap --template bootstrap/bootstrap-template.yaml` (see `cdk/mise.toml`). The generated template defines five inline `AWS::IAM::ManagedPolicy` resources that **replace** the default `AdministratorAccess` on the CloudFormation execution role; the `IaCRole-ABCA-Compute-ECS` policy is conditional on the `ComputeTypes` parameter including `ecs`. The policy sources are `cdk/src/bootstrap/policies/{infrastructure,application,observability,compute-agentcore,compute-ecs}.ts`, compiled to `cdk/bootstrap/policies/*.json` by `cdk/scripts/generate-bootstrap-artifacts.ts`. +Under the hood, `mise //cdk:bootstrap` runs `npx cdk bootstrap --template bootstrap/bootstrap-template.yaml` (see `cdk/mise.toml`). The generated template defines six inline `AWS::IAM::ManagedPolicy` resources that **replace** the default `AdministratorAccess` on the CloudFormation execution role; the `IaCRole-ABCA-Compute-ECS` and `IaCRole-ABCA-Compute-LambdaMicrovms` policies are conditional on the `ComputeTypes` parameter including their respective backend. The policy sources are `cdk/src/bootstrap/policies/{infrastructure,application,observability,compute-agentcore,compute-ecs,compute-lambda-microvm}.ts`, compiled to `cdk/bootstrap/policies/*.json` by `cdk/scripts/generate-bootstrap-artifacts.ts`. ## Trust policy @@ -82,7 +83,7 @@ Under the hood, `mise //cdk:bootstrap` runs `npx cdk bootstrap --template bootst For deploying the `backgroundagent-dev` stack. This single stack contains all platform resources including the AgentCore runtime, ECS compute (when enabled), API Gateway, Cognito, DynamoDB tables, VPC, DNS Firewall, and observability infrastructure. -> **IAM managed policy size limit**: A single managed policy cannot exceed 6,144 characters. The permissions below are split into five policies to stay under this limit (three always-applied, plus two compute-variant policies). They are wired into the CloudFormation execution role by the generated bootstrap template; see [Using these policies](#using-these-policies). +> **IAM managed policy size limit**: A single managed policy cannot exceed 6,144 characters. The permissions below are split into six policies to stay under this limit (three always-applied, plus three compute-variant policies). They are wired into the CloudFormation execution role by the generated bootstrap template; see [Using these policies](#using-these-policies). ### IaCRole-ABCA-Infrastructure @@ -738,6 +739,44 @@ When the ECS Fargate compute backend is enabled (set the `ComputeTypes` CFN para } ``` +### IaCRole-ABCA-Compute-LambdaMicrovms + +When the Lambda MicroVM compute backend is enabled (include `lambda-microvm` in the `ComputeTypes` CFN parameter on the `CDKToolkit` stack), the generated template conditionally attaches this policy to the CloudFormation execution role. It permits CloudFormation to manage MicroVM images and network connectors; runtime session lifecycle permissions remain on the orchestrator role. + +```json +{ + "Statement": [ + { + "Action": [ + "lambda:CreateMicrovmImage", + "lambda:GetMicrovmImage", + "lambda:UpdateMicrovmImage", + "lambda:DeleteMicrovmImage", + "lambda:ListMicrovmImages", + "lambda:GetMicrovmImageVersion", + "lambda:UpdateMicrovmImageVersion", + "lambda:DeleteMicrovmImageVersion", + "lambda:ListMicrovmImageVersions", + "lambda:GetMicrovmImageBuild", + "lambda:ListMicrovmImageBuilds", + "lambda:ListManagedMicrovmImages", + "lambda:ListManagedMicrovmImageVersions", + "lambda:CreateNetworkConnector", + "lambda:GetNetworkConnector", + "lambda:UpdateNetworkConnector", + "lambda:DeleteNetworkConnector", + "lambda:ListNetworkConnectors", + "lambda:PassNetworkConnector" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "LambdaMicrovms" + } + ], + "Version": "2012-10-17" +} +``` + ## Runtime IAM roles (created by the stack) These roles are created inside the CloudFormation stack at deploy time, not by the deployer. They are documented here for a complete picture of the IAM footprint. diff --git a/docs/src/content/docs/architecture/Deployment-roles.md b/docs/src/content/docs/architecture/Deployment-roles.md index a852a9b67..cdce53038 100644 --- a/docs/src/content/docs/architecture/Deployment-roles.md +++ b/docs/src/content/docs/architecture/Deployment-roles.md @@ -6,7 +6,7 @@ title: Deployment roles This document defines least-privilege IAM policies for the CloudFormation execution role used during `cdk deploy`. The default CDK bootstrap grants `AdministratorAccess` to this role; the policies below scope it to only what ABCA needs. -> **Origin**: These IAM policies were derived from a thorough review of the repository's CDK constructs, stacks, and handler code, then **validated against a live deployment** in `us-east-1` (create, update, task execution, and destroy). CloudTrail analysis identified 36 additional actions beyond the initial code review, and 7 deployment iterations refined the policies to their current form. The policies are split into five managed policies — three always-applied (Infrastructure, Application, Observability) plus two compute-variant policies (Compute-AgentCore, Compute-ECS) — to stay under the IAM 6,144-character limit and to support per-compute-backend bootstrap selection. The policy sources live in `cdk/src/bootstrap/policies/*.ts` and are compiled to `cdk/bootstrap/policies/*.json`. +> **Origin**: These IAM policies were derived from a thorough review of the repository's CDK constructs, stacks, and handler code, then **validated against a live deployment** in `us-east-1` (create, update, task execution, and destroy). CloudTrail analysis identified 36 additional actions beyond the initial code review, and 7 deployment iterations refined the policies to their current form. The policies are split into six managed policies — three always-applied (Infrastructure, Application, Observability) plus three compute-variant policies (Compute-AgentCore, Compute-ECS, Compute-LambdaMicrovms) — to stay under the IAM 6,144-character limit and to support per-compute-backend bootstrap selection. The policy sources live in `cdk/src/bootstrap/policies/*.ts` and are compiled to `cdk/bootstrap/policies/*.json`. ## How CDK deployment roles work @@ -21,7 +21,7 @@ The policy below is a **CloudFormation Execution Role** replacement. The other t ## Using these policies -The policies are split into five IAM managed policies (each under the 6,144-character limit): +The policies are split into six IAM managed policies (each under the 6,144-character limit): | Policy Name | Scope | When applied | |-------------|-------|--------------| @@ -30,10 +30,11 @@ The policies are split into five IAM managed policies (each under the 6,144-char | `IaCRole-ABCA-Observability` | Bedrock Guardrails, CloudWatch, X-Ray, S3, ECR, KMS, SSM, STS | Always | | `IaCRole-ABCA-Compute-Agentcore` | Bedrock AgentCore (`bedrock-agentcore:*`) | Always (default compute backend) | | `IaCRole-ABCA-Compute-ECS` | ECS cluster + task-definition operations | Only when `ecs` is in `ComputeTypes` | +| `IaCRole-ABCA-Compute-LambdaMicrovms` | Lambda MicroVM image + network-connector operations | Only when `lambda-microvm` is in `ComputeTypes` | > **Placeholder substitution**: Replace `ACCOUNT_ID` with your 12-digit AWS account ID and `REGION` with your deployment region (e.g., `us-east-1`) throughout this document. -These policies are not created or attached manually. The repository generates them — and a custom bootstrap template that wires all five into the CloudFormation execution role — from the TypeScript sources, then bootstraps with that template: +These policies are not created or attached manually. The repository generates them — and a custom bootstrap template that wires all six into the CloudFormation execution role — from the TypeScript sources, then bootstraps with that template: ```bash # Regenerate artifacts (policies JSON + template YAML) and bootstrap. @@ -51,7 +52,7 @@ aws cloudformation update-stack --stack-name CDKToolkit --use-previous-template aws cloudformation describe-stacks --stack-name CDKToolkit --query 'Stacks[0].Parameters' ``` -Under the hood, `mise //cdk:bootstrap` runs `npx cdk bootstrap --template bootstrap/bootstrap-template.yaml` (see `cdk/mise.toml`). The generated template defines five inline `AWS::IAM::ManagedPolicy` resources that **replace** the default `AdministratorAccess` on the CloudFormation execution role; the `IaCRole-ABCA-Compute-ECS` policy is conditional on the `ComputeTypes` parameter including `ecs`. The policy sources are `cdk/src/bootstrap/policies/{infrastructure,application,observability,compute-agentcore,compute-ecs}.ts`, compiled to `cdk/bootstrap/policies/*.json` by `cdk/scripts/generate-bootstrap-artifacts.ts`. +Under the hood, `mise //cdk:bootstrap` runs `npx cdk bootstrap --template bootstrap/bootstrap-template.yaml` (see `cdk/mise.toml`). The generated template defines six inline `AWS::IAM::ManagedPolicy` resources that **replace** the default `AdministratorAccess` on the CloudFormation execution role; the `IaCRole-ABCA-Compute-ECS` and `IaCRole-ABCA-Compute-LambdaMicrovms` policies are conditional on the `ComputeTypes` parameter including their respective backend. The policy sources are `cdk/src/bootstrap/policies/{infrastructure,application,observability,compute-agentcore,compute-ecs,compute-lambda-microvm}.ts`, compiled to `cdk/bootstrap/policies/*.json` by `cdk/scripts/generate-bootstrap-artifacts.ts`. ## Trust policy @@ -86,7 +87,7 @@ Under the hood, `mise //cdk:bootstrap` runs `npx cdk bootstrap --template bootst For deploying the `backgroundagent-dev` stack. This single stack contains all platform resources including the AgentCore runtime, ECS compute (when enabled), API Gateway, Cognito, DynamoDB tables, VPC, DNS Firewall, and observability infrastructure. -> **IAM managed policy size limit**: A single managed policy cannot exceed 6,144 characters. The permissions below are split into five policies to stay under this limit (three always-applied, plus two compute-variant policies). They are wired into the CloudFormation execution role by the generated bootstrap template; see [Using these policies](#using-these-policies). +> **IAM managed policy size limit**: A single managed policy cannot exceed 6,144 characters. The permissions below are split into six policies to stay under this limit (three always-applied, plus three compute-variant policies). They are wired into the CloudFormation execution role by the generated bootstrap template; see [Using these policies](#using-these-policies). ### IaCRole-ABCA-Infrastructure @@ -742,6 +743,44 @@ When the ECS Fargate compute backend is enabled (set the `ComputeTypes` CFN para } ``` +### IaCRole-ABCA-Compute-LambdaMicrovms + +When the Lambda MicroVM compute backend is enabled (include `lambda-microvm` in the `ComputeTypes` CFN parameter on the `CDKToolkit` stack), the generated template conditionally attaches this policy to the CloudFormation execution role. It permits CloudFormation to manage MicroVM images and network connectors; runtime session lifecycle permissions remain on the orchestrator role. + +```json +{ + "Statement": [ + { + "Action": [ + "lambda:CreateMicrovmImage", + "lambda:GetMicrovmImage", + "lambda:UpdateMicrovmImage", + "lambda:DeleteMicrovmImage", + "lambda:ListMicrovmImages", + "lambda:GetMicrovmImageVersion", + "lambda:UpdateMicrovmImageVersion", + "lambda:DeleteMicrovmImageVersion", + "lambda:ListMicrovmImageVersions", + "lambda:GetMicrovmImageBuild", + "lambda:ListMicrovmImageBuilds", + "lambda:ListManagedMicrovmImages", + "lambda:ListManagedMicrovmImageVersions", + "lambda:CreateNetworkConnector", + "lambda:GetNetworkConnector", + "lambda:UpdateNetworkConnector", + "lambda:DeleteNetworkConnector", + "lambda:ListNetworkConnectors", + "lambda:PassNetworkConnector" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "LambdaMicrovms" + } + ], + "Version": "2012-10-17" +} +``` + ## Runtime IAM roles (created by the stack) These roles are created inside the CloudFormation stack at deploy time, not by the deployer. They are documented here for a complete picture of the IAM footprint. diff --git a/docs/src/content/docs/decisions/Adr-021-lambda-microvms-compute-backend.md b/docs/src/content/docs/decisions/Adr-021-lambda-microvms-compute-backend.md index d94584952..a88f76fc5 100644 --- a/docs/src/content/docs/decisions/Adr-021-lambda-microvms-compute-backend.md +++ b/docs/src/content/docs/decisions/Adr-021-lambda-microvms-compute-backend.md @@ -25,25 +25,38 @@ ABCA selects a per-repo compute backend through the Blueprint's `compute_type` f | | AgentCore Runtime | ECS on Fargate | **Lambda MicroVMs** | |---|---|---|---| | Isolation | MicroVM (managed) | Task-level (Firecracker) | MicroVM (Firecracker) | -| Max duration | 8 h | No cap | 8 h (running + suspended) | -| Suspend/resume | No orchestrator-visible API | No | **Yes** — explicit API + idle policy, state preserved, no compute charge while suspended | -| Resources | AgentCore-managed | 16 vCPU / 120 GB / 20–200 GB disk | **16 vCPU / 32 GB RAM / 32 GB disk (hard ceiling)** | +| Max duration | 8 h | No cap | 8 h (running + suspended) — **verified** (`L-B430C318 = 8 hours`) | +| Suspend/resume | No orchestrator-visible API | No | **Yes** — explicit API + idle policy, state preserved, no compute charge while suspended. **Verified**: suspend reaches `SUSPENDED` in ~1 s with no `idlePolicy`; resume restores `RUNNING` in ~1 s with `microvmId` **and** `endpoint` byte-identical | +| Resources | AgentCore-managed | 16 vCPU / 120 GB / 20–200 GB disk | **Baseline 8 GiB RAM / 4 vCPU, auto-scaling to a 32 GiB / 16 vCPU peak**; 32 GB disk. `minimumMemoryInMiB` configures the BASELINE (max 8,192 MiB); the service scales vertically on demand — capacity is baseline-priced with 4× burst headroom | | Packaging | ECR image ≤ 2 GB | ECR image, no hard cap | **Zip + Dockerfile in S3 → service-built snapshot image** (versioned, storage billed) | -| Invocation | `InvokeAgentRuntime` (SigV4) | `RunTask` + container overrides | `RunMicrovm` → dedicated HTTPS endpoint + JWE token (`CreateMicrovmAuthToken`, ≤ 60 min TTL) | +| Invocation | `InvokeAgentRuntime` (SigV4) | `RunTask` + container overrides | `RunMicrovm` (image **ARN** required — a bare name is rejected) → dedicated HTTPS endpoint + JWE token (`CreateMicrovmAuthToken`, ≤ 60 min TTL) | | Liveness | Agent heartbeat + `/ping` | `DescribeTasks` | MicroVM state (RUNNING / SUSPENDED / TERMINATED) via control-plane API | | Session storage | `/mnt/workspace` FUSE (no `flock()`) | Ephemeral disk | Native disk in snapshot — **survives suspend/resume, `flock()` works** | | Architecture | ARM64 | ARM64 | ARM64 (Graviton) | | Regions (launch) | Broad | Broad | 5 (us-east-1/2, us-west-2, eu-west-1, ap-northeast-1) | -> Service facts in the Lambda MicroVMs column (APIs, limits, quota semantics, regions) are externally sourced from the AWS developer guide and the Agent Toolkit skill (see References) and cannot be corroborated from this repository; P1 re-verifies the load-bearing ones empirically (see Testing). +> Rows marked **verified** were discharged empirically on 2026-07-31 (us-east-1); see `docs/verification/645-p1-lambda-microvm-runbook.md`. Two originally documented constants were **refuted** by that run and are corrected throughout this ADR: the `runHookPayload` cap (16 KB → 4 KB) and the claim that `RunMicrovm` accepts a bare image name (it does not). +> +> **Source hierarchy for service facts.** Where sources disagree, the higher tier wins, and the tier is recorded next to the claim: +> +> 1. **Live boundary probe** — a request the service actually accepted or rejected in this account/Region. Strongest, and the only tier that can refute the others. (Both corrections above came from here.) +> 2. **Modeled constraints** — the CLI/SDK request schema and enumerated allowed values (`--generate-cli-skeleton`, `ARM_64`, `ENABLED|DISABLED`). Authoritative about request *shape*; silent about runtime behaviour. +> 3. **Service developer guide** — including its sizing/scaling tables. Authoritative about *semantics* a probe cannot see, which is exactly how the memory row was fixed: a probe can only show that 32,768 MiB is rejected as a `minimumMemoryInMiB` value; only the guide explains that the field is a BASELINE and that the service scales to a 32 GiB peak on its own. A boundary probe is the strongest evidence about a boundary and says nothing about what the boundary *means*. +> 4. **SDK docstrings** — generated, and demonstrably stale here: `runHookPayload` documents "Maximum: 16,384 bytes" against an enforced 4,096. +> 5. **Launch blogs / skills / toolkit material** — orientation only; never load-bearing on its own. +> +> **Omitted API fields mean "service default", never "none".** Two live findings drove this rule: leaving `ingressNetworkConnectors` unset attaches a PUBLIC `HTTP_INGRESS` connector, and leaving `/ready` out of `hooks` makes the image un-creatable once any lifecycle hook is enabled. So for any security-relevant field, the desired posture must be **requested explicitly** and the test must assert the **outcome** (the ARN present in the request, the hook enabled) rather than the omission (`expect(field).toBeUndefined()`) — an omission assertion passes just as happily when the service is silently choosing something wider. +> +> On the **memory** row specifically: `CreateMicrovmImage` enumerates the baseline sizes a base image supports — `[512, 1024, 2048, 4096, 8192]` MiB for `al2023-1` — and rejects anything else, which is why the construct validates against that list at synth. The 32 GiB / 16 vCPU peak is reached by the service's own vertical scaling, not by asking for it. The account memory quota (`L-CD1C0CC4`, 1024 GB, "burst up to 4×") is an aggregate across MicroVMs, not a per-VM limit; note that concurrency arithmetic should be done against the PEAK, not the baseline, since that is what a busy fleet can actually consume. ### Design tensions the strategy must resolve 1. **Idle detection is inbound-traffic-based; the ABCA agent is outbound-only.** MicroVM idle policies suspend when no traffic arrives at the *endpoint*. A busy agent running a 40-minute build receives no inbound traffic and would be suspended mid-work by a naive idle policy. Conversely, "no inbound traffic" is the agent's *normal* state. 2. **No self-suspend.** The agent cannot suspend its own MicroVM from inside; only an external `SuspendMicrovm` call can. Suspend decisions must be owned by the orchestrator — which aligns with the unified liveness model proposed in [#491](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/491). -3. **Snapshots bake state.** The image snapshot is captured once at build time; every MicroVM resumes from it. Secrets, tokens, and per-task identity must arrive at run time (`runHookPayload`, ≤ 16 KB, or fetched in the `/run` hook), never at image build. CSPRNGs must be reseeded on `/run` and `/resume`. +3. **Snapshots bake state.** The image snapshot is captured once at build time; every MicroVM resumes from it. Secrets, tokens, and per-task identity must arrive at run time (`runHookPayload`, ≤ 4 KB, or fetched in the `/run` hook), never at image build. CSPRNGs must be reseeded on `/run` and `/resume`. 4. **Auth tokens are short-lived.** JWE tokens max out at 60 minutes; any orchestrator→agent HTTP interaction over the endpoint needs token refresh, unlike AgentCore's SigV4 invoke or ECS's no-endpoint model. 5. **Identity delta — narrower than it looks.** Most AgentCore services ABCA uses are standalone and substrate-portable: Memory is already consumed from ECS via an IAM grant plus `MEMORY_ID` (`EcsAgentCluster`), and Gateway (ADR-019) is portable by design (SigV4 inbound). The genuinely Runtime-coupled piece is the workload-access-token **delivery mechanism** (`runtimeUserId` → `WorkloadAccessToken` request header → `BedrockAgentCoreContext`, used by `resolve_linear_api_token()`), which has no MicroVM equivalent. The ECS backend already lives with this delta (env-var token delivery); MicroVMs inherit the same posture until the pluggable identity work ([#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249), ADR-016) redesigns the seam. +6. **The service's defaults are not our posture.** Two of them, both discovered live: `RunMicrovm` attaches a **public** `HTTP_INGRESS` connector (and mints a public `*.lambda-microvm..on.aws` endpoint) when `ingressNetworkConnectors` is omitted, and `CreateMicrovmImage` **requires** the `/ready` hook whenever any lifecycle hook is enabled. Neither posture can be reached by leaving a field out — each needs an explicit control (see sub-decisions 3 and 4). ## Decision @@ -51,25 +64,52 @@ Adopt **AWS Lambda MicroVMs as a third, opt-in `ComputeStrategy` backend** named ### 1. Strategy shape: extend the interface with mandatory suspend/resume -`ComputeType` widens to `'agentcore' | 'ecs' | 'lambda-microvm'` (mirrored in `cli/src/types.ts` and the CLI's inline unions). `SessionHandle` gains a `{ strategyType: 'lambda-microvm', microvmId, endpoint }` variant — `microvmId` because every lifecycle API (`suspend-microvm`, `resume-microvm`, `terminate-microvm`, and `create-microvm-auth-token` — the latter not called in P1–P3, see sub-decision 3) takes only the MicroVM identifier, and `endpoint` because it is per-session (minted by `RunMicrovm`) and required for any orchestrator→agent HTTP interaction. The image ARN is deliberately **not** in the handle: like the ECS task definition ARN, it is deployment-time configuration consumed by `startSession` (from construct-injected environment) and recorded in the session-start log entry for diagnostics, not per-session lifecycle state. +`ComputeType` widens to `'agentcore' | 'ecs' | 'lambda-microvm'` (mirrored in `cli/src/types.ts` and the CLI's inline unions). `SessionHandle` gains a `{ strategyType: 'lambda-microvm', microvmId, endpoint }` variant — `microvmId` because every lifecycle API (`suspend-microvm`, `resume-microvm`, `terminate-microvm`, `get-microvm`, and `create-microvm-auth-token` — the latter not called in P1–P3, see sub-decision 3) takes only the MicroVM identifier, and `endpoint` because it is per-session (minted by `RunMicrovm`) and required for any orchestrator→agent HTTP interaction. Note the naming seam: the handle field is `microvmId` (matching `RunMicrovmResponse.microvmId`), while the request key on every lifecycle command is `microvmIdentifier` — the strategy is the only place that translates between the two. The image ARN is deliberately **not** in the handle: like the ECS task definition ARN, it is deployment-time configuration consumed by `startSession` (from construct-injected environment) and recorded in the session-start log entry for diagnostics, not per-session lifecycle state. + +**A second, sharper naming seam: `imageIdentifier` must be an ARN.** The name suggests a bare image name is acceptable — `create-microvm-image --name` takes one, and this ADR originally assumed `run-microvm --image-identifier` would too. It does not: a bare name is rejected with `ValidationException: Malformed ARN - doesn't start with 'arn:'`, and so is `list-microvm-image-builds --image-identifier ` (`Invalid ARN format`). The construct therefore resolves an operator-supplied name to its exact `arn:${Partition}:lambda:${Region}:${Account}:microvm-image:${Name}` ARN **once** — the same value it scopes the lifecycle IAM grant to — and injects THAT as `MICROVM_IMAGE_IDENTIFIER`. One derivation, two consumers, so a request field and an IAM resource can never disagree. The strategy validates the invariant and fails fast with the remedy, because the service's own error names neither the env var nor the fix. The `ComputeStrategy` interface gains **mandatory** `suspendSession(handle)` / `resumeSession(handle)` methods returning a typed result (`{ supported: false } | { supported: true }`-shaped, exact type at implementation time); the widening lands in P3 (see sub-decision 5), in one commit across all three strategies. Mandatory-with-explicit-stub is the codebase idiom, not optional methods: no behavioral interface in the codebase has an optional method, `AgentCoreComputeStrategy.pollSession` is already a mandatory explicit stub rather than an optional member, and the exhaustive-`never` switch culture means a fourth backend must make a compile-checked decision about its suspend semantics instead of silently falling through a `strategy.suspendSession?.()` feature-detection. The agentcore and ecs strategies return `unsupported` (not a silent success — a suspend that silently no-ops would let the orchestrator believe compute billing stopped when it did not); the orchestrator gates its suspend policy on the typed response, consistent with how `pollTaskStatus` already branches explicitly on `computeType`. -**Poll semantics — the strategy reports, the orchestrator interprets.** `pollSession(handle)` receives only the session handle and cannot see task state, so the health rules must live where the DynamoDB status lives. `SessionStatus` gains a `'suspended'` variant; the strategy maps `GetMicrovm` state mechanically (`RUNNING` → `running`, `SUSPENDED` → `suspended`, `TERMINATED` → `completed`) and the **orchestrator** cross-references against the task row — the same division of labor `finalPollState` already uses for ECS (substrate stopped + non-terminal DynamoDB status → failed) and `pollTaskStatus` uses for agentcore heartbeats: substrate `suspended` + task `AWAITING_APPROVAL` is healthy (orchestrator-intended suspend); `suspended` with any other task status is an anomaly to surface, not fail-fast; substrate terminal + non-terminal task status → classify failed. +**Poll semantics — the strategy reports, the orchestrator interprets.** `pollSession(handle)` receives only the session handle and cannot see task state, so the health rules must live where the DynamoDB status lives. `SessionStatus` gains a `'suspended'` variant; the strategy maps `GetMicrovm` state mechanically and the **orchestrator** cross-references against the task row — the same division of labor `finalPollState` already uses for ECS (substrate stopped + non-terminal DynamoDB status → failed) and `pollTaskStatus` uses for agentcore heartbeats: substrate `suspended` + task `AWAITING_APPROVAL` is healthy (orchestrator-intended suspend); `suspended` with any other task status is an anomaly to surface, not fail-fast; substrate terminal + non-terminal task status → classify failed. + +The service's `MicrovmState` enum has **six** members, not three, so the mapping is stated exhaustively (one line of rationale each, mirrored in the strategy's doc comment): + +| `MicrovmState` | `SessionStatus` | Why | +|---|---|---| +| `PENDING` | `running` | Still booting; the same way ECS's `PENDING`/`PROVISIONING` map to `running`. | +| `RUNNING` | `running` | — | +| `SUSPENDING` | `suspended` | Already on its way to frozen; reporting `running` would tell the orchestrator compute is still progressing when it is not. Both suspend states land on a report the orchestrator treats as benign-or-anomalous depending on task status, never as failure. **Never observable in practice** — suspend reached `SUSPENDED` in under 1 s live — so it is mapped for completeness and nothing may *wait* for it. | +| `SUSPENDED` | `suspended` | — | +| `TERMINATING` | `completed` | Terminal-bound and carries no exit code, so "the substrate is gone" is all the strategy can honestly say. | +| `TERMINATED` | `completed` | Success vs failure is the orchestrator's call — it cross-references the DynamoDB status. **This is the load-bearing terminal signal**, not `NotFound` (see below). | +| *unrecognized* | `running` | A future service enum addition must never fail a healthy task; the strategy warns and keeps polling. | + +**`GetMicrovm` `ResourceNotFoundException` → `completed`, but as a LATE fallback.** This deliberately diverges from `ecs-strategy`, where `DescribeTasks` returning no task maps to `failed`. ECS keeps stopped tasks describable for roughly an hour, so a missing task there really is anomalous; a MicroVM is eventually reaped from the control plane **by design**, so `failed` would fail tasks that finished cleanly. + +What the live run corrected is the *timing*, not the mapping. `NotFound` is **not the near-term terminal signal**: a terminated MicroVM reported `TERMINATING` at +1 s, `TERMINATED` at +3 s, and was **still `TERMINATED` ~10 minutes later** and at every subsequent checkpoint — `ResourceNotFoundException` was never observed in that window. So the branch that actually fires in practice is `TERMINATED → completed` in the table above; the `NotFound` rule covers only a VM reaped after a long gap (a poll resumed after a crash, a stranded-task reconciler sweep). Both are required and neither substitutes for the other: without the `TERMINATED` row the orchestrator would poll a finished VM until its safety net fired, and without the `NotFound` rule a late sweep would classify a cleanly-finished task as a poll error. + +The divergence is safe because it does not weaken detection: the orchestrator still fails the task when a terminal report lands while the DynamoDB status is non-terminal, so a genuine mid-run disappearance is caught — it simply receives the substrate-failure classification instead of a misleading poll error. Because that cross-check acts on a status read earlier in the same poll cycle, the orchestrator **re-reads the task row before failing** (the normal shutdown order is "agent writes terminal status → agent exits → VM terminates", which a stale read would otherwise turn into a spurious failure); ECS buys the same protection with a five-consecutive-poll patience counter instead. + +Neither the mapping nor the NotFound rule is a health decision: both are mechanical restatements of substrate state, which is what keeps the "strategy reports, orchestrator interprets" split intact. Normative requirements (EARS, per [ADR-020](/sample-autonomous-cloud-coding-agents/architecture/adr-020-ears-requirements-syntax)): - When a task's Blueprint sets `compute_type: 'lambda-microvm'`, the orchestrator shall resolve the `LambdaMicrovmComputeStrategy` via `resolveComputeStrategy`. - When `startSession` is invoked, the strategy shall call `RunMicrovm` with `maximumDurationInSeconds` set to 28 800 (the service maximum, matching AgentCore's 8-hour session cap and sitting inside the orchestrator's ~8.5 h safety-net poll window). +- When `startSession` is invoked, the strategy shall pass a fully-qualified MicroVM image ARN as `imageIdentifier`. +- If the configured image identifier is not an ARN, then the strategy shall fail the session start with an error naming the environment variable and the redeploy remedy, before performing any AWS call. - When `startSession` returns, the orchestrator shall persist the MicroVM handle (`microvmId`, `endpoint`) in the task row's `compute_metadata` (the field `cancel-task.ts` already reads ECS handles from). - The strategy shall omit `idlePolicy` on every `RunMicrovm` call, in every phase. - The orchestrator shall be the sole initiator of suspension, via `suspendSession`. -- When `pollSession` observes MicroVM state `SUSPENDED`, the strategy shall report `suspended` without interpreting task state. -- If the strategy reports a terminal substrate state while the task's DynamoDB status is non-terminal, then the orchestrator shall classify the task as failed with a substrate-failure remedy. +- When `pollSession` observes MicroVM state `SUSPENDED` or `SUSPENDING`, the strategy shall report `suspended` without interpreting task state. +- When `pollSession` observes MicroVM state `TERMINATED` or `TERMINATING`, the strategy shall report `completed` (the observable terminal state persists for at least ~10 minutes, so `completed` shall not depend on the MicroVM being reaped). +- When `pollSession` observes a MicroVM state it does not recognize, the strategy shall report `running`. +- If `GetMicrovm` reports that the MicroVM does not exist, then the strategy shall report `completed`. +- If the strategy reports a terminal substrate state while the task's DynamoDB status is non-terminal, then the orchestrator shall re-read the task row and, if it is still non-terminal, classify the task as failed with a substrate-failure remedy. - If the strategy reports `suspended` while the task's DynamoDB status is not `AWAITING_APPROVAL`, then the orchestrator shall surface an anomaly event and shall not fail-fast the task. - If `suspendSession` or `resumeSession` is invoked on a strategy that does not support suspension, then the strategy shall return an explicit unsupported result. - When the agent process reaches a terminal state, the agent shall exit. -- When the orchestrator finalizes a `lambda-microvm` task, the orchestrator shall call `terminate-microvm` (termination shall not rely on any substrate timeout). +- When the orchestrator finalizes a `lambda-microvm` task, the orchestrator shall call `terminate-microvm` (termination shall not rely on any substrate timeout, and shall not rely on the MicroVM self-terminating — it does not). *On the omitted `idlePolicy`:* if the block is present all three fields are required, so omission is the unambiguous disabled state the invariant test asserts. This deliberately forgoes `suspendedDurationSeconds` — it lives *inside* `idlePolicy` and cannot be set without re-enabling the traffic-idle machinery — so the suspended-state bound is `maximumDurationInSeconds` plus orchestrator termination and the stranded-approval reconciler (see sub-decision 2). A tighter substrate-level suspended-TTL remains available later as an additive `idlePolicy` change if operators want it. *On the fixed `maximumDurationInSeconds`:* no wall-clock task budget exists in the platform (budgets are `max_turns` / `max_budget_usd`), so the value is parity with AgentCore's 8 h cap rather than derived policy; a Blueprint override can be added later if a real need appears. @@ -84,8 +124,8 @@ The handshake must respect the existing approval mechanics: the agent **discover *Why inline rather than poll-only — codebase precedent:* resume-on-approve is structurally identical to task cancellation — a user-initiated, latency-sensitive action whose purpose is an immediate compute-lifecycle side effect. `cancel-task.ts` already resolves this exact tension: the API-plane handler invokes ECS `StopTask` / AgentCore `StopRuntimeSession` inline, best-effort (a failed stop logs a warning and the state transition stands; a `task_cancel_compute_orphan` event is written when no stoppable compute handle exists, `reason: missing_runtime_handle`) — with the conditional IAM wired in `task-api.ts`. The resume path goes one step further than the precedent by also writing the orphan event on *failed* resume calls, because a failed resume strands a suspended VM awaiting a decision — a stronger liveness consequence than a failed stop of an already-cancelled task. The alternative (orchestrator-poll-only resume) preserves single-owner lifecycle purity but pays up to a full poll interval (~30 s) of latency on every approval, and the purity argument was already litigated and declined for cancel. `approve-task.ts` is deliberately minimal today (security-critical ownership comparison, Cedar finding #6); the resume call is therefore added *after* the transaction commits, cannot alter the decision outcome, and carries one conditional `lambda:ResumeMicrovm` grant — the same blast-radius trade the cancel handler accepted in review. - **Timeout under freeze — the agent re-bases on the wall clock it already owns.** The agent's monotonic gate timer freezes while suspended, so resuming near the deadline is not enough: the frozen timer would still hold its remaining budget and fire the deny minutes *after* the user-visible window — colliding with the approval row's TTL (`created_at + timeout_s + 120s`) and triggering the "row reaped → stranded" fallback on a healthy gate. Instead, the gate expires at **`min(monotonic budget, created_at + timeout_s)`**, evaluated on each poll iteration and on `/resume`. This is not a new principle: Cedar decision #6 is already "min wins" for timeouts, the wall-clock deadline is already durable in the approval row the agent itself writes (`created_at` is in the agent's own clock domain — no skew), and §13.12's late-approval race fix already establishes that the durable row is authoritative over the agent's local timer. Deny authority stays agent-side (the conditional `TIMED_OUT` write + ConsistentRead re-read race protection is untouched); the orchestrator's resume at `deadline − margin` is purely the wake-up mechanism, with no correctness role. -- **Backstops, not mechanisms.** `maximumDurationInSeconds` (mandatory on every `RunMicrovm`, pinned at 28 800 s — see sub-decision 1) is the substrate kill switch bounding running **and** suspended time; the orchestrator's finalization `terminate-microvm` is the active cleanup path; the stranded-approval reconciler retains its role for orphaned waits. No `idlePolicy`-based bound is used in any phase — see sub-decision 1's omit-`idlePolicy` invariant. -- **Concurrency slot stays held** during suspend. Cedar decision #7's rationale ("container alive, consuming memory") weakens under suspend, but the conclusion survives for a harder reason: AWS counts `SUSPENDED` MicroVMs toward the account memory quota, so releasing ABCA's slot would not free real capacity — it would only invite admission of tasks the substrate cannot start. +- **Backstops, not mechanisms.** `maximumDurationInSeconds` (mandatory on every `RunMicrovm`, pinned at 28 800 s — see sub-decision 1) is the substrate kill switch bounding running **and** suspended time; the orchestrator's finalization `terminate-microvm` is the active cleanup path; the stranded-approval reconciler retains its role for orphaned waits. No `idlePolicy`-based bound is used in any phase — see sub-decision 1's omit-`idlePolicy` invariant. **The active terminate is mandatory, not belt-and-braces**: a MicroVM whose hook never ran reached `RUNNING` in 12 s and stayed `RUNNING` with no `stateReason` through every checkpoint (live). Nothing self-terminates on this substrate, so nothing cleans up — a leaked VM bills until the 8 h cap. +- **Concurrency slot stays held** during suspend. Cedar decision #7's rationale ("container alive, consuming memory") weakens under suspend, and the harder replacement rationale — "AWS counts `SUSPENDED` MicroVMs toward the account memory quota, so releasing ABCA's slot would not free real capacity" — is **undischarged**: the suspended VM stayed in `list-microvms` at every checkpoint, but that only proves *listed*. `L-CD1C0CC4` (1024 GB, account-scoped) exposes no `UsageMetric`, `AWS/Usage` carries only `CallCount` per API, and no MicroVM memory metric exists in any namespace, so consumption is **not observable safely** — proving it would need a large concurrent fleet. The conclusion (hold the slot) stands as the conservative choice, not as a verified fact. Size the arithmetic against the 32 GiB **peak** rather than the 8 GiB baseline: a busy fleet scales up, so peak is what actually competes for the account quota. The agent's `/suspend` hook flushes progress events (durable writes before returning 200, within the 60 s hook budget); `/resume` reseeds CSPRNGs and refreshes cached credentials. @@ -101,22 +141,55 @@ Normative requirements (EARS): ### 3. Packaging: same agent image source, new build path -The existing agent container (`agent/` Dockerfile, already ARM64) is repackaged as a zip + Dockerfile artifact in S3 and built into a versioned `MicrovmImage` via `CreateMicrovmImage` (with `/ready` and `/validate` build hooks for snapshot quality). The agent runs its existing FastAPI server (`agent/src/server.py`) — the MicroVM path uses the HTTP entrypoint like AgentCore, not ECS's batch bypass — plus the four runtime lifecycle hooks on a sidecar port. Runtime hooks are fast-notification only (1–60 s): `/run` validates the payload and starts the pipeline **asynchronously**, mirroring how the agent loop already runs in a background thread behind `/ping` on AgentCore. +The existing agent container (`agent/` Dockerfile, already ARM64) is repackaged as a zip + Dockerfile artifact in S3 and built into a versioned `MicrovmImage` via `CreateMicrovmImage`. The agent runs its existing FastAPI server (`agent/src/server.py`) — the MicroVM path uses the HTTP entrypoint like AgentCore, not ECS's batch bypass — plus the runtime lifecycle hooks (`/run`, `/suspend`, `/resume`, `/terminate`) and the `/ready` + `/validate` build hooks, all on the same port the server already listens on (8080, declared as the image's `hooks.port`). Runtime hooks are fast-notification only (1–60 s): `/run` validates the payload and starts the pipeline **asynchronously**, mirroring how the agent loop already runs in a background thread behind `/ping` on AgentCore. -**Payload delivery** reuses the ECS strategy's S3-pointer pattern, adapted to `runHookPayload` (≤ 16 KB): small payloads inline; larger ones uploaded by the strategy to a platform payload bucket (the ECS payload bucket pattern in `ecs-agent-cluster.ts`: orchestrator write access, compute-role read-only scoped to the bucket, lifecycle expiry on objects) with only the S3 URI in `runHookPayload` — the MicroVM **execution role** holds the read grant, exactly as the ECS task role does today. +**Hook phasing — corrected: `/ready` + `/run` are both P1.** The original plan split *declaring* a hook from *serving* it, putting `/run`'s declaration in P1 and its implementation in P2. Live verification proved that split is **not a reachable service state**, on two independent counts: -**No orchestrator→agent HTTP path exists in P1–P3**: payload arrives through the `/run` hook, all agent work is outbound, and therefore **no JWE auth tokens are minted at all** — token minting (and its ≤ 60 min TTL refresh problem) is deferred until a real consumer exists (e.g. operator shell access, [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391)). The `endpoint` stays in the `SessionHandle` because it is genuinely per-session state that becomes load-bearing the day such a consumer appears. **Constraint accepted:** 32 GB RAM / 32 GB disk (disk quota to be confirmed against current service quotas when COMPUTE.md is updated) means repos that motivated the 120 GB ECS sizing stay on `ecs`; MicroVMs target the default-sized workload with suspend economics, not the heavy-build niche. +- `CreateMicrovmImage` rejects an image that enables any lifecycle hook without `/ready`: *"The ready (/ready) MicroVM image hook must be enabled when any MicroVM lifecycle hook (run, resume, suspend, or terminate) is enabled."* So a P1 image declaring only `/run` is **not creatable**. +- With `/ready` added but unserved, both chipset builds fail: *"Ready hook check failed: the application returned a client error (HTTP 4xx) response."* So a declared hook must be served in the same phase. +- And an image with **no** hooks at all — the only other creatable shape — cannot receive a payload: *"The run hook must be enabled in the MicroVM image to pass the run hook payload."* So deferring hooks entirely also defers the whole payload-delivery channel. -- The image build shall not embed secrets, tokens, or per-task identity in the snapshot. -- The agent shall resolve credentials at `/run` time. -- When the `/run` hook receives the task payload, the agent shall validate it, start the pipeline asynchronously, and return HTTP 200 within the hook budget. -- The agent shall not execute the clone→verify→PR pipeline on the hook path. -- If the task payload exceeds the 16 KB `runHookPayload` limit, then the strategy shall upload the payload to the platform payload bucket and pass only its S3 URI in `runHookPayload`. -- The MicroVM execution role shall hold read-only access to the payload bucket, scoped to that bucket. +The phasing is therefore: + +| Hook | Declared by | Served by the agent | Notes | +|---|---|---|---| +| `/ready` | **P1** (construct sets `hooks.microvmImageHooks.ready`) | **P1** | MANDATORY, not a quality nicety — see above. A 200 once uvicorn is bound is the whole P1 contract: it also proves `server` imported cleanly (pulling in `pipeline` → `runner` → the policy engine), so a missing policy file fails the BUILD instead of the first task. | +| `/run` | **P1** (construct sets `hooks.microvmHooks.run`) | **P1** | The payload-delivery channel. Must be served in P1 because `/ready` forces hooks to exist at all, and a hook-less image cannot accept `runHookPayload`. | +| `/validate` | **P2** | **P2** | Build-time snapshot-quality hook. Still deliberately NOT declared: a `/validate` that 404s or reports failure fails every image build. Deeper warm-up assertions (Bedrock reachability, Memory access, tool availability) belong here. | +| `/suspend`, `/resume`, `/terminate` | **P3** (suspend/resume), P2 (`/terminate`) | P3 / P2 | Declaring a runtime hook the agent does not serve fails the corresponding lifecycle transition, so each is declared only in the phase that implements it. P1 termination is the orchestrator's `TerminateMicrovm`, which needs no in-guest cooperation. | + +Consequence to state plainly, replacing the original "a P1-built MicroVM image is not runnable end to end": **a P1 image is creatable, launchable and payload-deliverable, but carries no smoke-parity guarantee.** P1 delivers the strategy, the construct, the roles/buckets/connectors, the image resource, the packaging script, and the `/ready` + `/run` endpoints — so a `lambda-microvm` task can start a MicroVM and hand it a payload. What P1 has **not** established is anything P2 owns: AgentCore Memory grants and `MEMORY_ID` delivery, the agent's non-secret env parity inside the snapshot, egress specifics from a running MicroVM, and heartbeat/progress behaviour end to end. No clone → change → PR run has happened on this substrate. P2 ("smoke parity") is the phase that closes that gap. The construct and the packaging script both surface exactly this at synth/run time (`abca:microvm-image-p1-smoke-unverified`) so an operator cannot mistake a launchable substrate for a verified one. + +**Payload delivery** reuses the ECS strategy's S3-pointer pattern, adapted to `runHookPayload` (**≤ 4 KB** — measured, see below): payloads that fit ride inline; the rest are uploaded by the strategy to a platform payload bucket (the ECS payload bucket pattern in `ecs-agent-cluster.ts`: orchestrator write access, compute-role read-only scoped to the bucket, lifecycle expiry on objects) with only the S3 URI in `runHookPayload` — the MicroVM **execution role** holds the read grant, exactly as the ECS task role does today. + +The cap is **4 096 bytes**, not the 16 384 the SDK documents. Measured exactly: 4 096 passes, 4 097 is rejected with *"Value at 'runHookPayload' failed to satisfy constraint: Member must have length less than or equal to 4096"*. Two consequences follow. First, the original threshold would have inlined every envelope between 4 097 and 16 384 bytes and had the service reject all of them. Second, and more structurally: **the S3-pointer path is now the dominant one, and inline is the exception.** A hydrated task payload (prompt + issue thread + repo context) essentially always exceeds 4 KB, so "small payloads ride inline" describes tiny repo-less prompts rather than the common case. The payload bucket is therefore not a rarely-exercised overflow valve but a required part of every normal task, which raises its lifecycle rule (`MICROVM_PAYLOAD_TTL_DAYS`) and the execution role's read grant from edge-case plumbing to load-bearing. + +**No orchestrator→agent HTTP path exists in P1–P3**: payload arrives through the `/run` hook, all agent work is outbound, and therefore **no JWE auth tokens are minted at all** — token minting (and its ≤ 60 min TTL refresh problem) is deferred until a real consumer exists (e.g. operator shell access, [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391)). The `endpoint` stays in the `SessionHandle` because it is genuinely per-session state that becomes load-bearing the day such a consumer appears. But note the service does not agree by default: omitting `ingressNetworkConnectors` on `RunMicrovm` attaches a **public** `HTTP_INGRESS` connector, so the strategy passes the Lambda-managed `NO_INGRESS` connector explicitly on every launch (see sub-decision 4's security table). + +**Constraint accepted:** the configured **baseline is 8 GiB RAM / 4 vCPU** and the service scales vertically to a **32 GiB / 16 vCPU peak** on its own, with 32 GB of disk. So capacity is baseline-priced with 4× burst headroom — good for the bursty compile-and-test shape of an agent task — but the SUSTAINED ceiling is still 32 GiB, so repos that motivated the 120 GB ECS sizing stay on `ecs`. What the construct configures (and validates) is the baseline; the peak is not something a deployment asks for. + +Normative requirements (EARS). All of these are P1 now — the earlier P1/P2 split of this list existed only because the phasing table split declaring a hook from serving it, which the service does not permit: + +- (P1) The image build shall not embed secrets, tokens, or per-task identity in the snapshot. +- (P1) If the task payload exceeds the 4 KB `runHookPayload` limit, then the strategy shall upload the payload to the platform payload bucket and pass only its S3 URI in `runHookPayload`. +- (P1) The MicroVM execution role shall hold read-only access to the payload bucket, scoped to that bucket. +- (P1) Where no ingress is configured for a deployment, the strategy shall pass the Lambda-managed `NO_INGRESS` network connector on every `RunMicrovm` call (the field shall not be omitted). +- (P1) Where the image enables any MicroVM lifecycle hook, the image shall also enable the `/ready` hook and the agent shall serve it. +- (P1) When the `/run` hook receives the task payload, the agent shall validate it, start the pipeline asynchronously, and return HTTP 200 within the hook budget. +- (P1) If the `/run` hook payload cannot be resolved to a task payload, then the agent shall reject the hook with a client error and shall not start a pipeline. +- (P1) The agent shall not execute the clone→verify→PR pipeline on the hook path. +- (P1) The agent shall resolve credentials at `/run` time. +- (P1) Where a deployment configures a MicroVM image before smoke parity is verified, the platform shall warn that the backend has no smoke-parity guarantee. +- (P2) Where the image declares the `/validate` build hook, the agent shall serve it. ### 4. Infra and IAM: conditional resources behind bootstrap `ComputeTypes` -Mirroring the ECS pattern: a `compute-lambda-microvm` bootstrap policy (`cdk/src/bootstrap/policies/`) gated on the `ComputeTypes` CFN parameter; a CDK construct provisioning the build role, execution role (admitted to the per-session role via `AgentSessionRole.admitComputeRole`, which was designed for exactly this), the S3 artifact bucket wiring, and image build automation. Egress uses the platform VPC via an egress network connector so the DNS Firewall / security-group / flow-log stack in [COMPUTE.md](/sample-autonomous-cloud-coding-agents/architecture/compute) applies unchanged; ingress is restricted to the JWE-authenticated endpoint (no `SHELL_INGRESS` by default — it is noted as a candidate for [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391), operator session access, as a separate decision). +Mirroring the ECS pattern: a `compute-lambda-microvm` bootstrap policy (`cdk/src/bootstrap/policies/`) gated on the `ComputeTypes` CFN parameter; a CDK construct provisioning the build role, execution role (admitted to the per-session role via `AgentSessionRole.admitComputeRole`, which was designed for exactly this), the S3 artifact bucket wiring, and image build automation. Egress uses the platform VPC via egress network connectors so the DNS Firewall / security-group / flow-log stack in [COMPUTE.md](/sample-autonomous-cloud-coding-agents/architecture/compute) applies unchanged; ingress is suppressed with the Lambda-managed `NO_INGRESS` connector (no `SHELL_INGRESS` — it is noted as a candidate for [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391), operator session access, as a separate decision). + +Two networking facts the construct has to encode, both established live: + +- **A `VPC_EGRESS` connector requires an operator role.** CloudFormation's generated L1 types `operatorRole` as optional and this ADR originally assumed Lambda would manage the ENIs with its own service-linked role. It does not: the connector fails to create with *"NetworkConnectorOperatorRole is required for VPC_EGRESS connector type"*. The construct creates one role — trusting `lambda.amazonaws.com` with `aws:SourceAccount` pinned, carrying `AWSLambdaVPCAccessExecutionRole` plus the ENI / tag / private-IP actions that policy omits — and shares it across both connectors, since it manages interfaces rather than traffic. +- **Build-time egress needs port 80; runtime does not.** `agent/Dockerfile` installs Debian packages and `apt-get` fetches over plain HTTP, so a 443-only egress path fails every snapshot build (`Could not connect to deb.debian.org:80 … exit code: 100`). Rather than widen the runtime posture, the construct provisions a **second, build-only** connector on the same private subnets with a 443 + 80 security group, referenced solely by the image resource and the packaging script. The agent at run time still has 443-only egress. - Where the bootstrap `ComputeTypes` parameter includes `lambda-microvm`, the generated template shall attach the `IaCRole-ABCA-Compute-LambdaMicrovms` policy to the CloudFormation execution role. - The orchestrator role shall receive only the MicroVM lifecycle actions it calls (`lambda:RunMicrovm`, `lambda:SuspendMicrovm`, `lambda:ResumeMicrovm`, `lambda:TerminateMicrovm`, `lambda:GetMicrovm` for `pollSession`, and `lambda:PassNetworkConnector`, which is required even for the default connectors), scoped to platform-created images. @@ -132,14 +205,16 @@ Mirroring the ECS pattern: a `compute-lambda-microvm` bootstrap policy (`cdk/src | Control | AgentCore | ECS | Lambda MicroVMs | Delta | |---|---|---|---|---| -| Egress (DNS Firewall, TCP 443 SG, flow logs) | Platform VPC | Platform VPC | Platform VPC via egress network connector | None | +| Egress, runtime (DNS Firewall, TCP 443 SG, flow logs) | Platform VPC | Platform VPC | Platform VPC via egress network connector | None | +| Egress, image build | ECR build outside the platform VPC | ECR build outside the platform VPC | Platform VPC via a **separate build-only connector, TCP 443 + 80** (`apt-get` is plain HTTP) | New surface: build-time egress is wider than runtime egress by one port, on a connector no running MicroVM can use | | Tenant-data scoping | Per-session role (`admitComputeRole`) | Per-session role | Per-session role, execution role admitted identically | None | | Secrets delivery | Runtime env + Identity injection | Task env vars | Fetched at `/run`; never in snapshot | New surface: snapshot must stay secret-free (EARS req., sub-decision 3) | -| Inbound exposure | None (SigV4 invoke only) | None (no endpoint) | **Public HTTPS endpoint, JWE-gated, no unauthenticated access; no tokens minted in P1–P3** | New surface: endpoint exists but is unreachable without a minted token | +| Inbound exposure | None (SigV4 invoke only) | None (no endpoint) | **None — but only because the strategy passes `NO_INGRESS` explicitly.** The service default is a PUBLIC `HTTP_INGRESS` connector plus a public `*.lambda-microvm..on.aws` endpoint; no tokens are minted in P1–P3 either way | New surface **and** a new failure mode: "no inbound" is an active control, not an absence. Drop the `NO_INGRESS` argument and every agent MicroVM gets a public endpoint (EARS req., sub-decision 3) | | Session isolation | MicroVM | Task-level | MicroVM (Firecracker) | None (≥ ECS) | | State reuse | None | None | Snapshot shared across MicroVMs | New surface: CSPRNG reseed + credential refresh on `/run`/`/resume` (EARS req.) | | Workload-token injection | Yes (Runtime-coupled) | No (env-var posture) | No (env-var posture) | Shared with ECS; deferred to [#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249)/ADR-016 | | Operator shell access | No | No | Not enabled (`SHELL_INGRESS` omitted; candidate for [#391](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/391)) | None by default | +| Auth-token minting | n/a | n/a | `CreateMicrovmAuthToken` granted to no role in any phase | Verified static-only: any principal holding the action *can* mint a working JWE, including against a `SUSPENDED` MicroVM, so the posture rests entirely on the grant being absent | #### Regional availability enforcement @@ -158,21 +233,25 @@ Lambda MicroVMs launched in 5 regions (us-east-1/2, us-west-2, eu-west-1, ap-nor ### 5. Rollout: phased, default unchanged -- **P1 — strategy + infra:** `LambdaMicrovmComputeStrategy` (start/poll/stop), CDK construct, bootstrap policy, types sync, unit + CDK assertion tests. No suspend yet. -- **P2 — smoke parity:** agent completes clone → change → PR on the backend with progress visible to `bgagent watch`; failure classification entries in `error-classifier.ts`; **AgentCore Memory parity** (IAM grant + `MEMORY_ID` delivery, following the `EcsAgentCluster` pattern — Memory is a standalone service already consumed cross-substrate, and omitting the grant silently no-ops cross-session learning). +- **P1 — strategy + infra + minimal hook serving:** `LambdaMicrovmComputeStrategy` (start/poll/stop), CDK construct, bootstrap policy, types sync, unit + CDK assertion tests, and the agent's `/ready` + `/run` endpoints. No suspend yet. The image IS creatable and launchable and the payload DOES reach the agent — but there is **no smoke-parity guarantee** (sub-decision 3's phasing table). +- **P2 — smoke parity:** the agent serves the remaining hooks (`/terminate`, `/validate`); agent completes clone → change → PR on the backend with progress visible to `bgagent watch`; failure classification entries in `error-classifier.ts`; **AgentCore Memory parity** (IAM grant + `MEMORY_ID` delivery, following the `EcsAgentCluster` pattern — Memory is a standalone service already consumed cross-substrate, and omitting the grant silently no-ops cross-session learning); the agent's remaining non-secret env parity inside the snapshot. - **P3 — suspend/resume:** the interface widening from sub-decision 1 (mandatory methods, all three strategies in one commit), HITL-wait suspend policy, inline resume in the approve/deny Lambda with orchestrator-poll reconciliation (sub-decision 2), timeout-under-freeze wall-clock handling; coordinate with [#491](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/491)'s unified liveness model and update Cedar decision #7's rationale note. - **Out of scope:** replacing AgentCore as default; classic Lambda functions as a runtime; GPU; the Runtime-coupled workload-access-token injection path (delivery mechanism exists only on AgentCore Runtime; MicroVMs adopt the ECS env-var posture until [#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249)/ADR-016 redesign the seam). Gateway integration is orthogonal: ADR-019/[#641](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/641) is substrate-portable by design and applies to this backend when it lands. ## Consequences -- (+) **Suspend/resume economics.** Tasks idling on approval waits stop billing compute while preserving full state — bounded at ~1 h per gate under the current Cedar ceiling (decision #6), and the enabler for cheap off-hours gate-ceiling extensions later (§14.8). +- (+) **Suspend/resume economics.** Tasks idling on approval waits stop billing compute while preserving full state — bounded at ~1 h per gate under the current Cedar ceiling (decision #6), and the enabler for cheap off-hours gate-ceiling extensions later (§14.8). Verified end to end at the substrate level: suspend and resume each complete in ~1 s, and `microvmId` **and** `endpoint` survive the cycle byte-identical, so a stored `SessionHandle` remains valid. - (+) **VM-level isolation without cluster ops.** Firecracker isolation with no ECS cluster, task definition, or capacity management; one-session-per-MicroVM maps 1:1 onto ABCA's task model. -- (+) **Escapes AgentCore's 2 GB image limit and FUSE `flock()` workaround** — native disk in the snapshot supports `uv`/`mise` without the split-storage scheme. +- (+) **Escapes AgentCore's 2 GB image limit and FUSE `flock()` workaround** — native disk in the snapshot supports `uv`/`mise` without the split-storage scheme. Note the size comparison must say WHICH measure it means: the same agent tree is 1.799 GB as an OCI image (629.7 MB compressed, i.e. under AgentCore's limit) but reports `codeInstallSizeInBytes` of 2.17 GiB as a MicroVM snapshot (i.e. over it). The two straddle the limit and are not interchangeable; memory/disk snapshot sizes are a third thing again and must not be summed into the comparison. - (+) **Liveness becomes explicit.** Unlike AgentCore's stub `pollSession`, the strategy can report real substrate state, strengthening the [#491](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/491) unification. -- (−) **32 GB RAM / 32 GB disk hard ceiling** — not a successor to the ECS backend for heavy CI-parity builds; the platform now maintains three backends. -- (−) **New packaging pipeline.** Zip + Dockerfile + service-side image builds with versioned snapshots (storage billed per version) alongside the existing ECR flow; image versions need lifecycle cleanup. -- (−) **8-hour hard cap includes suspended time**, and with `idlePolicy` omitted there is no tighter substrate-level suspended-TTL — the suspended-state bound is `maximumDurationInSeconds` plus orchestrator termination and the stranded reconciler. Under today's 1 h gate ceiling this is comfortably sufficient, but any future extension of gate ceilings must revisit the bound (an additive `idlePolicy` change) and give the orchestrator a checkpoint-and-restart path (push branch, new session) beyond the cap. +- (−) **32 GiB sustained ceiling (8 GiB baseline + automatic 4× vertical scaling), 32 GB disk.** Not a successor to the ECS backend for heavy CI-parity builds; the platform now maintains three backends. +- (−) **Capacity is baseline-priced with burst headroom, which is a narrower promise than "32 GiB".** The deployment configures an 8 GiB / 4 vCPU baseline and the service scales to 32 GiB / 16 vCPU on demand — well matched to an agent task, which is idle-ish while waiting on the model and spiky during builds, and cheaper than reserving the peak. But it is *burst*, not a reservation: a workload that needs 32 GiB **sustained** is relying on scaling behaviour this ADR has not measured, and against ECS's 120 GB the gap for sustained-memory workloads is unchanged. So the value proposition remains the suspend economics, the observable control-plane state machine, and the absence of cluster ops — with capacity now a fair-to-good fit rather than a hard blocker. Repos with genuinely sustained heavy builds still belong on `ecs`. +- (−) **New packaging pipeline.** Zip + Dockerfile + service-side image builds with versioned snapshots (storage billed per version) alongside the existing ECR flow; image versions need lifecycle cleanup — including versions left behind by FAILED builds, and noting the last version of an image cannot be deleted individually (delete the image, which reaps it). +- (−) **The payload bucket is on the hot path, not the overflow path.** With a 4 KB `runHookPayload` cap, virtually every real task delivers its payload via S3, so the bucket, its TTL rule and the execution role's read grant are load-bearing for normal operation rather than an edge case (sub-decision 3). +- (−) **8-hour hard cap includes suspended time**, and with `idlePolicy` omitted there is no tighter substrate-level suspended-TTL — the suspended-state bound is `maximumDurationInSeconds` plus orchestrator termination and the stranded reconciler. A manually suspended VM was observed alive at 1 h with no TTL in sight (observation truncated there), so nothing contradicts this bound, but nothing narrows it either. Under today's 1 h gate ceiling it is comfortably sufficient; any future extension of gate ceilings must revisit the bound (an additive `idlePolicy` change) and give the orchestrator a checkpoint-and-restart path (push branch, new session) beyond the cap. - (!) **Idle-policy foot-gun.** Traffic-based auto-suspend would freeze a busy outbound-only agent; the decision to disable auto-suspend must be enforced in code and covered by tests, not left to configuration discipline. +- (!) **Service defaults are not the desired posture.** Two live-caught cases (public `HTTP_INGRESS` by default; `/ready` mandatory) mean an omitted field on this backend does not mean "off" — it can mean "the service picks, and it picks wider than we want". Every new `RunMicrovm` / `CreateMicrovmImage` field should be assumed to have an opinionated default until checked. +- (!) **Nothing self-terminates.** A MicroVM whose hook never ran still reaches `RUNNING` and stays there, billing, until the 8 h cap. The orchestrator's `TerminateMicrovm` on finalize is the only cleanup, so a leaked handle is a cost incident, not just an untidy state. - (!) **Snapshot uniqueness.** Shared memory snapshots require CSPRNG reseeding and credential refresh in `/run` / `/resume` hooks; missing this is a silent security defect. - (!) **Regional availability (5 regions at launch, expanding)** — enforced in layers (synth-time static check, onboarding + doctor live probes, orchestration-time classification; see sub-decision 4). The static CDK constant is the one piece that rots as AWS expands; its update path and context-flag escape hatch are deliberate. - (!) **Workload-token injection delta persists** (shared with the ECS backend) until [#249](https://github.com/aws-samples/sample-autonomous-cloud-coding-agents/issues/249)/ADR-016 land; document it in the security bar comparison rather than blocking on it. Memory and Gateway are explicitly *not* deltas — both are standalone services consumed via IAM from any substrate. @@ -181,11 +260,12 @@ Lambda MicroVMs launched in 5 regions (us-east-1/2, us-west-2, eu-west-1, ap-nor **P1 (start/poll/stop — no suspend):** -- Unit tests for the strategy: start/poll/stop mapping (including `SessionStatus` `'suspended'` reported mechanically, without task-state interpretation), payload-size branching (inline vs S3 pointer), error classification (`ServiceQuotaExceededException`, `ThrottlingException`, `ResourceNotFoundException`, regional-unavailability), the omit-`idlePolicy` invariant, and `maximumDurationInSeconds` fixed at 28 800. +- Unit tests for the strategy: start/poll/stop mapping (including `SessionStatus` `'suspended'` reported mechanically, without task-state interpretation), payload-size branching (inline vs S3 pointer, at the exact 4 096/4 097-byte boundary), the image-identifier-must-be-an-ARN guard, the explicit `NO_INGRESS` argument (including the blank-env-var fallback, which must never omit the field), error classification (`ServiceQuotaExceededException`, `ThrottlingException`, `ResourceNotFoundException`, regional-unavailability), the omit-`idlePolicy` invariant, and `maximumDurationInSeconds` fixed at 28 800. +- Agent tests: `/ready` returns 200 once the server is up and starts nothing; `/run` accepts both envelope shapes (inline and S3 pointer), starts the pipeline asynchronously through the same mapper `/invocations` uses, returns before the pipeline finishes, and rejects every unusable envelope with a named code before spawning; `/validate`, `/suspend`, `/resume` and `/terminate` are NOT served. - Orchestrator tests: substrate-terminal + non-terminal task status → failed classification; `suspended` + non-`AWAITING_APPROVAL` status → anomaly event, no fail-fast; `compute_metadata` persisted with `microvmId`/`endpoint` after `startSession`. -- CDK assertions: MicroVM resources present only when `ComputeTypes` includes the backend; synth failure for unsupported regions (plus the context-flag escape hatch); IAM actions scoped as specified (orchestrator lifecycle set; no `CreateMicrovmAuthToken` anywhere); payload-bucket grants (execution role read-only); backend cost-allocation tags; types-sync check covers the widened `ComputeType`. +- CDK assertions: MicroVM resources present only when `ComputeTypes` includes the backend; synth failure for unsupported regions (plus the context-flag escape hatch); memory size validated against the accepted list at synth; the connector operator role and its trust; two connectors with the build-only one carrying port 80 and the runtime one not; the `/ready` + `/run` hook declaration and the absence of the others; IAM actions scoped as specified (orchestrator lifecycle set; no `CreateMicrovmAuthToken` anywhere); payload-bucket grants (execution role read-only); backend cost-allocation tags; types-sync check covers the widened `ComputeType`. - CLI tests: onboarding rejection with remedy when the availability probe fails; doctor check present when a blueprint selects the backend. -- P1 verification items (external service facts): manual-suspend default TTL without `idlePolicy`; disk quota; `runHookPayload` 16 KB limit; IAM action names; region probe behavior; account-quota treatment of `SUSPENDED` MicroVMs — record answers in COMPUTE.md (`maximumDurationInSeconds` bounds the worst case either way). +- P1 verification items (external service facts) — **executed 2026-07-31, us-east-1**; see `docs/verification/645-p1-lambda-microvm-runbook.md` for the full evidence. Discharged: `runHookPayload` limit (**4 096**, not 16 KB), the accepted baseline memory sizes (`[512…8192]` MiB — note the developer guide, not the probe, is what establishes that this is a BASELINE with a 32 GiB peak), image-identifier ARN requirement, IAM action names and the observed image-ARN shape, region probe behaviour, manual suspend/resume without `idlePolicy`, terminate timing and the `TERMINATED`-persists-≥10-min finding, the default public `HTTP_INGRESS`, and the `/ready` requirement. **Not** discharged: account-quota treatment of `SUSPENDED` MicroVMs (not observable safely), suspended TTL beyond 1 h (truncated), the vertical-scaling behaviour itself (no workload here approached the baseline, so the 4× peak is documented rather than observed), and the `AWS::Lambda::MicrovmImage` CloudFormation value shapes (never exercised — the run used the out-of-band script path). Record the closed answers in COMPUTE.md. **P2 (smoke parity):** diff --git a/yarn.lock b/yarn.lock index a1b483392..a10463cde 100644 --- a/yarn.lock +++ b/yarn.lock @@ -492,6 +492,20 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/client-lambda-microvms@^3.1098.0": + version "3.1103.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-lambda-microvms/-/client-lambda-microvms-3.1103.0.tgz#0c3669d00e93efa4301f290f6819fca8eed8f8bc" + integrity sha512-orxyq0UvjaI/XPNUx/HQGXhcOnOk0AG1spNEHLKQKjlrY1Xkbr733ttzOYu319EZZJmsoIMkHercCfHJivbrVw== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/credential-provider-node" "^3.972.78" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/fetch-http-handler" "^5.6.13" + "@smithy/node-http-handler" "^4.9.13" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/client-lambda@^3.1014.0", "@aws-sdk/client-lambda@^3.1078.0": version "3.1081.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-lambda/-/client-lambda-3.1081.0.tgz#eed0c8ac8bb51fc1fad355cfdf33c62e9a4db2af" @@ -565,6 +579,20 @@ bowser "^2.11.0" tslib "^2.6.2" +"@aws-sdk/core@^3.977.6": + version "3.977.6" + resolved "https://registry.yarnpkg.com/@aws-sdk/core/-/core-3.977.6.tgz#4cc709e9299cfbbd4e5dfb83d82e3e67fb9a569c" + integrity sha512-QiaJV4/zDrB4ZY2mfeSXSzSTc36W16sZXcGz+SPFk0CJ26gziO0cS+4LjJUMAbdeeBOvS0k0Aq1cZpfGdUXxSw== + dependencies: + "@aws-sdk/types" "^3.974.2" + "@aws-sdk/xml-builder" "^3.972.37" + "@aws/lambda-invoke-store" "^0.3.0" + "@smithy/core" "^3.31.1" + "@smithy/signature-v4" "^5.6.12" + "@smithy/types" "^4.16.1" + bowser "^2.11.0" + tslib "^2.6.2" + "@aws-sdk/credential-provider-env@^3.972.55": version "3.972.55" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.55.tgz#1129acf8860db362a30a02a531387fd399448268" @@ -576,6 +604,17 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-env@^3.972.67": + version "3.972.67" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.67.tgz#7b369da235d280a3c7555eb67fdfc962fc6e6e90" + integrity sha512-rcIpk5kxUqDaaNa6Xk23pQ6ViY7jlqzmfFWCahQcBT97ddXaXYYwzCen9Tz1Jvo6aJft6wDl5bN44/Jw5B4oLA== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-http@^3.972.57": version "3.972.57" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.57.tgz#4f73bb2f0e03525ab11e001ac18c39cb120dd14c" @@ -589,6 +628,19 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-http@^3.972.69": + version "3.972.69" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.69.tgz#e6b73f06d8572215790a1facb171b67761bb3ba4" + integrity sha512-nggwJtZ4eeNsUw5IeWBMXsi1ryct5idi0K+/SCRF3kybLubOMaNTb3XCihXpWMiVpyzyPeIrl0zTkzhBH9porA== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/fetch-http-handler" "^5.6.13" + "@smithy/node-http-handler" "^4.9.13" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-ini@^3.972.62": version "3.972.62" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.62.tgz#a2ca7fc7a1899c0a4a38e501baa666cf1449f2e9" @@ -608,6 +660,25 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-ini@^3.973.12": + version "3.973.12" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.12.tgz#a3db28dd59fd948325ea4eb7f8efb9b499ed4a7e" + integrity sha512-pNEf/OeyN5X3VmLKlgSO6TqaWmW10CvI3TfwL1XhsuhYjSLT2VDaxFnCPHnOeQXSaFisMX4jNhpETriqN8DOmg== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/credential-provider-env" "^3.972.67" + "@aws-sdk/credential-provider-http" "^3.972.69" + "@aws-sdk/credential-provider-login" "^3.972.74" + "@aws-sdk/credential-provider-process" "^3.972.67" + "@aws-sdk/credential-provider-sso" "^3.973.11" + "@aws-sdk/credential-provider-web-identity" "^3.972.73" + "@aws-sdk/nested-clients" "^3.997.41" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/credential-provider-imds" "^4.4.16" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-login@^3.972.61": version "3.972.61" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.61.tgz#f5ba696bae9af3505ae5f3e2a70d7e216d0d37f6" @@ -620,6 +691,18 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-login@^3.972.74": + version "3.972.74" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.74.tgz#c942b01e5fe82d63fc149233d9ec1d7f4b527e15" + integrity sha512-0AQfDcf99TNmqVKv0owHrw/TQs6i4ZE5t9qmz6NvO53bE/sA/tpXhXL9AAcEP1qHc6Zzjd1UMb69+/9zdhvY3g== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/nested-clients" "^3.997.41" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-node@^3.972.61", "@aws-sdk/credential-provider-node@^3.972.64": version "3.972.64" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.64.tgz#282456c24bad616faefe2a6c68701913f8289c10" @@ -637,6 +720,23 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-node@^3.972.78": + version "3.972.78" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.78.tgz#8e9de78b6bec734781d29fcb3bde57983bd6e869" + integrity sha512-OgPAnfvbGAMWac6yvxJ1ihslrvDpPVwR68D2csospdNCCyPvHk9JLzYKwz48SNiS1T2znDwHauywRKRFfpyYng== + dependencies: + "@aws-sdk/credential-provider-env" "^3.972.67" + "@aws-sdk/credential-provider-http" "^3.972.69" + "@aws-sdk/credential-provider-ini" "^3.973.12" + "@aws-sdk/credential-provider-process" "^3.972.67" + "@aws-sdk/credential-provider-sso" "^3.973.11" + "@aws-sdk/credential-provider-web-identity" "^3.972.73" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/credential-provider-imds" "^4.4.16" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-process@^3.972.55": version "3.972.55" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.55.tgz#bebd30d0065ca34f64e14a870f54a09f23cf11e2" @@ -648,6 +748,17 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-process@^3.972.67": + version "3.972.67" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.67.tgz#818a86561e23cdc69449105ad2532b25fb6d3ba2" + integrity sha512-IlUEejorGTWKb4/Dm7K5Yw4QxUmXLThLhrvBmzVBqZFTbW72cv9LTcITmo1dsnYriALE4h68mOq4LB99x6sQ7Q== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-sso@^3.972.61": version "3.972.61" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.61.tgz#8c15846c65d2f03bfca3347626e1fcd1a0f40726" @@ -661,6 +772,19 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-sso@^3.973.11": + version "3.973.11" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.11.tgz#cae3068ae37db4f8429e9d67aa5b8d89d43d1ace" + integrity sha512-gAQBkBZxUB84d71+pPcI9L+jh2ujhuAVxc/4FgGiWFDjkPBlMKxzd5XDtkSXTFX8Ro7ansnT88+XadasxMeCRw== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/nested-clients" "^3.997.41" + "@aws-sdk/token-providers" "3.1103.0" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/credential-provider-web-identity@^3.972.61": version "3.972.61" resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.61.tgz#1938cc425e6156acd673484f10b437c5c4c85158" @@ -673,6 +797,18 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/credential-provider-web-identity@^3.972.73": + version "3.972.73" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.73.tgz#f1a007c0ee81c366d6646067e589db3a947e7940" + integrity sha512-SnlEmQa6SjOgs6iOPLUQl1Eyq4AKiAdPQlkOhFhqNfDtDCwibMGvL6QlkSmf3o6vAUSImzdPCxowT5dfQUZP1A== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/nested-clients" "^3.997.41" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/dynamodb-codec@^3.973.26", "@aws-sdk/dynamodb-codec@^3.973.29": version "3.973.29" resolved "https://registry.yarnpkg.com/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.29.tgz#ea72996b2e1f2c687254c69c357fa4278e42f1da" @@ -783,6 +919,20 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/nested-clients@^3.997.41": + version "3.997.41" + resolved "https://registry.yarnpkg.com/@aws-sdk/nested-clients/-/nested-clients-3.997.41.tgz#2a6de81c1405e0b1c2528db2cfebc0c4bae8e8d8" + integrity sha512-RDHqPGQWlF6tatA/Tp3rg6oIwtgN9IVderxE+9av2Y93Dfyu+mO1hZ5Bu2jpfZg2rwdNbsssnwM+sLafIczMlQ== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/signature-v4-multi-region" "^3.996.43" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/fetch-http-handler" "^5.6.13" + "@smithy/node-http-handler" "^4.9.13" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/s3-presigned-post@^3.1078.0": version "3.1081.0" resolved "https://registry.yarnpkg.com/@aws-sdk/s3-presigned-post/-/s3-presigned-post-3.1081.0.tgz#ed27961af8e772e23745e7ff34f34083bc69c291" @@ -818,6 +968,16 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/signature-v4-multi-region@^3.996.43": + version "3.996.43" + resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.43.tgz#f23113e6481741a110bb233a43d48aa40556d62b" + integrity sha512-lKekx8bLBXSv4O+cslk9Zfnw2XKSkWBs3uWL5QGhH2ZAQfNS7FE0vcSSN2vD/AhxX54ZTywWxR4STThoeOXlBA== + dependencies: + "@aws-sdk/types" "^3.974.2" + "@smithy/signature-v4" "^5.6.12" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/token-providers@3.1078.0": version "3.1078.0" resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1078.0.tgz#554be2f2c42f21191f31aead718bcedf2047926d" @@ -842,6 +1002,18 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/token-providers@3.1103.0": + version "3.1103.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.1103.0.tgz#afd08f067303c6cf39d0b94c88405ca27c5f2002" + integrity sha512-N4wy26MNn31ItGVHYHPrEuCIFY4MBBjC+C5v1lJKqIUSA7OZBdhleCY53zCCrXn27hsk7YNOaTuhQu807S4AfQ== + dependencies: + "@aws-sdk/core" "^3.977.6" + "@aws-sdk/nested-clients" "^3.997.41" + "@aws-sdk/types" "^3.974.2" + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/types@^3.222.0", "@aws-sdk/types@^3.973.15": version "3.973.15" resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.973.15.tgz#98a4860bed33c32c7088924d0ab52f9eabbdf7c3" @@ -850,6 +1022,14 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/types@^3.974.2": + version "3.974.2" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.974.2.tgz#05e7ccac417735e0786d430d2d5cc139047a08da" + integrity sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA== + dependencies: + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws-sdk/util-dynamodb@^3.996.5": version "3.996.5" resolved "https://registry.yarnpkg.com/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.5.tgz#2ad647e1532c20c76570a878e49f058a8d21e012" @@ -865,6 +1045,14 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@aws-sdk/xml-builder@^3.972.37": + version "3.972.37" + resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.972.37.tgz#669363721702d46a500c6ea485a44591e511f280" + integrity sha512-zKq4HQum8JwDyEuyfuI4bbiAcU0KxP6qy+9PR/IsR92IyE/DaBAikzAS50tjxip4bqIIANpCcG+Yyj6CVhXupg== + dependencies: + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@aws/durable-execution-sdk-js@^2.0.0": version "2.1.0" resolved "https://registry.yarnpkg.com/@aws/durable-execution-sdk-js/-/durable-execution-sdk-js-2.1.0.tgz#a020f8f3eae0fd3b577ad55397e454241a9e3029" @@ -2744,6 +2932,23 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@smithy/core@^3.31.1": + version "3.31.1" + resolved "https://registry.yarnpkg.com/@smithy/core/-/core-3.31.1.tgz#a57090db8997917d2f99eeb39db50b6283e60d3c" + integrity sha512-CyogUINxvi7C7LDsh8Syo6hVJOT9ckz4rG8dRZfTJ8r91HkMY59PnNooaj7WcHyxEkxPfBAmbgztZU+xTo76lg== + dependencies: + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + +"@smithy/credential-provider-imds@^4.4.16": + version "4.4.16" + resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.16.tgz#5197900c258a7c0b6fd2b9dbc11d5410f296d7c5" + integrity sha512-QfuLWAkLzptffFW980AFeHZFdqds2B64rpEd3uJ6lgs3xVn9QegGMUgUcj+4d7dRrAsya3r58ZKpku97WcFb4w== + dependencies: + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@smithy/credential-provider-imds@^4.4.5": version "4.4.6" resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.6.tgz#048ad961282016fc2d46d457f43beac8ca8f0e24" @@ -2753,6 +2958,15 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@smithy/fetch-http-handler@^5.6.13": + version "5.6.13" + resolved "https://registry.yarnpkg.com/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.13.tgz#688830a2d128aa95db5233d4ab827222a0bf4baa" + integrity sha512-4fW86pEUOMbrD5nkbyl/tTvPHHWJFbuB2odl6ps9lWfHoXf9HWh3Q/Smh59qH1g7+c/BSZghX6bbUk4gsiMs8A== + dependencies: + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@smithy/fetch-http-handler@^5.6.2": version "5.6.3" resolved "https://registry.yarnpkg.com/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.3.tgz#7b4aec28f89bd1e3aa00dfae0d0bd30079c58341" @@ -2769,6 +2983,15 @@ dependencies: tslib "^2.6.2" +"@smithy/node-http-handler@^4.9.13": + version "4.9.13" + resolved "https://registry.yarnpkg.com/@smithy/node-http-handler/-/node-http-handler-4.9.13.tgz#6c0811df72deb4890d07bf81a9a8bcdf782ba824" + integrity sha512-Nmd/Nl35zfYrd+a6OO2cDJb3GPh9bgTjIUhcM+JFfjpp8/osCgboDV5nCT1I01Pv6R13eSKDKLSoVa5ZB6Zsfw== + dependencies: + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@smithy/node-http-handler@^4.9.2": version "4.9.3" resolved "https://registry.yarnpkg.com/@smithy/node-http-handler/-/node-http-handler-4.9.3.tgz#f464b7f4ac242e8d6642dd8a602078bc88b0c53d" @@ -2795,6 +3018,15 @@ "@smithy/types" "^4.15.1" tslib "^2.6.2" +"@smithy/signature-v4@^5.6.12": + version "5.6.12" + resolved "https://registry.yarnpkg.com/@smithy/signature-v4/-/signature-v4-5.6.12.tgz#ee7a37e299de30e4d5c299b25485141b5b5d31c9" + integrity sha512-I6KLtq3H0qqSuV9vLglfi8puHqzygzWHOnI4z/Rdoo+q50vvo18vBRdPAvvEtcaKROz7Zn6qnPa14kRfPH6PcQ== + dependencies: + "@smithy/core" "^3.31.1" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + "@smithy/types@^4.15.1": version "4.15.1" resolved "https://registry.yarnpkg.com/@smithy/types/-/types-4.15.1.tgz#cdad0fdf4c5d1f5788dc21a3f1fb6af523d79da0" @@ -2802,6 +3034,13 @@ dependencies: tslib "^2.6.2" +"@smithy/types@^4.16.1": + version "4.16.1" + resolved "https://registry.yarnpkg.com/@smithy/types/-/types-4.16.1.tgz#19e199c234829a51c085caf63f0bb17bb80187e4" + integrity sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg== + dependencies: + tslib "^2.6.2" + "@smithy/util-buffer-from@^2.2.0": version "2.2.0" resolved "https://registry.yarnpkg.com/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz#6fc88585165ec73f8681d426d96de5d402021e4b"