From d5c30211647f4cf96a90686dd2b36c73ff80d1de Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Fri, 11 Sep 2026 23:20:42 +0200 Subject: [PATCH 01/14] add ls-on-firecracker: LocalStack on a Firecracker microVM demo Boots LocalStack inside a Firecracker microVM (Docker installed in the guest so LocalStack's Lambda executor works normally), with a self-describing Makefile (download -> rootfs -> up -> test -> down) and a GitHub Actions workflow that runs an S3 round-trip plus a real Lambda deploy/invoke assertion on an ubuntu-latest KVM runner. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- .github/workflows/test-ls-on-firecracker.yml | 52 +++++++++ ls-on-firecracker/.gitignore | 1 + ls-on-firecracker/Makefile | 44 ++++++++ ls-on-firecracker/README.md | 74 +++++++++++++ ls-on-firecracker/fixtures/handler.py | 3 + ls-on-firecracker/scripts/build-rootfs.sh | 85 +++++++++++++++ ls-on-firecracker/scripts/download-assets.sh | 72 ++++++++++++ ls-on-firecracker/scripts/run-vm.sh | 109 +++++++++++++++++++ ls-on-firecracker/scripts/smoke-test.sh | 78 +++++++++++++ ls-on-firecracker/scripts/teardown.sh | 38 +++++++ 10 files changed, 556 insertions(+) create mode 100644 .github/workflows/test-ls-on-firecracker.yml create mode 100644 ls-on-firecracker/.gitignore create mode 100644 ls-on-firecracker/Makefile create mode 100644 ls-on-firecracker/README.md create mode 100644 ls-on-firecracker/fixtures/handler.py create mode 100755 ls-on-firecracker/scripts/build-rootfs.sh create mode 100755 ls-on-firecracker/scripts/download-assets.sh create mode 100755 ls-on-firecracker/scripts/run-vm.sh create mode 100755 ls-on-firecracker/scripts/smoke-test.sh create mode 100755 ls-on-firecracker/scripts/teardown.sh diff --git a/.github/workflows/test-ls-on-firecracker.yml b/.github/workflows/test-ls-on-firecracker.yml new file mode 100644 index 0000000..fa40be8 --- /dev/null +++ b/.github/workflows/test-ls-on-firecracker.yml @@ -0,0 +1,52 @@ +name: LocalStack on Firecracker + +on: + pull_request: + branches: [main] + paths: + - 'ls-on-firecracker/**' + - '.github/workflows/test-ls-on-firecracker.yml' + push: + branches: [main] + paths: + - 'ls-on-firecracker/**' + - '.github/workflows/test-ls-on-firecracker.yml' + workflow_dispatch: + +env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.TEST_LOCALSTACK_AUTH_TOKEN }} + +jobs: + test-ls-on-firecracker: + name: Firecracker microVM + LocalStack + S3/Lambda smoke test + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + working-directory: ls-on-firecracker + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Enable KVM access + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls -l /dev/kvm + + - name: Boot the microVM + run: make up + + - name: Run S3 + Lambda smoke test + run: make test + + - name: Dump console/boot log on failure + if: failure() + run: cat work/firecracker.log 2>/dev/null || true + + - name: Tear down + if: always() + run: make down diff --git a/ls-on-firecracker/.gitignore b/ls-on-firecracker/.gitignore new file mode 100644 index 0000000..9d931c4 --- /dev/null +++ b/ls-on-firecracker/.gitignore @@ -0,0 +1 @@ +work/ diff --git a/ls-on-firecracker/Makefile b/ls-on-firecracker/Makefile new file mode 100644 index 0000000..2b20c15 --- /dev/null +++ b/ls-on-firecracker/Makefile @@ -0,0 +1,44 @@ +FC_VERSION := v1.10.1 +CI_TRACK := v1.10 +UNAME_M := $(shell uname -m) +# Firecracker's release/CI artifact names use "aarch64"/"x86_64", not the +# "arm64"/"amd64" spellings uname reports on macOS or Debian-flavored Linux. +ARCH := $(if $(filter arm64,$(UNAME_M)),aarch64,$(if $(filter amd64,$(UNAME_M)),x86_64,$(UNAME_M))) +WORK_DIR := work +TAP_DEV := fc-ls-tap0 +TAP_IP := 172.16.0.1 +VM_IP := 172.16.0.2 +VM_MASK := 255.255.255.0 +VCPUS := 2 +MEM_MB := 4096 +BUCKET := firecracker-demo +LAMBDA_FN := firecracker-demo-fn + +export WORK_DIR TAP_DEV TAP_IP VM_IP VM_MASK VCPUS MEM_MB BUCKET LAMBDA_FN FC_VERSION CI_TRACK ARCH LOCALSTACK_AUTH_TOKEN + +.PHONY: help download rootfs up test logs down clean + +help: ## Show available targets + @grep -E '^[a-zA-Z_-]+:.*##' $(MAKEFILE_LIST) | \ + awk 'BEGIN{FS=":.*##"}{printf " %-10s %s\n", $$1, $$2}' + +download: ## Fetch the firecracker binary, guest kernel and base rootfs + @bash scripts/download-assets.sh + +rootfs: download ## Build a LocalStack-flavored guest rootfs image + @bash scripts/build-rootfs.sh + +up: rootfs ## Create the tap device and boot the LocalStack microVM + @bash scripts/run-vm.sh + +test: ## Run S3 + Lambda smoke tests against the running LocalStack instance + @bash scripts/smoke-test.sh + +logs: ## Tail the microVM's console/boot log + @tail -n 100 -f $(WORK_DIR)/firecracker.log + +down: ## Stop the microVM and remove the tap device + @bash scripts/teardown.sh + +clean: down ## Remove all downloaded/built artifacts + rm -rf $(WORK_DIR) diff --git a/ls-on-firecracker/README.md b/ls-on-firecracker/README.md new file mode 100644 index 0000000..7509138 --- /dev/null +++ b/ls-on-firecracker/README.md @@ -0,0 +1,74 @@ +# LocalStack on Firecracker + +A demo that boots [LocalStack](https://localstack.cloud) inside a +[Firecracker](https://firecracker-microvm.github.io/) microVM — the same +technology AWS Lambda itself runs on — and exercises it over the network with +an S3 bucket and a real Lambda deploy + invoke. + +## How it works + +1. **`make download`** grabs the `firecracker` binary, a guest kernel and a + base Ubuntu rootfs from Firecracker's public CI artifacts. +2. **`make rootfs`** clones the base rootfs, grows it, and chroots in to + install Docker plus `pip install localstack awscli-local`, and registers + `systemd` units so Docker and then `localstack start --host` come up on + boot. Building happens on the host via a loop-mounted image, so the + customization step itself doesn't need the guest to be running. +3. **`make up`** creates a tap network device on the host, NATs the guest out + through the host's default interface (LocalStack needs to `docker pull` + the Lambda runtime image at invoke time), and boots the image with + Firecracker. It polls `http://:4566/_localstack/health` until + LocalStack is ready. +4. **`make test`** creates an S3 bucket and round-trips an object, then + deploys a small Python Lambda function, invokes it, and asserts the + response — all against the LocalStack instance running inside the + microVM. Lambda execution goes through the guest's own Docker daemon, + exactly like it would against a normal `docker run localstack` setup. +5. **`make down`** / **`make clean`** tear the VM, NAT rules, and tap device + down again. + +``` +make up # download -> rootfs -> boot the microVM +make test # exercise S3 + Lambda through it +make down # stop the VM +``` + +Run `make help` for the full target list. + +## Prerequisites + +- A **Linux host with KVM** (`/dev/kvm` present and accessible) — Firecracker + does not run on macOS or in most cloud VMs without nested virtualization. + Bare metal, an EC2 `.metal` instance, or a GitHub Actions `ubuntu-latest` + runner (which has KVM enabled) all work. See + `.github/workflows/test-ls-on-firecracker.yml` in the repo root for a + working CI setup. +- `curl`, `iproute2`, `iptables`, `e2fsprogs`, `zip`, `jq`, and the AWS CLI + (`aws`) on the host. +- `sudo` access — the scripts use it for loop-mounting the rootfs image, + managing the tap device and NAT rules, and launching `firecracker` itself. +- Optionally, a `LOCALSTACK_AUTH_TOKEN` environment variable on the host — + if set, it's passed through into the guest's `localstack.service` at boot. + +## What's actually running + +Docker runs *inside* the guest OS (installed at rootfs-build time), and +LocalStack uses it as its normal Docker-based Lambda executor. The guest +needs internet access at Lambda invoke time to pull the runtime image, which +is why `run-vm.sh` sets up NAT through the host rather than an isolated +host-only network. + +## Layout + +``` +Makefile self-describing entry point (make help) +scripts/ + download-assets.sh fetch firecracker + kernel + base rootfs + build-rootfs.sh install Docker + LocalStack into a working rootfs image + run-vm.sh set up networking (incl. NAT) and boot the microVM + smoke-test.sh S3 round-trip + Lambda deploy/invoke against it + teardown.sh stop the VM and remove the tap device / NAT rules +fixtures/ + handler.py the demo Lambda function +work/ downloaded/generated artifacts (git-ignored) +``` diff --git a/ls-on-firecracker/fixtures/handler.py b/ls-on-firecracker/fixtures/handler.py new file mode 100644 index 0000000..7dd5ecc --- /dev/null +++ b/ls-on-firecracker/fixtures/handler.py @@ -0,0 +1,3 @@ +def handler(event, context): + name = event.get("name", "world") + return {"message": f"hello {name}"} diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh new file mode 100755 index 0000000..7f070a5 --- /dev/null +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Turns the pristine CI rootfs into a "LocalStack appliance": grows the ext4 +# image, chroots into it, pip-installs LocalStack + awslocal, and registers a +# systemd unit that starts LocalStack on boot. +# +# The chroot shares the host's network namespace (it's just a mounted +# directory, not a container), so apt/pip work normally as long as the host +# has internet access. The guest VM itself does NOT get internet access at +# boot -- it doesn't need it, everything is already baked into the image. +set -euo pipefail + +: "${WORK_DIR:?run via 'make', not directly}" + +IMG_DIR="$WORK_DIR/images" +BASE_IMG="$IMG_DIR/base.ext4" +OUT_IMG="$IMG_DIR/localstack.ext4" +MNT="$WORK_DIR/rootfs-mnt" + +if [[ -f "$OUT_IMG" ]]; then + echo "[rootfs] $OUT_IMG already built, skipping (delete it, or 'make clean', to rebuild)" + exit 0 +fi + +echo "[rootfs] cloning base image and growing it to make room for LocalStack + Docker" +cp "$BASE_IMG" "$OUT_IMG" +truncate -s 8G "$OUT_IMG" +e2fsck -fy "$OUT_IMG" || true +resize2fs "$OUT_IMG" + +mkdir -p "$MNT" +LOOP_DEV=$(sudo losetup --find --show "$OUT_IMG") +cleanup() { + sudo umount -R "$MNT" 2>/dev/null || true + sudo losetup -d "$LOOP_DEV" 2>/dev/null || true +} +trap cleanup EXIT + +sudo mount "$LOOP_DEV" "$MNT" +sudo cp /etc/resolv.conf "$MNT/etc/resolv.conf" +sudo mount --bind /dev "$MNT/dev" +sudo mount --bind /proc "$MNT/proc" +sudo mount --bind /sys "$MNT/sys" + +echo "[rootfs] installing Docker + LocalStack inside the guest image (this can take a few minutes)" +sudo chroot "$MNT" /bin/bash -c ' + set -euo pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq python3-pip python3-venv docker.io >/dev/null + pip3 install --break-system-packages -q localstack awscli-local + systemctl enable docker.service +' + +# LocalStack uses the guest's own Docker daemon to run the Lambda executor +# container, exactly like real Lambda uses a container runtime inside its +# Firecracker microVM. It needs to start after (and depend on) Docker. +sudo tee "$MNT/etc/systemd/system/localstack.service" >/dev/null <<'UNIT' +[Unit] +Description=LocalStack +After=docker.service network.target +Requires=docker.service + +[Service] +Environment=LOCALSTACK_HOST=0.0.0.0 +ExecStart=/usr/local/bin/localstack start --host +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target +UNIT + +sudo chroot "$MNT" systemctl enable localstack.service + +# The chroot borrowed the host's /etc/resolv.conf (often a systemd-resolved +# stub at 127.0.0.53) to resolve apt/pip mirrors during the build above. That +# address is meaningless once the image boots as its own VM, so pin a public +# resolver for runtime -- the guest needs it to pull the Lambda runtime image +# from ECR public over the NAT'd link `run-vm.sh` sets up. +sudo tee "$MNT/etc/resolv.conf" >/dev/null <<'EOF' +nameserver 8.8.8.8 +nameserver 1.1.1.1 +EOF + +echo "[rootfs] built -> $OUT_IMG" diff --git a/ls-on-firecracker/scripts/download-assets.sh b/ls-on-firecracker/scripts/download-assets.sh new file mode 100755 index 0000000..a8a3bff --- /dev/null +++ b/ls-on-firecracker/scripts/download-assets.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Downloads the three things Firecracker needs to boot a microVM: +# 1. the firecracker binary itself +# 2. an uncompressed guest kernel (vmlinux) +# 3. a base guest rootfs (ext4 image) +# +# Kernel/rootfs are pulled from Firecracker's public CI bucket, which is the +# same source used in the project's own getting-started guide. We resolve +# "latest for this CI track" dynamically so the URLs don't go stale. +set -euo pipefail + +: "${WORK_DIR:?run via 'make', not directly}" +: "${FC_VERSION:?}" +: "${CI_TRACK:?}" +: "${ARCH:?}" + +if [[ "$(uname -s)" != "Linux" ]]; then + echo "[download] error: Firecracker requires Linux + KVM (/dev/kvm)." >&2 + echo " This host is $(uname -s), so it cannot run this demo." >&2 + echo " Try a Linux box with KVM, an EC2 .metal instance, or a" >&2 + echo " GitHub Actions ubuntu-latest runner instead." >&2 + exit 1 +fi + +if [[ ! -e /dev/kvm ]]; then + echo "[download] error: /dev/kvm not found. Firecracker needs KVM, which" >&2 + echo " usually means bare metal or a host with nested" >&2 + echo " virtualization enabled (not a typical cloud VM)." >&2 + exit 1 +fi + +BIN_DIR="$WORK_DIR/bin" +IMG_DIR="$WORK_DIR/images" +mkdir -p "$BIN_DIR" "$IMG_DIR" + +if [[ -x "$BIN_DIR/firecracker" ]]; then + echo "[download] firecracker binary already present, skipping" +else + echo "[download] fetching firecracker $FC_VERSION for $ARCH" + tmp=$(mktemp -d) + curl -fsSL "https://github.com/firecracker-microvm/firecracker/releases/download/${FC_VERSION}/firecracker-${FC_VERSION}-${ARCH}.tgz" \ + | tar -xz -C "$tmp" + find "$tmp" -type f -name "firecracker-${FC_VERSION}-${ARCH}" -exec cp {} "$BIN_DIR/firecracker" \; + chmod +x "$BIN_DIR/firecracker" + rm -rf "$tmp" +fi + +if [[ -f "$IMG_DIR/vmlinux.bin" ]]; then + echo "[download] kernel already present, skipping" +else + echo "[download] resolving latest CI kernel for track $CI_TRACK/$ARCH" + kernel_key=$(curl -fsSL "http://spec.ccfc.min.s3.amazonaws.com/?prefix=firecracker-ci/${CI_TRACK}/${ARCH}/vmlinux-&list-type=2" \ + | grep -oP '(?<=)[^<]+' \ + | grep -E "^firecracker-ci/${CI_TRACK}/${ARCH}/vmlinux-[0-9]+\.[0-9]+\.[0-9]+$" \ + | sort -V | tail -1) + [[ -n "$kernel_key" ]] || { echo "could not resolve a kernel for CI track $CI_TRACK/$ARCH" >&2; exit 1; } + curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${kernel_key}" -o "$IMG_DIR/vmlinux.bin" +fi + +if [[ -f "$IMG_DIR/base.ext4" ]]; then + echo "[download] base rootfs already present, skipping" +else + echo "[download] resolving latest CI rootfs for track $CI_TRACK/$ARCH" + rootfs_key=$(curl -fsSL "http://spec.ccfc.min.s3.amazonaws.com/?prefix=firecracker-ci/${CI_TRACK}/${ARCH}/ubuntu-&list-type=2" \ + | grep -oP '(?<=)[^<]+' \ + | grep -E "^firecracker-ci/${CI_TRACK}/${ARCH}/ubuntu-[0-9]+\.[0-9]+\.ext4$" \ + | sort -V | tail -1) + [[ -n "$rootfs_key" ]] || { echo "could not resolve a base rootfs for CI track $CI_TRACK/$ARCH" >&2; exit 1; } + curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${rootfs_key}" -o "$IMG_DIR/base.ext4" +fi + +echo "[download] done -> $BIN_DIR/firecracker, $IMG_DIR/vmlinux.bin, $IMG_DIR/base.ext4" diff --git a/ls-on-firecracker/scripts/run-vm.sh b/ls-on-firecracker/scripts/run-vm.sh new file mode 100755 index 0000000..0a6d976 --- /dev/null +++ b/ls-on-firecracker/scripts/run-vm.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Sets up a tap device on the host, writes a Firecracker VM config, and boots +# the microVM in the background. Waits until LocalStack answers its health +# endpoint over the tap network before returning. +set -euo pipefail + +: "${WORK_DIR:?run via 'make', not directly}" +: "${TAP_DEV:?}" "${TAP_IP:?}" "${VM_IP:?}" "${VM_MASK:?}" "${VCPUS:?}" "${MEM_MB:?}" + +FC_BIN="$WORK_DIR/bin/firecracker" +KERNEL="$WORK_DIR/images/vmlinux.bin" +ROOTFS="$WORK_DIR/images/localstack.ext4" +SOCKET="$WORK_DIR/firecracker.sock" +CONFIG="$WORK_DIR/vm-config.json" +LOG="$WORK_DIR/firecracker.log" +PIDFILE="$WORK_DIR/firecracker.pid" + +if [[ -f "$PIDFILE" ]] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then + echo "[run] a microVM is already running (pid $(cat "$PIDFILE")); run 'make down' first" + exit 0 +fi +rm -f "$SOCKET" + +echo "[run] configuring tap device $TAP_DEV ($TAP_IP <-> $VM_IP)" +if ! ip link show "$TAP_DEV" &>/dev/null; then + sudo ip tuntap add dev "$TAP_DEV" mode tap +fi +sudo ip addr flush dev "$TAP_DEV" +sudo ip addr add "${TAP_IP}/24" dev "$TAP_DEV" +sudo ip link set dev "$TAP_DEV" up +sudo sysctl -qw "net.ipv4.conf.${TAP_DEV}.proxy_arp=1" +sudo sysctl -qw "net.ipv6.conf.${TAP_DEV}.disable_ipv6=1" + +# NAT the guest out through the host's default interface. The guest needs +# real internet access for `docker pull` of the Lambda runtime image -- it's +# not just host<->guest traffic like the plain S3-only version of this demo. +NET_BASE="$(echo "$TAP_IP" | awk -F. '{print $1"."$2"."$3".0"}')/24" +HOST_IFACE="$(ip route show default 2>/dev/null | awk '/default/ {print $5; exit}')" +if [[ -n "$HOST_IFACE" ]]; then + echo "[run] enabling NAT via $HOST_IFACE so the guest can reach the internet" + sudo sysctl -qw net.ipv4.ip_forward=1 + sudo iptables -t nat -C POSTROUTING -s "$NET_BASE" -o "$HOST_IFACE" -j MASQUERADE 2>/dev/null \ + || sudo iptables -t nat -A POSTROUTING -s "$NET_BASE" -o "$HOST_IFACE" -j MASQUERADE + sudo iptables -C FORWARD -i "$TAP_DEV" -o "$HOST_IFACE" -j ACCEPT 2>/dev/null \ + || sudo iptables -A FORWARD -i "$TAP_DEV" -o "$HOST_IFACE" -j ACCEPT + sudo iptables -C FORWARD -i "$HOST_IFACE" -o "$TAP_DEV" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null \ + || sudo iptables -A FORWARD -i "$HOST_IFACE" -o "$TAP_DEV" -m state --state RELATED,ESTABLISHED -j ACCEPT +else + echo "[run] warning: no default route found on host; guest will have no internet access" >&2 +fi + +# Any KEY=VALUE on the kernel command line that systemd doesn't otherwise +# recognize is ignored unless it's spelled systemd.setenv=KEY=VALUE, in which +# case it becomes a manager-wide environment variable that every unit +# (including localstack.service) inherits. This is how LOCALSTACK_AUTH_TOKEN +# gets from the CI secret into the guest without baking it into the image. +AUTH_ENV="" +if [[ -n "${LOCALSTACK_AUTH_TOKEN:-}" ]]; then + AUTH_ENV=" systemd.setenv=LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN}" +fi + +BOOT_ARGS="console=ttyS0 reboot=k panic=1 pci=off ip=${VM_IP}::${TAP_IP}:${VM_MASK}::eth0:off${AUTH_ENV}" + +cat > "$CONFIG" < "$LOG" 2>&1 < /dev/null & +disown +echo $! | sudo tee "$PIDFILE" >/dev/null + +echo "[run] waiting for LocalStack to become healthy at http://${VM_IP}:4566 ..." +# Generous budget: guest boot + dockerd startup + LocalStack cold start. +for _ in $(seq 1 90); do + if curl -fsS "http://${VM_IP}:4566/_localstack/health" >/dev/null 2>&1; then + echo "[run] LocalStack is up: http://${VM_IP}:4566" + exit 0 + fi + sleep 5 +done + +echo "[run] timed out waiting for LocalStack; check $LOG" >&2 +exit 1 diff --git a/ls-on-firecracker/scripts/smoke-test.sh b/ls-on-firecracker/scripts/smoke-test.sh new file mode 100755 index 0000000..018ebb6 --- /dev/null +++ b/ls-on-firecracker/scripts/smoke-test.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Exercises the LocalStack instance running inside the microVM: +# - S3: create a bucket, put an object, read it back +# - Lambda: deploy a function, invoke it, assert the response +# Lambda execution happens via the guest's own Docker daemon, the same way +# it would against a normal `docker run localstack` setup. +set -euo pipefail + +: "${VM_IP:?run via 'make', not directly}" +: "${BUCKET:?}" +: "${LAMBDA_FN:?}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export AWS_ACCESS_KEY_ID=test +export AWS_SECRET_ACCESS_KEY=test +export AWS_DEFAULT_REGION=us-east-1 +ENDPOINT="http://${VM_IP}:4566" +AWS="aws --endpoint-url $ENDPOINT" + +echo "[test] --- S3 ---" +echo "[test] creating bucket s3://${BUCKET}" +$AWS s3 mb "s3://${BUCKET}" + +TMP_FILE=$(mktemp) +echo "hello from a firecracker microVM" > "$TMP_FILE" +$AWS s3 cp "$TMP_FILE" "s3://${BUCKET}/hello.txt" >/dev/null +rm -f "$TMP_FILE" + +$AWS s3 ls "s3://${BUCKET}/" +echo "[test] S3 OK" + +echo "[test] --- Lambda ---" +ZIP_FILE=$(mktemp -u --suffix=.zip) +(cd "$SCRIPT_DIR/../fixtures" && zip -q "$ZIP_FILE" handler.py) + +echo "[test] creating function $LAMBDA_FN" +$AWS lambda create-function \ + --function-name "$LAMBDA_FN" \ + --runtime python3.12 \ + --handler handler.handler \ + --role arn:aws:iam::000000000000:role/lambda-role \ + --zip-file "fileb://${ZIP_FILE}" >/dev/null +rm -f "$ZIP_FILE" + +echo "[test] waiting for $LAMBDA_FN to become active (this pulls the Lambda runtime image)" +state="" +for _ in $(seq 1 40); do + state=$($AWS lambda get-function --function-name "$LAMBDA_FN" \ + --query 'Configuration.State' --output text 2>/dev/null || echo "") + [[ "$state" == "Active" ]] && break + sleep 5 +done +if [[ "$state" != "Active" ]]; then + echo "[test] FAILED: function never became active (last state: ${state:-unknown})" >&2 + exit 1 +fi + +echo "[test] invoking $LAMBDA_FN" +OUT_FILE=$(mktemp) +$AWS lambda invoke \ + --function-name "$LAMBDA_FN" \ + --cli-binary-format raw-in-base64-out \ + --payload '{"name":"firecracker"}' \ + "$OUT_FILE" >/dev/null + +RESPONSE=$(cat "$OUT_FILE") +rm -f "$OUT_FILE" +echo "[test] response: $RESPONSE" + +MESSAGE=$(jq -r '.message' <<<"$RESPONSE") +if [[ "$MESSAGE" != "hello firecracker" ]]; then + echo "[test] FAILED: expected message 'hello firecracker', got '$MESSAGE'" >&2 + exit 1 +fi + +echo "[test] Lambda OK" +echo "[test] all checks passed" diff --git a/ls-on-firecracker/scripts/teardown.sh b/ls-on-firecracker/scripts/teardown.sh new file mode 100755 index 0000000..9245e40 --- /dev/null +++ b/ls-on-firecracker/scripts/teardown.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Stops the microVM and removes the tap device. Safe to run even if nothing +# is up. +set -euo pipefail + +: "${WORK_DIR:?run via 'make', not directly}" +: "${TAP_DEV:?}" +: "${TAP_IP:?}" + +PIDFILE="$WORK_DIR/firecracker.pid" + +if [[ -f "$PIDFILE" ]]; then + pid=$(cat "$PIDFILE") + if kill -0 "$pid" 2>/dev/null; then + echo "[down] stopping microVM (pid $pid)" + sudo kill "$pid" 2>/dev/null || true + sleep 1 + sudo kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$PIDFILE" +fi +rm -f "$WORK_DIR/firecracker.sock" + +NET_BASE="$(echo "$TAP_IP" | awk -F. '{print $1"."$2"."$3".0"}')/24" +HOST_IFACE="$(ip route show default 2>/dev/null | awk '/default/ {print $5; exit}')" +if [[ -n "$HOST_IFACE" ]]; then + echo "[down] removing NAT rules for $NET_BASE via $HOST_IFACE" + sudo iptables -t nat -D POSTROUTING -s "$NET_BASE" -o "$HOST_IFACE" -j MASQUERADE 2>/dev/null || true + sudo iptables -D FORWARD -i "$TAP_DEV" -o "$HOST_IFACE" -j ACCEPT 2>/dev/null || true + sudo iptables -D FORWARD -i "$HOST_IFACE" -o "$TAP_DEV" -m state --state RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true +fi + +if ip link show "$TAP_DEV" &>/dev/null; then + echo "[down] removing tap device $TAP_DEV" + sudo ip link del "$TAP_DEV" 2>/dev/null || true +fi + +echo "[down] done" From a17fbb1104e5f47906e90438e3c5f4a1bcb7b45b Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Fri, 11 Sep 2026 23:21:59 +0200 Subject: [PATCH 02/14] ls-on-firecracker: fix apt failure by mounting tmpfs on /tmp and /run in chroot CI run showed apt-get failing inside the rootfs chroot with confusing "No such file or directory" errors -- /tmp and /run are normally populated by systemd-tmpfiles at boot, which never runs since we chroot into a mounted image rather than booting it. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/scripts/build-rootfs.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index 7f070a5..d201b11 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -40,6 +40,13 @@ sudo cp /etc/resolv.conf "$MNT/etc/resolv.conf" sudo mount --bind /dev "$MNT/dev" sudo mount --bind /proc "$MNT/proc" sudo mount --bind /sys "$MNT/sys" +# The base image's /tmp and /run are normally populated by systemd-tmpfiles +# at boot; without that, apt/dpkg (which write scratch files there) fail +# with confusing "No such file or directory" errors inside the chroot. +sudo mkdir -p "$MNT/tmp" "$MNT/run" +sudo mount -t tmpfs tmpfs "$MNT/tmp" +sudo mount -t tmpfs tmpfs "$MNT/run" +sudo chmod 1777 "$MNT/tmp" echo "[rootfs] installing Docker + LocalStack inside the guest image (this can take a few minutes)" sudo chroot "$MNT" /bin/bash -c ' From 9eba3517f9dbf7b200e7fb71fc32284a145cd1f6 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Fri, 11 Sep 2026 23:43:42 +0200 Subject: [PATCH 03/14] ls-on-firecracker: bootstrap an empty dpkg database before apt-get Confirmed by loop-mounting the actual Firecracker CI base image: it ships apt/dpkg binaries but /var/lib/dpkg is completely empty (no status file, no info/updates/triggers dirs), causing apt-get to fail with "flAbsPath ... realpath: No such file or directory". This is the same bootstrap debootstrap itself does for a fresh root. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/scripts/build-rootfs.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index d201b11..623ab8b 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -48,6 +48,14 @@ sudo mount -t tmpfs tmpfs "$MNT/tmp" sudo mount -t tmpfs tmpfs "$MNT/run" sudo chmod 1777 "$MNT/tmp" +# This CI-provided base image ships apt/dpkg binaries but an empty +# /var/lib/dpkg (no `status` file, no info/updates/triggers dirs) -- it was +# stripped for Firecracker's own network-test use, not general package +# installs. Bootstrap a fresh, empty dpkg database, the same thing tools +# like debootstrap do, so apt has something to work from. +sudo mkdir -p "$MNT/var/lib/dpkg/info" "$MNT/var/lib/dpkg/updates" "$MNT/var/lib/dpkg/triggers" +sudo touch "$MNT/var/lib/dpkg/status" "$MNT/var/lib/dpkg/available" + echo "[rootfs] installing Docker + LocalStack inside the guest image (this can take a few minutes)" sudo chroot "$MNT" /bin/bash -c ' set -euo pipefail From c8b61520d68aa4425bede9ee0239614504522794 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Fri, 11 Sep 2026 23:46:32 +0200 Subject: [PATCH 04/14] ls-on-firecracker: recreate the full apt/dpkg directory skeleton Inspected the actual base image locally with debugfs: /var/cache/apt, /var/lib/apt and /var/log don't exist at all in this CI artifact (only /var/lib/{systemd,dpkg} survived whatever stripped the image down). Recreate the standard set of scratch dirs apt/dpkg expect before running any apt commands. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/scripts/build-rootfs.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index 623ab8b..63eb2a7 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -48,12 +48,20 @@ sudo mount -t tmpfs tmpfs "$MNT/tmp" sudo mount -t tmpfs tmpfs "$MNT/run" sudo chmod 1777 "$MNT/tmp" -# This CI-provided base image ships apt/dpkg binaries but an empty -# /var/lib/dpkg (no `status` file, no info/updates/triggers dirs) -- it was -# stripped for Firecracker's own network-test use, not general package -# installs. Bootstrap a fresh, empty dpkg database, the same thing tools -# like debootstrap do, so apt has something to work from. -sudo mkdir -p "$MNT/var/lib/dpkg/info" "$MNT/var/lib/dpkg/updates" "$MNT/var/lib/dpkg/triggers" +# This CI-provided base image ships apt/dpkg binaries, but it was stripped +# for Firecracker's own network-test use, not general package installs: +# /var/cache/apt, /var/lib/apt and /var/log don't exist at all, and +# /var/lib/dpkg is an empty directory with no status file. Recreate the +# standard skeleton apt/dpkg expect -- the same bootstrap debootstrap itself +# does for a fresh root -- before touching either. +sudo mkdir -p \ + "$MNT/var/cache/apt/archives/partial" \ + "$MNT/var/lib/apt/lists/partial" \ + "$MNT/var/log/apt" \ + "$MNT/var/lib/dpkg/info" \ + "$MNT/var/lib/dpkg/updates" \ + "$MNT/var/lib/dpkg/triggers" \ + "$MNT/var/backups" sudo touch "$MNT/var/lib/dpkg/status" "$MNT/var/lib/dpkg/available" echo "[rootfs] installing Docker + LocalStack inside the guest image (this can take a few minutes)" From 71a790bfb8dbcfc9568fc8acf0ddf1cc0814731b Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Fri, 11 Sep 2026 23:47:54 +0200 Subject: [PATCH 05/14] ls-on-firecracker: force dpkg conffile policy to avoid a stdin prompt apt-get install was hanging/failing on libpam-modules' postinst asking a Y/I/N/O/D/Z conffile-merge question on stdin (not a TTY in this chroot). --force-confdef + --force-confold resolve it the same way any non-interactive apt-get install does. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/scripts/build-rootfs.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index 63eb2a7..44c74a4 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -69,7 +69,13 @@ sudo chroot "$MNT" /bin/bash -c ' set -euo pipefail export DEBIAN_FRONTEND=noninteractive apt-get update -qq - apt-get install -y -qq python3-pip python3-venv docker.io >/dev/null + # --force-confdef/--force-confold: some packages (libpam-modules and + # friends) think their conffiles were locally modified in this base image + # and prompt for a merge decision on stdin, which is not a TTY here. + apt-get install -y -qq \ + -o Dpkg::Options::=--force-confdef \ + -o Dpkg::Options::=--force-confold \ + python3-pip python3-venv docker.io >/dev/null pip3 install --break-system-packages -q localstack awscli-local systemctl enable docker.service ' From 33d249e086bc1ef48037ba7be11f70ed43df3180 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Fri, 11 Sep 2026 23:49:54 +0200 Subject: [PATCH 06/14] ls-on-firecracker: drop --break-system-packages, unsupported on jammy's pip apt/docker install now completes cleanly (confirmed in CI). The base image is Ubuntu 22.04, whose bundled pip predates PEP 668 enforcement and doesn't recognize --break-system-packages at all. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/scripts/build-rootfs.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index 44c74a4..f1849f6 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -76,7 +76,10 @@ sudo chroot "$MNT" /bin/bash -c ' -o Dpkg::Options::=--force-confdef \ -o Dpkg::Options::=--force-confold \ python3-pip python3-venv docker.io >/dev/null - pip3 install --break-system-packages -q localstack awscli-local + # This base image is Ubuntu 22.04 (jammy), whose bundled pip predates + # PEP 668 "externally managed environment" enforcement, so it does not + # understand --break-system-packages -- and does not need it either. + pip3 install -q localstack awscli-local systemctl enable docker.service ' From 77cb102864e40162bb25ec3d532d7a0f193cf242 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 00:00:59 +0200 Subject: [PATCH 07/14] ls-on-firecracker: add SSH-based diagnostics for the microVM The microVM now boots fully (confirmed in CI: kernel, network, systemd, login all work), but docker.service fails to start and the serial console doesn't show why. The base image ships sshd with a pre-authorized root key; the matching private key is published alongside the rootfs in the same CI bucket. Fetch it, and add `make ssh` / `make diagnose` (systemctl status + journalctl for docker/localstack), wired into the CI failure step so the next run tells us the actual error instead of us guessing blind. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- .github/workflows/test-ls-on-firecracker.yml | 4 ++++ ls-on-firecracker/Makefile | 13 ++++++++++++- ls-on-firecracker/README.md | 10 +++++++--- ls-on-firecracker/scripts/download-assets.sh | 7 +++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-ls-on-firecracker.yml b/.github/workflows/test-ls-on-firecracker.yml index fa40be8..8fff70d 100644 --- a/.github/workflows/test-ls-on-firecracker.yml +++ b/.github/workflows/test-ls-on-firecracker.yml @@ -47,6 +47,10 @@ jobs: if: failure() run: cat work/firecracker.log 2>/dev/null || true + - name: Dump service diagnostics on failure + if: failure() + run: make diagnose 2>/dev/null || true + - name: Tear down if: always() run: make down diff --git a/ls-on-firecracker/Makefile b/ls-on-firecracker/Makefile index 2b20c15..3bd7d4d 100644 --- a/ls-on-firecracker/Makefile +++ b/ls-on-firecracker/Makefile @@ -16,7 +16,10 @@ LAMBDA_FN := firecracker-demo-fn export WORK_DIR TAP_DEV TAP_IP VM_IP VM_MASK VCPUS MEM_MB BUCKET LAMBDA_FN FC_VERSION CI_TRACK ARCH LOCALSTACK_AUTH_TOKEN -.PHONY: help download rootfs up test logs down clean +SSH := ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ + -i $(WORK_DIR)/images/id_rsa root@$(VM_IP) + +.PHONY: help download rootfs up test logs down clean ssh diagnose help: ## Show available targets @grep -E '^[a-zA-Z_-]+:.*##' $(MAKEFILE_LIST) | \ @@ -37,6 +40,14 @@ test: ## Run S3 + Lambda smoke tests against the running LocalStack instance logs: ## Tail the microVM's console/boot log @tail -n 100 -f $(WORK_DIR)/firecracker.log +ssh: ## SSH into the running microVM as root + @$(SSH) + +diagnose: ## Dump docker/localstack service status + logs from inside the microVM + @$(SSH) 'systemctl --no-pager status docker.service localstack.service; \ + echo ---docker.journal---; journalctl -u docker --no-pager -n 200; \ + echo ---localstack.journal---; journalctl -u localstack --no-pager -n 200' + down: ## Stop the microVM and remove the tap device @bash scripts/teardown.sh diff --git a/ls-on-firecracker/README.md b/ls-on-firecracker/README.md index 7509138..90c0ac1 100644 --- a/ls-on-firecracker/README.md +++ b/ls-on-firecracker/README.md @@ -33,7 +33,11 @@ make test # exercise S3 + Lambda through it make down # stop the VM ``` -Run `make help` for the full target list. +Run `make help` for the full target list. `make ssh` drops you into a root +shell on the running microVM (the base image ships a pre-authorized SSH key, +fetched by `download-assets.sh` alongside the kernel/rootfs); `make diagnose` +dumps `systemctl status` and `journalctl` for the `docker`/`localstack` +services, which is what the CI workflow does automatically on failure. ## Prerequisites @@ -43,8 +47,8 @@ Run `make help` for the full target list. runner (which has KVM enabled) all work. See `.github/workflows/test-ls-on-firecracker.yml` in the repo root for a working CI setup. -- `curl`, `iproute2`, `iptables`, `e2fsprogs`, `zip`, `jq`, and the AWS CLI - (`aws`) on the host. +- `curl`, `iproute2`, `iptables`, `e2fsprogs`, `zip`, `jq`, `ssh`, and the + AWS CLI (`aws`) on the host. - `sudo` access — the scripts use it for loop-mounting the rootfs image, managing the tap device and NAT rules, and launching `firecracker` itself. - Optionally, a `LOCALSTACK_AUTH_TOKEN` environment variable on the host — diff --git a/ls-on-firecracker/scripts/download-assets.sh b/ls-on-firecracker/scripts/download-assets.sh index a8a3bff..9bd46a8 100755 --- a/ls-on-firecracker/scripts/download-assets.sh +++ b/ls-on-firecracker/scripts/download-assets.sh @@ -67,6 +67,13 @@ else | sort -V | tail -1) [[ -n "$rootfs_key" ]] || { echo "could not resolve a base rootfs for CI track $CI_TRACK/$ARCH" >&2; exit 1; } curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${rootfs_key}" -o "$IMG_DIR/base.ext4" + + # This base image ships sshd running with a pre-authorized root key; the + # matching private key is published alongside it. Handy for `make ssh` + # and for pulling diagnostics (systemctl/journalctl) when a boot fails. + curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${rootfs_key}.id_rsa" -o "$IMG_DIR/id_rsa" \ + && chmod 600 "$IMG_DIR/id_rsa" \ + || echo "[download] warning: no matching SSH key found for this rootfs, 'make ssh' won't work" >&2 fi echo "[download] done -> $BIN_DIR/firecracker, $IMG_DIR/vmlinux.bin, $IMG_DIR/base.ext4" From 7e9b71712ea89c168ca295a1a47b039d00645d5e Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 00:14:38 +0200 Subject: [PATCH 08/14] ls-on-firecracker: fix SSH key path and stop swallowing diagnose stderr The id_rsa key is a sibling of the .ext4 file (ubuntu-22.04.id_rsa), not an appended suffix (ubuntu-22.04.ext4.id_rsa) -- the previous path 404'd, so 'make diagnose' had no key and its error went nowhere because the CI step redirected stderr to /dev/null. Fixed both. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- .github/workflows/test-ls-on-firecracker.yml | 2 +- ls-on-firecracker/Makefile | 1 + ls-on-firecracker/scripts/download-assets.sh | 8 +++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-ls-on-firecracker.yml b/.github/workflows/test-ls-on-firecracker.yml index 8fff70d..c9b4939 100644 --- a/.github/workflows/test-ls-on-firecracker.yml +++ b/.github/workflows/test-ls-on-firecracker.yml @@ -49,7 +49,7 @@ jobs: - name: Dump service diagnostics on failure if: failure() - run: make diagnose 2>/dev/null || true + run: make diagnose || true - name: Tear down if: always() diff --git a/ls-on-firecracker/Makefile b/ls-on-firecracker/Makefile index 3bd7d4d..a3a7b27 100644 --- a/ls-on-firecracker/Makefile +++ b/ls-on-firecracker/Makefile @@ -17,6 +17,7 @@ LAMBDA_FN := firecracker-demo-fn export WORK_DIR TAP_DEV TAP_IP VM_IP VM_MASK VCPUS MEM_MB BUCKET LAMBDA_FN FC_VERSION CI_TRACK ARCH LOCALSTACK_AUTH_TOKEN SSH := ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ + -o BatchMode=yes -o ConnectTimeout=10 \ -i $(WORK_DIR)/images/id_rsa root@$(VM_IP) .PHONY: help download rootfs up test logs down clean ssh diagnose diff --git a/ls-on-firecracker/scripts/download-assets.sh b/ls-on-firecracker/scripts/download-assets.sh index 9bd46a8..a330eea 100755 --- a/ls-on-firecracker/scripts/download-assets.sh +++ b/ls-on-firecracker/scripts/download-assets.sh @@ -69,9 +69,11 @@ else curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${rootfs_key}" -o "$IMG_DIR/base.ext4" # This base image ships sshd running with a pre-authorized root key; the - # matching private key is published alongside it. Handy for `make ssh` - # and for pulling diagnostics (systemctl/journalctl) when a boot fails. - curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${rootfs_key}.id_rsa" -o "$IMG_DIR/id_rsa" \ + # matching private key is published as a sibling of the .ext4 file (e.g. + # ubuntu-22.04.id_rsa next to ubuntu-22.04.ext4), not an appended suffix. + # Handy for `make ssh` and pulling diagnostics when a boot fails. + rsa_key="${rootfs_key%.ext4}.id_rsa" + curl -fsSL "https://s3.amazonaws.com/spec.ccfc.min/${rsa_key}" -o "$IMG_DIR/id_rsa" \ && chmod 600 "$IMG_DIR/id_rsa" \ || echo "[download] warning: no matching SSH key found for this rootfs, 'make ssh' won't work" >&2 fi From 116c04a4565e7b16cae6f8ac8d1c33edb91738fb Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 00:27:31 +0200 Subject: [PATCH 09/14] ls-on-firecracker: switch to lstk, fix Docker iptables backend Root cause of the docker.service failure (confirmed via SSH journalctl): Ubuntu 22.04's iptables defaults to the nftables backend, but Firecracker's CI kernel does not compile in nf_tables (only legacy x_tables, which we confirmed is present). Switch both iptables and ip6tables to the legacy alternative, which the kernel does support. Also switches LocalStack's startup from a bare `pip install localstack` to lstk, LocalStack's own CLI, which pulls and runs LocalStack as a container against the guest's Docker daemon -- matching how LocalStack is officially run today, and how real Lambda's own container-inside- Firecracker-microVM architecture is shaped. localstack.service now wraps `lstk start --non-interactive` as a oneshot unit (lstk blocks until the emulator is ready, then exits; the container is the actual long-running thing). Requires a LocalStack CI Auth Token in LOCALSTACK_AUTH_TOKEN, passed into the guest via a systemd.setenv= kernel boot arg. Also documents (README) that the base rootfs is Firecracker's own CI test artifact rather than a general-purpose image, which is the root cause of most of the earlier fixes, and points at the more idiomatic docker-export-based approach and firecracker-containerd as the production-grade alternative. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/README.md | 62 +++++++++++++------- ls-on-firecracker/scripts/build-rootfs.sh | 58 +++++++++++------- ls-on-firecracker/scripts/download-assets.sh | 22 ++++++- 3 files changed, 100 insertions(+), 42 deletions(-) diff --git a/ls-on-firecracker/README.md b/ls-on-firecracker/README.md index 90c0ac1..91cb7f1 100644 --- a/ls-on-firecracker/README.md +++ b/ls-on-firecracker/README.md @@ -7,23 +7,27 @@ an S3 bucket and a real Lambda deploy + invoke. ## How it works -1. **`make download`** grabs the `firecracker` binary, a guest kernel and a - base Ubuntu rootfs from Firecracker's public CI artifacts. +1. **`make download`** grabs the `firecracker` binary, LocalStack's own + [`lstk`](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) + CLI, a guest kernel and a base Ubuntu rootfs from Firecracker's public CI + artifacts. 2. **`make rootfs`** clones the base rootfs, grows it, and chroots in to - install Docker plus `pip install localstack awscli-local`, and registers - `systemd` units so Docker and then `localstack start --host` come up on - boot. Building happens on the host via a loop-mounted image, so the - customization step itself doesn't need the guest to be running. + install Docker and drop in the `lstk` binary, then registers `systemd` + units so Docker and then `lstk start` come up on boot. Building happens on + the host via a loop-mounted image, so the customization step itself + doesn't need the guest to be running. 3. **`make up`** creates a tap network device on the host, NATs the guest out - through the host's default interface (LocalStack needs to `docker pull` - the Lambda runtime image at invoke time), and boots the image with - Firecracker. It polls `http://:4566/_localstack/health` until - LocalStack is ready. + through the host's default interface (`lstk` needs to pull the LocalStack + image, and LocalStack itself needs to pull the Lambda runtime image at + invoke time), and boots the image with Firecracker. It polls + `http://:4566/_localstack/health` until LocalStack is ready. 4. **`make test`** creates an S3 bucket and round-trips an object, then deploys a small Python Lambda function, invokes it, and asserts the response — all against the LocalStack instance running inside the - microVM. Lambda execution goes through the guest's own Docker daemon, - exactly like it would against a normal `docker run localstack` setup. + microVM. `lstk` runs LocalStack as a container against the guest's own + Docker daemon, which is also what LocalStack itself uses to spawn the + Lambda executor container — the same two-layer shape real Lambda uses + (a container runtime inside a Firecracker microVM). 5. **`make down`** / **`make clean`** tear the VM, NAT rules, and tap device down again. @@ -51,24 +55,42 @@ services, which is what the CI workflow does automatically on failure. AWS CLI (`aws`) on the host. - `sudo` access — the scripts use it for loop-mounting the rootfs image, managing the tap device and NAT rules, and launching `firecracker` itself. -- Optionally, a `LOCALSTACK_AUTH_TOKEN` environment variable on the host — - if set, it's passed through into the guest's `localstack.service` at boot. +- A `LOCALSTACK_AUTH_TOKEN` environment variable on the host, set to a + LocalStack **CI Auth Token** (not a personal Developer Auth Token — `lstk` + rejects those non-interactively). It's passed through into the guest's + `localstack.service` at boot. Get one from + [your LocalStack workspace](https://app.localstack.cloud/workspace/auth-tokens). ## What's actually running Docker runs *inside* the guest OS (installed at rootfs-build time), and -LocalStack uses it as its normal Docker-based Lambda executor. The guest -needs internet access at Lambda invoke time to pull the runtime image, which -is why `run-vm.sh` sets up NAT through the host rather than an isolated -host-only network. +`lstk` uses it to pull and run the LocalStack container, which in turn uses +the same Docker daemon as its normal Docker-based Lambda executor. The guest +needs internet access both to pull the LocalStack image and, at Lambda +invoke time, the runtime image — which is why `run-vm.sh` sets up NAT +through the host rather than an isolated host-only network. + +## Caveats + +The base rootfs comes from Firecracker's own **CI test artifacts** — it's +what their integration tests boot, not a general-purpose image, and it's +stripped down accordingly (no `/var/cache/apt`, `/var/log`, or populated +dpkg database out of the box; `build-rootfs.sh` reconstructs what apt/Docker +need). It works, but a more idiomatic base for "run a Docker image as a +Firecracker rootfs" is `docker export`-ing a real image (e.g. `ubuntu:22.04`) +onto a formatted ext4 device. For running actual container workloads inside +Firecracker in production, see +[firecracker-containerd](https://github.com/firecracker-microvm/firecracker-containerd) +(what AWS Lambda/Fargate use) instead of a full Docker-in-VM setup like this +one. ## Layout ``` Makefile self-describing entry point (make help) scripts/ - download-assets.sh fetch firecracker + kernel + base rootfs - build-rootfs.sh install Docker + LocalStack into a working rootfs image + download-assets.sh fetch firecracker + lstk + kernel + base rootfs + build-rootfs.sh install Docker + lstk into a working rootfs image run-vm.sh set up networking (incl. NAT) and boot the microVM smoke-test.sh S3 round-trip + Lambda deploy/invoke against it teardown.sh stop the VM and remove the tap device / NAT rules diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index f1849f6..414a284 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -1,12 +1,15 @@ #!/usr/bin/env bash # Turns the pristine CI rootfs into a "LocalStack appliance": grows the ext4 -# image, chroots into it, pip-installs LocalStack + awslocal, and registers a -# systemd unit that starts LocalStack on boot. +# image, chroots into it to install Docker and drop in the lstk binary, and +# registers a systemd unit that runs `lstk start` (which pulls and runs the +# LocalStack container against the guest's own Docker daemon) on boot. # # The chroot shares the host's network namespace (it's just a mounted -# directory, not a container), so apt/pip work normally as long as the host -# has internet access. The guest VM itself does NOT get internet access at -# boot -- it doesn't need it, everything is already baked into the image. +# directory, not a container), so apt-get works normally here as long as the +# host has internet access. The guest VM itself needs its own internet +# access too, at boot time this time -- lstk has to pull the LocalStack +# image and Lambda invocations pull runtime images -- which is what the NAT +# setup in run-vm.sh is for. set -euo pipefail : "${WORK_DIR:?run via 'make', not directly}" @@ -75,28 +78,40 @@ sudo chroot "$MNT" /bin/bash -c ' apt-get install -y -qq \ -o Dpkg::Options::=--force-confdef \ -o Dpkg::Options::=--force-confold \ - python3-pip python3-venv docker.io >/dev/null - # This base image is Ubuntu 22.04 (jammy), whose bundled pip predates - # PEP 668 "externally managed environment" enforcement, so it does not - # understand --break-system-packages -- and does not need it either. - pip3 install -q localstack awscli-local + docker.io >/dev/null + # The Firecracker CI kernel does not compile in nf_tables (only the + # legacy x_tables framework), but Ubuntu 22.04 iptables defaults to the + # nftables backend -- dockerd then fails at startup with "Failed to + # initialize nft: Protocol not supported". Switch to the legacy backend, + # which the kernel does support. + update-alternatives --set iptables /usr/sbin/iptables-legacy + update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy systemctl enable docker.service ' +sudo cp "$WORK_DIR/bin/lstk" "$MNT/usr/local/bin/lstk" +sudo chmod +x "$MNT/usr/local/bin/lstk" -# LocalStack uses the guest's own Docker daemon to run the Lambda executor -# container, exactly like real Lambda uses a container runtime inside its -# Firecracker microVM. It needs to start after (and depend on) Docker. +# lstk pulls the LocalStack image and starts it as a container against the +# guest's own Docker daemon -- the container is what actually runs LocalStack +# and spawns Lambda executor containers, exactly like real Lambda uses a +# container runtime inside its Firecracker microVM. `lstk start` blocks +# until the emulator is ready and then exits, so the unit that "is" this +# service is really the container, not this process -- hence oneshot + +# RemainAfterExit rather than a long-running ExecStart. sudo tee "$MNT/etc/systemd/system/localstack.service" >/dev/null <<'UNIT' [Unit] -Description=LocalStack -After=docker.service network.target +Description=LocalStack (via lstk) +After=docker.service network-online.target +Wants=network-online.target Requires=docker.service [Service] -Environment=LOCALSTACK_HOST=0.0.0.0 -ExecStart=/usr/local/bin/localstack start --host +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/bin/lstk start --non-interactive --timeout 120s +TimeoutStartSec=200 Restart=on-failure -RestartSec=2 +RestartSec=5 [Install] WantedBy=multi-user.target @@ -105,10 +120,11 @@ UNIT sudo chroot "$MNT" systemctl enable localstack.service # The chroot borrowed the host's /etc/resolv.conf (often a systemd-resolved -# stub at 127.0.0.53) to resolve apt/pip mirrors during the build above. That +# stub at 127.0.0.53) to resolve apt mirrors during the build above. That # address is meaningless once the image boots as its own VM, so pin a public -# resolver for runtime -- the guest needs it to pull the Lambda runtime image -# from ECR public over the NAT'd link `run-vm.sh` sets up. +# resolver for runtime -- the guest needs it to pull the LocalStack image +# (lstk) and the Lambda runtime image (LocalStack itself) over the NAT'd +# link run-vm.sh sets up. sudo tee "$MNT/etc/resolv.conf" >/dev/null <<'EOF' nameserver 8.8.8.8 nameserver 1.1.1.1 diff --git a/ls-on-firecracker/scripts/download-assets.sh b/ls-on-firecracker/scripts/download-assets.sh index a330eea..f743067 100755 --- a/ls-on-firecracker/scripts/download-assets.sh +++ b/ls-on-firecracker/scripts/download-assets.sh @@ -45,6 +45,26 @@ else rm -rf "$tmp" fi +# lstk is LocalStack's own CLI: it pulls the LocalStack image, starts it as a +# container against the guest's Docker daemon, and waits for it to be ready. +# It ends up baked into the guest rootfs (see build-rootfs.sh), not run on +# the host, so we resolve its release for the *guest's* architecture. +GOARCH="$(if [[ "$ARCH" == "aarch64" ]]; then echo arm64; else echo amd64; fi)" +if [[ -x "$BIN_DIR/lstk" ]]; then + echo "[download] lstk binary already present, skipping" +else + echo "[download] fetching latest lstk for linux/$GOARCH" + lstk_tag=$(curl -fsSLI -o /dev/null -w '%{url_effective}' "https://github.com/localstack/lstk/releases/latest" | sed 's#.*/##') + [[ -n "$lstk_tag" ]] || { echo "could not resolve the latest lstk release" >&2; exit 1; } + lstk_ver="${lstk_tag#v}" + tmp=$(mktemp -d) + curl -fsSL "https://github.com/localstack/lstk/releases/download/${lstk_tag}/lstk_${lstk_ver}_linux_${GOARCH}.tar.gz" \ + | tar -xz -C "$tmp" lstk + mv "$tmp/lstk" "$BIN_DIR/lstk" + chmod +x "$BIN_DIR/lstk" + rm -rf "$tmp" +fi + if [[ -f "$IMG_DIR/vmlinux.bin" ]]; then echo "[download] kernel already present, skipping" else @@ -78,4 +98,4 @@ else || echo "[download] warning: no matching SSH key found for this rootfs, 'make ssh' won't work" >&2 fi -echo "[download] done -> $BIN_DIR/firecracker, $IMG_DIR/vmlinux.bin, $IMG_DIR/base.ext4" +echo "[download] done -> $BIN_DIR/firecracker, $BIN_DIR/lstk, $IMG_DIR/vmlinux.bin, $IMG_DIR/base.ext4" From 320f42f10d0b8a365039f68b917e682c82c0b162 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 00:41:21 +0200 Subject: [PATCH 10/14] ls-on-firecracker: fail fast on broken services, add Lambda/S3 interaction - run-vm.sh: bail out as soon as docker.service or localstack.service reaches a terminal "failed" state instead of waiting out the full boot budget every time (Docker itself now starts fine after the iptables-legacy fix, but lstk was failing and getting silently retried by systemd for the full 7 minutes). - build-rootfs.sh: localstack.service now has Restart=no -- repeated auto-restarts were burying the actual lstk failure under a wall of "Failed to start" lines with no visible error. - Makefile: `make diagnose` now prints the unit's exit code/result and the full (untruncated, single-attempt) journal instead of a possibly-truncated tail. - fixtures/handler.py + smoke-test.sh: the Lambda function now creates a bucket and lists all buckets via boto3 (LocalStack auto-injects AWS_ENDPOINT_URL, no endpoint code needed), and the test asserts it sees the CLI-created bucket and that the bucket it creates is visible back on the CLI afterward -- exercising real Lambda <-> S3 API interaction through the same LocalStack backend, not just a bare invoke/response round-trip. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/Makefile | 10 ++++++--- ls-on-firecracker/README.md | 16 ++++++++------ ls-on-firecracker/fixtures/handler.py | 16 +++++++++++++- ls-on-firecracker/scripts/build-rootfs.sh | 8 +++++-- ls-on-firecracker/scripts/run-vm.sh | 26 ++++++++++++++++++++--- ls-on-firecracker/scripts/smoke-test.sh | 26 +++++++++++++++++++---- 6 files changed, 83 insertions(+), 19 deletions(-) diff --git a/ls-on-firecracker/Makefile b/ls-on-firecracker/Makefile index a3a7b27..4a14cc8 100644 --- a/ls-on-firecracker/Makefile +++ b/ls-on-firecracker/Makefile @@ -13,8 +13,9 @@ VCPUS := 2 MEM_MB := 4096 BUCKET := firecracker-demo LAMBDA_FN := firecracker-demo-fn +LAMBDA_BUCKET := firecracker-demo-lambda-bucket -export WORK_DIR TAP_DEV TAP_IP VM_IP VM_MASK VCPUS MEM_MB BUCKET LAMBDA_FN FC_VERSION CI_TRACK ARCH LOCALSTACK_AUTH_TOKEN +export WORK_DIR TAP_DEV TAP_IP VM_IP VM_MASK VCPUS MEM_MB BUCKET LAMBDA_FN LAMBDA_BUCKET FC_VERSION CI_TRACK ARCH LOCALSTACK_AUTH_TOKEN SSH := ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ -o BatchMode=yes -o ConnectTimeout=10 \ @@ -46,8 +47,11 @@ ssh: ## SSH into the running microVM as root diagnose: ## Dump docker/localstack service status + logs from inside the microVM @$(SSH) 'systemctl --no-pager status docker.service localstack.service; \ - echo ---docker.journal---; journalctl -u docker --no-pager -n 200; \ - echo ---localstack.journal---; journalctl -u localstack --no-pager -n 200' + echo ---localstack.service exit info---; \ + systemctl show localstack.service -p Result -p ExecMainStatus -p ExecMainCode; \ + echo ---docker.journal---; journalctl -u docker --no-pager -n 100; \ + echo ---localstack.journal (full unit history, Restart=no so this is one attempt)---; \ + journalctl -u localstack --no-pager' down: ## Stop the microVM and remove the tap device @bash scripts/teardown.sh diff --git a/ls-on-firecracker/README.md b/ls-on-firecracker/README.md index 91cb7f1..22c838f 100644 --- a/ls-on-firecracker/README.md +++ b/ls-on-firecracker/README.md @@ -22,12 +22,16 @@ an S3 bucket and a real Lambda deploy + invoke. invoke time), and boots the image with Firecracker. It polls `http://:4566/_localstack/health` until LocalStack is ready. 4. **`make test`** creates an S3 bucket and round-trips an object, then - deploys a small Python Lambda function, invokes it, and asserts the - response — all against the LocalStack instance running inside the - microVM. `lstk` runs LocalStack as a container against the guest's own - Docker daemon, which is also what LocalStack itself uses to spawn the - Lambda executor container — the same two-layer shape real Lambda uses - (a container runtime inside a Firecracker microVM). + deploys a small Python Lambda function and invokes it. The function + itself creates a *second* bucket and lists all buckets via `boto3` + (LocalStack injects `AWS_ENDPOINT_URL` into the Lambda execution + environment automatically, so no endpoint code is needed) — proving the + Lambda's own AWS calls land on the same LocalStack backend as the CLI + calls above: it sees the first bucket, and the one it creates is visible + back on the CLI afterward. `lstk` runs LocalStack as a container against + the guest's own Docker daemon, which is also what LocalStack itself uses + to spawn the Lambda executor container — the same two-layer shape real + Lambda uses (a container runtime inside a Firecracker microVM). 5. **`make down`** / **`make clean`** tear the VM, NAT rules, and tap device down again. diff --git a/ls-on-firecracker/fixtures/handler.py b/ls-on-firecracker/fixtures/handler.py index 7dd5ecc..183dfe8 100644 --- a/ls-on-firecracker/fixtures/handler.py +++ b/ls-on-firecracker/fixtures/handler.py @@ -1,3 +1,17 @@ +import boto3 + +# LocalStack injects AWS_ENDPOINT_URL into the Lambda execution environment +# automatically, and boto3 has respected that variable since 1.28.0 -- no +# endpoint_url override needed here, on LocalStack or on real AWS. +s3 = boto3.client("s3") + + def handler(event, context): name = event.get("name", "world") - return {"message": f"hello {name}"} + bucket = event.get("bucket") + + if bucket: + s3.create_bucket(Bucket=bucket) + + buckets = [b["Name"] for b in s3.list_buckets()["Buckets"]] + return {"message": f"hello {name}", "buckets": buckets} diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index 414a284..179444c 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -110,8 +110,12 @@ Type=oneshot RemainAfterExit=yes ExecStart=/usr/local/bin/lstk start --non-interactive --timeout 120s TimeoutStartSec=200 -Restart=on-failure -RestartSec=5 +# No auto-restart: a single clear failure (visible via `systemctl is-failed` +# and its journal) is far more useful for a demo/CI than systemd silently +# retrying a broken command every few seconds for the entire boot budget, +# burying the real error under repeated "Failed to start" lines. Re-run +# `make up` (which boots a fresh VM) to retry. +Restart=no [Install] WantedBy=multi-user.target diff --git a/ls-on-firecracker/scripts/run-vm.sh b/ls-on-firecracker/scripts/run-vm.sh index 0a6d976..5fbfccd 100755 --- a/ls-on-firecracker/scripts/run-vm.sh +++ b/ls-on-firecracker/scripts/run-vm.sh @@ -95,15 +95,35 @@ sudo setsid "$FC_BIN" --api-sock "$SOCKET" --config-file "$CONFIG" \ disown echo $! | sudo tee "$PIDFILE" >/dev/null +SSH_KEY="$WORK_DIR/images/id_rsa" +SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR + -o BatchMode=yes -o ConnectTimeout=5 -i "$SSH_KEY" "root@${VM_IP}") + +# A cold `lstk` pull of the LocalStack image can legitimately take minutes, +# so the overall budget below stays generous -- but a broken docker.service +# or localstack.service (lstk itself failing to start, no auto-restart) +# reaches a terminal "failed" state within seconds of boot, so there is no +# reason to wait out the full budget for that case. Poll for it (after a +# short boot grace period) and bail immediately once either has failed. +is_service_broken() { + [[ -f "$SSH_KEY" ]] || return 1 + local states + states=$(ssh "${SSH_OPTS[@]}" 'systemctl is-active docker.service localstack.service' 2>/dev/null || echo "") + [[ "$states" == *failed* ]] +} + echo "[run] waiting for LocalStack to become healthy at http://${VM_IP}:4566 ..." -# Generous budget: guest boot + dockerd startup + LocalStack cold start. -for _ in $(seq 1 90); do +for i in $(seq 1 90); do if curl -fsS "http://${VM_IP}:4566/_localstack/health" >/dev/null 2>&1; then echo "[run] LocalStack is up: http://${VM_IP}:4566" exit 0 fi + if (( i > 12 )) && (( i % 6 == 0 )) && is_service_broken; then + echo "[run] docker.service or localstack.service failed inside the guest; not waiting further. Run 'make diagnose' for details." >&2 + exit 1 + fi sleep 5 done -echo "[run] timed out waiting for LocalStack; check $LOG" >&2 +echo "[run] timed out waiting for LocalStack; check $LOG or 'make diagnose'" >&2 exit 1 diff --git a/ls-on-firecracker/scripts/smoke-test.sh b/ls-on-firecracker/scripts/smoke-test.sh index 018ebb6..831e4a5 100755 --- a/ls-on-firecracker/scripts/smoke-test.sh +++ b/ls-on-firecracker/scripts/smoke-test.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash # Exercises the LocalStack instance running inside the microVM: # - S3: create a bucket, put an object, read it back -# - Lambda: deploy a function, invoke it, assert the response +# - Lambda: deploy a function that itself creates a bucket and lists all +# buckets, invoke it, and assert its response -- proving the Lambda's +# own AWS SDK calls land on the same LocalStack backend as the CLI calls +# above (it sees $BUCKET, and the bucket it creates is visible back here) # Lambda execution happens via the guest's own Docker daemon, the same way # it would against a normal `docker run localstack` setup. set -euo pipefail @@ -9,6 +12,7 @@ set -euo pipefail : "${VM_IP:?run via 'make', not directly}" : "${BUCKET:?}" : "${LAMBDA_FN:?}" +: "${LAMBDA_BUCKET:?}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -56,12 +60,12 @@ if [[ "$state" != "Active" ]]; then exit 1 fi -echo "[test] invoking $LAMBDA_FN" +echo "[test] invoking $LAMBDA_FN (it will create s3://${LAMBDA_BUCKET} and list all buckets)" OUT_FILE=$(mktemp) $AWS lambda invoke \ --function-name "$LAMBDA_FN" \ --cli-binary-format raw-in-base64-out \ - --payload '{"name":"firecracker"}' \ + --payload "{\"name\":\"firecracker\",\"bucket\":\"${LAMBDA_BUCKET}\"}" \ "$OUT_FILE" >/dev/null RESPONSE=$(cat "$OUT_FILE") @@ -74,5 +78,19 @@ if [[ "$MESSAGE" != "hello firecracker" ]]; then exit 1 fi -echo "[test] Lambda OK" +# The Lambda's own boto3 S3 calls must land on the same LocalStack backend +# as the CLI calls above: it should see the bucket created by this script +# ($BUCKET) and the one it just created itself ($LAMBDA_BUCKET). +mapfile -t LAMBDA_BUCKETS < <(jq -r '.buckets[]' <<<"$RESPONSE") +for expected in "$BUCKET" "$LAMBDA_BUCKET"; do + if [[ ! " ${LAMBDA_BUCKETS[*]} " == *" $expected "* ]]; then + echo "[test] FAILED: Lambda's bucket list did not include '$expected' (got: ${LAMBDA_BUCKETS[*]})" >&2 + exit 1 + fi +done + +# And the reverse: the bucket the Lambda created should be visible back here. +$AWS s3api head-bucket --bucket "$LAMBDA_BUCKET" + +echo "[test] Lambda OK (created s3://${LAMBDA_BUCKET}, saw both buckets, visible back on the CLI)" echo "[test] all checks passed" From 8a6024380537c08613ba45f16967604e86900a40 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 00:44:54 +0200 Subject: [PATCH 11/14] ls-on-firecracker: fix syntax error in make diagnose (unquoted parens) The previous run's diagnose step itself failed: an echo string with unquoted parentheses broke the remote bash command before it could print the actual lstk failure we need to see. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ls-on-firecracker/Makefile b/ls-on-firecracker/Makefile index 4a14cc8..6e8b80d 100644 --- a/ls-on-firecracker/Makefile +++ b/ls-on-firecracker/Makefile @@ -50,7 +50,7 @@ diagnose: ## Dump docker/localstack service status + logs from inside the microV echo ---localstack.service exit info---; \ systemctl show localstack.service -p Result -p ExecMainStatus -p ExecMainCode; \ echo ---docker.journal---; journalctl -u docker --no-pager -n 100; \ - echo ---localstack.journal (full unit history, Restart=no so this is one attempt)---; \ + echo "---localstack.journal (single attempt, Restart=no)---"; \ journalctl -u localstack --no-pager' down: ## Stop the microVM and remove the tap device From 13f132d1bbbd349a56d8ecf119eb957c92a6e653 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 00:48:28 +0200 Subject: [PATCH 12/14] ls-on-firecracker: set HOME for lstk under systemd Root cause finally visible thanks to the fast-fail + Restart=no fixes: "Error: failed to get user home directory: \$HOME is not defined". systemd services don't inherit HOME the way login shells do, but lstk needs it to resolve its config/cache directory. The unit runs as root, so point HOME at /root. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/scripts/build-rootfs.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index 179444c..a67538d 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -108,6 +108,10 @@ Requires=docker.service [Service] Type=oneshot RemainAfterExit=yes +# systemd services don't get $HOME the way login shells do, but lstk needs +# it to resolve its config/cache directory (~/.cache/lstk/...); this unit +# runs as root (no User= override), so point it at root's home. +Environment=HOME=/root ExecStart=/usr/local/bin/lstk start --non-interactive --timeout 120s TimeoutStartSec=200 # No auto-restart: a single clear failure (visible via `systemctl is-failed` From 8a91b9fa4786611a8d2c02e6a657dcd3f307ea97 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 00:53:47 +0200 Subject: [PATCH 13/14] ls-on-firecracker: opt out of Docker's iptables raw-table requirement Progress: the HOME fix worked -- lstk pulled the image, validated the auth token, and tried to start the container. New failure: "Unable to enable DIRECT ACCESS FILTERING - DROP rule ... iptables raw table: Table does not exist". Docker 28+ added this hardening rule, which needs CONFIG_IP_NF_RAW; our kernel has neither that config nor module loading at all, so it can never be satisfied. Docker 28.0.2 added DOCKER_INSECURE_NO_IPTABLES_RAW=1 as the documented opt-out for exactly this case (moby/moby#49621) -- set via a docker.service systemd drop-in. Documented the security tradeoff in the README. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/README.md | 10 ++++++++++ ls-on-firecracker/scripts/build-rootfs.sh | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/ls-on-firecracker/README.md b/ls-on-firecracker/README.md index 22c838f..4785ba3 100644 --- a/ls-on-firecracker/README.md +++ b/ls-on-firecracker/README.md @@ -88,6 +88,16 @@ Firecracker in production, see (what AWS Lambda/Fargate use) instead of a full Docker-in-VM setup like this one. +Relatedly: the kernel doesn't have `CONFIG_IP_NF_RAW` (and can't load it — +module loading is compiled out entirely), which Docker 28+ needs for a +hardening rule that stops a container from being reached directly, +bypassing its published-port restriction. `build-rootfs.sh` sets +`DOCKER_INSECURE_NO_IPTABLES_RAW=1` (Docker's documented opt-out for exactly +this case) to work around it. The tradeoff — a container published to +`127.0.0.1` becomes reachable from other hosts on the same network — is +acceptable for this single-tenant, ephemeral microVM reachable only over its +own host-only tap network, but wouldn't be on a shared or long-lived host. + ## Layout ``` diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index a67538d..4c6705c 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -88,6 +88,23 @@ sudo chroot "$MNT" /bin/bash -c ' update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy systemctl enable docker.service ' + +# Docker 28+ adds a DROP rule in the iptables "raw" table as a hardening +# measure (prevents reaching a container directly, bypassing its published +# port restriction). That table needs CONFIG_IP_NF_RAW, which this kernel +# does not have -- and can't load at runtime either, since module loading +# is compiled out entirely. DOCKER_INSECURE_NO_IPTABLES_RAW=1 (Docker +# 28.0.2+) is the documented opt-out for exactly this case. The tradeoff +# (a container published to 127.0.0.1 becomes reachable from other hosts +# on the same network) is acceptable here: this microVM is single-tenant, +# ephemeral, and only reachable via the host-only tap network run-vm.sh +# sets up, not by "other hosts on the local network". +sudo mkdir -p "$MNT/etc/systemd/system/docker.service.d" +sudo tee "$MNT/etc/systemd/system/docker.service.d/no-iptables-raw.conf" >/dev/null <<'UNIT' +[Service] +Environment=DOCKER_INSECURE_NO_IPTABLES_RAW=1 +UNIT + sudo cp "$WORK_DIR/bin/lstk" "$MNT/usr/local/bin/lstk" sudo chmod +x "$MNT/usr/local/bin/lstk" From 30661e52f84648a2181066726959957fb6f8308a Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 12 Sep 2026 01:06:17 +0200 Subject: [PATCH 14/14] ls-on-firecracker: fix lstk publishing to loopback only, split docs The previous run showed real progress: lstk pulled the image, the auth token validated, and the container started successfully -- but our host-side health check never got through. Checked lstk's own source (internal/container/gateway.go, start.go): it publishes the emulator's port bound to the guest's 127.0.0.1 by default, and the bind host only comes from a GATEWAY_LISTEN entry in a config.toml [env.*] profile -- a bare LOCALSTACK_GATEWAY_LISTEN process env var reaches the container too late, after the host-side Docker port binding is already decided. Added a config.toml at /root/.config/lstk/ (matches lstk's config search order once HOME is set) with GATEWAY_LISTEN=0.0.0.0:4566,0.0.0.0:443, so the port is reachable from outside the guest over the tap network. Also: - run-vm.sh: on a health-check timeout, SSH in and dump the actual listening sockets, docker port mappings, and an in-guest curl, so a future networking mismatch is visible immediately instead of requiring another round of guessing. - Split the README: kept it to quick start + prerequisites + layout, moved the "how it works" walkthrough and caveats (stripped-down CI rootfs, the iptables-legacy and DOCKER_INSECURE_NO_IPTABLES_RAW workarounds) into docs/NOTES.md. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Ny78ki4drCoyQgDxkSoSdC --- ls-on-firecracker/README.md | 100 ++++------------------ ls-on-firecracker/docs/NOTES.md | 66 ++++++++++++++ ls-on-firecracker/scripts/build-rootfs.sh | 19 ++++ ls-on-firecracker/scripts/run-vm.sh | 13 +++ 4 files changed, 115 insertions(+), 83 deletions(-) create mode 100644 ls-on-firecracker/docs/NOTES.md diff --git a/ls-on-firecracker/README.md b/ls-on-firecracker/README.md index 4785ba3..80db906 100644 --- a/ls-on-firecracker/README.md +++ b/ls-on-firecracker/README.md @@ -3,101 +3,31 @@ A demo that boots [LocalStack](https://localstack.cloud) inside a [Firecracker](https://firecracker-microvm.github.io/) microVM — the same technology AWS Lambda itself runs on — and exercises it over the network with -an S3 bucket and a real Lambda deploy + invoke. +an S3 bucket and a Lambda function that itself talks back to S3. -## How it works - -1. **`make download`** grabs the `firecracker` binary, LocalStack's own - [`lstk`](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) - CLI, a guest kernel and a base Ubuntu rootfs from Firecracker's public CI - artifacts. -2. **`make rootfs`** clones the base rootfs, grows it, and chroots in to - install Docker and drop in the `lstk` binary, then registers `systemd` - units so Docker and then `lstk start` come up on boot. Building happens on - the host via a loop-mounted image, so the customization step itself - doesn't need the guest to be running. -3. **`make up`** creates a tap network device on the host, NATs the guest out - through the host's default interface (`lstk` needs to pull the LocalStack - image, and LocalStack itself needs to pull the Lambda runtime image at - invoke time), and boots the image with Firecracker. It polls - `http://:4566/_localstack/health` until LocalStack is ready. -4. **`make test`** creates an S3 bucket and round-trips an object, then - deploys a small Python Lambda function and invokes it. The function - itself creates a *second* bucket and lists all buckets via `boto3` - (LocalStack injects `AWS_ENDPOINT_URL` into the Lambda execution - environment automatically, so no endpoint code is needed) — proving the - Lambda's own AWS calls land on the same LocalStack backend as the CLI - calls above: it sees the first bucket, and the one it creates is visible - back on the CLI afterward. `lstk` runs LocalStack as a container against - the guest's own Docker daemon, which is also what LocalStack itself uses - to spawn the Lambda executor container — the same two-layer shape real - Lambda uses (a container runtime inside a Firecracker microVM). -5. **`make down`** / **`make clean`** tear the VM, NAT rules, and tap device - down again. +## Quick start ``` -make up # download -> rootfs -> boot the microVM -make test # exercise S3 + Lambda through it +make up # download -> build a LocalStack-flavored rootfs -> boot the microVM +make test # S3 round-trip + Lambda deploy/invoke through it make down # stop the VM ``` -Run `make help` for the full target list. `make ssh` drops you into a root -shell on the running microVM (the base image ships a pre-authorized SSH key, -fetched by `download-assets.sh` alongside the kernel/rootfs); `make diagnose` -dumps `systemctl status` and `journalctl` for the `docker`/`localstack` -services, which is what the CI workflow does automatically on failure. +Run `make help` for the full target list, including `make ssh` and +`make diagnose` for poking around inside the running microVM. ## Prerequisites -- A **Linux host with KVM** (`/dev/kvm` present and accessible) — Firecracker - does not run on macOS or in most cloud VMs without nested virtualization. - Bare metal, an EC2 `.metal` instance, or a GitHub Actions `ubuntu-latest` - runner (which has KVM enabled) all work. See - `.github/workflows/test-ls-on-firecracker.yml` in the repo root for a - working CI setup. +- A **Linux host with KVM** (`/dev/kvm`) — doesn't run on macOS or most cloud + VMs without nested virtualization. Bare metal, an EC2 `.metal` instance, or + a GitHub Actions `ubuntu-latest` runner all work; see + `.github/workflows/test-ls-on-firecracker.yml` for a working CI setup. - `curl`, `iproute2`, `iptables`, `e2fsprogs`, `zip`, `jq`, `ssh`, and the - AWS CLI (`aws`) on the host. -- `sudo` access — the scripts use it for loop-mounting the rootfs image, - managing the tap device and NAT rules, and launching `firecracker` itself. -- A `LOCALSTACK_AUTH_TOKEN` environment variable on the host, set to a - LocalStack **CI Auth Token** (not a personal Developer Auth Token — `lstk` - rejects those non-interactively). It's passed through into the guest's - `localstack.service` at boot. Get one from + AWS CLI on the host, plus `sudo` access. +- A `LOCALSTACK_AUTH_TOKEN` environment variable set to a LocalStack **CI + Auth Token** — get one from [your LocalStack workspace](https://app.localstack.cloud/workspace/auth-tokens). -## What's actually running - -Docker runs *inside* the guest OS (installed at rootfs-build time), and -`lstk` uses it to pull and run the LocalStack container, which in turn uses -the same Docker daemon as its normal Docker-based Lambda executor. The guest -needs internet access both to pull the LocalStack image and, at Lambda -invoke time, the runtime image — which is why `run-vm.sh` sets up NAT -through the host rather than an isolated host-only network. - -## Caveats - -The base rootfs comes from Firecracker's own **CI test artifacts** — it's -what their integration tests boot, not a general-purpose image, and it's -stripped down accordingly (no `/var/cache/apt`, `/var/log`, or populated -dpkg database out of the box; `build-rootfs.sh` reconstructs what apt/Docker -need). It works, but a more idiomatic base for "run a Docker image as a -Firecracker rootfs" is `docker export`-ing a real image (e.g. `ubuntu:22.04`) -onto a formatted ext4 device. For running actual container workloads inside -Firecracker in production, see -[firecracker-containerd](https://github.com/firecracker-microvm/firecracker-containerd) -(what AWS Lambda/Fargate use) instead of a full Docker-in-VM setup like this -one. - -Relatedly: the kernel doesn't have `CONFIG_IP_NF_RAW` (and can't load it — -module loading is compiled out entirely), which Docker 28+ needs for a -hardening rule that stops a container from being reached directly, -bypassing its published-port restriction. `build-rootfs.sh` sets -`DOCKER_INSECURE_NO_IPTABLES_RAW=1` (Docker's documented opt-out for exactly -this case) to work around it. The tradeoff — a container published to -`127.0.0.1` becomes reachable from other hosts on the same network — is -acceptable for this single-tenant, ephemeral microVM reachable only over its -own host-only tap network, but wouldn't be on a shared or long-lived host. - ## Layout ``` @@ -110,5 +40,9 @@ scripts/ teardown.sh stop the VM and remove the tap device / NAT rules fixtures/ handler.py the demo Lambda function +docs/ + NOTES.md how it works under the hood, and known caveats work/ downloaded/generated artifacts (git-ignored) ``` + +See [docs/NOTES.md](docs/NOTES.md) for the details. diff --git a/ls-on-firecracker/docs/NOTES.md b/ls-on-firecracker/docs/NOTES.md new file mode 100644 index 0000000..66b3510 --- /dev/null +++ b/ls-on-firecracker/docs/NOTES.md @@ -0,0 +1,66 @@ +# How it works + +1. **`make download`** grabs the `firecracker` binary, LocalStack's own + [`lstk`](https://docs.localstack.cloud/aws/developer-tools/running-localstack/lstk/) + CLI, a guest kernel and a base Ubuntu rootfs from Firecracker's public CI + artifacts. +2. **`make rootfs`** clones the base rootfs, grows it, and chroots in to + install Docker and drop in the `lstk` binary, then registers `systemd` + units so Docker and then `lstk start` come up on boot. Building happens on + the host via a loop-mounted image, so the customization step itself + doesn't need the guest to be running. +3. **`make up`** creates a tap network device on the host, NATs the guest out + through the host's default interface (`lstk` needs to pull the LocalStack + image, and LocalStack itself needs to pull the Lambda runtime image at + invoke time), and boots the image with Firecracker. It polls + `http://:4566/_localstack/health` until LocalStack is ready. +4. **`make test`** creates an S3 bucket and round-trips an object, then + deploys a small Python Lambda function and invokes it. The function + itself creates a *second* bucket and lists all buckets via `boto3` + (LocalStack injects `AWS_ENDPOINT_URL` into the Lambda execution + environment automatically, so no endpoint code is needed) — proving the + Lambda's own AWS calls land on the same LocalStack backend as the CLI + calls above: it sees the first bucket, and the one it creates is visible + back on the CLI afterward. `lstk` runs LocalStack as a container against + the guest's own Docker daemon, which is also what LocalStack itself uses + to spawn the Lambda executor container — the same two-layer shape real + Lambda uses (a container runtime inside a Firecracker microVM). +5. **`make down`** / **`make clean`** tear the VM, NAT rules, and tap device + down again. + +`make ssh` drops you into a root shell on the running microVM (the base +image ships a pre-authorized SSH key, fetched by `download-assets.sh` +alongside the kernel/rootfs). `make diagnose` dumps `systemctl status` and +`journalctl` for the `docker`/`localstack` services — the same thing the CI +workflow does automatically on failure. + +## What's actually running + +Docker runs *inside* the guest OS (installed at rootfs-build time), and +`lstk` uses it to pull and run the LocalStack container, which in turn uses +the same Docker daemon as its normal Docker-based Lambda executor. The guest +needs internet access both to pull the LocalStack image and, at Lambda +invoke time, the runtime image — which is why `run-vm.sh` sets up NAT +through the host rather than an isolated host-only network. + +## Caveats + +The base rootfs comes from Firecracker's own **CI test artifacts** — what +their integration tests boot, not a general-purpose image. It's stripped +down accordingly (no `/var/cache/apt`, `/var/log`, or populated dpkg +database out of the box; `build-rootfs.sh` reconstructs what apt/Docker +need), and its kernel is minimal too (no loadable modules, no `nf_tables`, +no `CONFIG_IP_NF_RAW`) — a few of the fixes in `build-rootfs.sh` exist +specifically to work around that (switching Docker to the legacy iptables +backend, and opting out of a Docker 28+ hardening rule that needs a table +this kernel doesn't have via `DOCKER_INSECURE_NO_IPTABLES_RAW=1`, which is +fine for a single-tenant, ephemeral microVM but not something to carry into +a shared or long-lived host). + +A more idiomatic base for "run a Docker image as a Firecracker rootfs" would +be `docker export`-ing a real image (e.g. `ubuntu:22.04`) onto a formatted +ext4 device rather than patching Firecracker's CI artifact. For running +actual container workloads inside Firecracker in production, see +[firecracker-containerd](https://github.com/firecracker-microvm/firecracker-containerd) +(what AWS Lambda/Fargate use) instead of a full Docker-in-VM setup like this +one. diff --git a/ls-on-firecracker/scripts/build-rootfs.sh b/ls-on-firecracker/scripts/build-rootfs.sh index 4c6705c..f9ebe3b 100755 --- a/ls-on-firecracker/scripts/build-rootfs.sh +++ b/ls-on-firecracker/scripts/build-rootfs.sh @@ -108,6 +108,25 @@ UNIT sudo cp "$WORK_DIR/bin/lstk" "$MNT/usr/local/bin/lstk" sudo chmod +x "$MNT/usr/local/bin/lstk" +# lstk publishes the emulator's port bound to the *guest's* loopback by +# default (127.0.0.1) -- fine when lstk and its caller are on the same host, +# but we reach the emulator from outside the guest, over the tap network. +# The bind host comes from the first entry of GATEWAY_LISTEN, which lstk only +# reads from a config.toml [env.*] profile (a plain LOCALSTACK_GATEWAY_LISTEN +# process env var reaches the container too late, after the host-side Docker +# port binding is already decided). $HOME is /root (set above), so this is +# the second entry in lstk's config search order. +sudo mkdir -p "$MNT/root/.config/lstk" +sudo tee "$MNT/root/.config/lstk/config.toml" >/dev/null <<'TOML' +[[containers]] +type = "aws" +port = "4566" +env = ["net"] + +[env.net] +GATEWAY_LISTEN = "0.0.0.0:4566,0.0.0.0:443" +TOML + # lstk pulls the LocalStack image and starts it as a container against the # guest's own Docker daemon -- the container is what actually runs LocalStack # and spawns Lambda executor containers, exactly like real Lambda uses a diff --git a/ls-on-firecracker/scripts/run-vm.sh b/ls-on-firecracker/scripts/run-vm.sh index 5fbfccd..8a8e455 100755 --- a/ls-on-firecracker/scripts/run-vm.sh +++ b/ls-on-firecracker/scripts/run-vm.sh @@ -126,4 +126,17 @@ for i in $(seq 1 90); do done echo "[run] timed out waiting for LocalStack; check $LOG or 'make diagnose'" >&2 +# One more data point while the VM is still up: is the container actually +# listening, and is it reachable from *inside* the guest (loopback) even +# though it is not reachable from us (over the tap network)? That tells us +# whether this is a bind-address issue or a NAT/forwarding issue. +if [[ -f "$SSH_KEY" ]]; then + echo "[run] --- port binding + in-guest reachability ---" >&2 + ssh "${SSH_OPTS[@]}" ' + echo "listening sockets on 4566:"; ss -tlnp | grep -E ":(4566)\b" || echo "(none)"; + echo "docker port mappings:"; docker ps --format "{{.Names}}: {{.Ports}}" 2>/dev/null || echo "(docker ps failed)"; + echo "curl from inside the guest:"; + curl -fsS -m 5 http://localhost:4566/_localstack/health && echo || echo "(failed)" + ' >&2 2>&1 || echo "[run] (SSH diagnostic itself failed)" >&2 +fi exit 1