From cd25338cd3ea0402b68cf5f6f4ddc77b1f179a4f Mon Sep 17 00:00:00 2001 From: Nikita Kalyazin Date: Fri, 12 Jun 2026 10:33:36 +0100 Subject: [PATCH] feat: publish a debug-only firecracker binary (gdb-enabled) For each FC version, also build and publish a debug variant alongside the prod binary: a release (optimized) build with debug symbols kept and the upstream gdb feature enabled (`--features gdb`), plus its split DWARF companion. Used only for debugging guest kernels on dev nodes. Published as `firecracker-debug` (+ `firecracker-debug.debug`) next to `firecracker`. Client nodes resolve the FC binary at exactly `//firecracker`, so the differently-named debug artifacts are never pulled by prod; the dev debugging workflow fetches them explicitly, matched to a snapshot's firecracker version. - build.sh: second build with `--features gdb`, emit firecracker-debug + repoint the debuglink to the renamed DWARF companion. - release.yml: build skip-check requires both prod and debug assets; publish attaches firecracker-debug-[.debug]; deploy uploads them to each env's GCS bucket at the firecracker-debug[.debug] name. - validate.py: the has_new_artifacts gate (which decides whether publish downloads and uploads the freshly built artifacts) now requires both the prod and the debug binary, so a release that already has the prod binary but not the debug one still gets published. - upload-release-to-gcs.sh: also uploads the debug artifacts to the parallel path. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Nikita Kalyazin --- .github/workflows/release.yml | 62 ++++++++++++++++++++++++-------- build.sh | 29 +++++++++++++-- scripts/test_validate.py | 26 ++++++++++---- scripts/upload-release-to-gcs.sh | 9 +++++ scripts/validate.py | 23 ++++++++---- 5 files changed, 120 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb2745b9e..cffe559c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -84,13 +84,15 @@ jobs: run: | version_name="${{ needs.validate.outputs.version_name }}" arch="${{ matrix.arch }}" - asset_name="firecracker-${arch}" - if gh release view "$version_name" --json assets -q ".assets[].name" 2>/dev/null | grep -q "^${asset_name}$"; then - echo "Release: $arch artifact already exists, skipping build" + # Skip only if both the prod and the debug binaries already exist. + assets=$(gh release view "$version_name" --json assets -q ".assets[].name" 2>/dev/null || true) + if echo "$assets" | grep -q "^firecracker-${arch}$" \ + && echo "$assets" | grep -q "^firecracker-debug-${arch}$"; then + echo "Release: $arch artifacts (prod + debug) already exist, skipping build" echo "skip=true" >> $GITHUB_OUTPUT else - echo "Release: $arch artifact missing, will build" + echo "Release: $arch artifacts missing, will build" echo "skip=false" >> $GITHUB_OUTPUT fi @@ -201,6 +203,23 @@ jobs: rm -f "$tmp_file" fi fi + + # Debug-only artifacts: the gdb-enabled FC binary and its DWARF + # companion. Uploaded as firecracker-debug-[.debug]. + for dbg in firecracker-debug firecracker-debug.debug; do + dbg_path="./builds/$version_name/$arch/$dbg" + [[ -f "$dbg_path" ]] || continue + dbg_asset="${dbg/firecracker-debug/firecracker-debug-${arch}}" + if echo "$existing_assets" | grep -q "^${dbg_asset}$"; then + echo "Release: $dbg_asset already exists, skipping" + else + echo "Release: Uploading $dbg_asset..." + tmp_file=$(mktemp -d)/${dbg_asset} + cp "$dbg_path" "$tmp_file" + gh release upload "$version_name" "$tmp_file" + rm -f "$tmp_file" + fi + done done echo "Release URL: https://github.com/${{ github.repository }}/releases/tag/$version_name" @@ -229,12 +248,20 @@ jobs: version_name="${{ needs.validate.outputs.version_name }}" for arch in amd64 arm64; do - asset_name="firecracker-${arch}" mkdir -p ./builds/$version_name/$arch gh release download "$version_name" \ --repo "${{ github.repository }}" \ - --pattern "$asset_name" \ + --pattern "firecracker-${arch}" \ --output "./builds/$version_name/$arch/firecracker" 2>/dev/null || true + # Debug-only artifacts (gdb binary + DWARF companion). + gh release download "$version_name" \ + --repo "${{ github.repository }}" \ + --pattern "firecracker-debug-${arch}" \ + --output "./builds/$version_name/$arch/firecracker-debug" 2>/dev/null || true + gh release download "$version_name" \ + --repo "${{ github.repository }}" \ + --pattern "firecracker-debug-${arch}.debug" \ + --output "./builds/$version_name/$arch/firecracker-debug.debug" 2>/dev/null || true done - name: Upload to GCS @@ -244,15 +271,20 @@ jobs: version_name="${{ needs.validate.outputs.version_name }}" for arch in amd64 arm64; do - local_path="./builds/$version_name/$arch/firecracker" - [[ -f "$local_path" ]] || continue + # Upload the prod binary and the debug-only artifacts. The debug + # binary/symbols live at a different name ("firecracker-debug"), so + # client nodes — which resolve exactly "firecracker" — never pull them. + for name in firecracker firecracker-debug firecracker-debug.debug; do + local_path="./builds/$version_name/$arch/$name" + [[ -f "$local_path" ]] || continue - gcs_path="gs://${GCP_BUCKET_NAME}/firecrackers/${version_name}/${arch}/firecracker" + gcs_path="gs://${GCP_BUCKET_NAME}/firecrackers/${version_name}/${arch}/${name}" - if gcloud storage ls "$gcs_path" >/dev/null 2>&1; then - echo "GCS (${{ matrix.environment }}): $arch already exists, skipping" - else - echo "GCS (${{ matrix.environment }}): Uploading $arch..." - gcloud storage cp "$local_path" "$gcs_path" - fi + if gcloud storage ls "$gcs_path" >/dev/null 2>&1; then + echo "GCS (${{ matrix.environment }}): $name $arch already exists, skipping" + else + echo "GCS (${{ matrix.environment }}): Uploading $name $arch..." + gcloud storage cp "$local_path" "$gcs_path" + fi + done done diff --git a/build.sh b/build.sh index d59bbc7f4..5962d0bb2 100755 --- a/build.sh +++ b/build.sh @@ -32,12 +32,37 @@ git clone "https://x-access-token:${FIRECRACKER_REPO_TOKEN}@${FIRECRACKER_REPO_H cd firecracker git checkout "$commit_hash" +out_dir="../builds/${version_name}/${arch}" +mkdir -p "$out_dir" +release_bin="build/cargo_target/${rust_target}/release/firecracker" + echo "Building Firecracker $version_name for $arch ($rust_target)..." tools/devtool -y build --release -- --bin firecracker # Output goes into {version_name}/{arch}/firecracker -mkdir -p "../builds/${version_name}/${arch}" -cp "build/cargo_target/${rust_target}/release/firecracker" "../builds/${version_name}/${arch}/firecracker" +cp "$release_bin" "$out_dir/firecracker" + +# Also build a debug variant: same release (optimized) build with the gdb +# feature enabled and debug symbols kept. Used ONLY for debugging guest kernels +# on dev nodes; it is never deployed to prod client nodes, which resolve the FC +# binary at exactly "//firecracker" (a different name). +# +# devtool's release build strips and splits debug info into .debug, so we +# publish the binary plus its companion. The debuglink is repointed to the +# renamed companion so gdb auto-loads it when the two are colocated. +echo "Building debug Firecracker $version_name for $arch ($rust_target)..." +tools/devtool -y build --release -- --bin firecracker --features gdb +cp "$release_bin" "$out_dir/firecracker-debug" +if [[ -f "${release_bin}.debug" ]]; then + cp "${release_bin}.debug" "$out_dir/firecracker-debug.debug" + objcopy --remove-section .gnu_debuglink "$out_dir/firecracker-debug" 2>/dev/null || true + ( cd "$out_dir" && objcopy --add-gnu-debuglink=firecracker-debug.debug firecracker-debug ) +else + echo "Warning: ${release_bin}.debug not found; firecracker-debug ships without split DWARF" >&2 + # Drop any stale debuglink (the build points it at "firecracker.debug", which we + # do not ship next to firecracker-debug) so gdb doesn't chase a missing file. + objcopy --remove-section .gnu_debuglink "$out_dir/firecracker-debug" 2>/dev/null || true +fi cd .. rm -rf firecracker diff --git a/scripts/test_validate.py b/scripts/test_validate.py index 7890b37df..0b8c68bdd 100644 --- a/scripts/test_validate.py +++ b/scripts/test_validate.py @@ -499,23 +499,37 @@ def test_both_requested_arm64_missing(self): assert check_artifacts_needed("v1.0.0_abc1234", True, True) is True def test_both_requested_both_exist(self): - """Test returns False when both requested and both exist.""" - with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64", "firecracker-arm64"}): + """Test returns False when both requested and both prod + debug exist.""" + assets = { + "firecracker-amd64", "firecracker-arm64", + "firecracker-debug-amd64", "firecracker-debug-arm64", + } + with patch("validate.get_existing_release_assets", return_value=assets): assert check_artifacts_needed("v1.0.0_abc1234", True, True) is False + def test_both_requested_prod_exists_debug_missing(self): + """Test returns True when prod binaries exist but the debug ones do not.""" + with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64", "firecracker-arm64"}): + assert check_artifacts_needed("v1.0.0_abc1234", True, True) is True + def test_amd64_only_exists(self): - """Test returns False when only amd64 requested and it exists.""" - with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64"}): + """Test returns False when only amd64 requested and its prod + debug exist.""" + with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64", "firecracker-debug-amd64"}): assert check_artifacts_needed("v1.0.0_abc1234", True, False) is False + def test_amd64_only_prod_exists_debug_missing(self): + """Test returns True when amd64 prod exists but its debug binary does not.""" + with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64"}): + assert check_artifacts_needed("v1.0.0_abc1234", True, False) is True + def test_amd64_only_missing(self): """Test returns True when only amd64 requested and it's missing.""" with patch("validate.get_existing_release_assets", return_value=set()): assert check_artifacts_needed("v1.0.0_abc1234", True, False) is True def test_arm64_only_exists(self): - """Test returns False when only arm64 requested and it exists.""" - with patch("validate.get_existing_release_assets", return_value={"firecracker-arm64"}): + """Test returns False when only arm64 requested and its prod + debug exist.""" + with patch("validate.get_existing_release_assets", return_value={"firecracker-arm64", "firecracker-debug-arm64"}): assert check_artifacts_needed("v1.0.0_abc1234", False, True) is False def test_arm64_only_missing(self): diff --git a/scripts/upload-release-to-gcs.sh b/scripts/upload-release-to-gcs.sh index cf7dd91ac..d75fc60da 100755 --- a/scripts/upload-release-to-gcs.sh +++ b/scripts/upload-release-to-gcs.sh @@ -77,6 +77,15 @@ for asset in "${ASSETS[@]}"; do if [[ "$asset" =~ ^firecracker-(amd64|arm64)$ ]]; then arch="${BASH_REMATCH[1]}" dst="${BUCKET_URI}/${TAG}/${arch}/firecracker" + elif [[ "$asset" =~ ^firecracker-debug-(amd64|arm64)\.debug$ ]]; then + # Debug-symbols companion (DWARF) for the debug FC binary. Debug-only. + arch="${BASH_REMATCH[1]}" + dst="${BUCKET_URI}/${TAG}/${arch}/firecracker-debug.debug" + elif [[ "$asset" =~ ^firecracker-debug-(amd64|arm64)$ ]]; then + # gdb-enabled debug FC binary. Never the prod path ("firecracker"); fetched + # explicitly by the dev-node debugging workflow. + arch="${BASH_REMATCH[1]}" + dst="${BUCKET_URI}/${TAG}/${arch}/firecracker-debug" else continue fi diff --git a/scripts/validate.py b/scripts/validate.py index 38f4d6cd0..965efa9e5 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -312,16 +312,27 @@ def get_existing_release_assets(version_name: str) -> set[str]: def check_artifacts_needed(version_name: str, build_amd64: bool, build_arm64: bool) -> bool: """ - Check if any requested architectures are missing from the release. + Check if any requested architectures are missing an artifact from the release. - Returns True if at least one artifact needs to be built and uploaded. + Returns True if at least one artifact needs to be built and uploaded. Mirrors + the build job's skip-check: a release needs both the prod binary + (firecracker-) and the gdb-enabled debug binary (firecracker-debug-), + so a release that has the prod binary but not the debug one still has new + artifacts to publish. """ existing_assets = get_existing_release_assets(version_name) - if build_amd64 and "firecracker-amd64" not in existing_assets: - return True - if build_arm64 and "firecracker-arm64" not in existing_assets: - return True + archs = [] + if build_amd64: + archs.append("amd64") + if build_arm64: + archs.append("arm64") + + for arch in archs: + if f"firecracker-{arch}" not in existing_assets: + return True + if f"firecracker-debug-{arch}" not in existing_assets: + return True return False