feat: publish a debug-only firecracker binary (gdb-enabled)

.github/workflows/release.yml

@@ -84,13 +84,15 @@ jobs:
         run: |
           version_name="${{ needs.validate.outputs.version_name }}"
           arch="${{ matrix.arch }}"
-          asset_name="firecracker-${arch}"
 
-          if gh release view "$version_name" --json assets -q ".assets[].name" 2>/dev/null | grep -q "^${asset_name}$"; then
-            echo "Release: $arch artifact already exists, skipping build"
+          # Skip only if both the prod and the debug binaries already exist.
+          assets=$(gh release view "$version_name" --json assets -q ".assets[].name" 2>/dev/null || true)
+          if echo "$assets" | grep -q "^firecracker-${arch}$" \
+             && echo "$assets" | grep -q "^firecracker-debug-${arch}$"; then
+            echo "Release: $arch artifacts (prod + debug) already exist, skipping build"
             echo "skip=true" >> $GITHUB_OUTPUT
           else
-            echo "Release: $arch artifact missing, will build"
+            echo "Release: $arch artifacts missing, will build"
             echo "skip=false" >> $GITHUB_OUTPUT
           fi
 
@@ -201,6 +203,23 @@ jobs:
                 rm -f "$tmp_file"
               fi
             fi
+
+            # Debug-only artifacts: the gdb-enabled FC binary and its DWARF
+            # companion. Uploaded as firecracker-debug-<arch>[.debug].
+            for dbg in firecracker-debug firecracker-debug.debug; do
+              dbg_path="./builds/$version_name/$arch/$dbg"
+              [[ -f "$dbg_path" ]] || continue
+              dbg_asset="${dbg/firecracker-debug/firecracker-debug-${arch}}"
+              if echo "$existing_assets" | grep -q "^${dbg_asset}$"; then
+                echo "Release: $dbg_asset already exists, skipping"
+              else
+                echo "Release: Uploading $dbg_asset..."
+                tmp_file=$(mktemp -d)/${dbg_asset}
+                cp "$dbg_path" "$tmp_file"
+                gh release upload "$version_name" "$tmp_file"
+                rm -f "$tmp_file"
+              fi
+            done
           done
 
           echo "Release URL: https://github.com/${{ github.repository }}/releases/tag/$version_name"
@@ -229,12 +248,20 @@ jobs:
           version_name="${{ needs.validate.outputs.version_name }}"
 
           for arch in amd64 arm64; do
-            asset_name="firecracker-${arch}"
             mkdir -p ./builds/$version_name/$arch
             gh release download "$version_name" \
               --repo "${{ github.repository }}" \
-              --pattern "$asset_name" \
+              --pattern "firecracker-${arch}" \
               --output "./builds/$version_name/$arch/firecracker" 2>/dev/null || true
+            # Debug-only artifacts (gdb binary + DWARF companion).
+            gh release download "$version_name" \
+              --repo "${{ github.repository }}" \
+              --pattern "firecracker-debug-${arch}" \
+              --output "./builds/$version_name/$arch/firecracker-debug" 2>/dev/null || true
+            gh release download "$version_name" \
+              --repo "${{ github.repository }}" \
+              --pattern "firecracker-debug-${arch}.debug" \
+              --output "./builds/$version_name/$arch/firecracker-debug.debug" 2>/dev/null || true
           done
 
       - name: Upload to GCS
@@ -244,15 +271,20 @@ jobs:
           version_name="${{ needs.validate.outputs.version_name }}"
 
           for arch in amd64 arm64; do
-            local_path="./builds/$version_name/$arch/firecracker"
-            [[ -f "$local_path" ]] || continue
+            # Upload the prod binary and the debug-only artifacts. The debug
+            # binary/symbols live at a different name ("firecracker-debug"), so
+            # client nodes — which resolve exactly "firecracker" — never pull them.
+            for name in firecracker firecracker-debug firecracker-debug.debug; do
+              local_path="./builds/$version_name/$arch/$name"
+              [[ -f "$local_path" ]] || continue
 
-            gcs_path="gs://${GCP_BUCKET_NAME}/firecrackers/${version_name}/${arch}/firecracker"
+              gcs_path="gs://${GCP_BUCKET_NAME}/firecrackers/${version_name}/${arch}/${name}"
 
-            if gcloud storage ls "$gcs_path" >/dev/null 2>&1; then
-              echo "GCS (${{ matrix.environment }}): $arch already exists, skipping"
-            else
-              echo "GCS (${{ matrix.environment }}): Uploading $arch..."
-              gcloud storage cp "$local_path" "$gcs_path"
-            fi
+              if gcloud storage ls "$gcs_path" >/dev/null 2>&1; then
+                echo "GCS (${{ matrix.environment }}): $name $arch already exists, skipping"
+              else
+                echo "GCS (${{ matrix.environment }}): Uploading $name $arch..."
+                gcloud storage cp "$local_path" "$gcs_path"
+              fi
+            done
           done

build.sh

@@ -32,12 +32,37 @@ git clone "https://x-access-token:${FIRECRACKER_REPO_TOKEN}@${FIRECRACKER_REPO_H
 cd firecracker
 git checkout "$commit_hash"
 
+out_dir="../builds/${version_name}/${arch}"
+mkdir -p "$out_dir"
+release_bin="build/cargo_target/${rust_target}/release/firecracker"
+
 echo "Building Firecracker $version_name for $arch ($rust_target)..."
 tools/devtool -y build --release -- --bin firecracker
 
 # Output goes into {version_name}/{arch}/firecracker
-mkdir -p "../builds/${version_name}/${arch}"
-cp "build/cargo_target/${rust_target}/release/firecracker" "../builds/${version_name}/${arch}/firecracker"
+cp "$release_bin" "$out_dir/firecracker"
+
+# Also build a debug variant: same release (optimized) build with the gdb
+# feature enabled and debug symbols kept. Used ONLY for debugging guest kernels
+# on dev nodes; it is never deployed to prod client nodes, which resolve the FC
+# binary at exactly "<version>/<arch>/firecracker" (a different name).
+#
+# devtool's release build strips and splits debug info into <bin>.debug, so we
+# publish the binary plus its companion. The debuglink is repointed to the
+# renamed companion so gdb auto-loads it when the two are colocated.
+echo "Building debug Firecracker $version_name for $arch ($rust_target)..."
+tools/devtool -y build --release -- --bin firecracker --features gdb
+cp "$release_bin" "$out_dir/firecracker-debug"
+if [[ -f "${release_bin}.debug" ]]; then
+  cp "${release_bin}.debug" "$out_dir/firecracker-debug.debug"
+  objcopy --remove-section .gnu_debuglink "$out_dir/firecracker-debug" 2>/dev/null || true
+  ( cd "$out_dir" && objcopy --add-gnu-debuglink=firecracker-debug.debug firecracker-debug )
+else
+  echo "Warning: ${release_bin}.debug not found; firecracker-debug ships without split DWARF" >&2
+  # Drop any stale debuglink (the build points it at "firecracker.debug", which we
+  # do not ship next to firecracker-debug) so gdb doesn't chase a missing file.
+  objcopy --remove-section .gnu_debuglink "$out_dir/firecracker-debug" 2>/dev/null || true
+fi
 
 cd ..
 rm -rf firecracker

scripts/test_validate.py

@@ -499,23 +499,37 @@ def test_both_requested_arm64_missing(self):
             assert check_artifacts_needed("v1.0.0_abc1234", True, True) is True
 
     def test_both_requested_both_exist(self):
-        """Test returns False when both requested and both exist."""
-        with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64", "firecracker-arm64"}):
+        """Test returns False when both requested and both prod + debug exist."""
+        assets = {
+            "firecracker-amd64", "firecracker-arm64",
+            "firecracker-debug-amd64", "firecracker-debug-arm64",
+        }
+        with patch("validate.get_existing_release_assets", return_value=assets):
             assert check_artifacts_needed("v1.0.0_abc1234", True, True) is False
 
+    def test_both_requested_prod_exists_debug_missing(self):
+        """Test returns True when prod binaries exist but the debug ones do not."""
+        with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64", "firecracker-arm64"}):
+            assert check_artifacts_needed("v1.0.0_abc1234", True, True) is True
+
     def test_amd64_only_exists(self):
-        """Test returns False when only amd64 requested and it exists."""
-        with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64"}):
+        """Test returns False when only amd64 requested and its prod + debug exist."""
+        with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64", "firecracker-debug-amd64"}):
             assert check_artifacts_needed("v1.0.0_abc1234", True, False) is False
 
+    def test_amd64_only_prod_exists_debug_missing(self):
+        """Test returns True when amd64 prod exists but its debug binary does not."""
+        with patch("validate.get_existing_release_assets", return_value={"firecracker-amd64"}):
+            assert check_artifacts_needed("v1.0.0_abc1234", True, False) is True
+
     def test_amd64_only_missing(self):
         """Test returns True when only amd64 requested and it's missing."""
         with patch("validate.get_existing_release_assets", return_value=set()):
             assert check_artifacts_needed("v1.0.0_abc1234", True, False) is True
 
     def test_arm64_only_exists(self):
-        """Test returns False when only arm64 requested and it exists."""
-        with patch("validate.get_existing_release_assets", return_value={"firecracker-arm64"}):
+        """Test returns False when only arm64 requested and its prod + debug exist."""
+        with patch("validate.get_existing_release_assets", return_value={"firecracker-arm64", "firecracker-debug-arm64"}):
             assert check_artifacts_needed("v1.0.0_abc1234", False, True) is False
 
     def test_arm64_only_missing(self):

scripts/upload-release-to-gcs.sh

@@ -77,6 +77,15 @@ for asset in "${ASSETS[@]}"; do
   if [[ "$asset" =~ ^firecracker-(amd64|arm64)$ ]]; then
     arch="${BASH_REMATCH[1]}"
     dst="${BUCKET_URI}/${TAG}/${arch}/firecracker"
+  elif [[ "$asset" =~ ^firecracker-debug-(amd64|arm64)\.debug$ ]]; then
+    # Debug-symbols companion (DWARF) for the debug FC binary. Debug-only.
+    arch="${BASH_REMATCH[1]}"
+    dst="${BUCKET_URI}/${TAG}/${arch}/firecracker-debug.debug"
+  elif [[ "$asset" =~ ^firecracker-debug-(amd64|arm64)$ ]]; then
+    # gdb-enabled debug FC binary. Never the prod path ("firecracker"); fetched
+    # explicitly by the dev-node debugging workflow.
+    arch="${BASH_REMATCH[1]}"
+    dst="${BUCKET_URI}/${TAG}/${arch}/firecracker-debug"
   else
     continue
   fi

scripts/validate.py

@@ -312,16 +312,27 @@ def get_existing_release_assets(version_name: str) -> set[str]:
 
 def check_artifacts_needed(version_name: str, build_amd64: bool, build_arm64: bool) -> bool:
     """
-    Check if any requested architectures are missing from the release.
+    Check if any requested architectures are missing an artifact from the release.
 
-    Returns True if at least one artifact needs to be built and uploaded.
+    Returns True if at least one artifact needs to be built and uploaded. Mirrors
+    the build job's skip-check: a release needs both the prod binary
+    (firecracker-<arch>) and the gdb-enabled debug binary (firecracker-debug-<arch>),
+    so a release that has the prod binary but not the debug one still has new
+    artifacts to publish.
     """
     existing_assets = get_existing_release_assets(version_name)
 
-    if build_amd64 and "firecracker-amd64" not in existing_assets:
-        return True
-    if build_arm64 and "firecracker-arm64" not in existing_assets:
-        return True
+    archs = []
+    if build_amd64:
+        archs.append("amd64")
+    if build_arm64:
+        archs.append("arm64")
+
+    for arch in archs:
+        if f"firecracker-{arch}" not in existing_assets:
+            return True
+        if f"firecracker-debug-{arch}" not in existing_assets:
+            return True
 
     return False