Skip to content

Microsoft: build reproducible kernel in-place without a temp copy #152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
namancse merged 1 commit into from
Jul 8, 2026
+35 −101
Merged

Microsoft: build reproducible kernel in-place without a temp copy #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 16 additions & 63 deletions Microsoft/build-hcl-kernel-pipeline.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,63 +162,11 @@ BUILD_DIR=$(realpath "$BUILD_DIR")
# Call reproducible build setup before anything else
setup_reproducible_build

# For reproducible builds, copy source to a fixed path
# This ensures identical paths in binaries regardless of where source is located
# Build in-place in the kernel repository (no temporary copy). Path independence
# is achieved via -ffile-prefix-map below instead of copying to a fixed path.
ORIGINAL_SOURCE_DIR="$SOURCE_DIR"
ORIGINAL_BUILD_DIR="$BUILD_DIR"

setup_fixed_build_path() {
if [[ -n "$REPRODUCIBLE_BUILD" ]]; then
FIXED_BUILD_PATH="${FIXED_BUILD_PATH:-/tmp/ohcl-kernel-build}"
FIXED_SOURCE_DIR="${FIXED_BUILD_PATH}/src"
FIXED_BUILD_DIR="${FIXED_BUILD_PATH}/build"

echo ">>> Setting up fixed build path for reproducibility..."
echo " Original source: $ORIGINAL_SOURCE_DIR"
echo " Fixed source: $FIXED_SOURCE_DIR"

# Clean and create fixed build path
rm -rf "${FIXED_BUILD_PATH}"
mkdir -p "${FIXED_BUILD_PATH}"

# Copy source to fixed path (use anchored excludes to avoid matching subdirs like tools/build)
rsync -a --exclude='/.git' --exclude='/build' --exclude='/out' --exclude='/compare' \
"${ORIGINAL_SOURCE_DIR}/" "${FIXED_SOURCE_DIR}/"

# Update paths to use fixed locations
SOURCE_DIR="${FIXED_SOURCE_DIR}"
BUILD_DIR="${FIXED_BUILD_DIR}"

echo " Source copied to fixed path"
fi
}

# Cleanup function to remove temporary build directory
cleanup_fixed_build_path() {
if [[ -n "$REPRODUCIBLE_BUILD" ]] && [[ -n "${FIXED_BUILD_PATH:-}" ]] && [[ -d "${FIXED_BUILD_PATH:-}" ]]; then
echo ">>> Cleaning up temporary build directory: ${FIXED_BUILD_PATH}"
rm -rf "${FIXED_BUILD_PATH}"
fi
}

# Copy build artifacts back to original location
copy_artifacts_back() {
if [[ -n "$REPRODUCIBLE_BUILD" ]]; then
echo ">>> Copying build artifacts back to original location..."
mkdir -p "${ORIGINAL_BUILD_DIR}"
rsync -a "${FIXED_BUILD_DIR}/" "${ORIGINAL_BUILD_DIR}/"
echo " Artifacts copied to ${ORIGINAL_BUILD_DIR}"
fi
}

# Set trap to cleanup on exit (success or failure)
if [[ -n "$REPRODUCIBLE_BUILD" ]]; then
trap cleanup_fixed_build_path EXIT
fi

# Setup fixed build path for reproducible builds
setup_fixed_build_path

# Detect host architecture
HOST_ARCH="$(uname -m)"

Expand Down Expand Up @@ -418,9 +366,17 @@ build_kernel() {
make_args+=("CROSS_COMPILE=$CROSS_COMPILE_PREFIX")
fi

# Add reproducible build flags
# Add reproducible build flags.
# Map BOTH the source tree and the build dir (DWARF comp_dir = $KBUILD_OUTPUT)
# so absolute paths don't leak into DWARF info or the link-time GNU build-id.
# KAFLAGS is required because assembly (.S) sources use AFLAGS, not CFLAGS;
# without it their absolute path leaks into the build-id.
if [[ -n "$REPRODUCIBLE_BUILD" ]]; then
make_args+=("KCFLAGS=-fdebug-prefix-map=$SOURCE_DIR=.")
local repro_map="-ffile-prefix-map=$SOURCE_DIR=. -ffile-prefix-map=$BUILD_DIR=."
# Append (don't overwrite) so any caller-provided KCFLAGS/KAFLAGS are
# preserved; make command-line assignments override the environment.
make_args+=("KCFLAGS=${KCFLAGS:+${KCFLAGS} }$repro_map")
make_args+=("KAFLAGS=${KAFLAGS:+${KAFLAGS} }$repro_map")
fi

# Run olddefconfig
Expand Down Expand Up @@ -617,16 +573,13 @@ main() {
fi
echo "=============================================="

# Copy artifacts back to original location for reproducible builds
copy_artifacts_back

# Print sha256sum of vmlinux for reproducibility verification
# Check the STRIPPED vmlinux after it's been copied back to original location
if [[ -n "$REPRODUCIBLE_BUILD" ]] && [[ -f "$ORIGINAL_BUILD_DIR/vmlinux" ]]; then
# Check the STRIPPED vmlinux (final post-processed output)
if [[ -n "$REPRODUCIBLE_BUILD" ]] && [[ -f "$BUILD_DIR/vmlinux" ]]; then
echo ""
echo "Reproducibility verification:"
echo " vmlinux sha256sum: $(sha256sum "$ORIGINAL_BUILD_DIR/vmlinux" | cut -d' ' -f1)"
echo " (stripped vmlinux from $ORIGINAL_BUILD_DIR/vmlinux)"
echo " vmlinux sha256sum: $(sha256sum "$BUILD_DIR/vmlinux" | cut -d' ' -f1)"
echo " (stripped vmlinux from $BUILD_DIR/vmlinux)"
fi
}

Expand Down
6 changes: 3 additions & 3 deletions Microsoft/build-hcl-kernel.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,10 @@ set -e
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
SRC_DIR=`realpath ${SCRIPT_DIR}/..`

# For reproducible builds, add flags to normalize debug paths
# For reproducible builds, prevent the + suffix in the version string.
# The path-normalization flags (KCFLAGS/KAFLAGS -ffile-prefix-map) are supplied
# by the caller (Microsoft/nix-build.sh) through the environment.
if [ -n "$REPRODUCIBLE_BUILD" ]; then
makeargs+=("KCFLAGS=-fdebug-prefix-map=$SRC_DIR=.")
# Prevent + suffix from being added to version string
makeargs+=("LOCALVERSION=")
fi
Comment thread
namancse marked this conversation as resolved.

Expand Down
51 changes: 16 additions & 35 deletions Microsoft/nix-build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,8 @@ fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
KERNEL_ROOT_ORIGINAL="$(cd "${SCRIPT_DIR}/.." && pwd)"

# Fixed build path for reproducibility - ensures identical paths across machines
FIXED_BUILD_PATH="${FIXED_BUILD_PATH:-/tmp/ohcl-kernel-build}"
KERNEL_ROOT="${FIXED_BUILD_PATH}/src"
# Build in-place in the kernel repository (no temporary copy).
KERNEL_ROOT="${KERNEL_ROOT_ORIGINAL}"

# Colors for output
RED='\033[0;31m'
Expand Down Expand Up @@ -99,36 +98,28 @@ export KBUILD_BUILD_USER="${KBUILD_BUILD_USER:-builder}"
export KBUILD_BUILD_HOST="${KBUILD_BUILD_HOST:-nixos}"
export KBUILD_BUILD_VERSION="1"

# Cleanup function to remove temporary build directory
cleanup_build_path() {
if [ -n "${FIXED_BUILD_PATH}" ] && [ -d "${FIXED_BUILD_PATH}" ]; then
log_info "Cleaning up temporary build directory: ${FIXED_BUILD_PATH}"
rm -rf "${FIXED_BUILD_PATH}"
fi
}

# Set trap to cleanup on exit (success or failure)
trap cleanup_build_path EXIT
# Normalize embedded build paths so the resulting vmlinux is identical no matter
# where the kernel tree lives. Map BOTH the source tree and the out-of-tree build
# dir (the DWARF comp_dir = $KBUILD_OUTPUT) to '.'. -ffile-prefix-map covers
# __FILE__, DWARF paths and comp_dir, all of which feed the link-time GNU
# build-id. KAFLAGS is required because assembly (.S) sources use AFLAGS, not
# CFLAGS. These are passed via the environment so build-hcl-kernel.sh needs no
# reproducibility-specific flags of its own.
REPRO_SRC_DIR="$(realpath "${KERNEL_ROOT}")"
REPRO_BUILD_DIR="$(realpath -m "${KERNEL_ROOT}/../build")"
REPRO_MAP="-ffile-prefix-map=${REPRO_SRC_DIR}=. -ffile-prefix-map=${REPRO_BUILD_DIR}=."
# Append (don't overwrite) so any caller-provided KCFLAGS/KAFLAGS are preserved.
export KCFLAGS="${KCFLAGS:+${KCFLAGS} }${REPRO_MAP}"
export KAFLAGS="${KAFLAGS:+${KAFLAGS} }${REPRO_MAP}"

main() {
log_info "Starting reproducible kernel build..."
log_info "Original source: ${KERNEL_ROOT_ORIGINAL}"
log_info "Fixed build path: ${KERNEL_ROOT}"
log_info "Kernel source: ${KERNEL_ROOT}"
log_info "Reproducible environment:"
log_info " SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
log_info " KBUILD_BUILD_USER=${KBUILD_BUILD_USER}"
log_info " KBUILD_BUILD_HOST=${KBUILD_BUILD_HOST}"

# Copy source to fixed path for reproducibility
# This ensures the same paths are embedded in binaries regardless of where source is located
log_info "Copying source to fixed build path..."
rm -rf "${FIXED_BUILD_PATH}"
mkdir -p "${FIXED_BUILD_PATH}"
# Use anchored excludes (/) to avoid matching subdirectories like tools/build
rsync -a --exclude='/.git' --exclude='/build' --exclude='/out' \
"${KERNEL_ROOT_ORIGINAL}/" "${KERNEL_ROOT}/"
log_info "Source copied to ${KERNEL_ROOT}"

# Set environment for reproducibility
export KBUILD_BUILD_TIMESTAMP="${KBUILD_BUILD_TIMESTAMP}"
export KBUILD_BUILD_USER="${KBUILD_BUILD_USER}"
Expand All @@ -153,16 +144,6 @@ main() {
log_info "Invoking build-hcl-kernel.sh..."
"${KERNEL_ROOT}/Microsoft/build-hcl-kernel.sh" "${ARCH_TYPE}"

# Copy build artifacts back to original location
log_info "Copying build artifacts back to original location..."
mkdir -p "${KERNEL_ROOT_ORIGINAL}/out"
rsync -a "${KERNEL_ROOT}/out/" "${KERNEL_ROOT_ORIGINAL}/out/"

# Copy build directory back to original location
log_info "Copying build directory back to original location..."
mkdir -p "${KERNEL_ROOT_ORIGINAL}/../build"
rsync -a "${FIXED_BUILD_PATH}/build/" "${KERNEL_ROOT_ORIGINAL}/../build/"

log_info "Build completed successfully!"
log_info "Build artifacts are in: ${KERNEL_ROOT_ORIGINAL}/out"

Expand Down
Loading