From 60b251a260c4493bbb1c597df397789635a23d14 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 26 Jun 2026 12:50:05 +0000 Subject: [PATCH 1/7] Add composefs label and node image override to cluster test Use the Ginkgo Label("composefs") decorator so that the cluster creation test can be selected with --label-filter. Read BINK_NODE_IMAGE env var to allow overriding the default node image at runtime. Assisted-by: Claude Opus 4.6 (1M context) Signed-off-by: Alice Frosi --- test/integration/cluster_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/integration/cluster_test.go b/test/integration/cluster_test.go index 11cf63e..d9b537b 100644 --- a/test/integration/cluster_test.go +++ b/test/integration/cluster_test.go @@ -33,7 +33,7 @@ var _ = Describe("Cluster Lifecycle", func() { helpers.CleanupCluster(clusterName) }) - It("should create and initialize a complete Kubernetes cluster", func() { + It("should create and initialize a complete Kubernetes cluster", Label("composefs"), func() { customNodeName := "cp1" kubeconfigPath := fmt.Sprintf("../../kubeconfig-%s", clusterName) defer helpers.CleanupKubeconfig(kubeconfigPath) @@ -41,7 +41,11 @@ var _ = Describe("Cluster Lifecycle", func() { targetImgRef := "registry.cluster.local:5000/node:latest" By("Creating cluster with --expose, custom node name, memory ballooning, and target-imgref") - cmd := helpers.BinkCmd("cluster", "start", "--cluster-name", clusterName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-name", customNodeName, "--expose", kubeconfigPath, "--target-imgref", targetImgRef) + args := []string{"cluster", "start", "--cluster-name", clusterName, "--api-port", "0", "--memory", "1900", "--max-memory", "4096", "--node-name", customNodeName, "--expose", kubeconfigPath, "--target-imgref", targetImgRef} + if nodeImage := os.Getenv("BINK_NODE_IMAGE"); nodeImage != "" { + args = append(args, "--node-image", nodeImage) + } + cmd := helpers.BinkCmd(args...) session := helpers.RunCommand(cmd) By("Verifying cluster creation command succeeded") From 570aaf6d0a3c23af2c001917e6cd1850d3513d51 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 26 Jun 2026 12:50:09 +0000 Subject: [PATCH 2/7] Add test-integration-composefs Makefile target Provide a dedicated target to run only composefs-labeled tests using ginkgo --label-filter, mirroring the existing test targets. Assisted-by: Claude Opus 4.6 (1M context) Signed-off-by: Alice Frosi --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 5875fe3..5610851 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,11 @@ test-integration: @echo "=== Running Integration Tests ===" $(GINKGO) -v --procs=$(TEST_PROCS) $(GINKGO_FOCUS_FLAG) --fail-fast --randomize-all --randomize-suites test/integration/ +test-integration-composefs: + @test -f ./$(BINK_BINARY) || (echo "Error: bink binary not found. Run 'make build-bink' first" && exit 1) + @echo "=== Running Composefs Integration Tests ===" + $(GINKGO) -v --label-filter="composefs" --fail-fast test/integration/ + test-integration-quick: @test -f ./$(BINK_BINARY) || (echo "Error: bink binary not found. Run 'make build-bink' first" && exit 1) @echo "=== Running Quick Integration Tests ===" From 980102512e554b897ed83e2f0fcb157258939138 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 26 Jun 2026 12:50:14 +0000 Subject: [PATCH 3/7] Add composefs integration test GHA workflow Run the composefs-labeled subset of integration tests against the composefs node image to validate the composefs backend on every push and PR. Sets BINK_NODE_IMAGE to the composefs disk image so the test uses it instead of the default. Assisted-by: Claude Opus 4.6 (1M context) Signed-off-by: Alice Frosi --- .../workflows/integration-tests-composefs.yml | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 .github/workflows/integration-tests-composefs.yml diff --git a/.github/workflows/integration-tests-composefs.yml b/.github/workflows/integration-tests-composefs.yml new file mode 100644 index 0000000..ee7dec0 --- /dev/null +++ b/.github/workflows/integration-tests-composefs.yml @@ -0,0 +1,152 @@ +name: Integration Tests (composefs) + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: integration-composefs-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + BINK_IMAGES: >- + ghcr.io/bootc-dev/bink/cluster:latest + ghcr.io/bootc-dev/bink/node:v1.35-fedora-44-disk-composefs + ghcr.io/bootc-dev/bink/dns:latest + EXTERNAL_IMAGES: >- + quay.io/libpod/busybox:latest + +jobs: + integration-tests-composefs: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure kernel for nested containers + run: | + sudo aa-teardown 2>/dev/null || true + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + + - name: Enable KSM (Kernel Same-page Merging) + run: | + sudo sh -c 'echo 1 > /sys/kernel/mm/ksm/run' + sudo sh -c 'echo 5000 > /sys/kernel/mm/ksm/pages_to_scan' + cat /sys/kernel/mm/ksm/run + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + podman \ + libgpgme-dev \ + libbtrfs-dev \ + libdevmapper-dev \ + pkg-config + + - name: Set up KVM + run: | + sudo chmod 666 /dev/kvm + ls -la /dev/kvm + + - name: Configure Podman + run: | + podman --version + sudo mkdir -p /etc/containers + echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json + printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf + grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid + grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid + sudo systemctl start podman.socket + sudo podman info --format '{{.Store.GraphRoot}}' + + - name: Build bink binary + run: sudo make build-bink + + - name: Verify prerequisites + run: | + test -f ./bink + sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + df -h / + free -h + + - name: Get image digests + id: digests + run: | + ALL_DIGESTS="" + for img in $BINK_IMAGES $EXTERNAL_IMAGES; do + digest=$(skopeo inspect --no-creds "docker://${img}" --format '{{.Digest}}') + echo "${img}: ${digest}" + ALL_DIGESTS="${ALL_DIGESTS}${digest}" + done + echo "hash=$(echo -n "${ALL_DIGESTS}" | sha256sum | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" + + - name: Restore cached images + id: image-cache + uses: actions/cache/restore@v4 + with: + path: /tmp/podman-image-cache + key: podman-images-v2-composefs-${{ steps.digests.outputs.hash }} + + - name: Load cached images + if: steps.image-cache.outputs.cache-hit == 'true' + run: | + for f in /tmp/podman-image-cache/*.tar; do + sudo podman load -i "$f" + done + sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" + + - name: Pre-pull container images + if: steps.image-cache.outputs.cache-hit != 'true' + run: | + mkdir -p /tmp/podman-image-cache + for img in $BINK_IMAGES $EXTERNAL_IMAGES; do + sudo podman pull "$img" + name=$(echo "$img" | sed 's|[/:]|_|g') + sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img" + done + sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache + + - name: Save image cache + if: steps.image-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: /tmp/podman-image-cache + key: podman-images-v2-composefs-${{ steps.digests.outputs.hash }} + + - name: Run composefs integration tests + run: sudo make test-integration-composefs + timeout-minutes: 90 + env: + CONTAINER_HOST: unix:///run/podman/podman.sock + BINK_NODE_IMAGE: ghcr.io/bootc-dev/bink/node:v1.35-fedora-44-disk-composefs + + - name: Collect logs + if: failure() + run: .github/collect-logs.sh + + - name: Upload logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: test-logs-composefs + path: /tmp/bink-logs/ + + - name: Cleanup test clusters + if: always() + run: | + sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \ + xargs -r sudo podman rm -f 2>/dev/null || true + sudo podman volume prune -f 2>/dev/null || true From b3262b8f5cd9f6978b4b59cae3a8b1476d6ce892 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 26 Jun 2026 13:42:15 +0000 Subject: [PATCH 4/7] Guard /etc/fstab sed in cloud-init for composefs Composefs-based images do not have /etc/fstab, causing cloud-init runcmd to fail and report error status. Traditional bootc images use a regular filesystem where /etc/fstab lists mount points. Composefs-based images use a content-addressed, read-only erofs filesystem overlay where root mounting is handled by ostree deployment metadata and the initramfs boot logic, not by /etc/fstab. Since there are no traditional block-device mounts to describe, the file is simply absent. Guard the sed command with a file-existence check so cloud-init succeeds on both composefs and non-composefs images. Assisted-by: Claude Opus 4.6 (1M context) Signed-off-by: Alice Frosi --- internal/node/templates/user-data.yaml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/node/templates/user-data.yaml.tmpl b/internal/node/templates/user-data.yaml.tmpl index b7d63d1..3231bd4 100644 --- a/internal/node/templates/user-data.yaml.tmpl +++ b/internal/node/templates/user-data.yaml.tmpl @@ -68,7 +68,7 @@ write_files: runcmd: - swapoff -a - - sed -i '/swap/d' /etc/fstab + - "[ -f /etc/fstab ] && sed -i '/swap/d' /etc/fstab || true" - modprobe br_netfilter - echo 'br_netfilter' > /etc/modules-load.d/k8s-bridge.conf - sysctl -w net.ipv4.ip_forward=1 From cc646e6146e27b58cb4016b1d4ec9ea879d92892 Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 26 Jun 2026 13:42:37 +0000 Subject: [PATCH 5/7] Fix target-imgref override for composefs images The origin file path and format differ between ostree and composefs: - ostree: /ostree/deploy/default/deploy/..origin - composefs: /sysroot/state/deploy//.origin - composefs uses spaces around '=' in key-value pairs Use find to locate the origin file in both paths, and fix the sed pattern to match both formats. Assisted-by: Claude Opus 4.6 (1M context) Signed-off-by: Alice Frosi --- internal/node/templates/user-data.yaml.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/node/templates/user-data.yaml.tmpl b/internal/node/templates/user-data.yaml.tmpl index 3231bd4..ba51563 100644 --- a/internal/node/templates/user-data.yaml.tmpl +++ b/internal/node/templates/user-data.yaml.tmpl @@ -88,8 +88,8 @@ runcmd: {{- if .TargetImgRef}} - | mount -o remount,rw /sysroot - DEPLOY_PATH=$(bootc status --json | jq -r '.status.booted.ostree.deploySerial // empty') - CHECKSUM=$(bootc status --json | jq -r '.status.booted.ostree.checksum') - ORIGIN="/ostree/deploy/default/deploy/${CHECKSUM}.${DEPLOY_PATH}.origin" - sed -i 's|^container-image-reference=.*|container-image-reference=ostree-unverified-registry:{{.TargetImgRef}}|' "$ORIGIN" + ORIGIN=$(find /ostree/deploy/default/deploy/ /sysroot/state/deploy/ -name "*.origin" -type f 2>/dev/null | head -1) + if [ -n "$ORIGIN" ]; then + sed -i '/^container-image-reference/s|=.*|= ostree-unverified-registry:{{.TargetImgRef}}|' "$ORIGIN" + fi {{- end}} From bda47e92d7eca42c0cb1f68c6da01adacd668b7f Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Fri, 26 Jun 2026 13:42:55 +0000 Subject: [PATCH 6/7] Treat cloud-init error status as terminal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously WaitForCloudInit retried forever when cloud-init reported "error", but that is a terminal state — the status will never change to "done". Accept it with a warning instead of blocking for the full timeout. Assisted-by: Claude Opus 4.6 (1M context) Signed-off-by: Alice Frosi --- internal/cluster/cluster.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/cluster/cluster.go b/internal/cluster/cluster.go index fa686e2..0b94f1b 100644 --- a/internal/cluster/cluster.go +++ b/internal/cluster/cluster.go @@ -112,14 +112,18 @@ func (c *Cluster) WaitForCloudInit(ctx context.Context, nodeName string, timeout c.logger.Infof("cloud-init status: %s (attempt %d/%d)", status, i, maxRetries) - // Accept "done" (done with or without warnings is OK) - if status == "done" { + switch status { + case "done": c.logger.Info("✓ cloud-init completed") return nil + case "error": + c.logger.Warn("cloud-init finished with errors (non-critical modules may have failed)") + fullStatus, _ := sshClient.Exec(ctx, "cloud-init status --long") + c.logger.Debugf("cloud-init full status:\n%s", fullStatus) + return nil } if i == maxRetries { - // Get full status for debugging fullStatus, _ := sshClient.Exec(ctx, "cloud-init status --long") return fmt.Errorf("timeout waiting for cloud-init to complete on %s. Status: %s\nFull status:\n%s", nodeName, status, fullStatus) From 5245df46dd6a3b0236ae2298ac27d7e945a883ac Mon Sep 17 00:00:00 2001 From: Alice Frosi Date: Mon, 29 Jun 2026 13:07:26 +0000 Subject: [PATCH 7/7] Increase waitForAPI timeout from 2 to 5 minutes Signed-off-by: Alice Frosi --- internal/kube/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/kube/client.go b/internal/kube/client.go index 280f135..5b69a60 100644 --- a/internal/kube/client.go +++ b/internal/kube/client.go @@ -71,7 +71,7 @@ func waitForAPI(ctx context.Context, config *rest.Config) error { return err } - deadline := time.After(2 * time.Minute) + deadline := time.After(5 * time.Minute) ticker := time.NewTicker(2 * time.Second) defer ticker.Stop()