From bec8254969d39a5821eb4508cf180da7daaa40c6 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Mon, 15 Jun 2026 18:41:52 -0300 Subject: [PATCH 1/2] fix(snapshotter): atomic + complete layer manifest writeLayerManifest had two independent defects: 1. Non-atomic write: os.WriteFile could leave a truncated manifest on a crash mid-write, unlike the fsmeta/VMDK write path which already uses temp + rename. 2. Lost positional alignment: the digest-only format silently dropped fallback blobs (snapshot-.erofs, no content digest), so a fallback layer in the chain produced fewer lines than the device= options and desynced every layer after it. The manifest is the authoritative source for VMDK layer-order verification, so it must be 1:1 with device=. Changes: - New atomicWriteFile helper (temp in same dir + fsync + rename, temp cleaned up on any error). writeLayerManifest now uses it. - writeLayerManifest emits one line per layer in oldest-first order: "sha256:" for content-addressed blobs, "blob:" for fallback blobs. Line count now equals the device= count so consumers detect truncation/desync instead of misaligning silently. - Tests for both writeLayerManifest (digest-only, interleaved fallback, empty, no leftover temp) and atomicWriteFile. - readLayersManifest (integration) is forward-compatible: it already filters by the sha256: prefix and skips blob: lines; only its doc comment is updated to document the new format. Note: qemubox's manifest reader should be updated to recognize blob: lines to validate completeness by count; today it ignores them without breaking. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/snapshotter/commit.go | 81 +++++++++++---- internal/snapshotter/commit_test.go | 145 +++++++++++++++++++++++++++ test/integration/integration_test.go | 8 +- 3 files changed, 216 insertions(+), 18 deletions(-) diff --git a/internal/snapshotter/commit.go b/internal/snapshotter/commit.go index 6304463..8154923 100644 --- a/internal/snapshotter/commit.go +++ b/internal/snapshotter/commit.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "time" @@ -12,7 +13,6 @@ import ( "github.com/containerd/containerd/v2/core/snapshots/storage" "github.com/containerd/continuity/fs" "github.com/containerd/log" - "github.com/opencontainers/go-digest" "github.com/spin-stack/erofs-snapshotter/internal/erofs" "github.com/spin-stack/erofs-snapshotter/internal/mountutils" @@ -315,30 +315,77 @@ func fixVmdkPaths(vmdkFile, oldPath, newPath string) error { return nil } -// writeLayerManifest writes layer digests to a manifest file in VMDK/OCI order. -// Format: one digest per line (sha256:hex...), oldest/base layer first. -// This is the authoritative source for VMDK layer order verification. +// writeLayerManifest writes the layer manifest in VMDK/OCI order (oldest/base +// layer first). This is the authoritative source for VMDK layer-order +// verification, so it must be positionally 1:1 with the device= options the +// mount returns (mountFsMeta): one line per layer blob, in the same order, +// never dropping an entry. +// +// Format (one line per layer): +// - "sha256:" for content-addressed blobs (sha256-.erofs) +// - "blob:" for fallback blobs (snapshot-.erofs) which have no +// content digest. Emitting a placeholder keeps the line count equal to the +// device= count so consumers can detect a desync or a truncated file +// instead of silently misaligning every layer after the fallback. +// +// The file is written atomically (temp + fsync + rename) so a crash mid-write +// never leaves a truncated manifest, matching the fsmeta/VMDK write path. func (s *snapshotter) writeLayerManifest(manifestFile string, blobs []string) error { - var digests []digest.Digest + if len(blobs) == 0 { + return nil // Nothing to write + } + + var b strings.Builder for _, blob := range blobs { - d := erofs.DigestFromLayerBlobPath(blob) - if d != "" { - digests = append(digests, d) + if d := erofs.DigestFromLayerBlobPath(blob); d != "" { + b.WriteString(d.String()) + } else { + // Fallback blob (no content digest): record its identity so the + // entry is present and the line count stays aligned with device=. + b.WriteString("blob:") + b.WriteString(filepath.Base(blob)) } - // Skip non-digest-based blobs (e.g., snapshot-xxx.erofs fallback) + b.WriteByte('\n') } - if len(digests) == 0 { - return nil // No digests to write - } + return atomicWriteFile(manifestFile, []byte(b.String()), 0o644) +} - var lines []string - for _, d := range digests { - lines = append(lines, d.String()) +// atomicWriteFile writes data to path atomically: it writes to a temporary file +// in the same directory, fsyncs it, and renames it into place. A crash at any +// point leaves either the old file or the new file, never a truncated one. +func atomicWriteFile(path string, data []byte, perm os.FileMode) (retErr error) { + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".tmp-"+filepath.Base(path)+"-*") + if err != nil { + return fmt.Errorf("create temp file: %w", err) } + tmpName := tmp.Name() + // Remove the temp file on any error path; a no-op once the rename succeeds. + defer func() { + if retErr != nil { + _ = os.Remove(tmpName) + } + }() - content := strings.Join(lines, "\n") + "\n" - return os.WriteFile(manifestFile, []byte(content), 0o644) + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return fmt.Errorf("write temp file: %w", err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return fmt.Errorf("sync temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close temp file: %w", err) + } + if err := os.Chmod(tmpName, perm); err != nil { + return fmt.Errorf("chmod temp file: %w", err) + } + if err := os.Rename(tmpName, path); err != nil { + return fmt.Errorf("rename temp file: %w", err) + } + return nil } // Commit finalizes an active snapshot, converting it to EROFS format. diff --git a/internal/snapshotter/commit_test.go b/internal/snapshotter/commit_test.go index 6437a73..ae1c81c 100644 --- a/internal/snapshotter/commit_test.go +++ b/internal/snapshotter/commit_test.go @@ -4,9 +4,154 @@ import ( "context" "os" "path/filepath" + "strings" "testing" ) +func TestWriteLayerManifest(t *testing.T) { + const ( + digestA = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + digestB = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ) + blobA := "/snapshots/1/" + strings.Replace(digestA, ":", "-", 1) + ".erofs" + blobB := "/snapshots/2/" + strings.Replace(digestB, ":", "-", 1) + ".erofs" + fallback := "/snapshots/3/snapshot-3.erofs" + + tests := []struct { + name string + blobs []string + want string // empty means: file must not be created + }{ + { + name: "no blobs writes nothing", + blobs: nil, + want: "", + }, + { + name: "digest blobs only", + blobs: []string{blobA, blobB}, + want: digestA + "\n" + digestB + "\n", + }, + { + // A fallback blob (no content digest) must still produce a line so + // the manifest stays 1:1 with the device= options. Order preserved. + name: "fallback blob keeps positional alignment", + blobs: []string{blobA, fallback, blobB}, + want: digestA + "\n" + "blob:snapshot-3.erofs\n" + digestB + "\n", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + s := newTestSnapshotterWithRoot(t, root) //nolint:contextcheck // startup cleanup uses background context + manifestFile := filepath.Join(root, "layers.manifest") + + if err := s.writeLayerManifest(manifestFile, tc.blobs); err != nil { + t.Fatalf("writeLayerManifest: %v", err) + } + + data, err := os.ReadFile(manifestFile) + switch { + case tc.want == "": + if err == nil { + t.Fatalf("expected no manifest file, got content %q", data) + } + if !os.IsNotExist(err) { + t.Fatalf("unexpected error reading manifest: %v", err) + } + return + case err != nil: + t.Fatalf("read manifest: %v", err) + } + + if string(data) != tc.want { + t.Errorf("manifest content = %q, want %q", data, tc.want) + } + + // One line per blob: the manifest must be positionally 1:1 with + // the layers (and thus the device= options). + gotLines := strings.Count(string(data), "\n") + if gotLines != len(tc.blobs) { + t.Errorf("manifest line count = %d, want %d (one per layer)", gotLines, len(tc.blobs)) + } + + // No leftover temp files from the atomic write. + entries, err := os.ReadDir(root) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".tmp-") { + t.Errorf("leftover temp file after atomic write: %s", e.Name()) + } + } + }) + } +} + +func TestAtomicWriteFile(t *testing.T) { + t.Run("writes content and applies perm", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "out.txt") + + if err := atomicWriteFile(path, []byte("hello"), 0o600); err != nil { + t.Fatalf("atomicWriteFile: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(data) != "hello" { + t.Errorf("content = %q, want %q", data, "hello") + } + + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Errorf("perm = %o, want 600", info.Mode().Perm()) + } + }) + + t.Run("overwrites existing file atomically and leaves no temp", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "out.txt") + if err := os.WriteFile(path, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + + if err := atomicWriteFile(path, []byte("new content"), 0o644); err != nil { + t.Fatalf("atomicWriteFile: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(data) != "new content" { + t.Errorf("content = %q, want %q", data, "new content") + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Errorf("expected only the target file, got %d entries: %v", len(entries), entries) + } + }) + + t.Run("error when directory does not exist", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing-subdir", "out.txt") + if err := atomicWriteFile(path, []byte("x"), 0o644); err == nil { + t.Fatal("expected error for nonexistent directory") + } + }) +} + func TestGetCommitUpperDir(t *testing.T) { // This test verifies that getCommitUpperDir correctly determines // block vs overlay mode based on rwlayer.img existence and mount state. diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index cbb94f1..9680e46 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -1568,7 +1568,13 @@ func parseVMDKLayers(path string) ([]string, error) { return layers, scanner.Err() } -// readLayersManifest reads digests from a layers.manifest file. +// readLayersManifest reads the content digests from a layers.manifest file. +// +// The manifest has one line per layer in oldest-first order: "sha256:" +// for content-addressed layers and "blob:" for fallback layers that +// have no content digest. This reader returns only the sha256 digests; the +// "blob:" placeholder lines exist so the file stays positionally 1:1 with the +// device= options and are intentionally skipped here. func readLayersManifest(path string) ([]string, error) { f, err := os.Open(path) if err != nil { From a228e3c868ee8857b3163fd5828072961dace4e4 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Mon, 15 Jun 2026 18:50:19 -0300 Subject: [PATCH 2/2] test(differ): poll for async loop detach in checkNoLoopDevice checkNoLoopDevice did a single immediate FindByBackingFile check after the temp mount cleanup returned. The kernel releases loop devices asynchronously after LOOP_CLR_FD / autoclear, so the backing file can still resolve to a device for a short window, causing flaky failures: loop device /dev/loop1 still attached to .../layer.erofs (seen in TestWithStackedErofsTempMount and TestWithActiveSnapshotMountOverlay) Poll with a 5s deadline, matching the existing mountutils.verifyNoLoopDevice and loop.verifyDetached helpers, which already handle the same race. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/differ/compare_linux_test.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/internal/differ/compare_linux_test.go b/internal/differ/compare_linux_test.go index 04fbbfb..ca3d4eb 100644 --- a/internal/differ/compare_linux_test.go +++ b/internal/differ/compare_linux_test.go @@ -15,6 +15,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/diff" @@ -897,13 +898,25 @@ func populateExt4(t *testing.T, img string, fill func(root string)) { // the given backing files after the with*Mount helpers returned. func checkNoLoopDevice(t *testing.T, backingFiles ...string) { t.Helper() + // The kernel releases loop devices asynchronously after LOOP_CLR_FD / + // autoclear, so a backing file may still resolve to a device for a short + // window after cleanup returns. Poll with a deadline, matching + // mountutils.verifyNoLoopDevice and loop.verifyDetached. for _, f := range backingFiles { - dev, err := loop.FindByBackingFile(f) - if err != nil { - t.Fatalf("failed to look up loop device for %s: %v", f, err) - } - if dev != nil { - t.Errorf("loop device %s still attached to %s", dev.Path, f) + deadline := time.Now().Add(5 * time.Second) + for { + dev, err := loop.FindByBackingFile(f) + if err != nil { + t.Fatalf("failed to look up loop device for %s: %v", f, err) + } + if dev == nil { + break + } + if time.Now().After(deadline) { + t.Errorf("loop device %s still attached to %s", dev.Path, f) + break + } + time.Sleep(10 * time.Millisecond) } } }