diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index b67ae20c9..1e352d31a 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -118,11 +118,13 @@ staging the virtio-fs lower (micro-VM): legitimately list the same diffID twice — are collapsed to the topmost occurrence, which overlayfs otherwise rejects with `ELOOP`), `upperdir` / `workdir` are the bundle-local dirs, holding this actor's private writes. - The mount uses the new mount API (`fsopen` + one `fsconfig` `lowerdir+` - append per layer) rather than `mount(2)`, whose single-page option-string - cap the digest-derived layer paths would hit at ~34 layers. **Minimum - supported kernel: Linux 6.5** (`lowerdir+`); every current GKE channel - ships ≥ 6.6 (Stable: COS 121 LTS). + The mount prefers the new mount API (`fsopen` + one `fsconfig` `lowerdir+` + append per layer), which lifts `mount(2)`'s single-page option-string cap + that the digest-derived layer paths would hit at ~34 layers. `lowerdir+` + needs Linux 6.5+ (every current GKE channel ships >= 6.6, Stable: COS 121 + LTS); on older kernels the first `lowerdir+` attempt fails immediately and + ateom falls back to a plain `mount(2)` call, which keeps the ~34-layer + cap. 3. **ExtraDirs** are created through the mount (landing in the upper), again under `os.Root` confinement. 4. **Implicit-parent metadata repair.** A layer tar routinely omits entries diff --git a/internal/imagecache/bundle_linux.go b/internal/imagecache/bundle_linux.go index dc15a1460..a52cb1ede 100644 --- a/internal/imagecache/bundle_linux.go +++ b/internal/imagecache/bundle_linux.go @@ -22,10 +22,12 @@ package imagecache import ( "errors" "fmt" + "log/slog" "os" "path/filepath" "sort" "strings" + "sync" "golang.org/x/sys/unix" ) @@ -97,21 +99,40 @@ func SetupBundleRootfs(bundlePath string) error { return nil } +// lowerdirPlusOption is the fsconfig parameter name mountOverlay uses for +// each lowerdir. It is a var, not a literal, only so tests can force it to +// an unrecognized name and exercise the pre-6.5-kernel fallback below +// without needing an actual old kernel. +var lowerdirPlusOption = "lowerdir+" + +var warnLegacyMountOnce sync.Once + // mountOverlay attaches an overlay of lowers (top-most first) with the given -// upper/work dirs at mountpoint, using the new mount API rather than -// mount(2): appending lowerdirs one fsconfig(2) call at a time sidesteps -// mount(2)'s single-page option-string cap, which digest-derived layer paths -// (~114 bytes each) would hit at roughly 34 layers. +// upper/work dirs at mountpoint. It prefers the new mount API, appending +// lowerdirs one fsconfig(2) call at a time to sidestep mount(2)'s +// single-page option-string cap, which digest-derived layer paths (~114 +// bytes each) would hit at roughly 34 layers. // -// Minimum supported kernel: Linux 6.5, where overlayfs gained the -// incremental "lowerdir+" option. Every current GKE channel is at or above -// it (Stable runs COS 121 LTS on kernel 6.6; Regular and Rapid run COS -// 125/129 on 6.12). +// Kernels older than 6.5 do not recognize the incremental "lowerdir+" +// parameter at all, so the very first attempt to set it fails immediately +// with an unrecognized-parameter error, before any path is looked at or any +// superblock created. That failure is used directly as the fall-back +// signal, rather than a separate startup probe: a bad lowerdir path would +// only surface later, at FsconfigCreate, so this first call failing is an +// unambiguous kernel-version signal. func mountOverlay(mountpoint string, lowers []string, upper, work string) error { fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) if err != nil { return fmt.Errorf("while opening overlay fs context: %w", err) } + + if err := unix.FsconfigSetString(fsfd, lowerdirPlusOption, lowers[0]); err != nil { + unix.Close(fsfd) + warnLegacyMountOnce.Do(func() { + slog.Warn("kernel lacks overlayfs lowerdir+ (need >= 6.5); using legacy mount, images beyond ~34 layers will fail") + }) + return mountOverlayLegacy(mountpoint, lowers, upper, work) + } defer unix.Close(fsfd) set := func(key, val string) error { @@ -120,8 +141,8 @@ func mountOverlay(mountpoint string, lowers []string, upper, work string) error } return nil } - for _, lower := range lowers { - if err := set("lowerdir+", lower); err != nil { + for _, lower := range lowers[1:] { + if err := set(lowerdirPlusOption, lower); err != nil { return err } } @@ -146,6 +167,40 @@ func mountOverlay(mountpoint string, lowers []string, upper, work string) error return nil } +// legacyOptionsPageCap is mount(2)'s per-call option-string limit: the +// kernel copies the "data" argument through a single page. +const legacyOptionsPageCap = 4096 + +// overlayMountOptions builds the mount(2) overlayfs option string for +// kernels without "lowerdir+" support. lowers must already be top-first and +// deduplicated (see overlayLowerDirs). +func overlayMountOptions(lowers []string, upper, work string) (string, error) { + for _, p := range append([]string{upper, work}, lowers...) { + if strings.ContainsAny(p, ":,") { + return "", fmt.Errorf("overlay path %q contains mount option separators", p) + } + } + opts := "lowerdir=" + strings.Join(lowers, ":") + ",upperdir=" + upper + ",workdir=" + work + if len(opts) >= legacyOptionsPageCap { + return "", fmt.Errorf("overlay options for %d layers are %d bytes, at or over mount(2)'s %d-byte page cap; kernel lacks overlayfs lowerdir+, upgrade to Linux >= 6.5 to support longer layer chains", len(lowers), len(opts), legacyOptionsPageCap) + } + return opts, nil +} + +// mountOverlayLegacy mounts the overlay via a plain mount(2) call, for +// kernels older than 6.5 that do not support the incremental "lowerdir+" +// fsconfig option. +func mountOverlayLegacy(mountpoint string, lowers []string, upper, work string) error { + opts, err := overlayMountOptions(lowers, upper, work) + if err != nil { + return err + } + if err := unix.Mount("overlay", mountpoint, "overlay", 0, opts); err != nil { + return fmt.Errorf("while mounting overlay rootfs at %q (%s): %w", mountpoint, opts, err) + } + return nil +} + // fsContextLog drains the human-readable message log the kernel queues on an // fs context fd (one "e/w/i "-prefixed message per read, ENODATA when empty) // and renders it for appending to an error. mount(2) had no equivalent — a diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index 8a3d88c50..5bf21b62e 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "golang.org/x/sys/unix" @@ -289,3 +290,99 @@ func TestSetupBundleRootfs_ManyLayers(t *testing.T) { t.Fatalf("UnmountAllUnder: %v", err) } } + +// Legacy mount(2) path, exercised directly so it is covered even on kernels +// where mountOverlay's lowerdir+ attempt succeeds and never falls back to +// it. Needs CAP_SYS_ADMIN. +func TestMountOverlayLegacy(t *testing.T) { + roottest.Require(t, "mount/unmount") + layer := t.TempDir() + writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil) + + bundle := t.TempDir() + rootfs := filepath.Join(bundle, "rootfs") + upper := filepath.Join(bundle, "upper") + work := filepath.Join(bundle, "work") + for _, d := range []string{rootfs, upper, work} { + if err := os.MkdirAll(d, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + if err := FinalizeLayer(layer); err != nil { + t.Fatalf("FinalizeLayer: %v", err) + } + + if err := mountOverlayLegacy(rootfs, overlayLowerDirs([]string{layer}), upper, work); err != nil { + t.Fatalf("mountOverlayLegacy: %v", err) + } + t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + + if got, err := os.ReadFile(filepath.Join(rootfs, "from-layer.txt")); err != nil || string(got) != "hello" { + t.Errorf("layer content not visible through overlay: %q (%v)", got, err) + } +} + +// Forces mountOverlay's first fsconfig call to fail with an unrecognized +// parameter name, the same failure a pre-6.5 kernel gives for real, and +// checks it falls back to a working legacy mount rather than erroring out. +// Needs CAP_SYS_ADMIN. +func TestMountOverlay_FallsBackOnUnsupportedLowerdirPlus(t *testing.T) { + roottest.Require(t, "mount/unmount") + orig := lowerdirPlusOption + lowerdirPlusOption = "not-a-real-overlay-option+" + t.Cleanup(func() { lowerdirPlusOption = orig }) + + layer := t.TempDir() + writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil) + if err := FinalizeLayer(layer); err != nil { + t.Fatalf("FinalizeLayer: %v", err) + } + + bundle := t.TempDir() + rootfs := filepath.Join(bundle, "rootfs") + upper := filepath.Join(bundle, "upper") + work := filepath.Join(bundle, "work") + for _, d := range []string{rootfs, upper, work} { + if err := os.MkdirAll(d, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + + if err := mountOverlay(rootfs, overlayLowerDirs([]string{layer}), upper, work); err != nil { + t.Fatalf("mountOverlay: %v", err) + } + t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + + if got, err := os.ReadFile(filepath.Join(rootfs, "from-layer.txt")); err != nil || string(got) != "hello" { + t.Errorf("layer content not visible through overlay after fallback: %q (%v)", got, err) + } +} + +func TestOverlayMountOptions(t *testing.T) { + if _, err := overlayMountOptions([]string{"a:b"}, "/upper", "/work"); err == nil { + t.Error("lower path with separator: want error, got nil") + } + if _, err := overlayMountOptions([]string{"/a"}, "up,per", "/work"); err == nil { + t.Error("upper path with separator: want error, got nil") + } + + opts, err := overlayMountOptions([]string{"/a", "/b"}, "/upper", "/work") + if err != nil { + t.Fatalf("overlayMountOptions: %v", err) + } + if want := "lowerdir=/a:/b,upperdir=/upper,workdir=/work"; opts != want { + t.Errorf("opts = %q, want %q", opts, want) + } + + long := make([]string, 64) + for i := range long { + long[i] = filepath.Join("/sha256", fmt.Sprintf("%064d", i), "fs") + } + _, err = overlayMountOptions(long, "/upper", "/work") + if err == nil { + t.Fatal("over-page-cap options: want error, got nil") + } + if !strings.Contains(err.Error(), "lowerdir+") { + t.Errorf("over-page-cap error = %q, want a mention of lowerdir+ / kernel upgrade", err.Error()) + } +}