From 10538304ce4a7eb244681fca411bed108bdc5e4d Mon Sep 17 00:00:00 2001 From: mesutoezdil Date: Thu, 23 Jul 2026 20:15:05 +0200 Subject: [PATCH 1/3] imagecache: fall back to legacy mount on pre-6.5 kernels The new mount API needs overlayfs lowerdir+, which only exists on Linux 6.5+. Probe it once at first mount and use plain mount(2) when missing, so older kernels do not fail every actor mount with EINVAL. --- internal/imagecache/README.md | 11 ++-- internal/imagecache/bundle_linux.go | 72 +++++++++++++++++++++--- internal/imagecache/bundle_linux_test.go | 63 +++++++++++++++++++++ 3 files changed, 132 insertions(+), 14 deletions(-) diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 1b2b7118f..4a4fa2591 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -118,11 +118,12 @@ staging the virtio-fs lower (micro-VM): legitimately list the same diffID twice — are collapsed to the topmost occurrence, which overlayfs otherwise rejects with `ELOOP`), `upperdir` / `workdir` are the bundle-local dirs, holding this actor's private writes. - The mount uses the new mount API (`fsopen` + one `fsconfig` `lowerdir+` - append per layer) rather than `mount(2)`, whose single-page option-string - cap the digest-derived layer paths would hit at ~34 layers. **Minimum - supported kernel: Linux 6.5** (`lowerdir+`); every current GKE channel - ships ≥ 6.6 (Stable: COS 121 LTS). + The mount prefers the new mount API (`fsopen` + one `fsconfig` `lowerdir+` + append per layer), which lifts `mount(2)`'s single-page option-string cap + that the digest-derived layer paths would hit at ~34 layers. `lowerdir+` + needs Linux 6.5+ (every current GKE channel ships >= 6.6, Stable: COS 121 + LTS); on older kernels ateom probes this once and falls back to a plain + `mount(2)` call, which keeps the ~34-layer cap. 3. **ExtraDirs** are created through the mount (landing in the upper), again under `os.Root` confinement. 4. **Implicit-parent metadata repair.** A layer tar routinely omits entries diff --git a/internal/imagecache/bundle_linux.go b/internal/imagecache/bundle_linux.go index dc15a1460..a6bd389fe 100644 --- a/internal/imagecache/bundle_linux.go +++ b/internal/imagecache/bundle_linux.go @@ -22,10 +22,12 @@ package imagecache import ( "errors" "fmt" + "log/slog" "os" "path/filepath" "sort" "strings" + "sync" "golang.org/x/sys/unix" ) @@ -97,17 +99,35 @@ func SetupBundleRootfs(bundlePath string) error { return nil } +// lowerdirPlusSupported probes, once per process, whether the running +// kernel understands overlayfs's incremental "lowerdir+" fsconfig option +// (Linux >= 6.5). The probe opens a throwaway fs context and tries the +// option against a directory that always exists; it never creates a +// superblock or touches the filesystem otherwise. +var lowerdirPlusSupported = sync.OnceValue(func() bool { + fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) + if err != nil { + return false + } + defer unix.Close(fsfd) + if unix.FsconfigSetString(fsfd, "lowerdir+", "/") == nil { + return true + } + slog.Warn("kernel lacks overlayfs lowerdir+ (need >= 6.5); using legacy mount, images beyond ~34 layers will fail") + return false +}) + // mountOverlay attaches an overlay of lowers (top-most first) with the given -// upper/work dirs at mountpoint, using the new mount API rather than -// mount(2): appending lowerdirs one fsconfig(2) call at a time sidesteps -// mount(2)'s single-page option-string cap, which digest-derived layer paths -// (~114 bytes each) would hit at roughly 34 layers. -// -// Minimum supported kernel: Linux 6.5, where overlayfs gained the -// incremental "lowerdir+" option. Every current GKE channel is at or above -// it (Stable runs COS 121 LTS on kernel 6.6; Regular and Rapid run COS -// 125/129 on 6.12). +// upper/work dirs at mountpoint. It prefers the new mount API, appending +// lowerdirs one fsconfig(2) call at a time to sidestep mount(2)'s +// single-page option-string cap, which digest-derived layer paths (~114 +// bytes each) would hit at roughly 34 layers. On kernels older than 6.5, +// which lack the incremental "lowerdir+" option, it falls back to a plain +// mount(2) call. func mountOverlay(mountpoint string, lowers []string, upper, work string) error { + if !lowerdirPlusSupported() { + return mountOverlayLegacy(mountpoint, lowers, upper, work) + } fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) if err != nil { return fmt.Errorf("while opening overlay fs context: %w", err) @@ -146,6 +166,40 @@ func mountOverlay(mountpoint string, lowers []string, upper, work string) error return nil } +// legacyOptionsPageCap is mount(2)'s per-call option-string limit: the +// kernel copies the "data" argument through a single page. +const legacyOptionsPageCap = 4096 + +// overlayMountOptions builds the mount(2) overlayfs option string for +// kernels without "lowerdir+" support. lowers must already be top-first and +// deduplicated (see overlayLowerDirs). +func overlayMountOptions(lowers []string, upper, work string) (string, error) { + for _, p := range append([]string{upper, work}, lowers...) { + if strings.ContainsAny(p, ":,") { + return "", fmt.Errorf("overlay path %q contains mount option separators", p) + } + } + opts := "lowerdir=" + strings.Join(lowers, ":") + ",upperdir=" + upper + ",workdir=" + work + if len(opts) >= legacyOptionsPageCap { + return "", fmt.Errorf("overlay options for %d layers are %d bytes, at or over mount(2)'s %d-byte page cap; kernel lacks overlayfs lowerdir+, upgrade to Linux >= 6.5 to support longer layer chains", len(lowers), len(opts), legacyOptionsPageCap) + } + return opts, nil +} + +// mountOverlayLegacy mounts the overlay via a plain mount(2) call, for +// kernels older than 6.5 that do not support the incremental "lowerdir+" +// fsconfig option. +func mountOverlayLegacy(mountpoint string, lowers []string, upper, work string) error { + opts, err := overlayMountOptions(lowers, upper, work) + if err != nil { + return err + } + if err := unix.Mount("overlay", mountpoint, "overlay", 0, opts); err != nil { + return fmt.Errorf("while mounting overlay rootfs at %q (%s): %w", mountpoint, opts, err) + } + return nil +} + // fsContextLog drains the human-readable message log the kernel queues on an // fs context fd (one "e/w/i "-prefixed message per read, ENODATA when empty) // and renders it for appending to an error. mount(2) had no equivalent — a diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index 00fafae58..7c7ce14d8 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "golang.org/x/sys/unix" @@ -295,3 +296,65 @@ func TestSetupBundleRootfs_ManyLayers(t *testing.T) { t.Fatalf("UnmountAllUnder: %v", err) } } + +// Legacy mount(2) path, exercised directly so it is covered even on kernels +// where the fsconfig lowerdir+ probe succeeds and SetupBundleRootfs never +// takes this branch. Needs CAP_SYS_ADMIN. +func TestMountOverlayLegacy(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root (mount/unmount)") + } + layer := t.TempDir() + writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil) + + bundle := t.TempDir() + rootfs := filepath.Join(bundle, "rootfs") + upper := filepath.Join(bundle, "upper") + work := filepath.Join(bundle, "work") + for _, d := range []string{rootfs, upper, work} { + if err := os.MkdirAll(d, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + if err := FinalizeLayer(layer); err != nil { + t.Fatalf("FinalizeLayer: %v", err) + } + + if err := mountOverlayLegacy(rootfs, overlayLowerDirs([]string{layer}), upper, work); err != nil { + t.Fatalf("mountOverlayLegacy: %v", err) + } + t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + + if got, err := os.ReadFile(filepath.Join(rootfs, "from-layer.txt")); err != nil || string(got) != "hello" { + t.Errorf("layer content not visible through overlay: %q (%v)", got, err) + } +} + +func TestOverlayMountOptions(t *testing.T) { + if _, err := overlayMountOptions([]string{"a:b"}, "/upper", "/work"); err == nil { + t.Error("lower path with separator: want error, got nil") + } + if _, err := overlayMountOptions([]string{"/a"}, "up,per", "/work"); err == nil { + t.Error("upper path with separator: want error, got nil") + } + + opts, err := overlayMountOptions([]string{"/a", "/b"}, "/upper", "/work") + if err != nil { + t.Fatalf("overlayMountOptions: %v", err) + } + if want := "lowerdir=/a:/b,upperdir=/upper,workdir=/work"; opts != want { + t.Errorf("opts = %q, want %q", opts, want) + } + + long := make([]string, 64) + for i := range long { + long[i] = filepath.Join("/sha256", fmt.Sprintf("%064d", i), "fs") + } + _, err = overlayMountOptions(long, "/upper", "/work") + if err == nil { + t.Fatal("over-page-cap options: want error, got nil") + } + if !strings.Contains(err.Error(), "lowerdir+") { + t.Errorf("over-page-cap error = %q, want a mention of lowerdir+ / kernel upgrade", err.Error()) + } +} From 022dc768b60aa6d4e1058f737d595671fda62870 Mon Sep 17 00:00:00 2001 From: mesutoezdil Date: Thu, 23 Jul 2026 22:12:09 +0200 Subject: [PATCH 2/3] imagecache: react to the real lowerdir+ failure instead of probing Drop the separate startup probe. mountOverlay now tries lowerdir+ on the first layer directly; if the kernel does not recognize it, that call fails immediately (before any path is touched), and we fall back to legacy mount(2) from there. Simpler and the fallback is now covered by a test that forces the failure. --- internal/imagecache/README.md | 5 ++- internal/imagecache/bundle_linux.go | 51 ++++++++++++------------ internal/imagecache/bundle_linux_test.go | 42 ++++++++++++++++++- 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 4a4fa2591..25d512e7c 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -122,8 +122,9 @@ staging the virtio-fs lower (micro-VM): append per layer), which lifts `mount(2)`'s single-page option-string cap that the digest-derived layer paths would hit at ~34 layers. `lowerdir+` needs Linux 6.5+ (every current GKE channel ships >= 6.6, Stable: COS 121 - LTS); on older kernels ateom probes this once and falls back to a plain - `mount(2)` call, which keeps the ~34-layer cap. + LTS); on older kernels the first `lowerdir+` attempt fails immediately and + ateom falls back to a plain `mount(2)` call, which keeps the ~34-layer + cap. 3. **ExtraDirs** are created through the mount (landing in the upper), again under `os.Root` confinement. 4. **Implicit-parent metadata repair.** A layer tar routinely omits entries diff --git a/internal/imagecache/bundle_linux.go b/internal/imagecache/bundle_linux.go index a6bd389fe..a52cb1ede 100644 --- a/internal/imagecache/bundle_linux.go +++ b/internal/imagecache/bundle_linux.go @@ -99,39 +99,40 @@ func SetupBundleRootfs(bundlePath string) error { return nil } -// lowerdirPlusSupported probes, once per process, whether the running -// kernel understands overlayfs's incremental "lowerdir+" fsconfig option -// (Linux >= 6.5). The probe opens a throwaway fs context and tries the -// option against a directory that always exists; it never creates a -// superblock or touches the filesystem otherwise. -var lowerdirPlusSupported = sync.OnceValue(func() bool { - fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) - if err != nil { - return false - } - defer unix.Close(fsfd) - if unix.FsconfigSetString(fsfd, "lowerdir+", "/") == nil { - return true - } - slog.Warn("kernel lacks overlayfs lowerdir+ (need >= 6.5); using legacy mount, images beyond ~34 layers will fail") - return false -}) +// lowerdirPlusOption is the fsconfig parameter name mountOverlay uses for +// each lowerdir. It is a var, not a literal, only so tests can force it to +// an unrecognized name and exercise the pre-6.5-kernel fallback below +// without needing an actual old kernel. +var lowerdirPlusOption = "lowerdir+" + +var warnLegacyMountOnce sync.Once // mountOverlay attaches an overlay of lowers (top-most first) with the given // upper/work dirs at mountpoint. It prefers the new mount API, appending // lowerdirs one fsconfig(2) call at a time to sidestep mount(2)'s // single-page option-string cap, which digest-derived layer paths (~114 -// bytes each) would hit at roughly 34 layers. On kernels older than 6.5, -// which lack the incremental "lowerdir+" option, it falls back to a plain -// mount(2) call. +// bytes each) would hit at roughly 34 layers. +// +// Kernels older than 6.5 do not recognize the incremental "lowerdir+" +// parameter at all, so the very first attempt to set it fails immediately +// with an unrecognized-parameter error, before any path is looked at or any +// superblock created. That failure is used directly as the fall-back +// signal, rather than a separate startup probe: a bad lowerdir path would +// only surface later, at FsconfigCreate, so this first call failing is an +// unambiguous kernel-version signal. func mountOverlay(mountpoint string, lowers []string, upper, work string) error { - if !lowerdirPlusSupported() { - return mountOverlayLegacy(mountpoint, lowers, upper, work) - } fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) if err != nil { return fmt.Errorf("while opening overlay fs context: %w", err) } + + if err := unix.FsconfigSetString(fsfd, lowerdirPlusOption, lowers[0]); err != nil { + unix.Close(fsfd) + warnLegacyMountOnce.Do(func() { + slog.Warn("kernel lacks overlayfs lowerdir+ (need >= 6.5); using legacy mount, images beyond ~34 layers will fail") + }) + return mountOverlayLegacy(mountpoint, lowers, upper, work) + } defer unix.Close(fsfd) set := func(key, val string) error { @@ -140,8 +141,8 @@ func mountOverlay(mountpoint string, lowers []string, upper, work string) error } return nil } - for _, lower := range lowers { - if err := set("lowerdir+", lower); err != nil { + for _, lower := range lowers[1:] { + if err := set(lowerdirPlusOption, lower); err != nil { return err } } diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index 7c7ce14d8..acbc26f1b 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -298,8 +298,8 @@ func TestSetupBundleRootfs_ManyLayers(t *testing.T) { } // Legacy mount(2) path, exercised directly so it is covered even on kernels -// where the fsconfig lowerdir+ probe succeeds and SetupBundleRootfs never -// takes this branch. Needs CAP_SYS_ADMIN. +// where mountOverlay's lowerdir+ attempt succeeds and never falls back to +// it. Needs CAP_SYS_ADMIN. func TestMountOverlayLegacy(t *testing.T) { if os.Geteuid() != 0 { t.Skip("needs root (mount/unmount)") @@ -330,6 +330,44 @@ func TestMountOverlayLegacy(t *testing.T) { } } +// Forces mountOverlay's first fsconfig call to fail with an unrecognized +// parameter name, the same failure a pre-6.5 kernel gives for real, and +// checks it falls back to a working legacy mount rather than erroring out. +// Needs CAP_SYS_ADMIN. +func TestMountOverlay_FallsBackOnUnsupportedLowerdirPlus(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root (mount/unmount)") + } + orig := lowerdirPlusOption + lowerdirPlusOption = "not-a-real-overlay-option+" + t.Cleanup(func() { lowerdirPlusOption = orig }) + + layer := t.TempDir() + writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil) + if err := FinalizeLayer(layer); err != nil { + t.Fatalf("FinalizeLayer: %v", err) + } + + bundle := t.TempDir() + rootfs := filepath.Join(bundle, "rootfs") + upper := filepath.Join(bundle, "upper") + work := filepath.Join(bundle, "work") + for _, d := range []string{rootfs, upper, work} { + if err := os.MkdirAll(d, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + + if err := mountOverlay(rootfs, overlayLowerDirs([]string{layer}), upper, work); err != nil { + t.Fatalf("mountOverlay: %v", err) + } + t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + + if got, err := os.ReadFile(filepath.Join(rootfs, "from-layer.txt")); err != nil || string(got) != "hello" { + t.Errorf("layer content not visible through overlay after fallback: %q (%v)", got, err) + } +} + func TestOverlayMountOptions(t *testing.T) { if _, err := overlayMountOptions([]string{"a:b"}, "/upper", "/work"); err == nil { t.Error("lower path with separator: want error, got nil") From af258c76a12633c9df023a0e62bbf1232a384a3e Mon Sep 17 00:00:00 2001 From: mesutoezdil Date: Fri, 24 Jul 2026 08:47:59 +0200 Subject: [PATCH 3/3] imagecache: use roottest.Require in new mount tests --- internal/imagecache/bundle_linux_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index a23fcdeb9..5bf21b62e 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -295,9 +295,7 @@ func TestSetupBundleRootfs_ManyLayers(t *testing.T) { // where mountOverlay's lowerdir+ attempt succeeds and never falls back to // it. Needs CAP_SYS_ADMIN. func TestMountOverlayLegacy(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip("needs root (mount/unmount)") - } + roottest.Require(t, "mount/unmount") layer := t.TempDir() writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil) @@ -329,9 +327,7 @@ func TestMountOverlayLegacy(t *testing.T) { // checks it falls back to a working legacy mount rather than erroring out. // Needs CAP_SYS_ADMIN. func TestMountOverlay_FallsBackOnUnsupportedLowerdirPlus(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip("needs root (mount/unmount)") - } + roottest.Require(t, "mount/unmount") orig := lowerdirPlusOption lowerdirPlusOption = "not-a-real-overlay-option+" t.Cleanup(func() { lowerdirPlusOption = orig })