From 24d303a3c6194c8acdbdfa6fc499fd7b73336ccc Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 16:18:00 -0700 Subject: [PATCH 1/9] fix: read cgroup memory limit for cache sizing when GOMEMLIMIT is not set Extend containerMemoryMB() to try the cgroup memory limit (v2 at /sys/fs/cgroup/memory.max, then v1 at /sys/fs/cgroup/memory/memory.limit_in_bytes) before falling back to host RAM. This means the ristretto cache is correctly sized even in deployments that do not explicitly set GOMEMLIMIT. Co-Authored-By: Claude Sonnet 4.6 --- ...6058255-fix-cgroup-aware-cache-memory.yaml | 10 +++++ internal/pkg/config/env_defaults.go | 42 ++++++++++++++++-- internal/pkg/config/env_defaults_test.go | 43 +++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml diff --git a/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml b/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml new file mode 100644 index 0000000000..63f41665c0 --- /dev/null +++ b/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml @@ -0,0 +1,10 @@ +kind: enhancement + +summary: Read cgroup memory limit for cache sizing when GOMEMLIMIT is not set + +description: > + containerMemoryMB() now falls back to the cgroup memory limit (v2 then v1) + before host RAM. This ensures the ristretto cache is correctly sized for the + container even in deployments that do not explicitly set GOMEMLIMIT. + +component: fleet-server diff --git a/internal/pkg/config/env_defaults.go b/internal/pkg/config/env_defaults.go index ceb3790ab0..021818ac66 100644 --- a/internal/pkg/config/env_defaults.go +++ b/internal/pkg/config/env_defaults.go @@ -10,8 +10,10 @@ import ( "io" "io/fs" "math" + "os" "runtime" "runtime/debug" + "strconv" "strings" "time" @@ -323,17 +325,51 @@ func loadLimits(log *zerolog.Logger, agentLimit int) *envLimits { return defaultEnvLimits() } -// containerMemoryMB returns available memory in MiB, preferring the GOMEMLIMIT -// runtime setting over host RAM so the ristretto cache is sized for the container, -// not the node. Falls back to memory.TotalMemory() when GOMEMLIMIT is unset. +// containerMemoryMB returns available memory in MiB using this priority order: +// 1. GOMEMLIMIT, if explicitly set +// 2. cgroup memory limit (v2, then v1), for containers without an explicit GOMEMLIMIT +// 3. host total RAM, for non-containerised deployments func containerMemoryMB() uint64 { limit := debug.SetMemoryLimit(-1) if limit > 0 && limit != math.MaxInt64 { return uint64(limit) / 1024 / 1024 } + if mb, ok := cgroupMemoryLimitMB(); ok { + return mb + } return memory.TotalMemory() / 1024 / 1024 } +// cgroupMemoryLimitMB reads the container memory limit from cgroup files. +// It tries cgroup v2 first, then cgroup v1. Returns (0, false) when no +// applicable limit is found (unlimited, missing file, or parse error). +func cgroupMemoryLimitMB() (uint64, bool) { + if mb, ok := readCgroupMemoryFile("/sys/fs/cgroup/memory.max"); ok { + return mb, true + } + return readCgroupMemoryFile("/sys/fs/cgroup/memory/memory.limit_in_bytes") +} + +// readCgroupMemoryFile parses a cgroup memory limit file and returns the +// value in MiB. Returns (0, false) when the file doesn't exist, contains +// "max" (unlimited), or the value is at or above MaxInt64 (cgroup v1's +// sentinel for unlimited). +func readCgroupMemoryFile(path string) (uint64, bool) { + data, err := os.ReadFile(path) + if err != nil { + return 0, false + } + s := strings.TrimSpace(string(data)) + if s == "max" { + return 0, false + } + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || n >= math.MaxInt64 { + return 0, false + } + return n / 1024 / 1024, true +} + // memMB returns available memory in MiB. // It is a var so that unit tests can replace it. var memMB func() uint64 = containerMemoryMB diff --git a/internal/pkg/config/env_defaults_test.go b/internal/pkg/config/env_defaults_test.go index 3ec8794def..a49d5fee3e 100644 --- a/internal/pkg/config/env_defaults_test.go +++ b/internal/pkg/config/env_defaults_test.go @@ -8,6 +8,7 @@ import ( "io" "io/fs" "math" + "os" "reflect" "runtime/debug" "strings" @@ -110,3 +111,45 @@ func TestContainerMemoryMB(t *testing.T) { assert.Equal(t, memory.TotalMemory()/1024/1024, got) }) } + +func TestReadCgroupMemoryFile(t *testing.T) { + writeFile := func(t *testing.T, content string) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "cgroup-memory-*") + require.NoError(t, err) + _, err = f.WriteString(content) + require.NoError(t, err) + require.NoError(t, f.Close()) + return f.Name() + } + + t.Run("returns MiB for a valid byte limit", func(t *testing.T) { + path := writeFile(t, "134217728\n") // 128 MiB + mb, ok := readCgroupMemoryFile(path) + assert.True(t, ok) + assert.Equal(t, uint64(128), mb) + }) + + t.Run("returns false for 'max' (unlimited)", func(t *testing.T) { + path := writeFile(t, "max\n") + _, ok := readCgroupMemoryFile(path) + assert.False(t, ok) + }) + + t.Run("returns false for MaxInt64 sentinel (cgroup v1 unlimited)", func(t *testing.T) { + path := writeFile(t, "9223372036854775807\n") // math.MaxInt64 + _, ok := readCgroupMemoryFile(path) + assert.False(t, ok) + }) + + t.Run("returns false when file does not exist", func(t *testing.T) { + _, ok := readCgroupMemoryFile("/nonexistent/cgroup/memory.max") + assert.False(t, ok) + }) + + t.Run("returns false for invalid content", func(t *testing.T) { + path := writeFile(t, "not-a-number\n") + _, ok := readCgroupMemoryFile(path) + assert.False(t, ok) + }) +} From 650d1c3c992a5387db493c981d3dba8a5c742331 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 16:21:26 -0700 Subject: [PATCH 2/9] chore: remove ristretto-specific wording from comments and changelog containerMemoryMB is used for general memory-based sizing in fleet-server, not solely for the ristretto cache tier. Co-Authored-By: Claude Sonnet 4.6 --- .../fragments/1786058255-fix-cgroup-aware-cache-memory.yaml | 2 +- internal/pkg/config/env_defaults_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml b/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml index 63f41665c0..105b4c7b7b 100644 --- a/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml +++ b/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml @@ -4,7 +4,7 @@ summary: Read cgroup memory limit for cache sizing when GOMEMLIMIT is not set description: > containerMemoryMB() now falls back to the cgroup memory limit (v2 then v1) - before host RAM. This ensures the ristretto cache is correctly sized for the + before host RAM. This ensures fleet-server is correctly sized for the container even in deployments that do not explicitly set GOMEMLIMIT. component: fleet-server diff --git a/internal/pkg/config/env_defaults_test.go b/internal/pkg/config/env_defaults_test.go index a49d5fee3e..8df8018573 100644 --- a/internal/pkg/config/env_defaults_test.go +++ b/internal/pkg/config/env_defaults_test.go @@ -92,7 +92,7 @@ func TestDefaultLimitsYAMLKeys(t *testing.T) { } // TestContainerMemoryMB verifies that containerMemoryMB prefers GOMEMLIMIT over -// host RAM so the ristretto cache is sized for the container, not the node. +// host RAM so fleet-server is correctly sized for the container, not the node. func TestContainerMemoryMB(t *testing.T) { t.Run("uses GOMEMLIMIT when set", func(t *testing.T) { const setLimit = int64(256 * 1024 * 1024) // 256 MiB From cd62e593ffb132c97db5e19ebdb70ab893aca120 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 6 Aug 2026 16:24:27 -0700 Subject: [PATCH 3/9] chore: drop #7568 changelog fragment from this branch This fragment belongs to PR #7568. Removing it here so it doesn't appear twice in the diff against main. It will re-enter via main once #7568 merges. Co-Authored-By: Claude Sonnet 4.6 --- .../1786038287-fix-cache-container-memory.yaml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 changelog/fragments/1786038287-fix-cache-container-memory.yaml diff --git a/changelog/fragments/1786038287-fix-cache-container-memory.yaml b/changelog/fragments/1786038287-fix-cache-container-memory.yaml deleted file mode 100644 index cdbbc1733f..0000000000 --- a/changelog/fragments/1786038287-fix-cache-container-memory.yaml +++ /dev/null @@ -1,13 +0,0 @@ -kind: bug-fix - -summary: Fix ristretto cache sized from host RAM instead of container memory limit - -description: > - memEnvLimits() called memory.TotalMemory() to select the ristretto cache tier. - On a Kubernetes node with >=16 GB of RAM this picked a MaxCost of 256-512 MB -- - 2-4x the pod's GOMEMLIMIT -- causing OOMKills even at modest agent counts. - The fix reads GOMEMLIMIT via debug.SetMemoryLimit(-1) and uses that value to - select the cache tier, falling back to memory.TotalMemory() only when GOMEMLIMIT - is unset (non-containerised deployments). - -component: fleet-server From aee326c6593a5bbad7605f712175c04f9fee8308 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Aug 2026 07:25:43 -0700 Subject: [PATCH 4/9] fix: use MiB label in RAM warning log (values are 1024^2 bytes) --- internal/pkg/config/env_defaults.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/config/env_defaults.go b/internal/pkg/config/env_defaults.go index 021818ac66..40181177d0 100644 --- a/internal/pkg/config/env_defaults.go +++ b/internal/pkg/config/env_defaults.go @@ -316,7 +316,7 @@ func loadLimits(log *zerolog.Logger, agentLimit int) *envLimits { log.Info().Msgf("Using system limits for %d to %d agents for a configured value of %d agents", l.Agents.Min, l.Agents.Max, agentLimit) ramSize := memMB() if ramSize < l.RecommendedRAM { - log.Warn().Msgf("Detected %d MB of available memory, which is lower than the recommended amount (%d MB) for the configured agent limit", ramSize, l.RecommendedRAM) + log.Warn().Msgf("Detected %d MiB of available memory, which is lower than the recommended amount (%d MiB) for the configured agent limit", ramSize, l.RecommendedRAM) } return l } From 50e0dd3b01f562adc1ef84e4bf4fccac531f9fe0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Aug 2026 07:26:34 -0700 Subject: [PATCH 5/9] test: expose cgroupMemMB as a stubbable var for hermetic testing --- internal/pkg/config/env_defaults.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pkg/config/env_defaults.go b/internal/pkg/config/env_defaults.go index 40181177d0..1c3f4fdda5 100644 --- a/internal/pkg/config/env_defaults.go +++ b/internal/pkg/config/env_defaults.go @@ -325,6 +325,10 @@ func loadLimits(log *zerolog.Logger, agentLimit int) *envLimits { return defaultEnvLimits() } +// cgroupMemMB returns the cgroup memory limit in MiB. +// It is a var so that unit tests can replace it. +var cgroupMemMB func() (uint64, bool) = cgroupMemoryLimitMB + // containerMemoryMB returns available memory in MiB using this priority order: // 1. GOMEMLIMIT, if explicitly set // 2. cgroup memory limit (v2, then v1), for containers without an explicit GOMEMLIMIT @@ -334,7 +338,7 @@ func containerMemoryMB() uint64 { if limit > 0 && limit != math.MaxInt64 { return uint64(limit) / 1024 / 1024 } - if mb, ok := cgroupMemoryLimitMB(); ok { + if mb, ok := cgroupMemMB(); ok { return mb } return memory.TotalMemory() / 1024 / 1024 From 0cc44470c35cd71cf3c5ed412a6d55f1a0eafeb9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Aug 2026 07:26:59 -0700 Subject: [PATCH 6/9] test: stub cgroupMemMB in TestContainerMemoryMB for hermetic fallback tests --- internal/pkg/config/env_defaults_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/pkg/config/env_defaults_test.go b/internal/pkg/config/env_defaults_test.go index 8df8018573..d99a377f80 100644 --- a/internal/pkg/config/env_defaults_test.go +++ b/internal/pkg/config/env_defaults_test.go @@ -103,9 +103,23 @@ func TestContainerMemoryMB(t *testing.T) { assert.Equal(t, uint64(256), got) }) - t.Run("falls back to host RAM when GOMEMLIMIT is unset", func(t *testing.T) { + t.Run("uses cgroup limit when GOMEMLIMIT is unset", func(t *testing.T) { prev := debug.SetMemoryLimit(math.MaxInt64) t.Cleanup(func() { debug.SetMemoryLimit(prev) }) + prevCgroup := cgroupMemMB + cgroupMemMB = func() (uint64, bool) { return 128, true } + t.Cleanup(func() { cgroupMemMB = prevCgroup }) + + got := containerMemoryMB() + assert.Equal(t, uint64(128), got) + }) + + t.Run("falls back to host RAM when GOMEMLIMIT and cgroup are both unset", func(t *testing.T) { + prev := debug.SetMemoryLimit(math.MaxInt64) + t.Cleanup(func() { debug.SetMemoryLimit(prev) }) + prevCgroup := cgroupMemMB + cgroupMemMB = func() (uint64, bool) { return 0, false } + t.Cleanup(func() { cgroupMemMB = prevCgroup }) got := containerMemoryMB() assert.Equal(t, memory.TotalMemory()/1024/1024, got) From 0c4a12b1e6c68f46fd58ce951d7a2cb22b72299e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 7 Aug 2026 07:27:16 -0700 Subject: [PATCH 7/9] changelog: change kind from enhancement to bug-fix (fixes OOMKill-inducing cache sizing) --- .../fragments/1786058255-fix-cgroup-aware-cache-memory.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml b/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml index 105b4c7b7b..b655081ccc 100644 --- a/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml +++ b/changelog/fragments/1786058255-fix-cgroup-aware-cache-memory.yaml @@ -1,4 +1,4 @@ -kind: enhancement +kind: bug-fix summary: Read cgroup memory limit for cache sizing when GOMEMLIMIT is not set From 1b0247bcb84e3092f26dbdf6f8b138a9508517d4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 10 Aug 2026 09:49:15 -0700 Subject: [PATCH 8/9] fix: handle cgroup v1 unlimited sentinel --- internal/pkg/config/env_defaults.go | 13 ++++++++++--- internal/pkg/config/env_defaults_test.go | 5 +++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/internal/pkg/config/env_defaults.go b/internal/pkg/config/env_defaults.go index 1c3f4fdda5..2f83663649 100644 --- a/internal/pkg/config/env_defaults.go +++ b/internal/pkg/config/env_defaults.go @@ -356,8 +356,8 @@ func cgroupMemoryLimitMB() (uint64, bool) { // readCgroupMemoryFile parses a cgroup memory limit file and returns the // value in MiB. Returns (0, false) when the file doesn't exist, contains -// "max" (unlimited), or the value is at or above MaxInt64 (cgroup v1's -// sentinel for unlimited). +// "max" (unlimited), or the value is at or above cgroup v1's page-aligned +// unlimited sentinel. func readCgroupMemoryFile(path string) (uint64, bool) { data, err := os.ReadFile(path) if err != nil { @@ -368,12 +368,19 @@ func readCgroupMemoryFile(path string) (uint64, bool) { return 0, false } n, err := strconv.ParseUint(s, 10, 64) - if err != nil || n >= math.MaxInt64 { + if err != nil || n >= cgroupV1UnlimitedMemoryLimit() { return 0, false } return n / 1024 / 1024, true } +// cgroupV1UnlimitedMemoryLimit is the value cgroup v1 exposes for an +// unrestricted memory limit: MaxInt64 rounded down to the system page size. +func cgroupV1UnlimitedMemoryLimit() uint64 { + pageSize := int64(os.Getpagesize()) + return uint64(math.MaxInt64 / pageSize * pageSize) //nolint:gosec // the page-aligned result is always non-negative +} + // memMB returns available memory in MiB. // It is a var so that unit tests can replace it. var memMB func() uint64 = containerMemoryMB diff --git a/internal/pkg/config/env_defaults_test.go b/internal/pkg/config/env_defaults_test.go index d99a377f80..48e6eb68a7 100644 --- a/internal/pkg/config/env_defaults_test.go +++ b/internal/pkg/config/env_defaults_test.go @@ -11,6 +11,7 @@ import ( "os" "reflect" "runtime/debug" + "strconv" "strings" "testing" @@ -150,8 +151,8 @@ func TestReadCgroupMemoryFile(t *testing.T) { assert.False(t, ok) }) - t.Run("returns false for MaxInt64 sentinel (cgroup v1 unlimited)", func(t *testing.T) { - path := writeFile(t, "9223372036854775807\n") // math.MaxInt64 + t.Run("returns false for the cgroup v1 unlimited sentinel", func(t *testing.T) { + path := writeFile(t, strconv.FormatUint(cgroupV1UnlimitedMemoryLimit(), 10)+"\n") _, ok := readCgroupMemoryFile(path) assert.False(t, ok) }) From 628d16d2d3605907ec9dd1f3c12416029578ae8f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 10 Aug 2026 09:49:31 -0700 Subject: [PATCH 9/9] docs: note nested cgroup limitation --- internal/pkg/config/env_defaults.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/pkg/config/env_defaults.go b/internal/pkg/config/env_defaults.go index 2f83663649..ce7ec1a8a7 100644 --- a/internal/pkg/config/env_defaults.go +++ b/internal/pkg/config/env_defaults.go @@ -345,8 +345,10 @@ func containerMemoryMB() uint64 { } // cgroupMemoryLimitMB reads the container memory limit from cgroup files. -// It tries cgroup v2 first, then cgroup v1. Returns (0, false) when no -// applicable limit is found (unlimited, missing file, or parse error). +// It tries cgroup v2 first, then cgroup v1. This only checks the cgroup mount +// root, so it does not account for limits imposed by nested cgroups. Returns +// (0, false) when no applicable limit is found (unlimited, missing file, or +// parse error). func cgroupMemoryLimitMB() (uint64, bool) { if mb, ok := readCgroupMemoryFile("/sys/fs/cgroup/memory.max"); ok { return mb, true