From abead13160edacd20afdaa1693e5133be84907a8 Mon Sep 17 00:00:00 2001 From: Riadh <22998716+KiKoS0@users.noreply.github.com> Date: Mon, 4 May 2026 15:39:24 +0100 Subject: [PATCH] larger block size/cache & use bloom filters in pebble I noticed lots of cache thrashing when pods memory usage is nearing the limit, that puts the pod in a cycle of reading/invalidating indexes from disk and aggregate evaluation taking minutes sometimes. ``` 2 @ 0x485cee 0x46313d 0x463114 0x4877e5 0x14bb6cc 0x14bb6af 0x14bb025 0x14efb57 0x1559b14 0x1559a3b 0x1558d05 0x1647eb4 0x1647bb3 0x165f9cd 0x165e6a5 0x162266c 0x16220dd 0x236c350 0x2370a9e 0x236953f 0x2ab9086 0x2ab70df 0x50fe4b2 0x50fafb8 0x21ff766 0x48e821 2 @ 0x485cee 0x46313d 0x463114 0x487845 0x49a2cb 0x14bb8a6 0x14be4fc 0x14efe74 0x14efe64 0x155d2c5 0x155d1ee 0x155c539 0x155c036 0x1558e05 0x1647eb4 0x1647bb3 0x165f9cd 0x165e6a5 0x162266c 0x16220dd 0x236c350 0x2370a9e 0x236953f 0x2ab9086 0x2ab70df 0x50fe4b2 0x50fafb8 0x21ff766 0x48e821 ``` --- kvdb.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/kvdb.go b/kvdb.go index c50afc5..1a69c5c 100644 --- a/kvdb.go +++ b/kvdb.go @@ -5,6 +5,7 @@ import ( "sync/atomic" "github.com/cockroachdb/pebble/v2" + "github.com/cockroachdb/pebble/v2/bloom" "github.com/cockroachdb/pebble/v2/vfs" "github.com/google/uuid" ) @@ -38,15 +39,27 @@ func NewKV[T Evaluable](o KVOpts[T]) (KV[T], error) { o.FS = vfs.Default } - db, err := pebble.Open(o.Dir, &pebble.Options{ - FS: o.FS, - // cockroachdb defaults that should slightly help with faster writes - // https://github.com/cockroachdb/cockroach/blob/5a1f5da5bb3b2d962d8737848a4fca69f915dacb/pkg/storage/pebble.go#L668-L673 - L0CompactionThreshold: 2, + // closely following some of cockroachdb defaults + // https://github.com/cockroachdb/cockroach/blob/5a1f5da5bb3b2d962d8737848a4fca69f915dacb/pkg/storage/pebble.go#L668-L673 + opts := &pebble.Options{ + FS: o.FS, + Cache: pebble.NewCache(512 << 20), + DisableWAL: true, + L0CompactionThreshold: 8, L0StopWritesThreshold: 1000, MemTableSize: 64 << 20, // 64 MB MemTableStopWritesThreshold: 4, - }) + CompactionConcurrencyRange: func() (int, int) { return 1, 4 }, + } + opts.Levels[0] = pebble.LevelOptions{ + BlockSize: 32 << 10, + IndexBlockSize: 256 << 10, + FilterPolicy: bloom.FilterPolicy(10), + FilterType: pebble.TableFilter, + } + opts.Levels[0].EnsureL0Defaults() + + db, err := pebble.Open(o.Dir, opts) if err != nil { return nil, err }