From abe638ebdc8ce76d068cda989d87d2e4a4d5f525 Mon Sep 17 00:00:00 2001 From: Divya Mahendran Date: Mon, 24 Aug 2026 14:56:03 -0700 Subject: [PATCH 1/3] Add low-yield compaction backoff to stop wasteful copy-forward on all-live data Lookup/Scan compaction on all-live, larger-than-memory data copies nearly every live record forward to reclaim only segment-alignment slack, then repeats every cycle because the legacy "netReclaimed > 0" test misreads that alignment noise as progress. On a write-once dataset this pins the device at a high write duty even at idle, starving reads of NVMe bandwidth. This adds a yield-ratio gate: a cycle counts as productive only if it reclaims at least CompactionLowYieldReclaimPercent (default 20) of the begin-address range it advanced. After a low-yield cycle, compaction is parked until the log tail grows by CompactionLowYieldBackoffSegments segments (i.e. real new garbage arrives), then automatically resumes. Foreground writes re-arm it. - CompactionPolicy.cs: new CompactionState (skip/resume/RecordCycle) + helpers - DatabaseManagerBase.cs: gate the compaction loop + emit per-cycle telemetry (begin advance, tail growth, net reclaimed, ratio, duration) - GarnetServerOptions.cs: CompactionLowYieldBackoffSegments, CompactionLowYieldReclaimPercent - Options.cs + defaults.conf: expose --compaction-low-yield-backoff-segments (default 0 = disabled/opt-in) - GarnetDatabase.cs: per-database CompactionState - CompactionPolicyTests.cs: unit tests for the gate Co-authored-by: GitHub Copilot <223556219+Copilot@users.noreply.github.com> --- libs/host/Configuration/Options.cs | 5 + libs/host/defaults.conf | 3 + libs/server/Databases/CompactionPolicy.cs | 69 +++++++++++++ libs/server/Databases/DatabaseManagerBase.cs | 61 ++++++++++-- libs/server/GarnetDatabase.cs | 5 + libs/server/Servers/GarnetServerOptions.cs | 18 ++++ .../Garnet.test/CompactionPolicyTests.cs | 99 +++++++++++++++++++ 7 files changed, 253 insertions(+), 7 deletions(-) create mode 100644 libs/server/Databases/CompactionPolicy.cs create mode 100644 test/standalone/Garnet.test/CompactionPolicyTests.cs diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index a3cb5caadf7..f213a1664d4 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -267,6 +267,10 @@ internal sealed class Options : ICloneable [Option("compaction-max-segments", Required = false, HelpText = "Number of log segments created on disk before compaction triggers.")] public int CompactionMaxSegments { get; set; } + [IntRangeValidation(0, int.MaxValue)] + [Option("compaction-low-yield-backoff-segments", Required = false, HelpText = "Number of log segments the tail must grow before retrying after a compaction cycle that reclaimed no space. 0 = disabled.")] + public int CompactionLowYieldBackoffSegments { get; set; } + [OptionValidation] [Option("lua", Required = false, HelpText = "Enable Lua scripts on server.")] public bool? EnableLua { get; set; } @@ -905,6 +909,7 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno CompactionType = CompactionType, CompactionForceDelete = CompactionForceDelete.GetValueOrDefault(), CompactionMaxSegments = CompactionMaxSegments, + CompactionLowYieldBackoffSegments = CompactionLowYieldBackoffSegments, GossipSamplePercent = GossipSamplePercent, GossipDelay = GossipDelay, ClusterTimeout = ClusterTimeout, diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index a80d84ee38f..74d8bf52283 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -192,6 +192,9 @@ /* Number of log segments created on disk before compaction triggers. */ "CompactionMaxSegments" : 32, + /* Number of log segments the tail must grow before retrying after a compaction cycle that reclaimed no space. 0 = disabled. */ + "CompactionLowYieldBackoffSegments" : 0, + /* Enable Lua scripts on server. */ "EnableLua" : false, diff --git a/libs/server/Databases/CompactionPolicy.cs b/libs/server/Databases/CompactionPolicy.cs new file mode 100644 index 00000000000..2a42e334e79 --- /dev/null +++ b/libs/server/Databases/CompactionPolicy.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Garnet.server +{ + internal sealed class CompactionState + { + internal long RetryAfterTailAddress { get; private set; } + + internal bool ShouldSkip(long tailAddress) + => RetryAfterTailAddress > tailAddress; + + internal bool TryResume(long tailAddress) + { + if (RetryAfterTailAddress == 0 || tailAddress < RetryAfterTailAddress) + return false; + + RetryAfterTailAddress = 0; + return true; + } + + internal bool RecordCycle(long beginAddressBefore, long beginAddressAfter, long tailAddressBefore, + long tailAddressAfter, long retryBytes, int minReclaimPercent) + { + var beginAddressAdvance = beginAddressAfter - beginAddressBefore; + var tailAddressGrowth = tailAddressAfter - tailAddressBefore; + var netReclaimed = beginAddressAdvance - tailAddressGrowth; + + // A cycle is only "productive" if it reclaims at least minReclaimPercent of the range it + // scanned. A copy-forward pass over all-live data re-writes almost every byte it truncates, + // so netReclaimed sits at a few hundred bytes of segment-alignment noise even after moving + // tens of GB. The legacy "netReclaimed > 0" test treats that as progress and never backs + // off, so the wasteful loop runs forever. Requiring a meaningful reclaim floor parks it. + var minReclaim = minReclaimPercent > 0 + ? (long)Math.Ceiling(beginAddressAdvance * (minReclaimPercent / 100.0)) + : 1; + + if (beginAddressAdvance <= 0 || netReclaimed >= minReclaim) + { + RetryAfterTailAddress = 0; + return false; + } + + RetryAfterTailAddress = tailAddressAfter > long.MaxValue - retryBytes + ? long.MaxValue + : tailAddressAfter + retryBytes; + return true; + } + } + + internal static class CompactionPolicy + { + internal static long GetBackoffBytes(long segmentSize, int backoffSegments) + => backoffSegments > long.MaxValue / segmentSize + ? long.MaxValue + : segmentSize * backoffSegments; + + internal static long GetUntilAddress(long beginAddress, long readOnlyAddress, long segmentSize, + int maxSegments, int numSegmentsToCompact, bool boundCycle) + { + if (boundCycle) + return Math.Min(readOnlyAddress, beginAddress + segmentSize * numSegmentsToCompact); + + return readOnlyAddress - segmentSize * (maxSegments - numSegmentsToCompact); + } + } +} diff --git a/libs/server/Databases/DatabaseManagerBase.cs b/libs/server/Databases/DatabaseManagerBase.cs index 8e4d0b44622..30aca6d0004 100644 --- a/libs/server/Databases/DatabaseManagerBase.cs +++ b/libs/server/Databases/DatabaseManagerBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Garnet.common; @@ -427,17 +428,42 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg if (compactionType == LogCompactionType.None) return; var storeLog = db.Store.Log; + var segmentSize = 1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false); + var lowYieldBackoffSegments = StoreWrapper.serverOptions.CompactionLowYieldBackoffSegments; + var lowYieldBackoffEnabled = lowYieldBackoffSegments > 0 && + compactionType is LogCompactionType.Lookup or LogCompactionType.Scan; - var mainStoreMaxLogSize = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * mainStoreMaxSegments; + if (lowYieldBackoffEnabled) + { + if (db.CompactionState.ShouldSkip(storeLog.TailAddress)) + { + Logger?.LogDebug( + "Skipping low-yield compaction until tail reaches {retryTailAddress}; current Tail = {tailAddress}; DB ID = {id}", + db.CompactionState.RetryAfterTailAddress, storeLog.TailAddress, db.Id); + return; + } + + if (db.CompactionState.TryResume(storeLog.TailAddress)) + { + Logger?.LogInformation( + "Retrying compaction after foreground tail growth; Tail = {tailAddress}; DB ID = {id}", + storeLog.TailAddress, db.Id); + } + } + + var mainStoreMaxLogSize = segmentSize * mainStoreMaxSegments; if (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize) { + var beginAddressBefore = storeLog.BeginAddress; + var tailAddressBefore = storeLog.TailAddress; var readOnlyAddress = storeLog.ReadOnlyAddress; - var compactLength = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * (mainStoreMaxSegments - numSegmentsToCompact); - var untilAddress = readOnlyAddress - compactLength; + var untilAddress = CompactionPolicy.GetUntilAddress(beginAddressBefore, readOnlyAddress, segmentSize, + mainStoreMaxSegments, numSegmentsToCompact, lowYieldBackoffEnabled); + var stopwatch = Stopwatch.StartNew(); Logger?.LogInformation( - "Begin main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}", - untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress); + "Begin main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}; DB ID = {id}", + untilAddress, beginAddressBefore, readOnlyAddress, tailAddressBefore, db.Id); switch (compactionType) { @@ -464,9 +490,30 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg break; } + stopwatch.Stop(); + var beginAddressAfter = storeLog.BeginAddress; + var tailAddressAfter = storeLog.TailAddress; + var beginAddressAdvance = beginAddressAfter - beginAddressBefore; + var tailAddressGrowth = tailAddressAfter - tailAddressBefore; + var netReclaimedBytes = beginAddressAdvance - tailAddressGrowth; + var tailGrowthRatio = beginAddressAdvance > 0 ? tailAddressGrowth / (double)beginAddressAdvance : 0; + Logger?.LogInformation( - "End store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}", - untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress); + "End store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}; " + + "Begin advance = {beginAddressAdvance}; Tail growth = {tailAddressGrowth}; Net reclaimed = {netReclaimedBytes}; " + + "Tail growth ratio = {tailGrowthRatio}; Duration ms = {durationMs}; DB ID = {id}", + untilAddress, beginAddressAfter, readOnlyAddress, tailAddressAfter, beginAddressAdvance, tailAddressGrowth, + netReclaimedBytes, tailGrowthRatio, stopwatch.ElapsedMilliseconds, db.Id); + + if (lowYieldBackoffEnabled && + db.CompactionState.RecordCycle(beginAddressBefore, beginAddressAfter, tailAddressBefore, tailAddressAfter, + CompactionPolicy.GetBackoffBytes(segmentSize, lowYieldBackoffSegments), + StoreWrapper.serverOptions.CompactionLowYieldReclaimPercent)) + { + Logger?.LogWarning( + "Compaction reclaimed no space; pausing until tail reaches {retryTailAddress}; DB ID = {id}", + db.CompactionState.RetryAfterTailAddress, db.Id); + } } } diff --git a/libs/server/GarnetDatabase.cs b/libs/server/GarnetDatabase.cs index 5b099b5ae96..5e28a4b1ffc 100644 --- a/libs/server/GarnetDatabase.cs +++ b/libs/server/GarnetDatabase.cs @@ -69,6 +69,11 @@ public class GarnetDatabase : IDisposable /// public bool StoreIndexMaxedOut; + /// + /// State used to suppress repeated low-yield compaction cycles. + /// + internal readonly CompactionState CompactionState = new(); + /// /// Reader-Writer lock for database checkpointing /// diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs index 9fe08a38f96..ebdf4562f18 100644 --- a/libs/server/Servers/GarnetServerOptions.cs +++ b/libs/server/Servers/GarnetServerOptions.cs @@ -208,6 +208,24 @@ public class GarnetServerOptions : ServerOptions /// public int CompactionMaxSegments = 32; + /// + /// Number of log segments the tail must grow before retrying after a compaction cycle that + /// reclaimed no space. 0 disables low-yield backoff. + /// Default-enabled (32) so the protection is active even when a host wrapper constructs + /// GarnetServerOptions without forwarding this knob from configuration. + /// + public int CompactionLowYieldBackoffSegments = 32; + + /// + /// Minimum percent of the compacted (begin-address) range a cycle must reclaim to be + /// considered productive when low-yield backoff is enabled. A copy-forward pass over + /// all-live data re-writes almost every byte it truncates, so net reclaim hovers near + /// zero (only alignment noise); requiring a meaningful floor here prevents that wasteful + /// loop from being misclassified as productive. Only applies when + /// CompactionLowYieldBackoffSegments > 0. Range 0..100; default 20. + /// + public int CompactionLowYieldReclaimPercent = 20; + /// /// Percent of cluster nodes to gossip with at each gossip iteration. /// diff --git a/test/standalone/Garnet.test/CompactionPolicyTests.cs b/test/standalone/Garnet.test/CompactionPolicyTests.cs new file mode 100644 index 00000000000..453598da886 --- /dev/null +++ b/test/standalone/Garnet.test/CompactionPolicyTests.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.server; +using NUnit.Framework; + +namespace Garnet.test +{ + [TestFixture] + internal class CompactionPolicyTests + { + [Test] + public void LowYieldCycleBacksOffUntilForegroundGrowth() + { + const long segmentSize = 1024; + const int minReclaimPercent = 20; + var state = new CompactionState(); + + Assert.That(state.RecordCycle(0, segmentSize, 10 * segmentSize, 11 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.True); + Assert.That(state.RetryAfterTailAddress, Is.EqualTo(15 * segmentSize)); + Assert.That(state.ShouldSkip(14 * segmentSize), Is.True); + Assert.That(state.TryResume(14 * segmentSize), Is.False); + Assert.That(state.TryResume(15 * segmentSize), Is.True); + Assert.That(state.ShouldSkip(15 * segmentSize), Is.False); + } + + [Test] + public void TinyPositiveReclaimBacksOff() + { + // Regression for the all-live copy-forward loop: a cycle that truncates ~1000 segments + // but only reclaims 256 bytes (alignment noise) must be treated as low-yield and back + // off, even though net reclaim is strictly positive. The legacy "netReclaimed > 0" test + // misclassified this as productive and never parked. + const long segmentSize = 1024; + const int minReclaimPercent = 20; + var state = new CompactionState(); + + var beginAdvance = 1000 * segmentSize; + Assert.That(state.RecordCycle(0, beginAdvance, 0, beginAdvance - 256, 4 * segmentSize, minReclaimPercent), Is.True); + Assert.That(state.RetryAfterTailAddress, Is.EqualTo(beginAdvance - 256 + 4 * segmentSize)); + } + + [Test] + public void ProductiveCycleDoesNotBackOff() + { + const long segmentSize = 1024; + const int minReclaimPercent = 20; + var state = new CompactionState(); + + Assert.That(state.RecordCycle(0, segmentSize, 10 * segmentSize, 10 * segmentSize + 256, 4 * segmentSize, minReclaimPercent), Is.False); + Assert.That(state.RetryAfterTailAddress, Is.Zero); + Assert.That(state.ShouldSkip(10 * segmentSize + 256), Is.False); + } + + [Test] + public void LowYieldStateIsPerDatabase() + { + const long segmentSize = 1024; + const int minReclaimPercent = 20; + var first = new CompactionState(); + var second = new CompactionState(); + + Assert.That(first.RecordCycle(0, segmentSize, 0, segmentSize, segmentSize, minReclaimPercent), Is.True); + Assert.That(first.ShouldSkip(segmentSize), Is.True); + Assert.That(second.ShouldSkip(segmentSize), Is.False); + } + + [Test] + public void EnabledPolicyBoundsCycleToRequestedSegments() + { + const long segmentSize = 1024; + const long beginAddress = 2 * segmentSize; + const long readOnlyAddress = 40 * segmentSize; + + var untilAddress = CompactionPolicy.GetUntilAddress(beginAddress, readOnlyAddress, segmentSize, + maxSegments: 32, numSegmentsToCompact: 1, boundCycle: true); + + Assert.That(untilAddress, Is.EqualTo(3 * segmentSize)); + } + + [Test] + public void DisabledPolicyPreservesCatchUpBehavior() + { + const long segmentSize = 1024; + const long readOnlyAddress = 40 * segmentSize; + + var untilAddress = CompactionPolicy.GetUntilAddress(0, readOnlyAddress, segmentSize, + maxSegments: 32, numSegmentsToCompact: 1, boundCycle: false); + + Assert.That(untilAddress, Is.EqualTo(9 * segmentSize)); + } + + [Test] + public void BackoffByteCalculationSaturates() + { + Assert.That(CompactionPolicy.GetBackoffBytes(1L << 40, int.MaxValue), Is.EqualTo(long.MaxValue)); + } + } +} From f3c749fe3831b2feae78a663074096e99cd4cb01 Mon Sep 17 00:00:00 2001 From: Divya Mahendran Date: Mon, 24 Aug 2026 16:39:38 -0700 Subject: [PATCH 2/3] Address review feedback: clarify low-yield wording, add test coverage - Docs/help/log now describe the percentage-based low-yield gate instead of "reclaimed no space" (defaults.conf, Options.cs, GarnetServerOptions.cs, and the backoff LogWarning in DatabaseManagerBase.cs), since cycles with a small positive reclaim below CompactionLowYieldReclaimPercent also park. - CompactionPolicyTests now inherits TestBase so its cases are included in the repository's running-test diagnostics, matching the other fixtures. - Add CompactionLowYieldBackoffSegmentsParsing to GarnetServerConfigTests: covers the default (0), a positive CLI value, a positive JSON value, and rejection of a negative value, verifying each reaches GarnetServerOptions. Co-authored-by: GitHub Copilot <223556219+Copilot@users.noreply.github.com> --- libs/host/Configuration/Options.cs | 2 +- libs/host/defaults.conf | 2 +- libs/server/Databases/DatabaseManagerBase.cs | 2 +- libs/server/Servers/GarnetServerOptions.cs | 5 ++- .../Garnet.test/CompactionPolicyTests.cs | 2 +- .../Garnet.test/GarnetServerConfigTests.cs | 42 +++++++++++++++++++ 6 files changed, 49 insertions(+), 6 deletions(-) diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 9244ca7e223..86e91973f03 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -280,7 +280,7 @@ internal sealed class Options : ICloneable public int CompactionMaxSegments { get; set; } [IntRangeValidation(0, int.MaxValue)] - [Option("compaction-low-yield-backoff-segments", Required = false, HelpText = "Number of log segments the tail must grow before retrying after a compaction cycle that reclaimed no space. 0 = disabled.")] + [Option("compaction-low-yield-backoff-segments", Required = false, HelpText = "Number of log segments the tail must grow before retrying after a low-yield compaction cycle (one that reclaims less than CompactionLowYieldReclaimPercent of the range it scanned). 0 = disabled.")] public int CompactionLowYieldBackoffSegments { get; set; } [OptionValidation] diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index cc0b604fd99..e80cb5f93ef 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -210,7 +210,7 @@ /* Number of log segments created on disk before compaction triggers. */ "CompactionMaxSegments" : 32, - /* Number of log segments the tail must grow before retrying after a compaction cycle that reclaimed no space. 0 = disabled. */ + /* Number of log segments the tail must grow before retrying after a low-yield compaction cycle (one that reclaims less than CompactionLowYieldReclaimPercent of the range it scanned). 0 = disabled. */ "CompactionLowYieldBackoffSegments" : 0, /* Enable Lua scripts on server. */ diff --git a/libs/server/Databases/DatabaseManagerBase.cs b/libs/server/Databases/DatabaseManagerBase.cs index 30aca6d0004..2893c96ef4c 100644 --- a/libs/server/Databases/DatabaseManagerBase.cs +++ b/libs/server/Databases/DatabaseManagerBase.cs @@ -511,7 +511,7 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg StoreWrapper.serverOptions.CompactionLowYieldReclaimPercent)) { Logger?.LogWarning( - "Compaction reclaimed no space; pausing until tail reaches {retryTailAddress}; DB ID = {id}", + "Compaction was low-yield; pausing until tail reaches {retryTailAddress}; DB ID = {id}", db.CompactionState.RetryAfterTailAddress, db.Id); } } diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs index 7cbba72e825..0f64b8551b0 100644 --- a/libs/server/Servers/GarnetServerOptions.cs +++ b/libs/server/Servers/GarnetServerOptions.cs @@ -236,8 +236,9 @@ public class GarnetServerOptions : ServerOptions public int CompactionMaxSegments = 32; /// - /// Number of log segments the tail must grow before retrying after a compaction cycle that - /// reclaimed no space. 0 disables low-yield backoff. + /// Number of log segments the tail must grow before retrying after a low-yield compaction + /// cycle, i.e. one that reclaims less than of + /// the (begin-address) range it scanned. 0 disables low-yield backoff. /// Default-enabled (32) so the protection is active even when a host wrapper constructs /// GarnetServerOptions without forwarding this knob from configuration. /// diff --git a/test/standalone/Garnet.test/CompactionPolicyTests.cs b/test/standalone/Garnet.test/CompactionPolicyTests.cs index 453598da886..1d6d84adfdb 100644 --- a/test/standalone/Garnet.test/CompactionPolicyTests.cs +++ b/test/standalone/Garnet.test/CompactionPolicyTests.cs @@ -7,7 +7,7 @@ namespace Garnet.test { [TestFixture] - internal class CompactionPolicyTests + internal class CompactionPolicyTests : TestBase { [Test] public void LowYieldCycleBacksOffUntilForegroundGrowth() diff --git a/test/standalone/Garnet.test/GarnetServerConfigTests.cs b/test/standalone/Garnet.test/GarnetServerConfigTests.cs index 67d33446369..9df635698ef 100644 --- a/test/standalone/Garnet.test/GarnetServerConfigTests.cs +++ b/test/standalone/Garnet.test/GarnetServerConfigTests.cs @@ -2407,5 +2407,47 @@ public void InitialIORecordSizeParsing() ClassicAssert.AreEqual(4096, options.GetServerOptions().GetInitialIORecordSizeBytes()); } } + + [Test] + public void CompactionLowYieldBackoffSegmentsParsing() + { + // Default from defaults.conf is 0 (disabled) and must reach GarnetServerOptions. + { + var args = Array.Empty(); + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual(0, options.CompactionLowYieldBackoffSegments); + ClassicAssert.AreEqual(0, options.GetServerOptions().CompactionLowYieldBackoffSegments); + } + + // A positive CLI value is accepted and mapped onto GarnetServerOptions. + { + var args = new[] { "--compaction-low-yield-backoff-segments", "16" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual(16, options.CompactionLowYieldBackoffSegments); + ClassicAssert.AreEqual(16, options.GetServerOptions().CompactionLowYieldBackoffSegments); + } + + // A positive JSON value is accepted and mapped onto GarnetServerOptions. + { + const string JSON = @"{ ""CompactionLowYieldBackoffSegments"": 8 }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual(8, options.CompactionLowYieldBackoffSegments); + ClassicAssert.AreEqual(8, options.GetServerOptions().CompactionLowYieldBackoffSegments); + } + + // A negative value is rejected by IntRangeValidation. + { + var args = new[] { "--compaction-low-yield-backoff-segments", "-1" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out _, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsFalse(parseSuccessful, "A negative backoff segment count must be rejected"); + ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.CompactionLowYieldBackoffSegments))); + } + } } } \ No newline at end of file From bac84f092dbc349d67e2029b96477567a01331a7 Mon Sep 17 00:00:00 2001 From: Divya Mahendran Date: Mon, 24 Aug 2026 16:44:27 -0700 Subject: [PATCH 3/3] Drain the log fully with a per-chunk low-yield gate so productive workloads cannot outrun compaction When low-yield backoff is enabled, compaction previously ran a single bounded chunk per interval. A workload producing more than one chunk of garbage per interval could grow the log unbounded because compaction only ever reclaimed one chunk at a time. Convert the single-pass compaction block into a drain-fully loop: keep compacting bounded chunks while the log is over the configured limit. Productive chunks continue draining so high-churn workloads stay bounded; the first low-yield chunk parks compaction until the tail grows again (the all-live copy-forward case). The non-backoff path preserves the original single unbounded pass. Added a safety break for a productive chunk that fails to advance the begin address. Added ProductiveChunksDrainThenLowYieldParks documenting the drain-then-park state sequence the loop relies on. Co-authored-by: GitHub Copilot <223556219+Copilot@users.noreply.github.com> --- libs/server/Databases/DatabaseManagerBase.cs | 22 +++++++++++++++--- .../Garnet.test/CompactionPolicyTests.cs | 23 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/libs/server/Databases/DatabaseManagerBase.cs b/libs/server/Databases/DatabaseManagerBase.cs index 2893c96ef4c..370e33554f6 100644 --- a/libs/server/Databases/DatabaseManagerBase.cs +++ b/libs/server/Databases/DatabaseManagerBase.cs @@ -453,7 +453,12 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg var mainStoreMaxLogSize = segmentSize * mainStoreMaxSegments; - if (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize) + // Drain the log back under the configured limit. When low-yield backoff is disabled this + // runs a single unbounded pass (original behavior). When enabled, each iteration compacts a + // bounded chunk and measures its yield: productive chunks keep draining until the log is back + // under the limit, so a high-churn workload cannot outrun compaction, while the first low-yield + // chunk parks compaction until the tail grows again (the all-live copy-forward case). + while (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize) { var beginAddressBefore = storeLog.BeginAddress; var tailAddressBefore = storeLog.TailAddress; @@ -505,15 +510,26 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg untilAddress, beginAddressAfter, readOnlyAddress, tailAddressAfter, beginAddressAdvance, tailAddressGrowth, netReclaimedBytes, tailGrowthRatio, stopwatch.ElapsedMilliseconds, db.Id); - if (lowYieldBackoffEnabled && - db.CompactionState.RecordCycle(beginAddressBefore, beginAddressAfter, tailAddressBefore, tailAddressAfter, + if (!lowYieldBackoffEnabled) + { + // Original behavior: a single unbounded pass already brings the log under the limit. + break; + } + + if (db.CompactionState.RecordCycle(beginAddressBefore, beginAddressAfter, tailAddressBefore, tailAddressAfter, CompactionPolicy.GetBackoffBytes(segmentSize, lowYieldBackoffSegments), StoreWrapper.serverOptions.CompactionLowYieldReclaimPercent)) { Logger?.LogWarning( "Compaction was low-yield; pausing until tail reaches {retryTailAddress}; DB ID = {id}", db.CompactionState.RetryAfterTailAddress, db.Id); + break; } + + // Safety: if a chunk classified as productive did not actually advance the begin address, + // stop to avoid spinning; the next scheduled compaction will retry. + if (beginAddressAfter <= beginAddressBefore) + break; } } diff --git a/test/standalone/Garnet.test/CompactionPolicyTests.cs b/test/standalone/Garnet.test/CompactionPolicyTests.cs index 1d6d84adfdb..92c6c803998 100644 --- a/test/standalone/Garnet.test/CompactionPolicyTests.cs +++ b/test/standalone/Garnet.test/CompactionPolicyTests.cs @@ -65,6 +65,29 @@ public void LowYieldStateIsPerDatabase() Assert.That(second.ShouldSkip(segmentSize), Is.False); } + [Test] + public void ProductiveChunksDrainThenLowYieldParks() + { + // Models the drain-fully loop: several productive chunks in a row keep compaction running + // (no backoff, RetryAfterTailAddress stays cleared) and only the first low-yield chunk parks + // it. This is why a high-churn workload cannot outrun compaction while an all-live workload + // still backs off as soon as a chunk stops reclaiming. + const long segmentSize = 1024; + const int minReclaimPercent = 20; + var state = new CompactionState(); + + // Two productive chunks (reclaim >= 20% of the advanced range) → keep draining. + Assert.That(state.RecordCycle(0, 10 * segmentSize, 0, 5 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.False); + Assert.That(state.RetryAfterTailAddress, Is.Zero); + Assert.That(state.RecordCycle(10 * segmentSize, 20 * segmentSize, 5 * segmentSize, 12 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.False); + Assert.That(state.RetryAfterTailAddress, Is.Zero); + + // A low-yield chunk (advances 10 segments but reclaims only 256B of alignment noise) → park. + var tailAfter = 22 * segmentSize - 256; + Assert.That(state.RecordCycle(20 * segmentSize, 30 * segmentSize, 12 * segmentSize, tailAfter, 4 * segmentSize, minReclaimPercent), Is.True); + Assert.That(state.RetryAfterTailAddress, Is.EqualTo(tailAfter + 4 * segmentSize)); + } + [Test] public void EnabledPolicyBoundsCycleToRequestedSegments() {