diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs
index 5126af8e42a..86e91973f03 100644
--- a/libs/host/Configuration/Options.cs
+++ b/libs/host/Configuration/Options.cs
@@ -279,6 +279,10 @@ internal sealed class Options : ICloneable
[Option("compaction-max-segments", Required = false, HelpText = "Number of log segments created on disk before compaction triggers.")]
public int CompactionMaxSegments { get; set; }
+ [IntRangeValidation(0, int.MaxValue)]
+ [Option("compaction-low-yield-backoff-segments", Required = false, HelpText = "Number of log segments the tail must grow before retrying after a low-yield compaction cycle (one that reclaims less than CompactionLowYieldReclaimPercent of the range it scanned). 0 = disabled.")]
+ public int CompactionLowYieldBackoffSegments { get; set; }
+
[OptionValidation]
[Option("lua", Required = false, HelpText = "Enable Lua scripts on server.")]
public bool? EnableLua { get; set; }
@@ -939,6 +943,7 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno
CompactionType = CompactionType,
CompactionForceDelete = CompactionForceDelete.GetValueOrDefault(),
CompactionMaxSegments = CompactionMaxSegments,
+ CompactionLowYieldBackoffSegments = CompactionLowYieldBackoffSegments,
GossipSamplePercent = GossipSamplePercent,
GossipDelay = GossipDelay,
ClusterTimeout = ClusterTimeout,
diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf
index 7d72699d3c5..e80cb5f93ef 100644
--- a/libs/host/defaults.conf
+++ b/libs/host/defaults.conf
@@ -210,6 +210,9 @@
/* Number of log segments created on disk before compaction triggers. */
"CompactionMaxSegments" : 32,
+ /* Number of log segments the tail must grow before retrying after a low-yield compaction cycle (one that reclaims less than CompactionLowYieldReclaimPercent of the range it scanned). 0 = disabled. */
+ "CompactionLowYieldBackoffSegments" : 0,
+
/* Enable Lua scripts on server. */
"EnableLua" : false,
diff --git a/libs/server/Databases/CompactionPolicy.cs b/libs/server/Databases/CompactionPolicy.cs
new file mode 100644
index 00000000000..2a42e334e79
--- /dev/null
+++ b/libs/server/Databases/CompactionPolicy.cs
@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Garnet.server
+{
+ internal sealed class CompactionState
+ {
+ internal long RetryAfterTailAddress { get; private set; }
+
+ internal bool ShouldSkip(long tailAddress)
+ => RetryAfterTailAddress > tailAddress;
+
+ internal bool TryResume(long tailAddress)
+ {
+ if (RetryAfterTailAddress == 0 || tailAddress < RetryAfterTailAddress)
+ return false;
+
+ RetryAfterTailAddress = 0;
+ return true;
+ }
+
+ internal bool RecordCycle(long beginAddressBefore, long beginAddressAfter, long tailAddressBefore,
+ long tailAddressAfter, long retryBytes, int minReclaimPercent)
+ {
+ var beginAddressAdvance = beginAddressAfter - beginAddressBefore;
+ var tailAddressGrowth = tailAddressAfter - tailAddressBefore;
+ var netReclaimed = beginAddressAdvance - tailAddressGrowth;
+
+ // A cycle is only "productive" if it reclaims at least minReclaimPercent of the range it
+ // scanned. A copy-forward pass over all-live data re-writes almost every byte it truncates,
+ // so netReclaimed sits at a few hundred bytes of segment-alignment noise even after moving
+ // tens of GB. The legacy "netReclaimed > 0" test treats that as progress and never backs
+ // off, so the wasteful loop runs forever. Requiring a meaningful reclaim floor parks it.
+ var minReclaim = minReclaimPercent > 0
+ ? (long)Math.Ceiling(beginAddressAdvance * (minReclaimPercent / 100.0))
+ : 1;
+
+ if (beginAddressAdvance <= 0 || netReclaimed >= minReclaim)
+ {
+ RetryAfterTailAddress = 0;
+ return false;
+ }
+
+ RetryAfterTailAddress = tailAddressAfter > long.MaxValue - retryBytes
+ ? long.MaxValue
+ : tailAddressAfter + retryBytes;
+ return true;
+ }
+ }
+
+ internal static class CompactionPolicy
+ {
+ internal static long GetBackoffBytes(long segmentSize, int backoffSegments)
+ => backoffSegments > long.MaxValue / segmentSize
+ ? long.MaxValue
+ : segmentSize * backoffSegments;
+
+ internal static long GetUntilAddress(long beginAddress, long readOnlyAddress, long segmentSize,
+ int maxSegments, int numSegmentsToCompact, bool boundCycle)
+ {
+ if (boundCycle)
+ return Math.Min(readOnlyAddress, beginAddress + segmentSize * numSegmentsToCompact);
+
+ return readOnlyAddress - segmentSize * (maxSegments - numSegmentsToCompact);
+ }
+ }
+}
diff --git a/libs/server/Databases/DatabaseManagerBase.cs b/libs/server/Databases/DatabaseManagerBase.cs
index 8e4d0b44622..370e33554f6 100644
--- a/libs/server/Databases/DatabaseManagerBase.cs
+++ b/libs/server/Databases/DatabaseManagerBase.cs
@@ -2,6 +2,7 @@
// Licensed under the MIT license.
using System;
+using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Garnet.common;
@@ -427,17 +428,47 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg
if (compactionType == LogCompactionType.None) return;
var storeLog = db.Store.Log;
+ var segmentSize = 1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false);
+ var lowYieldBackoffSegments = StoreWrapper.serverOptions.CompactionLowYieldBackoffSegments;
+ var lowYieldBackoffEnabled = lowYieldBackoffSegments > 0 &&
+ compactionType is LogCompactionType.Lookup or LogCompactionType.Scan;
- var mainStoreMaxLogSize = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * mainStoreMaxSegments;
+ if (lowYieldBackoffEnabled)
+ {
+ if (db.CompactionState.ShouldSkip(storeLog.TailAddress))
+ {
+ Logger?.LogDebug(
+ "Skipping low-yield compaction until tail reaches {retryTailAddress}; current Tail = {tailAddress}; DB ID = {id}",
+ db.CompactionState.RetryAfterTailAddress, storeLog.TailAddress, db.Id);
+ return;
+ }
+
+ if (db.CompactionState.TryResume(storeLog.TailAddress))
+ {
+ Logger?.LogInformation(
+ "Retrying compaction after foreground tail growth; Tail = {tailAddress}; DB ID = {id}",
+ storeLog.TailAddress, db.Id);
+ }
+ }
+
+ var mainStoreMaxLogSize = segmentSize * mainStoreMaxSegments;
- if (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize)
+ // Drain the log back under the configured limit. When low-yield backoff is disabled this
+ // runs a single unbounded pass (original behavior). When enabled, each iteration compacts a
+ // bounded chunk and measures its yield: productive chunks keep draining until the log is back
+ // under the limit, so a high-churn workload cannot outrun compaction, while the first low-yield
+ // chunk parks compaction until the tail grows again (the all-live copy-forward case).
+ while (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize)
{
+ var beginAddressBefore = storeLog.BeginAddress;
+ var tailAddressBefore = storeLog.TailAddress;
var readOnlyAddress = storeLog.ReadOnlyAddress;
- var compactLength = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * (mainStoreMaxSegments - numSegmentsToCompact);
- var untilAddress = readOnlyAddress - compactLength;
+ var untilAddress = CompactionPolicy.GetUntilAddress(beginAddressBefore, readOnlyAddress, segmentSize,
+ mainStoreMaxSegments, numSegmentsToCompact, lowYieldBackoffEnabled);
+ var stopwatch = Stopwatch.StartNew();
Logger?.LogInformation(
- "Begin main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}",
- untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress);
+ "Begin main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}; DB ID = {id}",
+ untilAddress, beginAddressBefore, readOnlyAddress, tailAddressBefore, db.Id);
switch (compactionType)
{
@@ -464,9 +495,41 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg
break;
}
+ stopwatch.Stop();
+ var beginAddressAfter = storeLog.BeginAddress;
+ var tailAddressAfter = storeLog.TailAddress;
+ var beginAddressAdvance = beginAddressAfter - beginAddressBefore;
+ var tailAddressGrowth = tailAddressAfter - tailAddressBefore;
+ var netReclaimedBytes = beginAddressAdvance - tailAddressGrowth;
+ var tailGrowthRatio = beginAddressAdvance > 0 ? tailAddressGrowth / (double)beginAddressAdvance : 0;
+
Logger?.LogInformation(
- "End store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}",
- untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress);
+ "End store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}; " +
+ "Begin advance = {beginAddressAdvance}; Tail growth = {tailAddressGrowth}; Net reclaimed = {netReclaimedBytes}; " +
+ "Tail growth ratio = {tailGrowthRatio}; Duration ms = {durationMs}; DB ID = {id}",
+ untilAddress, beginAddressAfter, readOnlyAddress, tailAddressAfter, beginAddressAdvance, tailAddressGrowth,
+ netReclaimedBytes, tailGrowthRatio, stopwatch.ElapsedMilliseconds, db.Id);
+
+ if (!lowYieldBackoffEnabled)
+ {
+ // Original behavior: a single unbounded pass already brings the log under the limit.
+ break;
+ }
+
+ if (db.CompactionState.RecordCycle(beginAddressBefore, beginAddressAfter, tailAddressBefore, tailAddressAfter,
+ CompactionPolicy.GetBackoffBytes(segmentSize, lowYieldBackoffSegments),
+ StoreWrapper.serverOptions.CompactionLowYieldReclaimPercent))
+ {
+ Logger?.LogWarning(
+ "Compaction was low-yield; pausing until tail reaches {retryTailAddress}; DB ID = {id}",
+ db.CompactionState.RetryAfterTailAddress, db.Id);
+ break;
+ }
+
+ // Safety: if a chunk classified as productive did not actually advance the begin address,
+ // stop to avoid spinning; the next scheduled compaction will retry.
+ if (beginAddressAfter <= beginAddressBefore)
+ break;
}
}
diff --git a/libs/server/GarnetDatabase.cs b/libs/server/GarnetDatabase.cs
index 5b099b5ae96..5e28a4b1ffc 100644
--- a/libs/server/GarnetDatabase.cs
+++ b/libs/server/GarnetDatabase.cs
@@ -69,6 +69,11 @@ public class GarnetDatabase : IDisposable
///
public bool StoreIndexMaxedOut;
+ ///
+ /// State used to suppress repeated low-yield compaction cycles.
+ ///
+ internal readonly CompactionState CompactionState = new();
+
///
/// Reader-Writer lock for database checkpointing
///
diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs
index 3d89c1709c1..0f64b8551b0 100644
--- a/libs/server/Servers/GarnetServerOptions.cs
+++ b/libs/server/Servers/GarnetServerOptions.cs
@@ -235,6 +235,25 @@ public class GarnetServerOptions : ServerOptions
///
public int CompactionMaxSegments = 32;
+ ///
+ /// Number of log segments the tail must grow before retrying after a low-yield compaction
+ /// cycle, i.e. one that reclaims less than of
+ /// the (begin-address) range it scanned. 0 disables low-yield backoff.
+ /// Default-enabled (32) so the protection is active even when a host wrapper constructs
+ /// GarnetServerOptions without forwarding this knob from configuration.
+ ///
+ public int CompactionLowYieldBackoffSegments = 32;
+
+ ///
+ /// Minimum percent of the compacted (begin-address) range a cycle must reclaim to be
+ /// considered productive when low-yield backoff is enabled. A copy-forward pass over
+ /// all-live data re-writes almost every byte it truncates, so net reclaim hovers near
+ /// zero (only alignment noise); requiring a meaningful floor here prevents that wasteful
+ /// loop from being misclassified as productive. Only applies when
+ /// CompactionLowYieldBackoffSegments > 0. Range 0..100; default 20.
+ ///
+ public int CompactionLowYieldReclaimPercent = 20;
+
///
/// Percent of cluster nodes to gossip with at each gossip iteration.
///
diff --git a/test/standalone/Garnet.test/CompactionPolicyTests.cs b/test/standalone/Garnet.test/CompactionPolicyTests.cs
new file mode 100644
index 00000000000..92c6c803998
--- /dev/null
+++ b/test/standalone/Garnet.test/CompactionPolicyTests.cs
@@ -0,0 +1,122 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using Garnet.server;
+using NUnit.Framework;
+
+namespace Garnet.test
+{
+ [TestFixture]
+ internal class CompactionPolicyTests : TestBase
+ {
+ [Test]
+ public void LowYieldCycleBacksOffUntilForegroundGrowth()
+ {
+ const long segmentSize = 1024;
+ const int minReclaimPercent = 20;
+ var state = new CompactionState();
+
+ Assert.That(state.RecordCycle(0, segmentSize, 10 * segmentSize, 11 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.True);
+ Assert.That(state.RetryAfterTailAddress, Is.EqualTo(15 * segmentSize));
+ Assert.That(state.ShouldSkip(14 * segmentSize), Is.True);
+ Assert.That(state.TryResume(14 * segmentSize), Is.False);
+ Assert.That(state.TryResume(15 * segmentSize), Is.True);
+ Assert.That(state.ShouldSkip(15 * segmentSize), Is.False);
+ }
+
+ [Test]
+ public void TinyPositiveReclaimBacksOff()
+ {
+ // Regression for the all-live copy-forward loop: a cycle that truncates ~1000 segments
+ // but only reclaims 256 bytes (alignment noise) must be treated as low-yield and back
+ // off, even though net reclaim is strictly positive. The legacy "netReclaimed > 0" test
+ // misclassified this as productive and never parked.
+ const long segmentSize = 1024;
+ const int minReclaimPercent = 20;
+ var state = new CompactionState();
+
+ var beginAdvance = 1000 * segmentSize;
+ Assert.That(state.RecordCycle(0, beginAdvance, 0, beginAdvance - 256, 4 * segmentSize, minReclaimPercent), Is.True);
+ Assert.That(state.RetryAfterTailAddress, Is.EqualTo(beginAdvance - 256 + 4 * segmentSize));
+ }
+
+ [Test]
+ public void ProductiveCycleDoesNotBackOff()
+ {
+ const long segmentSize = 1024;
+ const int minReclaimPercent = 20;
+ var state = new CompactionState();
+
+ Assert.That(state.RecordCycle(0, segmentSize, 10 * segmentSize, 10 * segmentSize + 256, 4 * segmentSize, minReclaimPercent), Is.False);
+ Assert.That(state.RetryAfterTailAddress, Is.Zero);
+ Assert.That(state.ShouldSkip(10 * segmentSize + 256), Is.False);
+ }
+
+ [Test]
+ public void LowYieldStateIsPerDatabase()
+ {
+ const long segmentSize = 1024;
+ const int minReclaimPercent = 20;
+ var first = new CompactionState();
+ var second = new CompactionState();
+
+ Assert.That(first.RecordCycle(0, segmentSize, 0, segmentSize, segmentSize, minReclaimPercent), Is.True);
+ Assert.That(first.ShouldSkip(segmentSize), Is.True);
+ Assert.That(second.ShouldSkip(segmentSize), Is.False);
+ }
+
+ [Test]
+ public void ProductiveChunksDrainThenLowYieldParks()
+ {
+ // Models the drain-fully loop: several productive chunks in a row keep compaction running
+ // (no backoff, RetryAfterTailAddress stays cleared) and only the first low-yield chunk parks
+ // it. This is why a high-churn workload cannot outrun compaction while an all-live workload
+ // still backs off as soon as a chunk stops reclaiming.
+ const long segmentSize = 1024;
+ const int minReclaimPercent = 20;
+ var state = new CompactionState();
+
+ // Two productive chunks (reclaim >= 20% of the advanced range) → keep draining.
+ Assert.That(state.RecordCycle(0, 10 * segmentSize, 0, 5 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.False);
+ Assert.That(state.RetryAfterTailAddress, Is.Zero);
+ Assert.That(state.RecordCycle(10 * segmentSize, 20 * segmentSize, 5 * segmentSize, 12 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.False);
+ Assert.That(state.RetryAfterTailAddress, Is.Zero);
+
+ // A low-yield chunk (advances 10 segments but reclaims only 256B of alignment noise) → park.
+ var tailAfter = 22 * segmentSize - 256;
+ Assert.That(state.RecordCycle(20 * segmentSize, 30 * segmentSize, 12 * segmentSize, tailAfter, 4 * segmentSize, minReclaimPercent), Is.True);
+ Assert.That(state.RetryAfterTailAddress, Is.EqualTo(tailAfter + 4 * segmentSize));
+ }
+
+ [Test]
+ public void EnabledPolicyBoundsCycleToRequestedSegments()
+ {
+ const long segmentSize = 1024;
+ const long beginAddress = 2 * segmentSize;
+ const long readOnlyAddress = 40 * segmentSize;
+
+ var untilAddress = CompactionPolicy.GetUntilAddress(beginAddress, readOnlyAddress, segmentSize,
+ maxSegments: 32, numSegmentsToCompact: 1, boundCycle: true);
+
+ Assert.That(untilAddress, Is.EqualTo(3 * segmentSize));
+ }
+
+ [Test]
+ public void DisabledPolicyPreservesCatchUpBehavior()
+ {
+ const long segmentSize = 1024;
+ const long readOnlyAddress = 40 * segmentSize;
+
+ var untilAddress = CompactionPolicy.GetUntilAddress(0, readOnlyAddress, segmentSize,
+ maxSegments: 32, numSegmentsToCompact: 1, boundCycle: false);
+
+ Assert.That(untilAddress, Is.EqualTo(9 * segmentSize));
+ }
+
+ [Test]
+ public void BackoffByteCalculationSaturates()
+ {
+ Assert.That(CompactionPolicy.GetBackoffBytes(1L << 40, int.MaxValue), Is.EqualTo(long.MaxValue));
+ }
+ }
+}
diff --git a/test/standalone/Garnet.test/GarnetServerConfigTests.cs b/test/standalone/Garnet.test/GarnetServerConfigTests.cs
index 67d33446369..9df635698ef 100644
--- a/test/standalone/Garnet.test/GarnetServerConfigTests.cs
+++ b/test/standalone/Garnet.test/GarnetServerConfigTests.cs
@@ -2407,5 +2407,47 @@ public void InitialIORecordSizeParsing()
ClassicAssert.AreEqual(4096, options.GetServerOptions().GetInitialIORecordSizeBytes());
}
}
+
+ [Test]
+ public void CompactionLowYieldBackoffSegmentsParsing()
+ {
+ // Default from defaults.conf is 0 (disabled) and must reach GarnetServerOptions.
+ {
+ var args = Array.Empty();
+ var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true);
+ ClassicAssert.IsTrue(parseSuccessful);
+ ClassicAssert.AreEqual(0, invalidOptions.Count);
+ ClassicAssert.AreEqual(0, options.CompactionLowYieldBackoffSegments);
+ ClassicAssert.AreEqual(0, options.GetServerOptions().CompactionLowYieldBackoffSegments);
+ }
+
+ // A positive CLI value is accepted and mapped onto GarnetServerOptions.
+ {
+ var args = new[] { "--compaction-low-yield-backoff-segments", "16" };
+ var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true);
+ ClassicAssert.IsTrue(parseSuccessful);
+ ClassicAssert.AreEqual(0, invalidOptions.Count);
+ ClassicAssert.AreEqual(16, options.CompactionLowYieldBackoffSegments);
+ ClassicAssert.AreEqual(16, options.GetServerOptions().CompactionLowYieldBackoffSegments);
+ }
+
+ // A positive JSON value is accepted and mapped onto GarnetServerOptions.
+ {
+ const string JSON = @"{ ""CompactionLowYieldBackoffSegments"": 8 }";
+ var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out _);
+ ClassicAssert.IsTrue(parseSuccessful);
+ ClassicAssert.AreEqual(0, invalidOptions.Count);
+ ClassicAssert.AreEqual(8, options.CompactionLowYieldBackoffSegments);
+ ClassicAssert.AreEqual(8, options.GetServerOptions().CompactionLowYieldBackoffSegments);
+ }
+
+ // A negative value is rejected by IntRangeValidation.
+ {
+ var args = new[] { "--compaction-low-yield-backoff-segments", "-1" };
+ var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out _, out var invalidOptions, out _, out _, silentMode: true);
+ ClassicAssert.IsFalse(parseSuccessful, "A negative backoff segment count must be rejected");
+ ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.CompactionLowYieldBackoffSegments)));
+ }
+ }
}
}
\ No newline at end of file