Add low-yield compaction backoff to stop wasteful copy-forward on all-live data

libs/host/Configuration/Options.cs

@@ -279,6 +279,10 @@ internal sealed class Options : ICloneable
         [Option("compaction-max-segments", Required = false, HelpText = "Number of log segments created on disk before compaction triggers.")]
         public int CompactionMaxSegments { get; set; }
 
+        [IntRangeValidation(0, int.MaxValue)]
+        [Option("compaction-low-yield-backoff-segments", Required = false, HelpText = "Number of log segments the tail must grow before retrying after a low-yield compaction cycle (one that reclaims less than CompactionLowYieldReclaimPercent of the range it scanned). 0 = disabled.")]
+        public int CompactionLowYieldBackoffSegments { get; set; }
+
         [OptionValidation]
         [Option("lua", Required = false, HelpText = "Enable Lua scripts on server.")]
         public bool? EnableLua { get; set; }
@@ -939,6 +943,7 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno
                 CompactionType = CompactionType,
                 CompactionForceDelete = CompactionForceDelete.GetValueOrDefault(),
                 CompactionMaxSegments = CompactionMaxSegments,
+                CompactionLowYieldBackoffSegments = CompactionLowYieldBackoffSegments,
                 GossipSamplePercent = GossipSamplePercent,
                 GossipDelay = GossipDelay,
                 ClusterTimeout = ClusterTimeout,

libs/host/defaults.conf

@@ -210,6 +210,9 @@
 	/* Number of log segments created on disk before compaction triggers. */
 	"CompactionMaxSegments" : 32,
 
+	/* Number of log segments the tail must grow before retrying after a low-yield compaction cycle (one that reclaims less than CompactionLowYieldReclaimPercent of the range it scanned). 0 = disabled. */
+	"CompactionLowYieldBackoffSegments" : 0,
+
 	/* Enable Lua scripts on server. */
 	"EnableLua" : false,
 

libs/server/Databases/CompactionPolicy.cs

@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Garnet.server
+{
+    internal sealed class CompactionState
+    {
+        internal long RetryAfterTailAddress { get; private set; }
+
+        internal bool ShouldSkip(long tailAddress)
+            => RetryAfterTailAddress > tailAddress;
+
+        internal bool TryResume(long tailAddress)
+        {
+            if (RetryAfterTailAddress == 0 || tailAddress < RetryAfterTailAddress)
+                return false;
+
+            RetryAfterTailAddress = 0;
+            return true;
+        }
+
+        internal bool RecordCycle(long beginAddressBefore, long beginAddressAfter, long tailAddressBefore,
+            long tailAddressAfter, long retryBytes, int minReclaimPercent)
+        {
+            var beginAddressAdvance = beginAddressAfter - beginAddressBefore;
+            var tailAddressGrowth = tailAddressAfter - tailAddressBefore;
+            var netReclaimed = beginAddressAdvance - tailAddressGrowth;
+
+            // A cycle is only "productive" if it reclaims at least minReclaimPercent of the range it
+            // scanned. A copy-forward pass over all-live data re-writes almost every byte it truncates,
+            // so netReclaimed sits at a few hundred bytes of segment-alignment noise even after moving
+            // tens of GB. The legacy "netReclaimed > 0" test treats that as progress and never backs
+            // off, so the wasteful loop runs forever. Requiring a meaningful reclaim floor parks it.
+            var minReclaim = minReclaimPercent > 0
+                ? (long)Math.Ceiling(beginAddressAdvance * (minReclaimPercent / 100.0))
+                : 1;
+
+            if (beginAddressAdvance <= 0 || netReclaimed >= minReclaim)
+            {
+                RetryAfterTailAddress = 0;
+                return false;
+            }
+
+            RetryAfterTailAddress = tailAddressAfter > long.MaxValue - retryBytes
+                ? long.MaxValue
+                : tailAddressAfter + retryBytes;
+            return true;
+        }
+    }
+
+    internal static class CompactionPolicy
+    {
+        internal static long GetBackoffBytes(long segmentSize, int backoffSegments)
+            => backoffSegments > long.MaxValue / segmentSize
+                ? long.MaxValue
+                : segmentSize * backoffSegments;
+
+        internal static long GetUntilAddress(long beginAddress, long readOnlyAddress, long segmentSize,
+            int maxSegments, int numSegmentsToCompact, bool boundCycle)
+        {
+            if (boundCycle)
+                return Math.Min(readOnlyAddress, beginAddress + segmentSize * numSegmentsToCompact);
+
+            return readOnlyAddress - segmentSize * (maxSegments - numSegmentsToCompact);
+        }
+    }
+}

libs/server/Databases/DatabaseManagerBase.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using Garnet.common;
@@ -427,17 +428,47 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg
             if (compactionType == LogCompactionType.None) return;
 
             var storeLog = db.Store.Log;
+            var segmentSize = 1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false);
+            var lowYieldBackoffSegments = StoreWrapper.serverOptions.CompactionLowYieldBackoffSegments;
+            var lowYieldBackoffEnabled = lowYieldBackoffSegments > 0 &&
+                                         compactionType is LogCompactionType.Lookup or LogCompactionType.Scan;
 
-            var mainStoreMaxLogSize = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * mainStoreMaxSegments;
+            if (lowYieldBackoffEnabled)
+            {
+                if (db.CompactionState.ShouldSkip(storeLog.TailAddress))
+                {
+                    Logger?.LogDebug(
+                        "Skipping low-yield compaction until tail reaches {retryTailAddress}; current Tail = {tailAddress}; DB ID = {id}",
+                        db.CompactionState.RetryAfterTailAddress, storeLog.TailAddress, db.Id);
+                    return;
+                }
+
+                if (db.CompactionState.TryResume(storeLog.TailAddress))
+                {
+                    Logger?.LogInformation(
+                        "Retrying compaction after foreground tail growth; Tail = {tailAddress}; DB ID = {id}",
+                        storeLog.TailAddress, db.Id);
+                }
+            }
+
+            var mainStoreMaxLogSize = segmentSize * mainStoreMaxSegments;
 
-            if (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize)
+            // Drain the log back under the configured limit. When low-yield backoff is disabled this
+            // runs a single unbounded pass (original behavior). When enabled, each iteration compacts a
+            // bounded chunk and measures its yield: productive chunks keep draining until the log is back
+            // under the limit, so a high-churn workload cannot outrun compaction, while the first low-yield
+            // chunk parks compaction until the tail grows again (the all-live copy-forward case).
+            while (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize)
             {
+                var beginAddressBefore = storeLog.BeginAddress;
+                var tailAddressBefore = storeLog.TailAddress;
                 var readOnlyAddress = storeLog.ReadOnlyAddress;
-                var compactLength = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * (mainStoreMaxSegments - numSegmentsToCompact);
-                var untilAddress = readOnlyAddress - compactLength;
+                var untilAddress = CompactionPolicy.GetUntilAddress(beginAddressBefore, readOnlyAddress, segmentSize,
+                    mainStoreMaxSegments, numSegmentsToCompact, lowYieldBackoffEnabled);
+                var stopwatch = Stopwatch.StartNew();
                 Logger?.LogInformation(
-                    "Begin main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}",
-                    untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress);
+                    "Begin main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}; DB ID = {id}",
+                    untilAddress, beginAddressBefore, readOnlyAddress, tailAddressBefore, db.Id);
 
                 switch (compactionType)
                 {
@@ -464,9 +495,41 @@ private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSeg
                         break;
                 }
 
+                stopwatch.Stop();
+                var beginAddressAfter = storeLog.BeginAddress;
+                var tailAddressAfter = storeLog.TailAddress;
+                var beginAddressAdvance = beginAddressAfter - beginAddressBefore;
+                var tailAddressGrowth = tailAddressAfter - tailAddressBefore;
+                var netReclaimedBytes = beginAddressAdvance - tailAddressGrowth;
+                var tailGrowthRatio = beginAddressAdvance > 0 ? tailAddressGrowth / (double)beginAddressAdvance : 0;
+
                 Logger?.LogInformation(
-                    "End store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}",
-                    untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress);
+                    "End store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}; " +
+                    "Begin advance = {beginAddressAdvance}; Tail growth = {tailAddressGrowth}; Net reclaimed = {netReclaimedBytes}; " +
+                    "Tail growth ratio = {tailGrowthRatio}; Duration ms = {durationMs}; DB ID = {id}",
+                    untilAddress, beginAddressAfter, readOnlyAddress, tailAddressAfter, beginAddressAdvance, tailAddressGrowth,
+                    netReclaimedBytes, tailGrowthRatio, stopwatch.ElapsedMilliseconds, db.Id);
+
+                if (!lowYieldBackoffEnabled)
+                {
+                    // Original behavior: a single unbounded pass already brings the log under the limit.
+                    break;
+                }
+
+                if (db.CompactionState.RecordCycle(beginAddressBefore, beginAddressAfter, tailAddressBefore, tailAddressAfter,
+                        CompactionPolicy.GetBackoffBytes(segmentSize, lowYieldBackoffSegments),
+                        StoreWrapper.serverOptions.CompactionLowYieldReclaimPercent))
+                {
+                    Logger?.LogWarning(
+                        "Compaction was low-yield; pausing until tail reaches {retryTailAddress}; DB ID = {id}",
+                        db.CompactionState.RetryAfterTailAddress, db.Id);
+                    break;
+                }
+
+                // Safety: if a chunk classified as productive did not actually advance the begin address,
+                // stop to avoid spinning; the next scheduled compaction will retry.
+                if (beginAddressAfter <= beginAddressBefore)
+                    break;
             }
         }
 

libs/server/GarnetDatabase.cs

@@ -69,6 +69,11 @@ public class GarnetDatabase : IDisposable
         /// </summary>
         public bool StoreIndexMaxedOut;
 
+        /// <summary>
+        /// State used to suppress repeated low-yield compaction cycles.
+        /// </summary>
+        internal readonly CompactionState CompactionState = new();
+
         /// <summary>
         /// Reader-Writer lock for database checkpointing
         /// </summary>

libs/server/Servers/GarnetServerOptions.cs

@@ -235,6 +235,25 @@ public class GarnetServerOptions : ServerOptions
         /// </summary>
         public int CompactionMaxSegments = 32;
 
+        /// <summary>
+        /// Number of log segments the tail must grow before retrying after a low-yield compaction
+        /// cycle, i.e. one that reclaims less than <see cref="CompactionLowYieldReclaimPercent"/> of
+        /// the (begin-address) range it scanned. 0 disables low-yield backoff.
+        /// Default-enabled (32) so the protection is active even when a host wrapper constructs
+        /// GarnetServerOptions without forwarding this knob from configuration.
+        /// </summary>
+        public int CompactionLowYieldBackoffSegments = 32;
+
+        /// <summary>
+        /// Minimum percent of the compacted (begin-address) range a cycle must reclaim to be
+        /// considered productive when low-yield backoff is enabled. A copy-forward pass over
+        /// all-live data re-writes almost every byte it truncates, so net reclaim hovers near
+        /// zero (only alignment noise); requiring a meaningful floor here prevents that wasteful
+        /// loop from being misclassified as productive. Only applies when
+        /// CompactionLowYieldBackoffSegments &gt; 0. Range 0..100; default 20.
+        /// </summary>
+        public int CompactionLowYieldReclaimPercent = 20;
+
         /// <summary>
         /// Percent of cluster nodes to gossip with at each gossip iteration.
         /// </summary>

test/standalone/Garnet.test/CompactionPolicyTests.cs

@@ -0,0 +1,122 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using Garnet.server;
+using NUnit.Framework;
+
+namespace Garnet.test
+{
+    [TestFixture]
+    internal class CompactionPolicyTests : TestBase
+    {
+        [Test]
+        public void LowYieldCycleBacksOffUntilForegroundGrowth()
+        {
+            const long segmentSize = 1024;
+            const int minReclaimPercent = 20;
+            var state = new CompactionState();
+
+            Assert.That(state.RecordCycle(0, segmentSize, 10 * segmentSize, 11 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.True);
+            Assert.That(state.RetryAfterTailAddress, Is.EqualTo(15 * segmentSize));
+            Assert.That(state.ShouldSkip(14 * segmentSize), Is.True);
+            Assert.That(state.TryResume(14 * segmentSize), Is.False);
+            Assert.That(state.TryResume(15 * segmentSize), Is.True);
+            Assert.That(state.ShouldSkip(15 * segmentSize), Is.False);
+        }
+
+        [Test]
+        public void TinyPositiveReclaimBacksOff()
+        {
+            // Regression for the all-live copy-forward loop: a cycle that truncates ~1000 segments
+            // but only reclaims 256 bytes (alignment noise) must be treated as low-yield and back
+            // off, even though net reclaim is strictly positive. The legacy "netReclaimed > 0" test
+            // misclassified this as productive and never parked.
+            const long segmentSize = 1024;
+            const int minReclaimPercent = 20;
+            var state = new CompactionState();
+
+            var beginAdvance = 1000 * segmentSize;
+            Assert.That(state.RecordCycle(0, beginAdvance, 0, beginAdvance - 256, 4 * segmentSize, minReclaimPercent), Is.True);
+            Assert.That(state.RetryAfterTailAddress, Is.EqualTo(beginAdvance - 256 + 4 * segmentSize));
+        }
+
+        [Test]
+        public void ProductiveCycleDoesNotBackOff()
+        {
+            const long segmentSize = 1024;
+            const int minReclaimPercent = 20;
+            var state = new CompactionState();
+
+            Assert.That(state.RecordCycle(0, segmentSize, 10 * segmentSize, 10 * segmentSize + 256, 4 * segmentSize, minReclaimPercent), Is.False);
+            Assert.That(state.RetryAfterTailAddress, Is.Zero);
+            Assert.That(state.ShouldSkip(10 * segmentSize + 256), Is.False);
+        }
+
+        [Test]
+        public void LowYieldStateIsPerDatabase()
+        {
+            const long segmentSize = 1024;
+            const int minReclaimPercent = 20;
+            var first = new CompactionState();
+            var second = new CompactionState();
+
+            Assert.That(first.RecordCycle(0, segmentSize, 0, segmentSize, segmentSize, minReclaimPercent), Is.True);
+            Assert.That(first.ShouldSkip(segmentSize), Is.True);
+            Assert.That(second.ShouldSkip(segmentSize), Is.False);
+        }
+
+        [Test]
+        public void ProductiveChunksDrainThenLowYieldParks()
+        {
+            // Models the drain-fully loop: several productive chunks in a row keep compaction running
+            // (no backoff, RetryAfterTailAddress stays cleared) and only the first low-yield chunk parks
+            // it. This is why a high-churn workload cannot outrun compaction while an all-live workload
+            // still backs off as soon as a chunk stops reclaiming.
+            const long segmentSize = 1024;
+            const int minReclaimPercent = 20;
+            var state = new CompactionState();
+
+            // Two productive chunks (reclaim >= 20% of the advanced range) → keep draining.
+            Assert.That(state.RecordCycle(0, 10 * segmentSize, 0, 5 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.False);
+            Assert.That(state.RetryAfterTailAddress, Is.Zero);
+            Assert.That(state.RecordCycle(10 * segmentSize, 20 * segmentSize, 5 * segmentSize, 12 * segmentSize, 4 * segmentSize, minReclaimPercent), Is.False);
+            Assert.That(state.RetryAfterTailAddress, Is.Zero);
+
+            // A low-yield chunk (advances 10 segments but reclaims only 256B of alignment noise) → park.
+            var tailAfter = 22 * segmentSize - 256;
+            Assert.That(state.RecordCycle(20 * segmentSize, 30 * segmentSize, 12 * segmentSize, tailAfter, 4 * segmentSize, minReclaimPercent), Is.True);
+            Assert.That(state.RetryAfterTailAddress, Is.EqualTo(tailAfter + 4 * segmentSize));
+        }
+
+        [Test]
+        public void EnabledPolicyBoundsCycleToRequestedSegments()
+        {
+            const long segmentSize = 1024;
+            const long beginAddress = 2 * segmentSize;
+            const long readOnlyAddress = 40 * segmentSize;
+
+            var untilAddress = CompactionPolicy.GetUntilAddress(beginAddress, readOnlyAddress, segmentSize,
+                maxSegments: 32, numSegmentsToCompact: 1, boundCycle: true);
+
+            Assert.That(untilAddress, Is.EqualTo(3 * segmentSize));
+        }
+
+        [Test]
+        public void DisabledPolicyPreservesCatchUpBehavior()
+        {
+            const long segmentSize = 1024;
+            const long readOnlyAddress = 40 * segmentSize;
+
+            var untilAddress = CompactionPolicy.GetUntilAddress(0, readOnlyAddress, segmentSize,
+                maxSegments: 32, numSegmentsToCompact: 1, boundCycle: false);
+
+            Assert.That(untilAddress, Is.EqualTo(9 * segmentSize));
+        }
+
+        [Test]
+        public void BackoffByteCalculationSaturates()
+        {
+            Assert.That(CompactionPolicy.GetBackoffBytes(1L << 40, int.MaxValue), Is.EqualTo(long.MaxValue));
+        }
+    }
+}