From 3c098439882054d93d4429eaf9a3169c42c4a0cb Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 20:35:12 +0000 Subject: [PATCH 1/3] SOLR-18506: Fix TestThinCache.testSimple flakiness from Caffeine eviction notification race --- .../org/apache/solr/search/TestThinCache.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/solr/core/src/test/org/apache/solr/search/TestThinCache.java b/solr/core/src/test/org/apache/solr/search/TestThinCache.java index 7d668a8e9ed5..83f926b9d7fb 100644 --- a/solr/core/src/test/org/apache/solr/search/TestThinCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestThinCache.java @@ -150,6 +150,18 @@ public void testSimple() throws IOException { assertNull(lfuCache.get(1)); // first item put in should be the first out + // Everything after this point used to run against a backing cache that stayed full: + // warming copies 25 entries in and the test then puts key 103, and every put into the + // full backing evicts an entry. An eviction whose victim belongs to the second cache's + // scope is counted in that cache's evictions metric, and Caffeine's lazy eviction + // maintenance decides whether such an eviction has been delivered by the time the + // metric is sampled at the end of the test; that race is the flake in SOLR-18506, + // where the second cache read 1 eviction instead of 0. Enlarging the backing to 200 + // (both phases together hold at most 126 distinct entries) means nothing is evicted + // from here on, under any interleaving. As a side effect setMaxSize runs Caffeine + // cleanUp, settling any maintenance still pending from the puts above. + backing.setMaxSize(200); + // Test autowarming newLFUCache.init(Map.of("autowarmCount", "25"), initObj, regenerator); newLFUCache.warm(null, lfuCache); @@ -167,6 +179,10 @@ public void testSimple() throws IOException { assertEquals(7L, newhits + newmiss); assertEquals(4L, newhits); assertEquals(102L, newinserts); + // The eviction metric is cumulative and prior-inclusive, like the hits, inserts, and + // lookups asserted above, and the exact value here is 0: the priors copied from the + // first cache contribute no evictions, and the enlarged backing means the warming + // phase above evicted nothing either. assertEquals(0L, evictions); solrMetricsContext.close(); From 77c019e1ff0e922e3c2350379dece841651b1d44 Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Mon, 5 Oct 2026 22:43:10 +0000 Subject: [PATCH 2/3] SOLR-18506: Correct the TestThinCache eviction-flake explanation in the test comments --- .../org/apache/solr/search/TestThinCache.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/search/TestThinCache.java b/solr/core/src/test/org/apache/solr/search/TestThinCache.java index 83f926b9d7fb..792469b1ea7d 100644 --- a/solr/core/src/test/org/apache/solr/search/TestThinCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestThinCache.java @@ -150,16 +150,11 @@ public void testSimple() throws IOException { assertNull(lfuCache.get(1)); // first item put in should be the first out - // Everything after this point used to run against a backing cache that stayed full: - // warming copies 25 entries in and the test then puts key 103, and every put into the - // full backing evicts an entry. An eviction whose victim belongs to the second cache's - // scope is counted in that cache's evictions metric, and Caffeine's lazy eviction - // maintenance decides whether such an eviction has been delivered by the time the - // metric is sampled at the end of the test; that race is the flake in SOLR-18506, - // where the second cache read 1 eviction instead of 0. Enlarging the backing to 200 - // (both phases together hold at most 126 distinct entries) means nothing is evicted - // from here on, under any interleaving. As a side effect setMaxSize runs Caffeine - // cleanUp, settling any maintenance still pending from the puts above. + // Warming and the put below used to run against a full backing cache, so each put evicted + // an entry. Which entry is evicted depends on key hashes, and the scopes here hash by + // identity, so occasionally the victim was one of the second cache's own entries and its + // evictions metric read 1 instead of 0 (SOLR-18506). Size the backing above everything + // this test puts so that nothing is evicted from here on. backing.setMaxSize(200); // Test autowarming @@ -179,10 +174,8 @@ public void testSimple() throws IOException { assertEquals(7L, newhits + newmiss); assertEquals(4L, newhits); assertEquals(102L, newinserts); - // The eviction metric is cumulative and prior-inclusive, like the hits, inserts, and - // lookups asserted above, and the exact value here is 0: the priors copied from the - // first cache contribute no evictions, and the enlarged backing means the warming - // phase above evicted nothing either. + // 0: the first cache is never registered for removals, and nothing is evicted after the + // resize. assertEquals(0L, evictions); solrMetricsContext.close(); From 5262d15678d0f21ec1b0a9cfcbf968252a0de5aa Mon Sep 17 00:00:00 2001 From: Nick Shanin Date: Fri, 23 Oct 2026 16:35:00 +0000 Subject: [PATCH 3/3] SOLR-18506: Close TestThinCache metrics context on assertion failure --- .../org/apache/solr/search/TestThinCache.java | 141 +++++++++--------- 1 file changed, 70 insertions(+), 71 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/search/TestThinCache.java b/solr/core/src/test/org/apache/solr/search/TestThinCache.java index 792469b1ea7d..e734fe880e44 100644 --- a/solr/core/src/test/org/apache/solr/search/TestThinCache.java +++ b/solr/core/src/test/org/apache/solr/search/TestThinCache.java @@ -107,78 +107,77 @@ public void testSimple() throws IOException { ThinCache lfuCache = new ThinCache<>(); String lfuCacheName = "lfu_cache"; lfuCache.setBacking(cacheScope, backing); - var solrMetricsContext = new SolrMetricsContext(metricManager, registry); - solrMetricsContext.registerCloseable(lfuCache); - lfuCache.initializeMetrics( - solrMetricsContext, - Attributes.of( - CATEGORY_ATTR, SolrInfoBean.Category.CACHE.toString(), NAME_ATTR, lfuCacheName), - "solr_node_cache"); - - Object cacheScope2 = new Object(); - ThinCache newLFUCache = new ThinCache<>(); - solrMetricsContext.registerCloseable(newLFUCache); - String newLfuCacheName = "new_lfu_cache"; - newLFUCache.setBacking(cacheScope2, backing); - newLFUCache.initializeMetrics( - solrMetricsContext, - Attributes.of( - CATEGORY_ATTR, SolrInfoBean.Category.CACHE.toString(), NAME_ATTR, newLfuCacheName), - "solr_node_cache"); - - Map params = new HashMap<>(); - params.put("size", "100"); - params.put("initialSize", "10"); - - NoOpRegenerator regenerator = new NoOpRegenerator(); - backing.init(params, null, null); - Object initObj = lfuCache.init(Map.of("autowarmCount", "25"), null, regenerator); - lfuCache.setState(SolrCache.State.LIVE); - for (int i = 0; i < 101; i++) { - lfuCache.put(i + 1, Integer.toString(i + 1)); + try (var solrMetricsContext = new SolrMetricsContext(metricManager, registry)) { + solrMetricsContext.registerCloseable(lfuCache); + lfuCache.initializeMetrics( + solrMetricsContext, + Attributes.of( + CATEGORY_ATTR, SolrInfoBean.Category.CACHE.toString(), NAME_ATTR, lfuCacheName), + "solr_node_cache"); + + Object cacheScope2 = new Object(); + ThinCache newLFUCache = new ThinCache<>(); + solrMetricsContext.registerCloseable(newLFUCache); + String newLfuCacheName = "new_lfu_cache"; + newLFUCache.setBacking(cacheScope2, backing); + newLFUCache.initializeMetrics( + solrMetricsContext, + Attributes.of( + CATEGORY_ATTR, SolrInfoBean.Category.CACHE.toString(), NAME_ATTR, newLfuCacheName), + "solr_node_cache"); + + Map params = new HashMap<>(); + params.put("size", "100"); + params.put("initialSize", "10"); + + NoOpRegenerator regenerator = new NoOpRegenerator(); + backing.init(params, null, null); + Object initObj = lfuCache.init(Map.of("autowarmCount", "25"), null, regenerator); + lfuCache.setState(SolrCache.State.LIVE); + for (int i = 0; i < 101; i++) { + lfuCache.put(i + 1, Integer.toString(i + 1)); + } + assertEquals("15", lfuCache.get(15)); + assertEquals("75", lfuCache.get(75)); + assertNull(lfuCache.get(110)); + + var hits = getNodeCacheLookups(metricManager, registry, lfuCacheName, "hit"); + var miss = getNodeCacheLookups(metricManager, registry, lfuCacheName, "miss"); + var inserts = getNodeCacheOp(metricManager, registry, lfuCacheName, "inserts"); + assertEquals(3L, hits + miss); + assertEquals(2L, hits); + assertEquals(101L, inserts); + + assertNull(lfuCache.get(1)); // first item put in should be the first out + + // Warming and the put below used to run against a full backing cache, so each put evicted + // an entry. Which entry is evicted depends on key hashes, and the scopes here hash by + // identity, so occasionally the victim was one of the second cache's own entries and its + // evictions metric read 1 instead of 0 (SOLR-18506). Size the backing above everything + // this test puts so that nothing is evicted from here on. + backing.setMaxSize(200); + + // Test autowarming + newLFUCache.init(Map.of("autowarmCount", "25"), initObj, regenerator); + newLFUCache.warm(null, lfuCache); + newLFUCache.setState(SolrCache.State.LIVE); + + newLFUCache.put(103, "103"); + assertEquals("15", newLFUCache.get(15)); + assertEquals("75", newLFUCache.get(75)); + assertNull(newLFUCache.get(50)); + + var newhits = getNodeCacheLookups(metricManager, registry, newLfuCacheName, "hit"); + var newmiss = getNodeCacheLookups(metricManager, registry, newLfuCacheName, "miss"); + var newinserts = getNodeCacheOp(metricManager, registry, newLfuCacheName, "inserts"); + var evictions = getNodeCacheOp(metricManager, registry, newLfuCacheName, "evictions"); + assertEquals(7L, newhits + newmiss); + assertEquals(4L, newhits); + assertEquals(102L, newinserts); + // 0: the first cache is never registered for removals, and nothing is evicted after the + // resize. + assertEquals(0L, evictions); } - assertEquals("15", lfuCache.get(15)); - assertEquals("75", lfuCache.get(75)); - assertNull(lfuCache.get(110)); - - var hits = getNodeCacheLookups(metricManager, registry, lfuCacheName, "hit"); - var miss = getNodeCacheLookups(metricManager, registry, lfuCacheName, "miss"); - var inserts = getNodeCacheOp(metricManager, registry, lfuCacheName, "inserts"); - assertEquals(3L, hits + miss); - assertEquals(2L, hits); - assertEquals(101L, inserts); - - assertNull(lfuCache.get(1)); // first item put in should be the first out - - // Warming and the put below used to run against a full backing cache, so each put evicted - // an entry. Which entry is evicted depends on key hashes, and the scopes here hash by - // identity, so occasionally the victim was one of the second cache's own entries and its - // evictions metric read 1 instead of 0 (SOLR-18506). Size the backing above everything - // this test puts so that nothing is evicted from here on. - backing.setMaxSize(200); - - // Test autowarming - newLFUCache.init(Map.of("autowarmCount", "25"), initObj, regenerator); - newLFUCache.warm(null, lfuCache); - newLFUCache.setState(SolrCache.State.LIVE); - - newLFUCache.put(103, "103"); - assertEquals("15", newLFUCache.get(15)); - assertEquals("75", newLFUCache.get(75)); - assertNull(newLFUCache.get(50)); - - var newhits = getNodeCacheLookups(metricManager, registry, newLfuCacheName, "hit"); - var newmiss = getNodeCacheLookups(metricManager, registry, newLfuCacheName, "miss"); - var newinserts = getNodeCacheOp(metricManager, registry, newLfuCacheName, "inserts"); - var evictions = getNodeCacheOp(metricManager, registry, newLfuCacheName, "evictions"); - assertEquals(7L, newhits + newmiss); - assertEquals(4L, newhits); - assertEquals(102L, newinserts); - // 0: the first cache is never registered for removals, and nothing is evicted after the - // resize. - assertEquals(0L, evictions); - - solrMetricsContext.close(); } @Test