From 639e57037225394c6e8039191faaca90a0bc814d Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 21 Nov 2026 02:13:31 -0500
Subject: [PATCH 1/3] Lite: cheaper High Impact read, FinOps loads wait for the
first show, one skip line per cycle (#9233)
The High Impact statement picked each hash's grant with a per-hash LATERAL that sorted the whole window, and its sample text with a ROW_NUMBER sort over the text of every row of the kept hashes. Both are now one grouped pass with the same rows (equality test against the old statement).
The FinOps tab ran its whole per-server load at window start, twice, for a tab nobody had opened. A selection that moves while the tab is hidden now loads nothing, and the first show runs the load.
A burst of skipped database-state maintenance sweeps writes one line naming every server it skipped.
---
.../FinOpsFirstVisitAndBaselineTests.cs | 14 +
Lite.Tests/HighImpactReadEqualityTests.cs | 272 ++++++++++++++++++
Lite.Tests/StoreLockTimeoutLogTests.cs | 64 ++++-
Lite/Controls/FinOpsTab.xaml.cs | 13 +-
Lite/Services/FinOpsShowReloadPolicy.cs | 7 +
.../LocalDataService.DatabaseStates.cs | 66 ++++-
.../LocalDataService.FinOps.Workload.cs | 40 ++-
7 files changed, 453 insertions(+), 23 deletions(-)
create mode 100644 Lite.Tests/HighImpactReadEqualityTests.cs
diff --git a/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs b/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs
index cf42cf7f86..dbfd8a5459 100644
--- a/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs
+++ b/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs
@@ -59,6 +59,20 @@ public void ACollectionFinishingWhileTheTabIsVisible_ReloadsOnceTheLastLoadIsAMi
[Fact]
public void ACollectionBeforeAnyLoad_Reloads() => Assert.True(FinOpsShowReloadPolicy.ShouldReloadAfterCollection(null, Now, Now));
+ ///
+ /// #9233: window start selects the first server on a tab nobody has opened. That selection must not run the fifteen store reads
+ /// (High Impact the heaviest) against a store the first collection has not written to. The tab's first show runs the whole load
+ /// whatever the age of any earlier one, so nothing is lost by waiting for it.
+ ///
+ [Theory]
+ [InlineData(false, false)]
+ [InlineData(true, true)]
+ public void ASelectionChange_LoadsOnlyWhileTheTabIsShown(bool tabVisible, bool loads)
+ {
+ Assert.Equal(loads, FinOpsShowReloadPolicy.ShouldLoadOnSelectionChange(tabVisible));
+ Assert.True(FinOpsShowReloadPolicy.ShouldReloadOnShow(null, Now, firstShowSinceStart: true));
+ }
+
[Theory]
[InlineData(0, false)]
[InlineData(5, false)]
diff --git a/Lite.Tests/HighImpactReadEqualityTests.cs b/Lite.Tests/HighImpactReadEqualityTests.cs
new file mode 100644
index 0000000000..69682e2394
--- /dev/null
+++ b/Lite.Tests/HighImpactReadEqualityTests.cs
@@ -0,0 +1,272 @@
+/*
+ * Copyright (c) 2026 Erik Darling, Darling Data LLC
+ *
+ * This file is part of the SQL Server Performance Monitor.
+ *
+ * Licensed under the MIT License. See LICENSE file in the project root for full license information.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Threading.Tasks;
+using DuckDB.NET.Data;
+using PerformanceMonitorLite.Database;
+using PerformanceMonitorLite.Services;
+using Xunit;
+
+namespace PerformanceMonitorLite.Tests;
+
+///
+/// #9233: the High Impact read picked each hash's grant with a per-hash LATERAL that sorted the whole window, and its sample
+/// text with a ROW_NUMBER sort that carried the text of every row of the kept hashes. Both are now one grouped pass. The new
+/// statement must return the same rows. This runs the statement the app ships and the one it shipped before (kept here as the
+/// oracle) over one seeded window and compares every column of every row. The seed holds the cases where a NULL sort key
+/// decides the pick: a NULL creation_time and a NULL delta_worker_time (the newest row's grant), a NULL count (the sample
+/// text), ties on collection_time, hashes with several plans and several databases, an empty and a NULL text.
+///
+public sealed class HighImpactReadEqualityTests : IClassFixture
+{
+ private const int ServerId = -9233;
+ private const string ServerName = "lite-high-impact-equality";
+
+ private readonly DuckDbInitializer _duckDb;
+
+ public HighImpactReadEqualityTests(SharedDuckDbFixture fixture)
+ {
+ fixture.ResetData();
+ _duckDb = fixture.DuckDb;
+ }
+
+ /// The statement as it shipped before #9233: the grant from a per-hash LATERAL, the sample text from a ROW_NUMBER over the rows.
+ private const string OracleSql = @"
+WITH win AS MATERIALIZED
+(
+ SELECT query_hash, database_name, delta_execution_count, delta_worker_time, delta_elapsed_time,
+ delta_logical_reads, delta_logical_writes, max_grant_kb, collection_time, creation_time
+ FROM v_query_stats
+ WHERE server_id = $1
+ AND collection_time >= $2
+ AND query_hash IS NOT NULL AND query_hash != ''
+),
+per_database AS
+(
+ SELECT query_hash, database_name,
+ SUM(delta_execution_count) AS executions,
+ SUM(delta_worker_time) AS worker_time,
+ SUM(delta_elapsed_time) AS elapsed_time,
+ SUM(delta_logical_reads) AS logical_reads,
+ SUM(delta_logical_writes) AS logical_writes
+ FROM win
+ WHERE delta_execution_count > 0
+ GROUP BY query_hash, database_name
+),
+per_hash AS
+(
+ SELECT
+ query_hash,
+ (ARRAY_AGG(database_name ORDER BY worker_time DESC NULLS LAST, LOWER(database_name), database_name)
+ FILTER (WHERE database_name IS NOT NULL))[1] AS database_name,
+ SUM(executions) AS total_executions,
+ SUM(worker_time) AS total_worker_time,
+ SUM(elapsed_time) AS total_elapsed_time,
+ SUM(logical_reads) AS total_reads,
+ SUM(logical_writes) AS total_writes,
+ COUNT(DISTINCT database_name) AS database_count
+ FROM per_database
+ GROUP BY query_hash
+ HAVING SUM(executions) > 0
+),
+ranked AS
+(
+ SELECT h.*, COALESCE(g.max_grant_kb, 0) AS peak_grant_kb,
+ ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_worker_time, 0) DESC, h.total_worker_time DESC, h.query_hash) AS cpu_rank,
+ ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_elapsed_time, 0) DESC, h.total_worker_time DESC, h.query_hash) AS duration_rank,
+ ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_reads, 0) DESC, h.total_worker_time DESC, h.query_hash) AS reads_rank,
+ ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_writes, 0) DESC, h.total_worker_time DESC, h.query_hash) AS writes_rank,
+ ROW_NUMBER() OVER (ORDER BY COALESCE(g.max_grant_kb, 0) DESC, h.total_worker_time DESC, h.query_hash) AS memory_rank,
+ ROW_NUMBER() OVER (ORDER BY h.total_executions DESC, h.total_worker_time DESC, h.query_hash) AS executions_rank
+ FROM per_hash AS h
+ LEFT JOIN LATERAL
+ (
+ SELECT qs.max_grant_kb
+ FROM win AS qs
+ WHERE qs.query_hash = h.query_hash
+ AND qs.delta_execution_count > 0
+ ORDER BY qs.collection_time DESC, qs.creation_time DESC NULLS LAST, qs.delta_worker_time DESC NULLS LAST
+ LIMIT 1
+ ) AS g ON true
+),
+kept AS
+(
+ SELECT *
+ FROM ranked
+ WHERE cpu_rank <= $3 OR duration_rank <= $3 OR reads_rank <= $3
+ OR writes_rank <= $3 OR memory_rank <= $3 OR executions_rank <= $3
+),
+sample_text AS
+(
+ SELECT qs2.query_hash, qs2.query_text
+ FROM v_query_stats AS qs2
+ JOIN kept AS kk ON kk.query_hash = qs2.query_hash
+ WHERE qs2.server_id = $1
+ AND qs2.collection_time >= $2
+ AND qs2.query_text IS NOT NULL AND qs2.query_text != ''
+ QUALIFY ROW_NUMBER() OVER (PARTITION BY qs2.query_hash ORDER BY qs2.delta_execution_count DESC NULLS LAST, qs2.collection_time DESC, qs2.creation_time DESC NULLS LAST, qs2.query_text) = 1
+)
+SELECT
+ k.query_hash,
+ k.database_name,
+ k.total_executions,
+ k.total_worker_time / 1000.0 AS total_cpu_ms,
+ k.total_elapsed_time / 1000.0 AS total_duration_ms,
+ k.total_reads,
+ k.total_writes,
+ k.peak_grant_kb / 1024.0 AS peak_memory_mb,
+ k.database_count,
+ LEFT(t.query_text, 200) AS sample_query_text,
+ t.query_text AS full_query_text
+FROM kept AS k
+LEFT JOIN sample_text AS t ON t.query_hash = k.query_hash
+ORDER BY k.total_worker_time DESC, k.query_hash";
+
+ private static readonly DateTime Now = DateTime.UtcNow;
+ private static readonly DateTime T1 = Now.AddHours(-5);
+ private static readonly DateTime Plan1 = Now.AddDays(-9);
+ private static readonly DateTime Plan2 = Now.AddDays(-4);
+
+ private static readonly object Null = DBNull.Value;
+
+ private async Task SeedAsync()
+ {
+ using var readLock = _duckDb.AcquireReadLock();
+ using var connection = _duckDb.CreateConnection();
+ await connection.OpenAsync();
+ long id = -9_233_000;
+
+ async Task Row(string hash, string db, object? text, DateTime at, object creation, object count, object cpu, object grant)
+ {
+ using var command = connection.CreateCommand();
+ command.CommandText = @"
+INSERT INTO query_stats
+ (collection_id, collection_time, server_id, server_name, database_name, query_hash, query_text, creation_time,
+ delta_execution_count, delta_worker_time, delta_elapsed_time, delta_logical_reads, delta_logical_writes, max_grant_kb)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)";
+ foreach (var value in new object[] { id--, at, ServerId, ServerName, db, hash, text ?? Null, creation, count, cpu, 1_000L, 10L, 1L, grant })
+ {
+ command.Parameters.Add(new DuckDBParameter { Value = value });
+ }
+
+ await command.ExecuteNonQueryAsync();
+ }
+
+ /* Grant: the newest collection, then the newest plan creation (a NULL creation last), then the most worker time (a NULL last). */
+ await Row("0xA", "Db1", "SELECT a", T1, Null, 5L, 100L, 100L); // a NULL creation_time loses to a real one: 200
+ await Row("0xA", "Db1", "SELECT a", T1, Plan1, 5L, 100L, 200L);
+ await Row("0xB", "Db1", "SELECT b", T1, Plan1, 5L, Null, 300L); // a NULL delta_worker_time loses to a real one: 400
+ await Row("0xB", "Db1", "SELECT b", T1, Plan1, 5L, 7L, 400L);
+ await Row("0xC", "Db1", "SELECT c", T1, Plan2, 5L, 100L, 900L); // the newer collection wins whatever the grant: 50
+ await Row("0xC", "Db1", "SELECT c", T1.AddMinutes(1), Plan1, 5L, 1L, 50L);
+ await Row("0xD", "Db1", "SELECT d", T1, Plan1, 5L, 100L, 600L); // a newest row with no executions is no reading: 600
+ await Row("0xD", "Db1", "SELECT d", T1.AddMinutes(1), Plan2, 0L, 100L, 9_000L);
+ await Row("0xE", "Db1", "SELECT e", T1, Plan1, 5L, 100L, 700L); // the newest row's grant is NULL: 0
+ await Row("0xE", "Db1", "SELECT e", T1.AddMinutes(1), Plan2, 5L, 100L, Null);
+ await Row("0xF", "Db1", "SELECT f", T1, Null, 5L, Null, 11L); // both keys NULL on one row, a real key on the other: 22
+ await Row("0xF", "Db1", "SELECT f", T1, Plan2, 5L, 3L, 22L);
+ await Row("0xG", "Db2", "SELECT g", T1, Plan1, 5L, 100L, 33L); // the same figures in two databases: the same grant either way
+ await Row("0xG", "Db3", "SELECT g", T1, Plan1, 5L, 100L, 33L);
+
+ /* Sample text: the most executions (a NULL count last), then the newest collection, then the newest creation (a NULL last), then the smallest text. */
+ await Row("0xH", "Db1", "SELECT h-null-count", T1.AddMinutes(5), Plan2, Null, 10L, 1L); // a NULL count ranks last: "SELECT h-few"
+ await Row("0xH", "Db1", "SELECT h-few", T1, Plan1, 1L, 10L, 1L);
+ await Row("0xI", "Db1", "SELECT i-null-creation", T1, Null, 4L, 10L, 1L); // a NULL creation ranks last: "SELECT i-real"
+ await Row("0xI", "Db1", "SELECT i-real", T1, Plan1, 4L, 10L, 1L);
+ await Row("0xJ", "Db1", "SELECT j-b", T1, Plan1, 4L, 10L, 1L); // a full tie: the smaller text, "SELECT j-a"
+ await Row("0xJ", "Db2", "SELECT j-a", T1, Plan1, 4L, 10L, 1L);
+ await Row("0xK", "Db1", "SELECT k-old", T1, Plan1, 4L, 10L, 1L); // the newer collection wins a count tie: "SELECT k-new"
+ await Row("0xK", "Db1", "SELECT k-new", T1.AddMinutes(1), Plan1, 4L, 10L, 1L);
+ await Row("0xL", "Db1", "", T1.AddMinutes(1), Plan1, 9L, 10L, 1L); // an empty and a NULL text are no sample: "SELECT l"
+ await Row("0xL", "Db1", null, T1.AddMinutes(2), Plan1, 9L, 10L, 1L);
+ await Row("0xL", "Db1", "SELECT l", T1, Plan1, 1L, 10L, 1L);
+ await Row("0xM", "Db1", null, T1, Plan1, 3L, 10L, 1L); // no text anywhere: a NULL sample
+
+ /* Filler, so the keep count cuts the window: each measure puts a different hash on top. */
+ for (var i = 0; i < 40; i++)
+ {
+ await Row("0xZ" + i.ToString("D2", CultureInfo.InvariantCulture), "Db" + (i % 3), "SELECT z" + i, T1.AddMinutes(i % 4),
+ i % 5 == 0 ? Null : Plan1, (i * 7) % 13 + 1L, i % 6 == 0 ? Null : (i * 11) % 17 * 100L, (i * 13) % 19 * 1024L);
+ }
+
+ /* A row outside the window is not read. */
+ await Row("0xOLD", "Db1", "SELECT old", Now.AddDays(-3), Plan1, 99L, 999L, 99_999L);
+ }
+
+ private async Task> RunAsync(string sql, int keep)
+ {
+ using var readLock = _duckDb.AcquireReadLock();
+ using var connection = _duckDb.CreateConnection();
+ await connection.OpenAsync();
+ using var command = connection.CreateCommand();
+ command.CommandText = sql.Replace("v_query_stats", "query_stats", StringComparison.Ordinal);
+ command.Parameters.Add(new DuckDBParameter { Value = ServerId });
+ command.Parameters.Add(new DuckDBParameter { Value = Now.AddHours(-24) });
+ command.Parameters.Add(new DuckDBParameter { Value = keep });
+
+ var rows = new List();
+ using var reader = await command.ExecuteReaderAsync();
+ while (await reader.ReadAsync())
+ {
+ var cells = new string[reader.FieldCount];
+ for (var c = 0; c < cells.Length; c++)
+ {
+ cells[c] = reader.IsDBNull(c) ? "" : Convert.ToString(reader.GetValue(c), CultureInfo.InvariantCulture) ?? "";
+ }
+
+ rows.Add(string.Join("|", cells));
+ }
+
+ return rows;
+ }
+
+ [Theory]
+ [InlineData(1000)]
+ [InlineData(10)]
+ [InlineData(3)]
+ public async Task TheGroupedPasses_ReturnTheRowsTheLateralAndTheRowNumberSortReturned(int keep)
+ {
+ await SeedAsync();
+
+ var oracle = await RunAsync(OracleSql, keep);
+ var shipped = await RunAsync(LocalDataService.HighImpactQueriesSql, keep);
+
+ Assert.True(oracle.Count >= 3, $"the seed must keep a spread of hashes, the oracle read {oracle.Count}");
+ Assert.Equal(oracle, shipped);
+ }
+
+ /// The picks the seed was built around, read from the shipped statement, so a seed that stopped exercising a NULL key fails here and not silently.
+ [Fact]
+ public async Task TheShippedStatement_PicksTheGrantAndTheSampleTheNullKeysCallFor()
+ {
+ await SeedAsync();
+
+ var shipped = await RunAsync(LocalDataService.HighImpactQueriesSql, 1000);
+ string Cell(string hash, int column) => shipped.Single(r => r.StartsWith(hash + "|", StringComparison.Ordinal)).Split('|')[column];
+
+ /* Column 7 is the peak memory in MB, the grant in KB over 1024. */
+ Assert.Equal(200.0 / 1024, double.Parse(Cell("0xA", 7), CultureInfo.InvariantCulture), 1e-9);
+ Assert.Equal(400.0 / 1024, double.Parse(Cell("0xB", 7), CultureInfo.InvariantCulture), 1e-9);
+ Assert.Equal(50.0 / 1024, double.Parse(Cell("0xC", 7), CultureInfo.InvariantCulture), 1e-9);
+ Assert.Equal(600.0 / 1024, double.Parse(Cell("0xD", 7), CultureInfo.InvariantCulture), 1e-9);
+ Assert.Equal(0.0, double.Parse(Cell("0xE", 7), CultureInfo.InvariantCulture), 1e-9);
+ Assert.Equal(22.0 / 1024, double.Parse(Cell("0xF", 7), CultureInfo.InvariantCulture), 1e-9);
+
+ /* Column 10 is the full sample text. */
+ Assert.Equal("SELECT h-few", Cell("0xH", 10));
+ Assert.Equal("SELECT i-real", Cell("0xI", 10));
+ Assert.Equal("SELECT j-a", Cell("0xJ", 10));
+ Assert.Equal("SELECT k-new", Cell("0xK", 10));
+ Assert.Equal("SELECT l", Cell("0xL", 10));
+ Assert.Equal("", Cell("0xM", 10));
+ }
+}
diff --git a/Lite.Tests/StoreLockTimeoutLogTests.cs b/Lite.Tests/StoreLockTimeoutLogTests.cs
index d0c7673021..d3d9872ed4 100644
--- a/Lite.Tests/StoreLockTimeoutLogTests.cs
+++ b/Lite.Tests/StoreLockTimeoutLogTests.cs
@@ -8,6 +8,7 @@
using System;
using System.Collections.Generic;
+using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
@@ -102,12 +103,60 @@ public async Task TheSkippedMaintenanceLine_KeepsTheStorageId_ForAServerNobodyCa
private async Task SkipMaintenanceAndReadTheLineAsync(string? displayName)
{
- var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100));
- if (displayName is not null)
+ var names = displayName is null ? null : new Dictionary { [ServerId] = displayName };
+ return Assert.Single(await SkipMaintenanceForAsync(names, ServerId));
+ }
+
+ /// #9233: three servers' sweeps meet the same held write lock, one after another as the alert engine runs them. Their one line names all three.
+ [Fact]
+ public async Task ASkippedCycle_WritesOneLine_NamingEveryServerItSkipped()
+ {
+ var names = new Dictionary { [ServerId] = "Display One", [ServerId - 1] = "Display Two", [ServerId - 2] = "Display Three" };
+
+ var lines = await SkipMaintenanceForAsync(names, ServerId, ServerId - 1, ServerId - 2);
+
+ var line = Assert.Single(lines);
+ Assert.Contains("'Display One'", line, StringComparison.Ordinal);
+ Assert.Contains("'Display Two'", line, StringComparison.Ordinal);
+ Assert.Contains("'Display Three'", line, StringComparison.Ordinal);
+ }
+
+ /// The skips of a cycle that has been written do not leak into the next cycle's line.
+ [Fact]
+ public async Task TheNextCycleOfSkips_WritesItsOwnLine_WithoutTheEarlierServers()
+ {
+ var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100))
{
- service.DisplayNames = () => new Dictionary { [ServerId] = displayName };
+ MaintenanceSkipLogQuietPeriodOverride = TimeSpan.FromHours(1),
+ DisplayNames = () => new Dictionary { [ServerId] = "Display One", [ServerId - 1] = "Display Two" },
+ };
+
+ var first = await SkipUnderAHeldLockAsync(service, ServerId);
+ var second = await SkipUnderAHeldLockAsync(service, ServerId - 1);
+
+ Assert.Contains("'Display One'", Assert.Single(first), StringComparison.Ordinal);
+ var later = Assert.Single(second);
+ Assert.Contains("'Display Two'", later, StringComparison.Ordinal);
+ Assert.DoesNotContain("'Display One'", later, StringComparison.Ordinal);
+ }
+
+ private async Task> SkipMaintenanceForAsync(Dictionary? displayNames, params int[] serverIds)
+ {
+ var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100))
+ {
+ MaintenanceSkipLogQuietPeriodOverride = TimeSpan.FromHours(1),
+ };
+ if (displayNames is not null)
+ {
+ service.DisplayNames = () => displayNames;
}
+ return await SkipUnderAHeldLockAsync(service, serverIds);
+ }
+
+ /// Sweeps each server under a read held on another thread, then writes the burst's line, and returns the maintenance-skip lines the log buffered.
+ private async Task> SkipUnderAHeldLockAsync(LocalDataService service, params int[] serverIds)
+ {
/* A read held on another thread: the maintenance wants the write lock, waits its 100 ms and skips the cycle. */
var held = new ManualResetEventSlim(false);
var release = new ManualResetEventSlim(false);
@@ -126,7 +175,12 @@ private async Task SkipMaintenanceAndReadTheLineAsync(string? displayNam
{
Assert.True(held.Wait(TimeSpan.FromSeconds(10)), "the read never took the lock");
AppLogger.DrainBufferedLines();
- await service.GetDatabaseStateDeviationsAsync(ServerId);
+ foreach (var serverId in serverIds)
+ {
+ await service.GetDatabaseStateDeviationsAsync(serverId);
+ }
+
+ service.FlushMaintenanceSkipLog();
}
finally
{
@@ -134,7 +188,7 @@ private async Task SkipMaintenanceAndReadTheLineAsync(string? displayNam
holder.Join(TimeSpan.FromSeconds(10));
}
- return Assert.Single(AppLogger.DrainBufferedLines(), l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal));
+ return AppLogger.DrainBufferedLines().Where(l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal)).ToList();
}
private sealed class CapturingLogger : ILogger
diff --git a/Lite/Controls/FinOpsTab.xaml.cs b/Lite/Controls/FinOpsTab.xaml.cs
index 7df1354f2c..ba36f98635 100644
--- a/Lite/Controls/FinOpsTab.xaml.cs
+++ b/Lite/Controls/FinOpsTab.xaml.cs
@@ -160,8 +160,11 @@ public void Initialize(LocalDataService dataService, ServerManager serverManager
_openTabClock = openTabClock;
_credentialResolver = serverManager.CredentialResolver;
+ /* #9233: the per-server load (fifteen store reads, High Impact the heaviest) runs only while the tab is shown. Window start used to run
+ it twice for a tab nobody had opened (the selector's first selection, then RefreshData), against a store the first collection
+ has not written to yet, and the first show re-runs it whole anyway (FinOpsShowReloadPolicy). Start-up keeps the fleet inventory. */
PopulateServerSelector();
- RefreshData();
+ _ = LoadServerInventoryAsync();
}
///
@@ -1440,6 +1443,14 @@ private async void ServerSelector_SelectionChanged(object sender, SelectionChang
selected server B with A's row. */
ClearPerServerGrids();
+ /* #9233: a selection that moves while the tab is hidden (the first one at start-up, a server list that changed) loads nothing. The
+ next show runs the whole load, as the first show always does. */
+ if (!FinOpsShowReloadPolicy.ShouldLoadOnSelectionChange(IsVisible))
+ {
+ _firstShowPending = true;
+ return;
+ }
+
await LoadPerServerDataAsync();
}
diff --git a/Lite/Services/FinOpsShowReloadPolicy.cs b/Lite/Services/FinOpsShowReloadPolicy.cs
index 5a8fecc155..1c779f877d 100644
--- a/Lite/Services/FinOpsShowReloadPolicy.cs
+++ b/Lite/Services/FinOpsShowReloadPolicy.cs
@@ -35,6 +35,13 @@ internal static class FinOpsShowReloadPolicy
internal static bool ShouldReloadOnShow(DateTime? lastLoadStartedUtc, DateTime nowUtc, bool firstShowSinceStart = false) =>
firstShowSinceStart || lastLoadStartedUtc is not DateTime last || nowUtc - last >= MinAge;
+ ///
+ /// True when a selection change should run the whole per-server load now (#9233): only while the tab is shown. A hidden tab
+ /// leaves the load to its first show, which always runs it (), so window start does not run
+ /// fifteen store reads, High Impact the heaviest, for a tab nobody has opened.
+ ///
+ internal static bool ShouldLoadOnSelectionChange(bool tabVisible) => tabVisible;
+
///
/// True when a collection for the selected server that finished at should reload the
/// visible tab: it finished after the last whole load began (that load may have read the window before it), and that load began
diff --git a/Lite/Services/LocalDataService.DatabaseStates.cs b/Lite/Services/LocalDataService.DatabaseStates.cs
index 8e0703eacb..32cfaabab1 100644
--- a/Lite/Services/LocalDataService.DatabaseStates.cs
+++ b/Lite/Services/LocalDataService.DatabaseStates.cs
@@ -9,6 +9,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
+using System.Linq;
using System.Threading.Tasks;
using DuckDB.NET.Data;
using PerformanceMonitor.Alerting;
@@ -34,6 +35,56 @@ public partial class LocalDataService
/// says nothing about another's. Never pruned — one bool per server ever swept.
///
private readonly System.Collections.Concurrent.ConcurrentDictionary _lastMaintenanceSkipped = new();
+
+ ///
+ /// #9233: the servers whose maintenance skipped in the current burst, with the holder each named, waiting for ONE line. The alert
+ /// engine sweeps the servers one at a time and a held lock skips every one of them, so a line per server read as six identical
+ /// lines about one event. A skip joins this list and (re)starts . The line is written once the burst
+ /// has been quiet for , naming every server in it.
+ ///
+ private readonly List<(string Server, string Holder)> _skipsThisCycle = new();
+ private readonly object _skipLogGate = new();
+ private System.Threading.Timer? _skipLogTimer;
+
+ ///
+ /// How long no further server may skip before the burst's line is written. One sweep waits up to the write-lock budget before it
+ /// skips, so the default is that budget plus five seconds. A test sets a long time and calls .
+ ///
+ internal TimeSpan? MaintenanceSkipLogQuietPeriodOverride { get; set; }
+
+ private TimeSpan MaintenanceSkipLogQuietPeriod =>
+ MaintenanceSkipLogQuietPeriodOverride ?? _backgroundWriteLockBudget + TimeSpan.FromSeconds(5);
+
+ private void NoteMaintenanceSkipped(string server, string holder)
+ {
+ lock (_skipLogGate)
+ {
+ _skipsThisCycle.Add((server, holder));
+ _skipLogTimer?.Dispose();
+ _skipLogTimer = new System.Threading.Timer(_ => FlushMaintenanceSkipLog(), null, MaintenanceSkipLogQuietPeriod, System.Threading.Timeout.InfiniteTimeSpan);
+ }
+ }
+
+ /// Writes the one line for the servers waiting in , if any, and empties the list.
+ internal void FlushMaintenanceSkipLog()
+ {
+ (string Server, string Holder)[] skipped;
+ lock (_skipLogGate)
+ {
+ _skipLogTimer?.Dispose();
+ _skipLogTimer = null;
+ skipped = _skipsThisCycle.ToArray();
+ _skipsThisCycle.Clear();
+ }
+
+ if (skipped.Length == 0)
+ {
+ return;
+ }
+
+ AppLogger.Warn(nameof(GetDatabaseStateDeviationsAsync),
+ MaintenanceSkippedWarning(skipped.Select(x => x.Server).ToList(), skipped.Select(x => x.Holder).Distinct(StringComparer.Ordinal).ToList(), _backgroundWriteLockBudget));
+ }
/* Effective state = STANDBY for a read-only log-shipping secondary (is_in_standby), else the raw
state_desc. A standby secondary reports state_desc = ONLINE with is_in_standby = 1 and flips through
RESTORING on every log restore; collapsing it to a single stable STANDBY token means it baselines as
@@ -144,8 +195,16 @@ internal string ServerLabelForLog(int serverId)
/// had (5 s in the shipped app, two minutes in Lite.Tests), not a number written into the text.
///
internal static string MaintenanceSkippedWarning(string server, string holder, TimeSpan budget) =>
- $"server {server}: skipped this cycle's database-state maintenance: could not acquire the " +
- $"store write lock within {budget.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture)}s ({holder}). Baselines are not being " +
+ MaintenanceSkippedWarning(new[] { server }, new[] { holder }, budget);
+
+ ///
+ /// #9233: the one line for a skipped cycle, naming every server it skipped. The holders are the distinct ones the skipped servers
+ /// saw (usually one), joined with a bar.
+ ///
+ internal static string MaintenanceSkippedWarning(IReadOnlyList servers, IReadOnlyList holders, TimeSpan budget) =>
+ (servers.Count == 1 ? $"server {servers[0]}" : $"servers {string.Join(", ", servers)}") +
+ ": skipped this cycle's database-state maintenance: could not acquire the " +
+ $"store write lock within {budget.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture)}s ({string.Join(" | ", holders)}). Baselines are not being " +
"seeded or healed while this persists (#2189/#2203); deviations are still read. Expected " +
"occasionally. If it repeats, the write lock is contended.";
@@ -233,8 +292,7 @@ SUSTAINED run of them that means baselines have stopped being seeded and healed.
if (!_lastMaintenanceSkipped.TryGetValue(serverId, out var wasSkipped) || !wasSkipped)
{
_lastMaintenanceSkipped[serverId] = true;
- AppLogger.Warn(nameof(GetDatabaseStateDeviationsAsync),
- MaintenanceSkippedWarning(ServerLabelForLog(serverId), skipHolder, _backgroundWriteLockBudget));
+ NoteMaintenanceSkipped(ServerLabelForLog(serverId), skipHolder);
}
}
else if (_lastMaintenanceSkipped.TryGetValue(serverId, out var hadSkipped) && hadSkipped)
diff --git a/Lite/Services/LocalDataService.FinOps.Workload.cs b/Lite/Services/LocalDataService.FinOps.Workload.cs
index be7dafc16f..4baf9a2fec 100644
--- a/Lite/Services/LocalDataService.FinOps.Workload.cs
+++ b/Lite/Services/LocalDataService.FinOps.Workload.cs
@@ -589,17 +589,23 @@ ranked AS
FROM per_hash AS h
-- #7166: max_grant_kb is a per-plan high-water mark since that plan was cached, so MAX across a hash's plans is
-- the largest grant any plan ever had (the max_dop trap, #3648). The reading is the hash's newest row's, the same
- -- choice max_dop makes: newest collection first, newest plan first within it. Explicit NULLS LAST: DuckDB and
- -- Postgres default DESC null placement differently.
- LEFT JOIN LATERAL
+ -- choice max_dop makes: newest collection first, newest plan first within it. #9233: one grouped pass over win,
+ -- where a per-hash LATERAL with ORDER BY ... LIMIT 1 sorted the whole window (the statement's costliest operator
+ -- after the scan). arg_max over a struct ranks a NULL field ABOVE every value (#9195), so each key that sorted
+ -- NULLS LAST maps to a value below every real one. The grant is the last key, so rows tied on the other three
+ -- agree on the largest grant, not on whichever the sort met first.
+ LEFT JOIN
(
- SELECT qs.max_grant_kb
- FROM win AS qs
- WHERE qs.query_hash = h.query_hash
- AND qs.delta_execution_count > 0
- ORDER BY qs.collection_time DESC, qs.creation_time DESC NULLS LAST, qs.delta_worker_time DESC NULLS LAST
- LIMIT 1
- ) AS g ON true
+ SELECT query_hash,
+ arg_max(COALESCE(max_grant_kb, 0),
+ (collection_time,
+ COALESCE(creation_time, '-infinity'::TIMESTAMP),
+ COALESCE(delta_worker_time, -9223372036854775807),
+ COALESCE(max_grant_kb, 0))) AS max_grant_kb
+ FROM win
+ WHERE delta_execution_count > 0
+ GROUP BY query_hash
+ ) AS g ON g.query_hash = h.query_hash
),
kept AS
(
@@ -613,14 +619,22 @@ sample_text AS
-- The sample text for the kept hashes only (#6803), from each hash's busiest row in the window. #9058: one pass for
-- the whole set. The two per-hash lookups this replaces each scanned the window again. A NULL count ranks last. A tie
-- on the count goes to the newest collection, then the newest plan creation, then the text itself, so the pick is
- -- the same on every read even for rows of one collection (several plans or databases).
- SELECT qs2.query_hash, qs2.query_text
+ -- the same on every read even for rows of one collection (several plans or databases). #9233: the pick is an
+ -- arg_min over the key as a struct, not a ROW_NUMBER sort that carried the text of every row of the kept hashes.
+ -- Each DESC key is negated and its NULL maps to the largest value, so a NULL still ranks last, and the text is the
+ -- struct's last member, so a full tie still goes to the smallest text.
+ SELECT qs2.query_hash,
+ arg_min(qs2.query_text,
+ (COALESCE(-qs2.delta_execution_count, 9223372036854775807),
+ -epoch_us(qs2.collection_time),
+ COALESCE(-epoch_us(qs2.creation_time), 9223372036854775807),
+ qs2.query_text)) AS query_text
FROM v_query_stats AS qs2
JOIN kept AS kk ON kk.query_hash = qs2.query_hash
WHERE qs2.server_id = $1
AND qs2.collection_time >= $2
AND qs2.query_text IS NOT NULL AND qs2.query_text != ''
- QUALIFY ROW_NUMBER() OVER (PARTITION BY qs2.query_hash ORDER BY qs2.delta_execution_count DESC NULLS LAST, qs2.collection_time DESC, qs2.creation_time DESC NULLS LAST, qs2.query_text) = 1
+ GROUP BY qs2.query_hash
)
SELECT
k.query_hash,
From f7bf6e11b0e77d253f57c40c167f7aa3e8a02e9b Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 21 Nov 2026 02:25:56 -0500
Subject: [PATCH 2/3] Open the skip-log window at the first skip of a burst and
leave it fixed (#9233)
A skip used to restart the quiet period. A stuck lock keeps skipping, so with enough servers the skips arrived less than a period apart and the line never logged. The first skip now starts the window, later skips only join it, and the next skip after the line starts a new one.
---
Lite.Tests/StoreLockTimeoutLogTests.cs | 70 +++++++++++++++++++
.../LocalDataService.DatabaseStates.cs | 12 ++--
2 files changed, 76 insertions(+), 6 deletions(-)
diff --git a/Lite.Tests/StoreLockTimeoutLogTests.cs b/Lite.Tests/StoreLockTimeoutLogTests.cs
index d3d9872ed4..9b97e9862a 100644
--- a/Lite.Tests/StoreLockTimeoutLogTests.cs
+++ b/Lite.Tests/StoreLockTimeoutLogTests.cs
@@ -140,6 +140,76 @@ public async Task TheNextCycleOfSkips_WritesItsOwnLine_WithoutTheEarlierServers(
Assert.DoesNotContain("'Display One'", later, StringComparison.Ordinal);
}
+ ///
+ /// #9233: a stuck lock keeps skipping servers, each one less than a quiet period after the last. The window opens at the first
+ /// skip and is not pushed back by the later ones, so the line still logs and each window's line names its own servers.
+ ///
+ [Fact]
+ public async Task SkipsThatKeepArriving_StillLogWithinOneWindowOfTheFirst_EachLineNamingItsOwnServers()
+ {
+ var quiet = TimeSpan.FromSeconds(1);
+ var serverIds = Enumerable.Range(0, 9).Select(i => ServerId - i).ToArray();
+ var names = serverIds.ToDictionary(id => id, id => $"Display {ServerId - id}");
+ var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100))
+ {
+ MaintenanceSkipLogQuietPeriodOverride = quiet,
+ DisplayNames = () => names,
+ };
+
+ var held = new ManualResetEventSlim(false);
+ var release = new ManualResetEventSlim(false);
+ var holder = new Thread(() =>
+ {
+ using (_duckDb.AcquireReadLock())
+ {
+ held.Set();
+ release.Wait(TimeSpan.FromSeconds(30));
+ }
+ })
+ { IsBackground = true };
+ holder.Start();
+
+ var lines = new List();
+ TimeSpan? firstLineAfter = null;
+ try
+ {
+ Assert.True(held.Wait(TimeSpan.FromSeconds(10)), "the read never took the lock");
+ AppLogger.DrainBufferedLines();
+ var started = System.Diagnostics.Stopwatch.StartNew();
+ foreach (var serverId in serverIds)
+ {
+ /* Each skip waits the 100 ms budget, then the pause puts the skips half a quiet period apart: four and a
+ half quiet periods in all, and no gap between skips as long as one quiet period. */
+ await service.GetDatabaseStateDeviationsAsync(serverId);
+ var seen = AppLogger.DrainBufferedLines().Where(l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal)).ToList();
+ if (seen.Count > 0 && firstLineAfter is null)
+ {
+ firstLineAfter = started.Elapsed;
+ }
+
+ lines.AddRange(seen);
+ await Task.Delay(quiet / 2 - TimeSpan.FromMilliseconds(100));
+ }
+
+ service.FlushMaintenanceSkipLog();
+ lines.AddRange(AppLogger.DrainBufferedLines().Where(l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal)));
+ }
+ finally
+ {
+ release.Set();
+ holder.Join(TimeSpan.FromSeconds(10));
+ }
+
+ /* With a window that slid, nothing logged until the final flush and the one line held all nine servers. */
+ Assert.NotNull(firstLineAfter);
+ Assert.True(firstLineAfter < quiet * 2.5, $"the first line came {firstLineAfter} after the first skip");
+ Assert.True(lines.Count >= 3, $"{lines.Count} lines for a burst that ran several windows");
+ foreach (var id in serverIds)
+ {
+ Assert.Single(lines, l => l.Contains($"'{names[id]}'", StringComparison.Ordinal));
+ }
+ }
+
private async Task> SkipMaintenanceForAsync(Dictionary? displayNames, params int[] serverIds)
{
var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100))
diff --git a/Lite/Services/LocalDataService.DatabaseStates.cs b/Lite/Services/LocalDataService.DatabaseStates.cs
index 32cfaabab1..f4096c4582 100644
--- a/Lite/Services/LocalDataService.DatabaseStates.cs
+++ b/Lite/Services/LocalDataService.DatabaseStates.cs
@@ -39,16 +39,17 @@ public partial class LocalDataService
///
/// #9233: the servers whose maintenance skipped in the current burst, with the holder each named, waiting for ONE line. The alert
/// engine sweeps the servers one at a time and a held lock skips every one of them, so a line per server read as six identical
- /// lines about one event. A skip joins this list and (re)starts . The line is written once the burst
- /// has been quiet for , naming every server in it.
+ /// lines about one event. The first skip of a burst joins this list and starts , a fixed window. A later
+ /// skip only joins the list. The line is written when the window ends, naming every server in it, and the next skip starts a
+ /// new window. The window is not reset by later skips: a stuck lock keeps skipping, and a window that slid would never log.
///
private readonly List<(string Server, string Holder)> _skipsThisCycle = new();
private readonly object _skipLogGate = new();
private System.Threading.Timer? _skipLogTimer;
///
- /// How long no further server may skip before the burst's line is written. One sweep waits up to the write-lock budget before it
- /// skips, so the default is that budget plus five seconds. A test sets a long time and calls .
+ /// How long after the first skip of a burst the line is written. One sweep waits up to the write-lock budget before it skips,
+ /// so the default is that budget plus five seconds, long enough for every server of the sweep to join. A test sets a long time and calls .
///
internal TimeSpan? MaintenanceSkipLogQuietPeriodOverride { get; set; }
@@ -60,8 +61,7 @@ private void NoteMaintenanceSkipped(string server, string holder)
lock (_skipLogGate)
{
_skipsThisCycle.Add((server, holder));
- _skipLogTimer?.Dispose();
- _skipLogTimer = new System.Threading.Timer(_ => FlushMaintenanceSkipLog(), null, MaintenanceSkipLogQuietPeriod, System.Threading.Timeout.InfiniteTimeSpan);
+ _skipLogTimer ??= new System.Threading.Timer(_ => FlushMaintenanceSkipLog(), null, MaintenanceSkipLogQuietPeriod, System.Threading.Timeout.InfiniteTimeSpan);
}
}
From 2cb1dac4961005a1776727f0e000ce8be0a8d013 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 21 Nov 2026 02:31:36 -0500
Subject: [PATCH 3/3] Count skips instead of timing them in the skip-log window
test (#9233)
---
Lite.Tests/StoreLockTimeoutLogTests.cs | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/Lite.Tests/StoreLockTimeoutLogTests.cs b/Lite.Tests/StoreLockTimeoutLogTests.cs
index 9b97e9862a..ac0481b859 100644
--- a/Lite.Tests/StoreLockTimeoutLogTests.cs
+++ b/Lite.Tests/StoreLockTimeoutLogTests.cs
@@ -170,21 +170,22 @@ public async Task SkipsThatKeepArriving_StillLogWithinOneWindowOfTheFirst_EachLi
holder.Start();
var lines = new List();
- TimeSpan? firstLineAfter = null;
+ int? firstLineAtSkip = null;
try
{
Assert.True(held.Wait(TimeSpan.FromSeconds(10)), "the read never took the lock");
AppLogger.DrainBufferedLines();
- var started = System.Diagnostics.Stopwatch.StartNew();
+ var skipNumber = 0;
foreach (var serverId in serverIds)
{
/* Each skip waits the 100 ms budget, then the pause puts the skips half a quiet period apart: four and a
half quiet periods in all, and no gap between skips as long as one quiet period. */
await service.GetDatabaseStateDeviationsAsync(serverId);
var seen = AppLogger.DrainBufferedLines().Where(l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal)).ToList();
- if (seen.Count > 0 && firstLineAfter is null)
+ skipNumber++;
+ if (seen.Count > 0 && firstLineAtSkip is null)
{
- firstLineAfter = started.Elapsed;
+ firstLineAtSkip = skipNumber;
}
lines.AddRange(seen);
@@ -201,8 +202,8 @@ public async Task SkipsThatKeepArriving_StillLogWithinOneWindowOfTheFirst_EachLi
}
/* With a window that slid, nothing logged until the final flush and the one line held all nine servers. */
- Assert.NotNull(firstLineAfter);
- Assert.True(firstLineAfter < quiet * 2.5, $"the first line came {firstLineAfter} after the first skip");
+ Assert.NotNull(firstLineAtSkip);
+ Assert.True(firstLineAtSkip <= 5, $"the first line came after skip {firstLineAtSkip}, skips being half a window apart");
Assert.True(lines.Count >= 3, $"{lines.Count} lines for a burst that ran several windows");
foreach (var id in serverIds)
{