diff --git a/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs b/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs index cf42cf7f86..dbfd8a5459 100644 --- a/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs +++ b/Lite.Tests/FinOpsFirstVisitAndBaselineTests.cs @@ -59,6 +59,20 @@ public void ACollectionFinishingWhileTheTabIsVisible_ReloadsOnceTheLastLoadIsAMi [Fact] public void ACollectionBeforeAnyLoad_Reloads() => Assert.True(FinOpsShowReloadPolicy.ShouldReloadAfterCollection(null, Now, Now)); + /// + /// #9233: window start selects the first server on a tab nobody has opened. That selection must not run the fifteen store reads + /// (High Impact the heaviest) against a store the first collection has not written to. The tab's first show runs the whole load + /// whatever the age of any earlier one, so nothing is lost by waiting for it. + /// + [Theory] + [InlineData(false, false)] + [InlineData(true, true)] + public void ASelectionChange_LoadsOnlyWhileTheTabIsShown(bool tabVisible, bool loads) + { + Assert.Equal(loads, FinOpsShowReloadPolicy.ShouldLoadOnSelectionChange(tabVisible)); + Assert.True(FinOpsShowReloadPolicy.ShouldReloadOnShow(null, Now, firstShowSinceStart: true)); + } + [Theory] [InlineData(0, false)] [InlineData(5, false)] diff --git a/Lite.Tests/HighImpactReadEqualityTests.cs b/Lite.Tests/HighImpactReadEqualityTests.cs new file mode 100644 index 0000000000..69682e2394 --- /dev/null +++ b/Lite.Tests/HighImpactReadEqualityTests.cs @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; +using DuckDB.NET.Data; +using PerformanceMonitorLite.Database; +using PerformanceMonitorLite.Services; +using Xunit; + +namespace PerformanceMonitorLite.Tests; + +/// +/// #9233: the High Impact read picked each hash's grant with a per-hash LATERAL that sorted the whole window, and its sample +/// text with a ROW_NUMBER sort that carried the text of every row of the kept hashes. Both are now one grouped pass. The new +/// statement must return the same rows. This runs the statement the app ships and the one it shipped before (kept here as the +/// oracle) over one seeded window and compares every column of every row. The seed holds the cases where a NULL sort key +/// decides the pick: a NULL creation_time and a NULL delta_worker_time (the newest row's grant), a NULL count (the sample +/// text), ties on collection_time, hashes with several plans and several databases, an empty and a NULL text. +/// +public sealed class HighImpactReadEqualityTests : IClassFixture +{ + private const int ServerId = -9233; + private const string ServerName = "lite-high-impact-equality"; + + private readonly DuckDbInitializer _duckDb; + + public HighImpactReadEqualityTests(SharedDuckDbFixture fixture) + { + fixture.ResetData(); + _duckDb = fixture.DuckDb; + } + + /// The statement as it shipped before #9233: the grant from a per-hash LATERAL, the sample text from a ROW_NUMBER over the rows. + private const string OracleSql = @" +WITH win AS MATERIALIZED +( + SELECT query_hash, database_name, delta_execution_count, delta_worker_time, delta_elapsed_time, + delta_logical_reads, delta_logical_writes, max_grant_kb, collection_time, creation_time + FROM v_query_stats + WHERE server_id = $1 + AND collection_time >= $2 + AND query_hash IS NOT NULL AND query_hash != '' +), +per_database AS +( + SELECT query_hash, database_name, + SUM(delta_execution_count) AS executions, + SUM(delta_worker_time) AS worker_time, + SUM(delta_elapsed_time) AS elapsed_time, + SUM(delta_logical_reads) AS logical_reads, + SUM(delta_logical_writes) AS logical_writes + FROM win + WHERE delta_execution_count > 0 + GROUP BY query_hash, database_name +), +per_hash AS +( + SELECT + query_hash, + (ARRAY_AGG(database_name ORDER BY worker_time DESC NULLS LAST, LOWER(database_name), database_name) + FILTER (WHERE database_name IS NOT NULL))[1] AS database_name, + SUM(executions) AS total_executions, + SUM(worker_time) AS total_worker_time, + SUM(elapsed_time) AS total_elapsed_time, + SUM(logical_reads) AS total_reads, + SUM(logical_writes) AS total_writes, + COUNT(DISTINCT database_name) AS database_count + FROM per_database + GROUP BY query_hash + HAVING SUM(executions) > 0 +), +ranked AS +( + SELECT h.*, COALESCE(g.max_grant_kb, 0) AS peak_grant_kb, + ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_worker_time, 0) DESC, h.total_worker_time DESC, h.query_hash) AS cpu_rank, + ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_elapsed_time, 0) DESC, h.total_worker_time DESC, h.query_hash) AS duration_rank, + ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_reads, 0) DESC, h.total_worker_time DESC, h.query_hash) AS reads_rank, + ROW_NUMBER() OVER (ORDER BY COALESCE(h.total_writes, 0) DESC, h.total_worker_time DESC, h.query_hash) AS writes_rank, + ROW_NUMBER() OVER (ORDER BY COALESCE(g.max_grant_kb, 0) DESC, h.total_worker_time DESC, h.query_hash) AS memory_rank, + ROW_NUMBER() OVER (ORDER BY h.total_executions DESC, h.total_worker_time DESC, h.query_hash) AS executions_rank + FROM per_hash AS h + LEFT JOIN LATERAL + ( + SELECT qs.max_grant_kb + FROM win AS qs + WHERE qs.query_hash = h.query_hash + AND qs.delta_execution_count > 0 + ORDER BY qs.collection_time DESC, qs.creation_time DESC NULLS LAST, qs.delta_worker_time DESC NULLS LAST + LIMIT 1 + ) AS g ON true +), +kept AS +( + SELECT * + FROM ranked + WHERE cpu_rank <= $3 OR duration_rank <= $3 OR reads_rank <= $3 + OR writes_rank <= $3 OR memory_rank <= $3 OR executions_rank <= $3 +), +sample_text AS +( + SELECT qs2.query_hash, qs2.query_text + FROM v_query_stats AS qs2 + JOIN kept AS kk ON kk.query_hash = qs2.query_hash + WHERE qs2.server_id = $1 + AND qs2.collection_time >= $2 + AND qs2.query_text IS NOT NULL AND qs2.query_text != '' + QUALIFY ROW_NUMBER() OVER (PARTITION BY qs2.query_hash ORDER BY qs2.delta_execution_count DESC NULLS LAST, qs2.collection_time DESC, qs2.creation_time DESC NULLS LAST, qs2.query_text) = 1 +) +SELECT + k.query_hash, + k.database_name, + k.total_executions, + k.total_worker_time / 1000.0 AS total_cpu_ms, + k.total_elapsed_time / 1000.0 AS total_duration_ms, + k.total_reads, + k.total_writes, + k.peak_grant_kb / 1024.0 AS peak_memory_mb, + k.database_count, + LEFT(t.query_text, 200) AS sample_query_text, + t.query_text AS full_query_text +FROM kept AS k +LEFT JOIN sample_text AS t ON t.query_hash = k.query_hash +ORDER BY k.total_worker_time DESC, k.query_hash"; + + private static readonly DateTime Now = DateTime.UtcNow; + private static readonly DateTime T1 = Now.AddHours(-5); + private static readonly DateTime Plan1 = Now.AddDays(-9); + private static readonly DateTime Plan2 = Now.AddDays(-4); + + private static readonly object Null = DBNull.Value; + + private async Task SeedAsync() + { + using var readLock = _duckDb.AcquireReadLock(); + using var connection = _duckDb.CreateConnection(); + await connection.OpenAsync(); + long id = -9_233_000; + + async Task Row(string hash, string db, object? text, DateTime at, object creation, object count, object cpu, object grant) + { + using var command = connection.CreateCommand(); + command.CommandText = @" +INSERT INTO query_stats + (collection_id, collection_time, server_id, server_name, database_name, query_hash, query_text, creation_time, + delta_execution_count, delta_worker_time, delta_elapsed_time, delta_logical_reads, delta_logical_writes, max_grant_kb) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)"; + foreach (var value in new object[] { id--, at, ServerId, ServerName, db, hash, text ?? Null, creation, count, cpu, 1_000L, 10L, 1L, grant }) + { + command.Parameters.Add(new DuckDBParameter { Value = value }); + } + + await command.ExecuteNonQueryAsync(); + } + + /* Grant: the newest collection, then the newest plan creation (a NULL creation last), then the most worker time (a NULL last). */ + await Row("0xA", "Db1", "SELECT a", T1, Null, 5L, 100L, 100L); // a NULL creation_time loses to a real one: 200 + await Row("0xA", "Db1", "SELECT a", T1, Plan1, 5L, 100L, 200L); + await Row("0xB", "Db1", "SELECT b", T1, Plan1, 5L, Null, 300L); // a NULL delta_worker_time loses to a real one: 400 + await Row("0xB", "Db1", "SELECT b", T1, Plan1, 5L, 7L, 400L); + await Row("0xC", "Db1", "SELECT c", T1, Plan2, 5L, 100L, 900L); // the newer collection wins whatever the grant: 50 + await Row("0xC", "Db1", "SELECT c", T1.AddMinutes(1), Plan1, 5L, 1L, 50L); + await Row("0xD", "Db1", "SELECT d", T1, Plan1, 5L, 100L, 600L); // a newest row with no executions is no reading: 600 + await Row("0xD", "Db1", "SELECT d", T1.AddMinutes(1), Plan2, 0L, 100L, 9_000L); + await Row("0xE", "Db1", "SELECT e", T1, Plan1, 5L, 100L, 700L); // the newest row's grant is NULL: 0 + await Row("0xE", "Db1", "SELECT e", T1.AddMinutes(1), Plan2, 5L, 100L, Null); + await Row("0xF", "Db1", "SELECT f", T1, Null, 5L, Null, 11L); // both keys NULL on one row, a real key on the other: 22 + await Row("0xF", "Db1", "SELECT f", T1, Plan2, 5L, 3L, 22L); + await Row("0xG", "Db2", "SELECT g", T1, Plan1, 5L, 100L, 33L); // the same figures in two databases: the same grant either way + await Row("0xG", "Db3", "SELECT g", T1, Plan1, 5L, 100L, 33L); + + /* Sample text: the most executions (a NULL count last), then the newest collection, then the newest creation (a NULL last), then the smallest text. */ + await Row("0xH", "Db1", "SELECT h-null-count", T1.AddMinutes(5), Plan2, Null, 10L, 1L); // a NULL count ranks last: "SELECT h-few" + await Row("0xH", "Db1", "SELECT h-few", T1, Plan1, 1L, 10L, 1L); + await Row("0xI", "Db1", "SELECT i-null-creation", T1, Null, 4L, 10L, 1L); // a NULL creation ranks last: "SELECT i-real" + await Row("0xI", "Db1", "SELECT i-real", T1, Plan1, 4L, 10L, 1L); + await Row("0xJ", "Db1", "SELECT j-b", T1, Plan1, 4L, 10L, 1L); // a full tie: the smaller text, "SELECT j-a" + await Row("0xJ", "Db2", "SELECT j-a", T1, Plan1, 4L, 10L, 1L); + await Row("0xK", "Db1", "SELECT k-old", T1, Plan1, 4L, 10L, 1L); // the newer collection wins a count tie: "SELECT k-new" + await Row("0xK", "Db1", "SELECT k-new", T1.AddMinutes(1), Plan1, 4L, 10L, 1L); + await Row("0xL", "Db1", "", T1.AddMinutes(1), Plan1, 9L, 10L, 1L); // an empty and a NULL text are no sample: "SELECT l" + await Row("0xL", "Db1", null, T1.AddMinutes(2), Plan1, 9L, 10L, 1L); + await Row("0xL", "Db1", "SELECT l", T1, Plan1, 1L, 10L, 1L); + await Row("0xM", "Db1", null, T1, Plan1, 3L, 10L, 1L); // no text anywhere: a NULL sample + + /* Filler, so the keep count cuts the window: each measure puts a different hash on top. */ + for (var i = 0; i < 40; i++) + { + await Row("0xZ" + i.ToString("D2", CultureInfo.InvariantCulture), "Db" + (i % 3), "SELECT z" + i, T1.AddMinutes(i % 4), + i % 5 == 0 ? Null : Plan1, (i * 7) % 13 + 1L, i % 6 == 0 ? Null : (i * 11) % 17 * 100L, (i * 13) % 19 * 1024L); + } + + /* A row outside the window is not read. */ + await Row("0xOLD", "Db1", "SELECT old", Now.AddDays(-3), Plan1, 99L, 999L, 99_999L); + } + + private async Task> RunAsync(string sql, int keep) + { + using var readLock = _duckDb.AcquireReadLock(); + using var connection = _duckDb.CreateConnection(); + await connection.OpenAsync(); + using var command = connection.CreateCommand(); + command.CommandText = sql.Replace("v_query_stats", "query_stats", StringComparison.Ordinal); + command.Parameters.Add(new DuckDBParameter { Value = ServerId }); + command.Parameters.Add(new DuckDBParameter { Value = Now.AddHours(-24) }); + command.Parameters.Add(new DuckDBParameter { Value = keep }); + + var rows = new List(); + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + var cells = new string[reader.FieldCount]; + for (var c = 0; c < cells.Length; c++) + { + cells[c] = reader.IsDBNull(c) ? "" : Convert.ToString(reader.GetValue(c), CultureInfo.InvariantCulture) ?? ""; + } + + rows.Add(string.Join("|", cells)); + } + + return rows; + } + + [Theory] + [InlineData(1000)] + [InlineData(10)] + [InlineData(3)] + public async Task TheGroupedPasses_ReturnTheRowsTheLateralAndTheRowNumberSortReturned(int keep) + { + await SeedAsync(); + + var oracle = await RunAsync(OracleSql, keep); + var shipped = await RunAsync(LocalDataService.HighImpactQueriesSql, keep); + + Assert.True(oracle.Count >= 3, $"the seed must keep a spread of hashes, the oracle read {oracle.Count}"); + Assert.Equal(oracle, shipped); + } + + /// The picks the seed was built around, read from the shipped statement, so a seed that stopped exercising a NULL key fails here and not silently. + [Fact] + public async Task TheShippedStatement_PicksTheGrantAndTheSampleTheNullKeysCallFor() + { + await SeedAsync(); + + var shipped = await RunAsync(LocalDataService.HighImpactQueriesSql, 1000); + string Cell(string hash, int column) => shipped.Single(r => r.StartsWith(hash + "|", StringComparison.Ordinal)).Split('|')[column]; + + /* Column 7 is the peak memory in MB, the grant in KB over 1024. */ + Assert.Equal(200.0 / 1024, double.Parse(Cell("0xA", 7), CultureInfo.InvariantCulture), 1e-9); + Assert.Equal(400.0 / 1024, double.Parse(Cell("0xB", 7), CultureInfo.InvariantCulture), 1e-9); + Assert.Equal(50.0 / 1024, double.Parse(Cell("0xC", 7), CultureInfo.InvariantCulture), 1e-9); + Assert.Equal(600.0 / 1024, double.Parse(Cell("0xD", 7), CultureInfo.InvariantCulture), 1e-9); + Assert.Equal(0.0, double.Parse(Cell("0xE", 7), CultureInfo.InvariantCulture), 1e-9); + Assert.Equal(22.0 / 1024, double.Parse(Cell("0xF", 7), CultureInfo.InvariantCulture), 1e-9); + + /* Column 10 is the full sample text. */ + Assert.Equal("SELECT h-few", Cell("0xH", 10)); + Assert.Equal("SELECT i-real", Cell("0xI", 10)); + Assert.Equal("SELECT j-a", Cell("0xJ", 10)); + Assert.Equal("SELECT k-new", Cell("0xK", 10)); + Assert.Equal("SELECT l", Cell("0xL", 10)); + Assert.Equal("", Cell("0xM", 10)); + } +} diff --git a/Lite.Tests/StoreLockTimeoutLogTests.cs b/Lite.Tests/StoreLockTimeoutLogTests.cs index d0c7673021..ac0481b859 100644 --- a/Lite.Tests/StoreLockTimeoutLogTests.cs +++ b/Lite.Tests/StoreLockTimeoutLogTests.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -102,12 +103,131 @@ public async Task TheSkippedMaintenanceLine_KeepsTheStorageId_ForAServerNobodyCa private async Task SkipMaintenanceAndReadTheLineAsync(string? displayName) { - var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100)); - if (displayName is not null) + var names = displayName is null ? null : new Dictionary { [ServerId] = displayName }; + return Assert.Single(await SkipMaintenanceForAsync(names, ServerId)); + } + + /// #9233: three servers' sweeps meet the same held write lock, one after another as the alert engine runs them. Their one line names all three. + [Fact] + public async Task ASkippedCycle_WritesOneLine_NamingEveryServerItSkipped() + { + var names = new Dictionary { [ServerId] = "Display One", [ServerId - 1] = "Display Two", [ServerId - 2] = "Display Three" }; + + var lines = await SkipMaintenanceForAsync(names, ServerId, ServerId - 1, ServerId - 2); + + var line = Assert.Single(lines); + Assert.Contains("'Display One'", line, StringComparison.Ordinal); + Assert.Contains("'Display Two'", line, StringComparison.Ordinal); + Assert.Contains("'Display Three'", line, StringComparison.Ordinal); + } + + /// The skips of a cycle that has been written do not leak into the next cycle's line. + [Fact] + public async Task TheNextCycleOfSkips_WritesItsOwnLine_WithoutTheEarlierServers() + { + var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100)) + { + MaintenanceSkipLogQuietPeriodOverride = TimeSpan.FromHours(1), + DisplayNames = () => new Dictionary { [ServerId] = "Display One", [ServerId - 1] = "Display Two" }, + }; + + var first = await SkipUnderAHeldLockAsync(service, ServerId); + var second = await SkipUnderAHeldLockAsync(service, ServerId - 1); + + Assert.Contains("'Display One'", Assert.Single(first), StringComparison.Ordinal); + var later = Assert.Single(second); + Assert.Contains("'Display Two'", later, StringComparison.Ordinal); + Assert.DoesNotContain("'Display One'", later, StringComparison.Ordinal); + } + + /// + /// #9233: a stuck lock keeps skipping servers, each one less than a quiet period after the last. The window opens at the first + /// skip and is not pushed back by the later ones, so the line still logs and each window's line names its own servers. + /// + [Fact] + public async Task SkipsThatKeepArriving_StillLogWithinOneWindowOfTheFirst_EachLineNamingItsOwnServers() + { + var quiet = TimeSpan.FromSeconds(1); + var serverIds = Enumerable.Range(0, 9).Select(i => ServerId - i).ToArray(); + var names = serverIds.ToDictionary(id => id, id => $"Display {ServerId - id}"); + var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100)) + { + MaintenanceSkipLogQuietPeriodOverride = quiet, + DisplayNames = () => names, + }; + + var held = new ManualResetEventSlim(false); + var release = new ManualResetEventSlim(false); + var holder = new Thread(() => + { + using (_duckDb.AcquireReadLock()) + { + held.Set(); + release.Wait(TimeSpan.FromSeconds(30)); + } + }) + { IsBackground = true }; + holder.Start(); + + var lines = new List(); + int? firstLineAtSkip = null; + try { - service.DisplayNames = () => new Dictionary { [ServerId] = displayName }; + Assert.True(held.Wait(TimeSpan.FromSeconds(10)), "the read never took the lock"); + AppLogger.DrainBufferedLines(); + var skipNumber = 0; + foreach (var serverId in serverIds) + { + /* Each skip waits the 100 ms budget, then the pause puts the skips half a quiet period apart: four and a + half quiet periods in all, and no gap between skips as long as one quiet period. */ + await service.GetDatabaseStateDeviationsAsync(serverId); + var seen = AppLogger.DrainBufferedLines().Where(l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal)).ToList(); + skipNumber++; + if (seen.Count > 0 && firstLineAtSkip is null) + { + firstLineAtSkip = skipNumber; + } + + lines.AddRange(seen); + await Task.Delay(quiet / 2 - TimeSpan.FromMilliseconds(100)); + } + + service.FlushMaintenanceSkipLog(); + lines.AddRange(AppLogger.DrainBufferedLines().Where(l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal))); + } + finally + { + release.Set(); + holder.Join(TimeSpan.FromSeconds(10)); } + /* With a window that slid, nothing logged until the final flush and the one line held all nine servers. */ + Assert.NotNull(firstLineAtSkip); + Assert.True(firstLineAtSkip <= 5, $"the first line came after skip {firstLineAtSkip}, skips being half a window apart"); + Assert.True(lines.Count >= 3, $"{lines.Count} lines for a burst that ran several windows"); + foreach (var id in serverIds) + { + Assert.Single(lines, l => l.Contains($"'{names[id]}'", StringComparison.Ordinal)); + } + } + + private async Task> SkipMaintenanceForAsync(Dictionary? displayNames, params int[] serverIds) + { + var service = new LocalDataService(_duckDb, TimeSpan.FromMilliseconds(100), TimeSpan.FromMilliseconds(100)) + { + MaintenanceSkipLogQuietPeriodOverride = TimeSpan.FromHours(1), + }; + if (displayNames is not null) + { + service.DisplayNames = () => displayNames; + } + + return await SkipUnderAHeldLockAsync(service, serverIds); + } + + /// Sweeps each server under a read held on another thread, then writes the burst's line, and returns the maintenance-skip lines the log buffered. + private async Task> SkipUnderAHeldLockAsync(LocalDataService service, params int[] serverIds) + { /* A read held on another thread: the maintenance wants the write lock, waits its 100 ms and skips the cycle. */ var held = new ManualResetEventSlim(false); var release = new ManualResetEventSlim(false); @@ -126,7 +246,12 @@ private async Task SkipMaintenanceAndReadTheLineAsync(string? displayNam { Assert.True(held.Wait(TimeSpan.FromSeconds(10)), "the read never took the lock"); AppLogger.DrainBufferedLines(); - await service.GetDatabaseStateDeviationsAsync(ServerId); + foreach (var serverId in serverIds) + { + await service.GetDatabaseStateDeviationsAsync(serverId); + } + + service.FlushMaintenanceSkipLog(); } finally { @@ -134,7 +259,7 @@ private async Task SkipMaintenanceAndReadTheLineAsync(string? displayNam holder.Join(TimeSpan.FromSeconds(10)); } - return Assert.Single(AppLogger.DrainBufferedLines(), l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal)); + return AppLogger.DrainBufferedLines().Where(l => l.Contains("skipped this cycle's database-state maintenance", StringComparison.Ordinal)).ToList(); } private sealed class CapturingLogger : ILogger diff --git a/Lite/Controls/FinOpsTab.xaml.cs b/Lite/Controls/FinOpsTab.xaml.cs index 7df1354f2c..ba36f98635 100644 --- a/Lite/Controls/FinOpsTab.xaml.cs +++ b/Lite/Controls/FinOpsTab.xaml.cs @@ -160,8 +160,11 @@ public void Initialize(LocalDataService dataService, ServerManager serverManager _openTabClock = openTabClock; _credentialResolver = serverManager.CredentialResolver; + /* #9233: the per-server load (fifteen store reads, High Impact the heaviest) runs only while the tab is shown. Window start used to run + it twice for a tab nobody had opened (the selector's first selection, then RefreshData), against a store the first collection + has not written to yet, and the first show re-runs it whole anyway (FinOpsShowReloadPolicy). Start-up keeps the fleet inventory. */ PopulateServerSelector(); - RefreshData(); + _ = LoadServerInventoryAsync(); } /// @@ -1440,6 +1443,14 @@ private async void ServerSelector_SelectionChanged(object sender, SelectionChang selected server B with A's row. */ ClearPerServerGrids(); + /* #9233: a selection that moves while the tab is hidden (the first one at start-up, a server list that changed) loads nothing. The + next show runs the whole load, as the first show always does. */ + if (!FinOpsShowReloadPolicy.ShouldLoadOnSelectionChange(IsVisible)) + { + _firstShowPending = true; + return; + } + await LoadPerServerDataAsync(); } diff --git a/Lite/Services/FinOpsShowReloadPolicy.cs b/Lite/Services/FinOpsShowReloadPolicy.cs index 5a8fecc155..1c779f877d 100644 --- a/Lite/Services/FinOpsShowReloadPolicy.cs +++ b/Lite/Services/FinOpsShowReloadPolicy.cs @@ -35,6 +35,13 @@ internal static class FinOpsShowReloadPolicy internal static bool ShouldReloadOnShow(DateTime? lastLoadStartedUtc, DateTime nowUtc, bool firstShowSinceStart = false) => firstShowSinceStart || lastLoadStartedUtc is not DateTime last || nowUtc - last >= MinAge; + /// + /// True when a selection change should run the whole per-server load now (#9233): only while the tab is shown. A hidden tab + /// leaves the load to its first show, which always runs it (), so window start does not run + /// fifteen store reads, High Impact the heaviest, for a tab nobody has opened. + /// + internal static bool ShouldLoadOnSelectionChange(bool tabVisible) => tabVisible; + /// /// True when a collection for the selected server that finished at should reload the /// visible tab: it finished after the last whole load began (that load may have read the window before it), and that load began diff --git a/Lite/Services/LocalDataService.DatabaseStates.cs b/Lite/Services/LocalDataService.DatabaseStates.cs index 8e0703eacb..f4096c4582 100644 --- a/Lite/Services/LocalDataService.DatabaseStates.cs +++ b/Lite/Services/LocalDataService.DatabaseStates.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; using System.Threading.Tasks; using DuckDB.NET.Data; using PerformanceMonitor.Alerting; @@ -34,6 +35,56 @@ public partial class LocalDataService /// says nothing about another's. Never pruned — one bool per server ever swept. /// private readonly System.Collections.Concurrent.ConcurrentDictionary _lastMaintenanceSkipped = new(); + + /// + /// #9233: the servers whose maintenance skipped in the current burst, with the holder each named, waiting for ONE line. The alert + /// engine sweeps the servers one at a time and a held lock skips every one of them, so a line per server read as six identical + /// lines about one event. The first skip of a burst joins this list and starts , a fixed window. A later + /// skip only joins the list. The line is written when the window ends, naming every server in it, and the next skip starts a + /// new window. The window is not reset by later skips: a stuck lock keeps skipping, and a window that slid would never log. + /// + private readonly List<(string Server, string Holder)> _skipsThisCycle = new(); + private readonly object _skipLogGate = new(); + private System.Threading.Timer? _skipLogTimer; + + /// + /// How long after the first skip of a burst the line is written. One sweep waits up to the write-lock budget before it skips, + /// so the default is that budget plus five seconds, long enough for every server of the sweep to join. A test sets a long time and calls . + /// + internal TimeSpan? MaintenanceSkipLogQuietPeriodOverride { get; set; } + + private TimeSpan MaintenanceSkipLogQuietPeriod => + MaintenanceSkipLogQuietPeriodOverride ?? _backgroundWriteLockBudget + TimeSpan.FromSeconds(5); + + private void NoteMaintenanceSkipped(string server, string holder) + { + lock (_skipLogGate) + { + _skipsThisCycle.Add((server, holder)); + _skipLogTimer ??= new System.Threading.Timer(_ => FlushMaintenanceSkipLog(), null, MaintenanceSkipLogQuietPeriod, System.Threading.Timeout.InfiniteTimeSpan); + } + } + + /// Writes the one line for the servers waiting in , if any, and empties the list. + internal void FlushMaintenanceSkipLog() + { + (string Server, string Holder)[] skipped; + lock (_skipLogGate) + { + _skipLogTimer?.Dispose(); + _skipLogTimer = null; + skipped = _skipsThisCycle.ToArray(); + _skipsThisCycle.Clear(); + } + + if (skipped.Length == 0) + { + return; + } + + AppLogger.Warn(nameof(GetDatabaseStateDeviationsAsync), + MaintenanceSkippedWarning(skipped.Select(x => x.Server).ToList(), skipped.Select(x => x.Holder).Distinct(StringComparer.Ordinal).ToList(), _backgroundWriteLockBudget)); + } /* Effective state = STANDBY for a read-only log-shipping secondary (is_in_standby), else the raw state_desc. A standby secondary reports state_desc = ONLINE with is_in_standby = 1 and flips through RESTORING on every log restore; collapsing it to a single stable STANDBY token means it baselines as @@ -144,8 +195,16 @@ internal string ServerLabelForLog(int serverId) /// had (5 s in the shipped app, two minutes in Lite.Tests), not a number written into the text. /// internal static string MaintenanceSkippedWarning(string server, string holder, TimeSpan budget) => - $"server {server}: skipped this cycle's database-state maintenance: could not acquire the " + - $"store write lock within {budget.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture)}s ({holder}). Baselines are not being " + + MaintenanceSkippedWarning(new[] { server }, new[] { holder }, budget); + + /// + /// #9233: the one line for a skipped cycle, naming every server it skipped. The holders are the distinct ones the skipped servers + /// saw (usually one), joined with a bar. + /// + internal static string MaintenanceSkippedWarning(IReadOnlyList servers, IReadOnlyList holders, TimeSpan budget) => + (servers.Count == 1 ? $"server {servers[0]}" : $"servers {string.Join(", ", servers)}") + + ": skipped this cycle's database-state maintenance: could not acquire the " + + $"store write lock within {budget.TotalSeconds.ToString("0.##", CultureInfo.InvariantCulture)}s ({string.Join(" | ", holders)}). Baselines are not being " + "seeded or healed while this persists (#2189/#2203); deviations are still read. Expected " + "occasionally. If it repeats, the write lock is contended."; @@ -233,8 +292,7 @@ SUSTAINED run of them that means baselines have stopped being seeded and healed. if (!_lastMaintenanceSkipped.TryGetValue(serverId, out var wasSkipped) || !wasSkipped) { _lastMaintenanceSkipped[serverId] = true; - AppLogger.Warn(nameof(GetDatabaseStateDeviationsAsync), - MaintenanceSkippedWarning(ServerLabelForLog(serverId), skipHolder, _backgroundWriteLockBudget)); + NoteMaintenanceSkipped(ServerLabelForLog(serverId), skipHolder); } } else if (_lastMaintenanceSkipped.TryGetValue(serverId, out var hadSkipped) && hadSkipped) diff --git a/Lite/Services/LocalDataService.FinOps.Workload.cs b/Lite/Services/LocalDataService.FinOps.Workload.cs index be7dafc16f..4baf9a2fec 100644 --- a/Lite/Services/LocalDataService.FinOps.Workload.cs +++ b/Lite/Services/LocalDataService.FinOps.Workload.cs @@ -589,17 +589,23 @@ ranked AS FROM per_hash AS h -- #7166: max_grant_kb is a per-plan high-water mark since that plan was cached, so MAX across a hash's plans is -- the largest grant any plan ever had (the max_dop trap, #3648). The reading is the hash's newest row's, the same - -- choice max_dop makes: newest collection first, newest plan first within it. Explicit NULLS LAST: DuckDB and - -- Postgres default DESC null placement differently. - LEFT JOIN LATERAL + -- choice max_dop makes: newest collection first, newest plan first within it. #9233: one grouped pass over win, + -- where a per-hash LATERAL with ORDER BY ... LIMIT 1 sorted the whole window (the statement's costliest operator + -- after the scan). arg_max over a struct ranks a NULL field ABOVE every value (#9195), so each key that sorted + -- NULLS LAST maps to a value below every real one. The grant is the last key, so rows tied on the other three + -- agree on the largest grant, not on whichever the sort met first. + LEFT JOIN ( - SELECT qs.max_grant_kb - FROM win AS qs - WHERE qs.query_hash = h.query_hash - AND qs.delta_execution_count > 0 - ORDER BY qs.collection_time DESC, qs.creation_time DESC NULLS LAST, qs.delta_worker_time DESC NULLS LAST - LIMIT 1 - ) AS g ON true + SELECT query_hash, + arg_max(COALESCE(max_grant_kb, 0), + (collection_time, + COALESCE(creation_time, '-infinity'::TIMESTAMP), + COALESCE(delta_worker_time, -9223372036854775807), + COALESCE(max_grant_kb, 0))) AS max_grant_kb + FROM win + WHERE delta_execution_count > 0 + GROUP BY query_hash + ) AS g ON g.query_hash = h.query_hash ), kept AS ( @@ -613,14 +619,22 @@ sample_text AS -- The sample text for the kept hashes only (#6803), from each hash's busiest row in the window. #9058: one pass for -- the whole set. The two per-hash lookups this replaces each scanned the window again. A NULL count ranks last. A tie -- on the count goes to the newest collection, then the newest plan creation, then the text itself, so the pick is - -- the same on every read even for rows of one collection (several plans or databases). - SELECT qs2.query_hash, qs2.query_text + -- the same on every read even for rows of one collection (several plans or databases). #9233: the pick is an + -- arg_min over the key as a struct, not a ROW_NUMBER sort that carried the text of every row of the kept hashes. + -- Each DESC key is negated and its NULL maps to the largest value, so a NULL still ranks last, and the text is the + -- struct's last member, so a full tie still goes to the smallest text. + SELECT qs2.query_hash, + arg_min(qs2.query_text, + (COALESCE(-qs2.delta_execution_count, 9223372036854775807), + -epoch_us(qs2.collection_time), + COALESCE(-epoch_us(qs2.creation_time), 9223372036854775807), + qs2.query_text)) AS query_text FROM v_query_stats AS qs2 JOIN kept AS kk ON kk.query_hash = qs2.query_hash WHERE qs2.server_id = $1 AND qs2.collection_time >= $2 AND qs2.query_text IS NOT NULL AND qs2.query_text != '' - QUALIFY ROW_NUMBER() OVER (PARTITION BY qs2.query_hash ORDER BY qs2.delta_execution_count DESC NULLS LAST, qs2.collection_time DESC, qs2.creation_time DESC NULLS LAST, qs2.query_text) = 1 + GROUP BY qs2.query_hash ) SELECT k.query_hash,