diff --git a/Darling/Darling.Tests/CurrentDatabaseScopeLoggerThreadingTests.cs b/Darling/Darling.Tests/CurrentDatabaseScopeLoggerThreadingTests.cs index 6180470386..8ca70f97a4 100644 --- a/Darling/Darling.Tests/CurrentDatabaseScopeLoggerThreadingTests.cs +++ b/Darling/Darling.Tests/CurrentDatabaseScopeLoggerThreadingTests.cs @@ -24,8 +24,8 @@ namespace Darling.Tests; /// failure at Warning, but only when the caller hands it a logger. These pins hold the two FinOps tools that were left passing none /// (get_finops index_analysis and get_finops_recommendations) to the same shape as get_wait_stats (ILogger? logger = null just /// before the cancellation token), and hold that no non-test caller of a logger-taking CurrentDatabaseScope method leaves the -/// logger off, so a new caller cannot reopen the gap. The Viewer reaches the same readers with no host logger; it calls only the -/// readers, whose logger parameter is optional, never CurrentDatabaseScope directly. +/// logger off, so a new caller cannot reopen the gap. The Viewer has no host logger: it reaches most of these through the readers, whose +/// logger parameter is optional, and the one direct call (the regressions grid, #6654) passes its own ViewerStoreLogger. /// [Trait("Stage", "Guard")] // #5798: walks the repo tree, so the guard stage runs it public sealed class CurrentDatabaseScopeLoggerThreadingTests diff --git a/Darling/Darling.Tests/DarlingSelfQueryMarkerTests.cs b/Darling/Darling.Tests/DarlingSelfQueryMarkerTests.cs index 46b13a1cc2..c6e78255d7 100644 --- a/Darling/Darling.Tests/DarlingSelfQueryMarkerTests.cs +++ b/Darling/Darling.Tests/DarlingSelfQueryMarkerTests.cs @@ -18,7 +18,9 @@ namespace Darling.Tests; /// /// Release walk V8, Darling side: the service's own statements against a monitored SQL Server carry the -/// PerformanceMonitorLite marker the collectors' self-exclusion reads, so they never top a Top Queries or Query Store list. +/// PerformanceMonitorLite marker the collectors' self-exclusion reads. On Darling the Query Store collector's drop of the +/// statement does not fire (the text is fetched separately, so the row it tests is empty); the readers that rank Query Store +/// queries hide it instead (QueryStoreSelfStatement, #6665). /// Darling runs the shared Query Store collector in its own shape (plan XML and query text fetched by id), which Lite's census /// does not build, so the by-id fetches, the database probe and the connect-time detection query are checked here. /// diff --git a/Darling/Darling.Tests/InvestigationEvidenceFamiliesLiveTests.cs b/Darling/Darling.Tests/InvestigationEvidenceFamiliesLiveTests.cs index 74388e9279..9576276b50 100644 --- a/Darling/Darling.Tests/InvestigationEvidenceFamiliesLiveTests.cs +++ b/Darling/Darling.Tests/InvestigationEvidenceFamiliesLiveTests.cs @@ -230,7 +230,15 @@ await RunAsync(async (setup, ct) => await PlantQueryStoreAsync(setup.Connection, ct, 511, 22, 2, cpuUs: 10_000, lastExec: AlertAt.AddMinutes(-1)); await PlantQueryStoreAsync(setup.Connection, ct, 512, 23, 1, cpuUs: 1_000, lastExec: AlertAt.AddMinutes(-4)); await PlantQueryStoreAsync(setup.Connection, ct, 512, 24, 2, cpuUs: 100_000, lastExec: AlertAt.AddMinutes(-1)); - foreach (var (queryId, text) in new[] { (511L, "SELECT * FROM dbo.Orders"), (512L, "(@csn bigint,@batch_size int)delete top(@batch_size) from sys.[change_tracking_1228027148] where sys_change_xdes_id < @csn") }) + /* #6665: the monitor's own statement regressed 200 times over, stored only in the text table; it is no target either. */ + await PlantQueryStoreAsync(setup.Connection, ct, 513, 25, 1, cpuUs: 1_000, lastExec: AlertAt.AddMinutes(-4)); + await PlantQueryStoreAsync(setup.Connection, ct, 513, 26, 2, cpuUs: 200_000, lastExec: AlertAt.AddMinutes(-1)); + foreach (var (queryId, text) in new[] + { + (511L, "SELECT * FROM dbo.Orders"), + (512L, "(@csn bigint,@batch_size int)delete top(@batch_size) from sys.[change_tracking_1228027148] where sys_change_xdes_id < @csn"), + (513L, "(@cutoff_time datetime2(7))SELECT /* " + PerformanceMonitor.Collectors.QueryStoreCollector.SelfQueryMarker + " */ qsrs.plan_id"), + }) { await DarlingMcpTestData.ExecAsync(setup.Connection, ct, "INSERT INTO collect.query_store_text (server_id, database_name, query_id, query_sql_text, last_seen) VALUES ($1, $2, $3, $4, $5)", diff --git a/Darling/Darling.Tests/PlanRegressionBuiltDaysFallbackLiveTests.cs b/Darling/Darling.Tests/PlanRegressionBuiltDaysFallbackLiveTests.cs index 09a9c037a9..ddcce31be7 100644 --- a/Darling/Darling.Tests/PlanRegressionBuiltDaysFallbackLiveTests.cs +++ b/Darling/Darling.Tests/PlanRegressionBuiltDaysFallbackLiveTests.cs @@ -367,6 +367,28 @@ public async Task ARegressionOfSqlServersOwnCleanup_IsNotAnOffender_AndALookalik } Assert.Null(await ReadAsync()); + + /* #6665: the monitor's own statement, stored only in the text table (the shape a live Darling store has), is no offender + either; the same row with an ordinary text is. */ + await using (var restore = new NpgsqlCommand( + "UPDATE collect.query_store_text SET query_sql_text = $1 WHERE server_id = $2 AND database_name = 'fallbackdb' AND query_id = 1", connection)) + { + restore.Parameters.AddWithValue("SELECT ordinary"); + restore.Parameters.AddWithValue(ServerId); + await restore.ExecuteNonQueryAsync(ct); + } + + Assert.NotNull(await ReadAsync()); + + await using (var own = new NpgsqlCommand( + "UPDATE collect.query_store_text SET query_sql_text = $1 WHERE server_id = $2 AND database_name = 'fallbackdb' AND query_id = 1", connection)) + { + own.Parameters.AddWithValue("(@cutoff_time datetime2(7))SELECT /* " + PerformanceMonitor.Collectors.QueryStoreCollector.SelfQueryMarker + " */ qsrs.plan_id"); + own.Parameters.AddWithValue(ServerId); + await own.ExecuteNonQueryAsync(ct); + } + + Assert.Null(await ReadAsync()); } private static List PlanRegressionFailures(AnalysisContext context) => diff --git a/Darling/Darling.Tests/PlanRegressionDrillDownDailyParityLiveTests.cs b/Darling/Darling.Tests/PlanRegressionDrillDownDailyParityLiveTests.cs index 25dc4b79ab..3cf6e10477 100644 --- a/Darling/Darling.Tests/PlanRegressionDrillDownDailyParityLiveTests.cs +++ b/Darling/Darling.Tests/PlanRegressionDrillDownDailyParityLiveTests.cs @@ -151,6 +151,14 @@ loses the plan-cache flag. */ Assert.Contains(unrestricted, r => r.GetProperty("query_id").GetInt64() == NotAnOffender); Assert.Equal(unrestricted.Select(r => r.GetRawText()), (await DrillDownAsync(postgres, Array.Empty())).Select(r => r.GetRawText())); Assert.Equal(await TableRowsAsync(connection, ct), await RowsOfAsync(unrestricted)); + + /* ---- #6665: the monitor's own statement, stored only in the text table, is left out of both routes ---- */ + await ExecAsync(connection, $"UPDATE collect.query_store_text SET query_sql_text = '/* PerformanceMonitorLite */ SELECT own' WHERE server_id = {ServerId} AND database_name = '{Db}' AND query_id = 1", ct); + Assert.DoesNotContain(OnPairs(await DrillDownAsync(postgres, Pairs)), r => r.GetProperty("query_id").GetInt64() == 1); + await ExecAsync(connection, $"DELETE FROM collect.plan_regression_daily_built WHERE server_id = {ServerId}", ct); + var tableWithoutOwn = OnPairs(await DrillDownAsync(postgres, Pairs)); + Assert.DoesNotContain(tableWithoutOwn, r => r.GetProperty("query_id").GetInt64() == 1); + Assert.Contains(tableWithoutOwn, r => r.GetProperty("query_id").GetInt64() == 2); } /// diff --git a/Darling/Darling.Tests/QueryStoreRegressionsDailyBaselineLiveTests.cs b/Darling/Darling.Tests/QueryStoreRegressionsDailyBaselineLiveTests.cs index bc5547d915..466218c170 100644 --- a/Darling/Darling.Tests/QueryStoreRegressionsDailyBaselineLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreRegressionsDailyBaselineLiveTests.cs @@ -182,6 +182,7 @@ private static async Task> RunAsync( command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Date, Value = plan.RunDays }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.LowEnd }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.HighStart }); + command.Parameters.Add(DatabaseFilter.All.ExcludeParameter()); var rows = new List(); await using var reader = await command.ExecuteReaderAsync(ct); while (await reader.ReadAsync(ct)) diff --git a/Darling/Darling.Tests/QueryStoreRegressionsFailureGroupsLiveTests.cs b/Darling/Darling.Tests/QueryStoreRegressionsFailureGroupsLiveTests.cs index f0aa7b127f..ca8b6d52db 100644 --- a/Darling/Darling.Tests/QueryStoreRegressionsFailureGroupsLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreRegressionsFailureGroupsLiveTests.cs @@ -38,12 +38,12 @@ own database through ScratchPostgres and works entirely inside it. It shares the [Collection("gap-cache-serial")] public sealed class QueryStoreRegressionsFailureGroupsLiveTests { - private const int ServerId = QueryStoreIntervalWideBelowFloorLiveTests.ServerId; - private const string Db = "qsFail"; + internal const int ServerId = QueryStoreIntervalWideBelowFloorLiveTests.ServerId; + internal const string Db = "qsFail"; private static string? BaseConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); - private sealed class Rig : IAsyncDisposable + internal sealed class Rig : IAsyncDisposable { public required ScratchPostgres Scratch { get; init; } public required NpgsqlConnection Connection { get; init; } @@ -62,13 +62,14 @@ public async ValueTask DisposeAsync() /// One interval of one query and outcome, written through the real collector write path. Its last execution is 30 /// minutes after its first, so its day and its side of the window start are those of first + 30 minutes. - private static async Task WriteAsync( + internal static async Task WriteAsync( DarlingCollectorRunner runner, ServerRuntime server, CollectorContext context, string type, long intervalId, - DateTime first, long queryId, long executions, long cpuUs, long durationUs, CancellationToken ct) + DateTime first, long queryId, long executions, long cpuUs, long durationUs, CancellationToken ct, + string? databaseName = null, string? queryText = null) { var row = new QueryStoreCollector.Row { - DatabaseName = Db, + DatabaseName = databaseName ?? Db, QueryId = queryId, PlanId = queryId * 10, ExecutionTypeDesc = type, @@ -76,7 +77,7 @@ private static async Task WriteAsync( LastExecutionTime = first.AddMinutes(30), QueryHash = "0x" + queryId.ToString("X8", CultureInfo.InvariantCulture), QueryPlanHash = "0x" + queryId.ToString("X8", CultureInfo.InvariantCulture), - QueryText = "SELECT " + queryId.ToString(CultureInfo.InvariantCulture), + QueryText = queryText ?? "SELECT " + queryId.ToString(CultureInfo.InvariantCulture), ExecutionCount = executions, AvgCpuTimeUs = cpuUs, AvgDurationUs = durationUs, @@ -88,7 +89,7 @@ private static async Task WriteAsync( await runner.WriteBackfillBatchAsync(QueryStoreCollector.Instance, [row], server, first.AddMinutes(40), context, ct); } - private static async Task OpenAsync(Func seed, CancellationToken ct) + internal static async Task OpenAsync(Func seed, CancellationToken ct) { var baseCs = BaseConnectionString; var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); @@ -208,6 +209,7 @@ private static Task OpenFixtureAsync(CancellationToken ct) => command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Date, Value = plan.RunDays }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.LowEnd }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.HighStart }); + command.Parameters.Add(DatabaseFilter.All.ExcludeParameter()); await using var reader = await command.ExecuteReaderAsync(ct); return await DarlingQueryStoreRegressionReader.ReadRowsAsync(reader, ct); } @@ -221,6 +223,123 @@ private static string Describe(DarlingQueryStoreRegressionReader.RegressionRow r private static DarlingQueryStoreRegressionReader.RegressionRow Row(IEnumerable rows, long queryId) => rows.Single(r => r.QueryId == queryId); + /// + /// #6654: the monitor's own statement and a dropped database's queries are not regressions. Four queries each triple their CPU: + /// 20 (a normal statement), 21 and 22 (their text carries the collector's self-query marker: 21 as an inline sample, 22 only in + /// query_store_text with a NULL inline sample, the shape a live Darling store has) and 30 in a + /// database an earlier database_states snapshot lists and the newest does not. With no database chosen only 20 is listed and the + /// dropped database is named in databases_left_out; naming the dropped database reads it as asked, and the marker stays out. + /// + [Fact] + public async Task TheMonitorsOwnStatementAndADroppedDatabase_AreNotListed_AndTheDroppedOneIsNamed() + { + Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the #6654 live tests."); + var ct = TestContext.Current.CancellationToken; + const string Gone = "qsGone"; + var now = LiveClock.Now(); + await using var rig = await OpenAsync(async (runner, server, context, today, start) => + { + long id = 9500; + async Task W(DateTime first, long q, long cpu, string database, string text) + { + id++; + await WriteAsync(runner, server, context, "Regular", id, first, q, 100, cpu, cpu * 2, ct, database, text); + } + + foreach (var (q, database, text) in new[] + { + (20L, Db, "SELECT normal"), + (21L, Db, "/* " + QueryStoreCollector.SelfQueryMarker + " */ SELECT own"), + (22L, Db, "SELECT own_in_text_table"), + (30L, Gone, "SELECT in_dropped"), + }) + { + await W(now.AddDays(-5), q, 1_000, database, text); + await W(now.AddHours(-3), q, 3_000, database, text); + } + + await W(start.AddDays(-12), 90, 1_000, Db, "SELECT floor"); + }, ct); + + /* Query 22 is the shape a live Darling store has (#6665): the text fetch is separate, so the stats rows carry NO inline text + and the marker sits only in collect.query_store_text, which is the arm the predicate's lookup reads. Query 21 keeps the + inline sample, the fallback arm for a query the text table has no row for. */ + await using (var blank = new NpgsqlCommand("UPDATE query_store_stats SET query_text = NULL WHERE server_id = $1 AND query_id = 22", rig.Connection)) + { + blank.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId }); + await blank.ExecuteNonQueryAsync(ct); + } + + await using (var production = new NpgsqlCommand( + "INSERT INTO query_store_text (server_id, database_name, query_id, query_sql_text, last_seen) VALUES ($1, $2, 22, $3, $4)", + rig.Connection)) + { + production.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = Db }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = "/* " + QueryStoreCollector.SelfQueryMarker + " */ SELECT own_in_text_table" }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = DateTime.SpecifyKind(now, DateTimeKind.Unspecified) }); + await production.ExecuteNonQueryAsync(ct); + } + + foreach (var (age, names) in new[] { (TimeSpan.FromHours(3.5), new[] { Db, Gone }), (TimeSpan.FromHours(1), new[] { Db, Gone }), (TimeSpan.FromMinutes(5), new[] { Db }) }) + { + var at = DateTime.SpecifyKind(DateTime.UtcNow - age, DateTimeKind.Unspecified); + var n = 1; + foreach (var name in names) + { + await using var insert = new NpgsqlCommand( + "INSERT INTO database_states (collection_id, collection_time, server_id, server_name, database_name, database_id, state_desc, is_in_standby) VALUES ($1,$2,$3,'qsfail-host',$4,$5,'ONLINE',FALSE)", + rig.Connection); + insert.Parameters.Add(new NpgsqlParameter { TypedValue = CollectionIdGenerator.Next() }); + insert.Parameters.Add(new NpgsqlParameter { TypedValue = at }); + insert.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId }); + insert.Parameters.Add(new NpgsqlParameter { TypedValue = name }); + insert.Parameters.Add(new NpgsqlParameter { TypedValue = n++ }); + await insert.ExecuteNonQueryAsync(ct); + } + } + + static (List Ids, List LeftOut) Parse(string json) + { + using var doc = System.Text.Json.JsonDocument.Parse(json); + var ids = doc.RootElement.GetProperty("regressions").EnumerateArray().Select(r => r.GetProperty("query_id").GetInt64()).OrderBy(x => x).ToList(); + var left = doc.RootElement.TryGetProperty("databases_left_out", out var l) && l.ValueKind == System.Text.Json.JsonValueKind.Array + ? l.EnumerateArray().Select(x => x.GetString()!).ToList() + : []; + return (ids, left); + } + + var all = Parse(await DarlingMcpQueryStoreRegressionTools.GetQueryStoreRegressions(rig.Postgres, "qsfail", 24, DatabaseFilter.All, 30, false, null, NullLogger.Instance, ct)); + Assert.Equal([20L], all.Ids); + Assert.Equal([Gone], all.LeftOut); + + var named = Parse(await DarlingMcpQueryStoreRegressionTools.GetQueryStoreRegressions(rig.Postgres, "qsfail", 24, DatabaseFilter.One(Gone), 30, false, null, NullLogger.Instance, ct)); + Assert.Equal([30L], named.Ids); + Assert.Empty(named.LeftOut); + + var db = Parse(await DarlingMcpQueryStoreRegressionTools.GetQueryStoreRegressions(rig.Postgres, "qsfail", 24, DatabaseFilter.One(Db), 30, false, null, NullLogger.Instance, ct)); + Assert.Equal([20L], db.Ids); + + /* The Viewer grid reads the same way and hands the left-out names back for its banner (#6654 review): with no database chosen + the dropped one is named and its query is not listed; naming it reads it as asked and names nothing. */ + await using var viewer = new ViewerDataService(rig.Scratch.ConnectionString); + var viewerStart = now.AddHours(-24); + var viewerAll = await viewer.GetQueryStoreRegressionsWithScopeAsync(ServerId, viewerStart, now, cancellationToken: ct); + Assert.Equal([20L], viewerAll.Rows.Select(r => r.QueryId).OrderBy(x => x).ToList()); + Assert.Equal([Gone], viewerAll.DatabasesLeftOut); + var viewerNamed = await viewer.GetQueryStoreRegressionsWithScopeAsync(ServerId, viewerStart, now, [Gone], cancellationToken: ct); + Assert.Equal([30L], viewerNamed.Rows.Select(r => r.QueryId).OrderBy(x => x).ToList()); + Assert.Null(viewerNamed.DatabasesLeftOut); + + /* #6665: a custom range that ended BEFORE the drop is judged by the databases the server had then (the dropped set is + anchored at the range's end, as get_query_store_regressions does with as_of), so the database dropped later still shows + its rows for that range. The newest snapshot at or before the range's end (3.5 hours ago) lists both databases. */ + var rangeEnd = now.AddHours(-2); + var viewerHistorical = await viewer.GetQueryStoreRegressionsWithScopeAsync(ServerId, rangeEnd.AddHours(-24), rangeEnd, literalEndUtc: rangeEnd, cancellationToken: ct); + Assert.Equal([20L, 30L], viewerHistorical.Rows.Select(r => r.QueryId).OrderBy(x => x).ToList()); + Assert.Null(viewerHistorical.DatabasesLeftOut); + } + [Fact] public async Task EachGroupsCountAndPerExecutionFigures_AreTheHandWorkedOnes_OnTheWideArmAndTheRawArm() { diff --git a/Darling/Darling.Tests/QueryStoreRegressionsIntervalTableLiveTests.cs b/Darling/Darling.Tests/QueryStoreRegressionsIntervalTableLiveTests.cs index 00917da9c8..7cda276234 100644 --- a/Darling/Darling.Tests/QueryStoreRegressionsIntervalTableLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreRegressionsIntervalTableLiveTests.cs @@ -155,6 +155,7 @@ private static void AddParameters(NpgsqlCommand command, DateTime start, DateTim command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Date, Value = plan.RunDays }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.LowEnd }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.HighStart }); + command.Parameters.Add(DatabaseFilter.All.ExcludeParameter()); } /// @@ -179,6 +180,7 @@ private static void AddParameters(NpgsqlCommand command, DateTime start, DateTim command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Date, Value = plan.RunDays }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.LowEnd }); command.Parameters.Add(new NpgsqlParameter { TypedValue = plan.HighStart }); + command.Parameters.Add(DatabaseFilter.All.ExcludeParameter()); } else { @@ -292,7 +294,7 @@ INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major /* The viewer's grid takes the same route to the same rows. */ await using (var viewer = new ViewerDataService(scratch.ConnectionString)) { - var viewerTable = await viewer.TryGetQueryStoreRegressionsFromTableAsync(ServerId, start, end, null, [Db], baselineStart, ct); + var viewerTable = await viewer.TryGetQueryStoreRegressionsFromTableAsync(ServerId, start, end, null, [Db], baselineStart, null, ct); Assert.NotNull(viewerTable); Assert.Equal(raw, viewerTable.Select(Describe).ToList()); var viewerRows = await viewer.GetQueryStoreRegressionsAsync(ServerId, start, end, [Db], cancellationToken: ct); @@ -321,7 +323,7 @@ INSERT INTO servers (server_id, server_name, display_name, is_enabled, sql_major Assert.Equal("raw", fallback.Source); await using (var viewer = new ViewerDataService(scratch.ConnectionString)) { - Assert.Null(await viewer.TryGetQueryStoreRegressionsFromTableAsync(ServerId, start, end, null, [Db], baselineStart, ct)); + Assert.Null(await viewer.TryGetQueryStoreRegressionsFromTableAsync(ServerId, start, end, null, [Db], baselineStart, null, ct)); var viewerRows = await viewer.GetQueryStoreRegressionsAsync(ServerId, start, end, [Db], cancellationToken: ct); Assert.Equal(raw, viewerRows.Select(Describe).ToList()); } @@ -459,7 +461,7 @@ public async Task AServerWithAMeasuredMargin_ReadsTheNarrowFloor_AndReturnsTheFi Assert.Equal(fixedBefore, served.Rows.Select(Describe).ToList()); await using var viewer = new ViewerDataService(rig.Scratch.ConnectionString); - var viewerTable = await viewer.TryGetQueryStoreRegressionsFromTableAsync(ServerId, rig.Start, rig.End, null, [Db], rig.BaselineStart, ct); + var viewerTable = await viewer.TryGetQueryStoreRegressionsFromTableAsync(ServerId, rig.Start, rig.End, null, [Db], rig.BaselineStart, null, ct); Assert.NotNull(viewerTable); Assert.Equal(fixedBefore, viewerTable.Select(Describe).ToList()); var viewerRows = await viewer.GetQueryStoreRegressionsAsync(ServerId, rig.Start, rig.End, [Db], cancellationToken: ct); diff --git a/Darling/Darling.Tests/QueryStoreSelfStatementReadersLiveTests.cs b/Darling/Darling.Tests/QueryStoreSelfStatementReadersLiveTests.cs new file mode 100644 index 0000000000..3faf164507 --- /dev/null +++ b/Darling/Darling.Tests/QueryStoreSelfStatementReadersLiveTests.cs @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using PerformanceMonitor.Darling.Viewer; +using Xunit; + +namespace Darling.Tests; + +/// +/// #6665: the monitor's own Query Store statement is hidden from every reader that ranks Query Store queries. On Darling the +/// collector's drop of it never fires (the statement text arrives separately, so the row it tests is empty), so the statement is a +/// stored fact row and its marker is only in collect.query_store_text. Seeded the way a live store is: the stats rows carry +/// NO inline text, both queries' text is in query_store_text, and one of them carries the marker. Each ranked reader lists +/// the normal query (20) and not the monitor's own (23): the MCP Query Store top, the Viewer's Query Store grid, the aborted and +/// failed queries read, and the Viewer's current-vs-baseline comparison. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Every test reaches DARLING_TEST_PG only to CREATE and DROP its + own database through ScratchPostgres (the rig of QueryStoreRegressionsFailureGroupsLiveTests) and works entirely inside it. It + shares the gap-cache collection with the other regressions table tests, because the gate's cadence check reads a process-wide + cache. */ +[Collection("gap-cache-serial")] +public sealed class QueryStoreSelfStatementReadersLiveTests +{ + private const int ServerId = QueryStoreRegressionsFailureGroupsLiveTests.ServerId; + private const string Db = QueryStoreRegressionsFailureGroupsLiveTests.Db; + private const long Normal = 20; + private const long Own = 23; + + [Fact] + public async Task TheMonitorsOwnStatement_StoredOnlyInTheTextTable_IsHiddenFromEveryRankedQueryStoreReader() + { + Assert.SkipWhen(string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DARLING_TEST_PG")), "Set DARLING_TEST_PG to a Postgres connection string to run the #6665 live tests."); + var ct = TestContext.Current.CancellationToken; + var now = LiveClock.Now(); + await using var rig = await QueryStoreRegressionsFailureGroupsLiveTests.OpenAsync(async (runner, server, context, today, start) => + { + long id = 9700; + foreach (var query in new[] { Normal, Own }) + { + foreach (var type in new[] { "Regular", "Aborted" }) + { + id++; + await QueryStoreRegressionsFailureGroupsLiveTests.WriteAsync(runner, server, context, type, id, now.AddHours(-3), query, 100, 2_000, 4_000, ct); + } + } + }, ct); + + /* The live shape: no inline text on the stats rows, the text of both queries in collect.query_store_text. */ + await using (var blank = new NpgsqlCommand("UPDATE query_store_stats SET query_text = NULL WHERE server_id = $1", rig.Connection)) + { + blank.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId }); + await blank.ExecuteNonQueryAsync(ct); + } + + await using (var production = new NpgsqlCommand( + "INSERT INTO query_store_text (server_id, database_name, query_id, query_sql_text, last_seen) VALUES ($1, $2, $3, 'SELECT normal', $5), ($1, $2, $4, $6, $5)", + rig.Connection)) + { + production.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = Db }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = Normal }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = Own }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = DateTime.SpecifyKind(now, DateTimeKind.Unspecified) }); + production.Parameters.Add(new NpgsqlParameter { TypedValue = "(@cutoff_time datetime2(7))SELECT /* " + QueryStoreCollector.SelfQueryMarker + " */ qsrs.plan_id" }); + await production.ExecuteNonQueryAsync(ct); + } + + /* Not vacuous: both queries' rows are stored, so what hides query 23 is the readers. */ + await using (var count = new NpgsqlCommand("SELECT count(DISTINCT query_id) FROM query_store_stats WHERE server_id = $1 AND query_text IS NULL", rig.Connection)) + { + count.Parameters.Add(new NpgsqlParameter { TypedValue = ServerId }); + Assert.Equal(2L, await count.ExecuteScalarAsync(ct)); + } + + var start = now.AddHours(-24); + var end = now.AddMinutes(1); + + var top = await DarlingDataReader.GetQueryStoreTopAsync(rig.Postgres, ServerId, start, end, 10, DatabaseFilter.All, ct); + Assert.Equal([Normal], top.Select(r => r.QueryId).Distinct().ToList()); + + await using var viewer = new ViewerDataService(rig.Scratch.ConnectionString); + var grid = await viewer.GetQueryStoreTopQueriesAsync(ServerId, start, end, 10, null, null, null, ct); + Assert.Equal([Normal], grid.Select(r => r.QueryId).Distinct().ToList()); + + var aborts = await DarlingQueryStoreAbortsReader.ReadAsync(rig.Postgres, ServerId, start, end, 10, DatabaseFilter.All, ct); + Assert.Equal([Normal], aborts.Rows.Select(r => r.QueryId).ToList()); + + var comparison = await viewer.GetQueryStoreComparisonAsync(ServerId, start, end, start.AddHours(-24), start, null, ct); + var hash = "0x" + Normal.ToString("X8", System.Globalization.CultureInfo.InvariantCulture); + Assert.Equal([hash], comparison.Select(r => r.QueryHash).ToList()); + } +} diff --git a/Darling/Darling.Tests/QueryStoreTopDailyReadLiveTests.cs b/Darling/Darling.Tests/QueryStoreTopDailyReadLiveTests.cs index a3346f22c5..77d39f495a 100644 --- a/Darling/Darling.Tests/QueryStoreTopDailyReadLiveTests.cs +++ b/Darling/Darling.Tests/QueryStoreTopDailyReadLiveTests.cs @@ -10,8 +10,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using System.Security.Cryptography; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; @@ -47,26 +45,6 @@ public sealed class QueryStoreTopDailyReadLiveTests private static string? ConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); - /// SHA-256 of each statement, line endings normalized. They were taken from the pre-split text, before - /// QueryStoreTopSuffix was divided into its ranked head and its tail, and re-taken once for #5313, which - /// deliberately changed both statements: the over-fetch LIMIT $4 + 5 became the round's candidate limit and - /// the tail gained the page / count-row wrapper, and re-taken again for #5420, which deliberately bounded the tail's - /// inline-text fallback to the read's window (collection_time >= $2 and <= $3), and re-taken for #5523, which deliberately added the table read's upper first_execution_time bound (<= $3 + the slack, twelve hours). The split pin is the - /// tail-sharing assert below. Re-taken for #5815, which deliberately sorted the page's candidates before the text lateral - /// so LIMIT $4 stops the text lookups at a full page (rows and order unchanged). Re-taken for #5984, which deliberately added the query-tag filter's comment token (a no-op until a read names a tag). - private const string RawSqlHash = "DD0DD535EFADA1E258360AE5C8CBFDCD5E6EC0D21F87BCECEAE7BA76411CF827"; - private const string TableSqlHash = "0CFFEA04CFB2F5FD8C6C9AE9A21E186C1A6349B3772799B909DB39BC9DCE89D3"; - - private static string Hash(string sql) => - Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(sql.ReplaceLineEndings("\n")))); - - [Fact] - public void TheRawAndTableStatements_AreByteIdenticalToTheirPreSplitText() - { - Assert.Equal(RawSqlHash, Hash(DarlingDataReader.QueryStoreTopSql)); - Assert.Equal(TableSqlHash, Hash(DarlingDataReader.QueryStoreTopTableSql)); - } - [Fact] public void TheDailyStatement_SharesTheTableReadsTail_AndItsPredicates() { diff --git a/Darling/Darling.Tests/ViewerQueryStoreRegressionsDataStartTests.cs b/Darling/Darling.Tests/ViewerQueryStoreRegressionsDataStartTests.cs index 6b08bba837..a8c1e10ee9 100644 --- a/Darling/Darling.Tests/ViewerQueryStoreRegressionsDataStartTests.cs +++ b/Darling/Darling.Tests/ViewerQueryStoreRegressionsDataStartTests.cs @@ -99,12 +99,35 @@ public void QueryStoreRegressions_AsksAboutTheBaselineWindow_AndComparesWithItsS step); } + /* #6654 review: the grid has no envelope, so the dropped databases it left out reach the user through this text. */ + [Fact] + public void TheLeftOutBanner_NamesTheDatabases_InTheWordsOfDatabasesLeftOut() + { + Assert.Null(ViewerServerTab.DatabasesLeftOutBannerText(null)); + Assert.Null(ViewerServerTab.DatabasesLeftOutBannerText(Array.Empty())); + Assert.Null(ViewerServerTab.DatabasesLeftOutBannerText(new[] { " ", "" })); + + var one = ViewerServerTab.DatabasesLeftOutBannerText(new[] { "Gone" })!; + Assert.StartsWith("Left out of this grid: Gone. ", one, StringComparison.Ordinal); + Assert.Contains("no longer lists it", one, StringComparison.Ordinal); + Assert.DoesNotContain("in all", one, StringComparison.Ordinal); + + var two = ViewerServerTab.DatabasesLeftOutBannerText(new[] { "b", "a", "b" })!; + Assert.StartsWith("Left out of this grid: a, b. ", two, StringComparison.Ordinal); + Assert.Contains("no longer lists them", two, StringComparison.Ordinal); + + var many = ViewerServerTab.DatabasesLeftOutBannerText(Enumerable.Range(0, 25).Select(i => "db" + i.ToString("00", System.Globalization.CultureInfo.InvariantCulture)).ToArray())!; + Assert.Contains("(25 in all; the first 20 are listed)", many, StringComparison.Ordinal); + Assert.Contains("db19", many, StringComparison.Ordinal); + Assert.DoesNotContain("db20", many, StringComparison.Ordinal); + } + [Fact] public void EveryTabRead_OfTheGrid_HasItsProbe() { var tabs = string.Join("\n", Directory.GetFiles(PathTo("Darling", "PerformanceMonitor.Darling.Viewer"), "ViewerServerTab*.cs").Select(File.ReadAllText)); - Assert.Equal(1, Matches(tabs, @"_dataService\.GetQueryStoreRegressionsAsync\(")); + Assert.Equal(1, Matches(tabs, @"_dataService\.GetQueryStoreRegressionsWithScopeAsync\(")); Assert.Equal(1, Matches(tabs, @"_dataService\.GetQueryStoreRegressionsDataStartAsync\(")); Assert.Equal(1, Matches(tabs, @"ShowQueryStoreRegressionsDataStartAsync\(QueryStoreRegressionsTruncationBanner,")); } @@ -118,7 +141,7 @@ public void TheBanner_SitsAboveTheGrid_InTheExistingStyle() Assert.True(banner.Success, "QueryStoreRegressionsTruncationBanner is not declared in ViewerServerTab.xaml, in the row above the grid"); Assert.Contains("Visibility=\"Collapsed\"", banner.Value, StringComparison.Ordinal); Assert.Contains("Background=\"#22FFAA00\"", banner.Value, StringComparison.Ordinal); - Assert.Matches(@" new FakeNode(t), createTextNode: ( globalThis.location = { hash: "#/server/a/events" }; const root = process.argv[2]; -const { windowFloorStrip, cappedReadNote, partialWindowSubtitle, markPartialSubtitle, localizeIsoText, localizeAlertDetail, emptyStrip, noticeStrip, el, localTime } = await import(pathToFileURL(root + "/util.js").href); +const { windowFloorStrip, leftOutStrip, cappedReadNote, partialWindowSubtitle, markPartialSubtitle, localizeIsoText, localizeAlertDetail, emptyStrip, noticeStrip, el, localTime } = await import(pathToFileURL(root + "/util.js").href); const text = (n) => (n ? n.textContent : null); const OLDEST = "2026-10-12T13:29:59.5027090Z"; @@ -62,6 +62,15 @@ out.noOldest = text(windowFloorStrip({ truncated: true, order: "collection_time_ out.serverNoteWins = text(windowFloorStrip({ ...active, ...serverCapped }, { viz: "table" })); out.ownNote = text(windowFloorStrip(active, { viz: "table", noteKey: "truncation_note" })); +/* #6654: the left-out note, drawn on an empty answer only for a grid that names it */ +const leftOutNote = "These databases were left out of this read because the server's newest database snapshot no longer lists them."; +out.leftOutNamed = text(leftOutStrip({ databases_left_out_note: leftOutNote }, { viz: "table", noteKey: "databases_left_out_note" })); +out.leftOutMore = text(leftOutStrip({ databases_left_out_note: leftOutNote }, { viz: "table", moreNoteKeys: ["databases_left_out_note"] })); +out.leftOutUnnamed = text(leftOutStrip({ databases_left_out_note: leftOutNote }, { viz: "table" })); +out.leftOutOtherKey = text(leftOutStrip({ databases_left_out_note: leftOutNote }, { viz: "table", noteKey: "note" })); +out.leftOutAbsent = text(leftOutStrip({}, { viz: "table", noteKey: "databases_left_out_note" })); +out.leftOutBlank = text(leftOutStrip({ databases_left_out_note: " " }, { viz: "table", noteKey: "databases_left_out_note" })); + /* the heading: a capped panel drops its window label, any other keeps it */ out.subtitleActive = partialWindowSubtitle(active, { viz: "table" }); out.subtitleServer = partialWindowSubtitle(serverCapped, { viz: "line" }); diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs index 298fcbf24d..c5bd5aa3a6 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgDrillDownCollector.Queries.cs @@ -992,7 +992,9 @@ AND NOT EXISTS (SELECT 1 FROM query_store_text AS x WHERE x.server_id = $1 AND x.database_name = scored.database_name AND x.query_id = scored.query_id - AND lower(x.query_sql_text) ~ '" + PerformanceMonitor.Common.InternalStatements.LowerTextPattern + @"') + AND (lower(x.query_sql_text) ~ '" + PerformanceMonitor.Common.InternalStatements.LowerTextPattern + @"' + -- #6665: or the monitor's own statement (its collector-side drop never fires on Darling) + OR " + PerformanceMonitor.Darling.Storage.QueryStoreSelfStatement.SelfOfStoredText + @")) ORDER BY regression_factor DESC LIMIT 5"; @@ -1165,7 +1167,9 @@ AND NOT EXISTS (SELECT 1 FROM query_store_text AS x WHERE x.server_id = $1 AND x.database_name = scored.database_name AND x.query_id = scored.query_id - AND lower(x.query_sql_text) ~ '" + PerformanceMonitor.Common.InternalStatements.LowerTextPattern + @"') + AND (lower(x.query_sql_text) ~ '" + PerformanceMonitor.Common.InternalStatements.LowerTextPattern + @"' + -- #6665: or the monitor's own statement (its collector-side drop never fires on Darling) + OR " + PerformanceMonitor.Darling.Storage.QueryStoreSelfStatement.SelfOfStoredText + @")) ORDER BY regression_factor DESC LIMIT 5 ) diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs index 7028855a03..f31f87ec54 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgFactCollector.QueryPerf.cs @@ -727,7 +727,9 @@ AND NOT EXISTS (SELECT 1 FROM query_store_text AS x WHERE x.server_id = $1 AND x.database_name = compared.database_name AND x.query_id = compared.query_id - AND lower(x.query_sql_text) ~ '" + PerformanceMonitor.Common.InternalStatements.LowerTextPattern + @"') + AND (lower(x.query_sql_text) ~ '" + PerformanceMonitor.Common.InternalStatements.LowerTextPattern + @"' + -- #6665: or the monitor's own statement (its collector-side drop never fires on Darling) + OR " + PerformanceMonitor.Darling.Storage.QueryStoreSelfStatement.SelfOfStoredText + @")) -- #5630: 100, not 20. The caller drops the queries whose two plans were compiled for different inputs -- (PlanInputComparison) and keeps the worst 20 of what is left, so the cut moved after that check. ORDER BY regression_factor DESC diff --git a/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationEvidenceCapture.Families.cs b/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationEvidenceCapture.Families.cs index 35aea43e51..a65f3deb58 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationEvidenceCapture.Families.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Investigations/InvestigationEvidenceCapture.Families.cs @@ -163,7 +163,9 @@ AND NOT EXISTS (SELECT 1 FROM query_store_text AS x WHERE x.server_id = $1 AND x.database_name = b.database_name AND x.query_id = b.query_id - AND lower(x.query_sql_text) ~ '" + InternalStatements.LowerTextPattern + @"') + AND (lower(x.query_sql_text) ~ '" + InternalStatements.LowerTextPattern + @"' + -- #6665: or the monitor's own statement (its collector-side drop never fires on Darling) + OR " + PerformanceMonitor.Darling.Storage.QueryStoreSelfStatement.SelfOfStoredText + @")) ORDER BY b.cpu_per_exec_us / g.cpu_per_exec_us DESC, b.database_name, b.query_id LIMIT $6"; diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs index 7e93f781d6..3c24870107 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs @@ -2733,7 +2733,7 @@ ORDER BY SUM(execution_count) * AVG(CAST(avg_duration_us AS double precision)) D /// the final ORDER BY and LIMIT $4. It reads only ranked's column list, so /// (the raw and the interval-table reads) and /// (the daily-summary read) share this one text and cannot drift apart below ranked. - private const string QueryStoreTopTail = """ + private const string QueryStoreTopTail = $$""" page AS ( SELECT p.database_name, @@ -2801,7 +2801,11 @@ LIMIT 1 ) ) AS query_text ) AS t ON TRUE - WHERE t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%' + /* #6665: the monitor's own statement is left out here too, before the LIMIT, by the shared read-side predicate + (QueryStoreSelfStatement): on Darling its row is stored, because the collector's drop tests an inline text column + that the separate text fetch leaves empty, and the marker is only in the text resolved above. */ + WHERE (t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%') + AND {{QueryStoreSelfStatement.NotSelfOfLateralText}} ORDER BY r.sort_cost DESC, r.database_name, r.query_id, r.plan_id, r.query_hash, r.execution_type_desc, r.replica_role LIMIT $4 ) AS p diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs index d3b1af078c..6064d3f6dd 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpQueryStoreRegressionTools.cs @@ -92,6 +92,13 @@ internal static async Task GetQueryStoreRegressions( "every retained row before it", whose cost tracked retention rather than the window asked for. */ var baselineStart = start.AddDays(-DarlingQueryStoreRegressionReader.BaselineLookbackDays); + /* #6654: with no database chosen the read leaves out the databases the server DROPPED (named by an earlier database_states + snapshot, absent from the newest), as get_top_procedures_by_cpu does (#5633 item 4): their queries would otherwise fill the + page. A named database is read as asked; an unknown set (no snapshot) leaves out nothing. The names ride the payload. */ + databaseFilter = await CurrentDatabaseScope.NarrowAsync(postgres, resolved.ServerId, databaseFilter, + asOfUtc: string.IsNullOrWhiteSpace(as_of) ? null : windowEnd, windowStartUtc: baselineStart, logger: logger, cancellationToken: cancellationToken); + var leftOut = DatabasesLeftOut.Distinct(databaseFilter.ExcludedNames); + /* Over-fetch by one. Comparing the row count to the cap reports truncation for a server that happens to have exactly `limit` regressions and nothing more, which is a false positive in @@ -112,7 +119,8 @@ negative. Unfiltered by database_name (the floor is a property of the table). */ emptyAnswer: rows.Count == 0, logger: logger, cancellationToken: cancellationToken); if (rows.Count == 0) - return await EmptyAsync(postgres, resolved.DisplayName, resolved.ServerId, start, end, baselineStart, hours_back, databaseFilter, notice, cancellationToken); + return McpHelpers.WithDatabasesLeftOut( + await EmptyAsync(postgres, resolved.DisplayName, resolved.ServerId, start, end, baselineStart, hours_back, databaseFilter, notice, cancellationToken), leftOut)!; var truncated = rows.Count > limit; var shown = rows.Take(limit); @@ -148,6 +156,9 @@ before it" - the baseline no longer grows with retention. */ returned_count = shown.Count(), truncated, regressions = shown.Select(r => RegressionRowJson(r, full_text)), + /* #6654: the dropped databases this read left out, in the words get_top_procedures_by_cpu uses. */ + databases_left_out = DatabasesLeftOut.Named(leftOut), + databases_left_out_note = DatabasesLeftOut.Note(leftOut), }, McpHelpers.JsonOptions); /* A failed probe costs the notice, never the rows (see DarlingMcpWindowNotice.ReadAsync). */ diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreAbortsReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreAbortsReader.cs index 5f21c5de0e..d34e5f55a5 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreAbortsReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreAbortsReader.cs @@ -78,7 +78,7 @@ FROM query_store_interval_wide /// aborted or exception run does not pass the HAVING, so a Regular-only query never appears. The page is cut before the text /// lookup, so the lateral runs for the rows kept only. /// - private const string Tail = """ + private const string Tail = $$""" ranked AS ( SELECT database_name, @@ -102,6 +102,15 @@ page AS ( r.aborted_runs + r.exception_runs AS failed_runs, CAST(r.aborted_runs + r.exception_runs AS double precision) / NULLIF(r.total_runs, 0) AS failed_share FROM ranked AS r + /* #6665: the monitor's own statement is left out before the cap, by the shared read-side predicate. Its text is only in + query_store_text on Darling (the collector's drop never fires), and only the few queries with an aborted or failed run + reach this point, so the key lookup is cheap. Rows from before the separate text fetch carry inline text the collector + already filtered, so there is no inline fallback here. */ + WHERE NOT EXISTS (SELECT 1 FROM query_store_text AS x + WHERE x.server_id = $1 + AND x.database_name = r.database_name + AND x.query_id = r.query_id + AND {{QueryStoreSelfStatement.SelfOfStoredText}}) ORDER BY r.aborted_runs + r.exception_runs DESC, CAST(r.aborted_runs + r.exception_runs AS double precision) / NULLIF(r.total_runs, 0) DESC NULLS LAST, r.database_name, r.query_id, r.query_hash diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs index aedbb9d685..8fde4b75de 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingQueryStoreRegressionReader.cs @@ -107,14 +107,14 @@ public sealed record RunGroup( /// ), the aggregates () /// and , bound to this read's parameters: $1 server_id, $2 window start, $3 window /// end, $4 database filter (text[] or NULL), $5 row cap, $6 baseline start, $7 the baseline's days read from the daily - /// totals (date[]), $8 and $9 the live half's first_execution_time bounds. false + /// totals (date[]), $8 and $9 the live half's first_execution_time bounds, $10 the databases left out (#6654, text[] or NULL). false /// names no daily table (no run day was found, or the store has none). /// internal static string BuildSql(bool wide, bool useDailyTotals) => QueryStoreRegressionBaseline.Bind( (wide ? QueryStoreRegressionBaseline.WideArmsSql : QueryStoreRegressionBaseline.RawArmsSql) + "\n" + QueryStoreRegressionBaseline.AggregatesSql(useDailyTotals) + "\n" + QueryStoreRegressionBaseline.SuffixSql, - "$6", "$7", "$8", "$9", "$5"); + "$6", "$7", "$8", "$9", "$5", "$10"); /// /// The viewer's regression read over raw query_store_stats, with the baseline's closed days from the daily totals @@ -329,9 +329,10 @@ internal static void LogTableFault(ILogger? logger, Exception ex) /// Re-arms 's once-per-process Warning. Tests only. internal static void ResetTableFaultWarningForTests() => Interlocked.Exchange(ref s_tableFaultWarned, 0); - /// The nine parameters both forms of the read take, in their order: $1 server_id, $2 window start, $3 window + /// The ten parameters both forms of the read take, in their order: $1 server_id, $2 window start, $3 window /// end, $4 database filter, $5 row cap, $6 baseline start, $7 the baseline days read from the daily totals (#5506), $8 the - /// end of the live half's low first_execution_time range, $9 the start of its high range. + /// end of the live half's low first_execution_time range, $9 the start of its high range, $10 the databases left out + /// (, #6654). private static void AddParameters( NpgsqlCommand command, int serverId, DateTime startUtc, DateTime endUtc, DatabaseFilter databases, int limit, DateTime baselineStartUtc, QueryStoreRegressionBaseline.Plan plan) @@ -343,6 +344,7 @@ private static void AddParameters( command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Date, Value = plan.RunDays }); command.Parameters.Add(new NpgsqlParameter { TypedValue = DateTime.SpecifyKind(plan.LowEnd, DateTimeKind.Unspecified) }); command.Parameters.Add(new NpgsqlParameter { TypedValue = DateTime.SpecifyKind(plan.HighStart, DateTimeKind.Unspecified) }); + command.Parameters.Add(databases.ExcludeParameter()); } private static double? NullableDouble(NpgsqlDataReader reader, int ordinal) => diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js index 361c97daf2..b6280ea74b 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/pages/server-tabs.js @@ -2135,7 +2135,8 @@ export const SERVER_TABS = [ ctx.label, "No query regressed against its baseline in this window. A server with no earlier history has nothing to compare against.", 2, - null, + /* #6654: the read leaves out databases the server dropped; the note names them (drawn on an empty answer too, panels.js). */ + "databases_left_out_note", null, QUERY_STORE_REGRESSION_GROUPS ), diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js index fc0f22c173..093697508d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/panels.js @@ -31,6 +31,7 @@ import { noticeStrip, keptWindowStrip, windowFloorStrip, + leftOutStrip, readTool, readWithinKeptHistory, apiGet, @@ -204,7 +205,7 @@ async function loadPanelBody(desc, body, signal) { window's start (the server adds the note to that envelope only, never to an unavailable or not_collected one), so a new server's empty week does not read as a quiet one. windowFloorStrip is null for a chart and for an envelope without the note. */ - mount(body, [kept, windowFloorStrip(res.data, desc), emptyStrip(emptyHeadline(desc, res))]); + mount(body, [kept, windowFloorStrip(res.data, desc), leftOutStrip(res.data, desc), emptyStrip(emptyHeadline(desc, res))]); return; } diff --git a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js index 401156da11..d943f38b60 100644 --- a/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js +++ b/Darling/PerformanceMonitor.Darling.Service/wwwroot/js/util.js @@ -390,6 +390,20 @@ export function keptWindowStrip(res) { return noticeStrip(readLimitText(res.keptHours) + ", so it shows the last " + daysText(res.keptHours) + "."); } +/** + * #6654: the strip over a grid that names `databases_left_out_note` as its `noteKey` (or in `moreNoteKeys`), drawn on an EMPTY answer too. The + * populated path already draws a named note; the empty path draws only the read's message, and "no regressions" over a read that left + * out a dropped database is not the same statement as "nothing regressed". Null for any other panel, so a note that is not about + * omitted databases is not pulled onto an empty answer by this. + */ +export function leftOutStrip(data, desc) { + if (!desc || !data) return null; + const key = "databases_left_out_note"; + if (desc.noteKey !== key && !(desc.moreNoteKeys || []).includes(key)) return null; + const note = data[key]; + return typeof note === "string" && note.trim() ? noticeStrip(note) : null; +} + /** * The notice for a grid whose table starts covering the server after the window does (#4966): the response says so with * `window_truncated: true` and a `truncation_note` naming where the data starts (in the browser's zone by the time a diff --git a/Darling/PerformanceMonitor.Darling.Storage/CurrentDatabaseScope.cs b/Darling/PerformanceMonitor.Darling.Storage/CurrentDatabaseScope.cs index 8c6d20a337..3d5394fa3f 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/CurrentDatabaseScope.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/CurrentDatabaseScope.cs @@ -213,7 +213,7 @@ public static DatabaseFilter Narrow(DatabaseFilter filter, IReadOnlySet? => filter.IsAll && dropped is { Count: > 0 } ? DatabaseFilter.Excluding(dropped) : filter; /// after reading the server's dropped set - /// (). The reader the filter goes to must apply . + /// (). The reader the filter goes to must apply . public static async Task NarrowAsync( NpgsqlDataSource postgres, int serverId, DatabaseFilter filter, DateTime? nowUtc = null, DateTime? asOfUtc = null, DateTime? windowStartUtc = null, ILogger? logger = null, CancellationToken cancellationToken = default) diff --git a/Darling/PerformanceMonitor.Darling.Storage/DatabaseFilter.cs b/Darling/PerformanceMonitor.Darling.Storage/DatabaseFilter.cs index 60874986a8..c17be6d694 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DatabaseFilter.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DatabaseFilter.cs @@ -45,7 +45,7 @@ public readonly record struct DatabaseFilter private readonly string[]? _names; /// #5633 review L2 leftover: names a read must NOT cover, set only by on an - /// filter. Null when nothing is excluded. Only the readers that carry apply it. + /// filter. Null when nothing is excluded. Only the readers that carry apply it. private readonly string[]? _excluded; private DatabaseFilter(string[] names) => _names = names; @@ -90,8 +90,9 @@ public static DatabaseFilter Of(IEnumerable? names) /// /// #5633 review: every database EXCEPT (blank names dropped, repeats kept once). It is still /// (no database is chosen), so a caller that does not apply the exclusion reads every database as before. - /// A list with no name left is plain . Only the top-procedures ranking applies it today (see - /// ); CurrentDatabaseScope.NarrowAsync is its one producer. + /// A list with no name left is plain . The top-procedures ranking and the Query Store regressions read + /// (the MCP tool and the Viewer grid, #6654) apply it (see ); CurrentDatabaseScope.NarrowAsync is + /// its one producer. /// public static DatabaseFilter Excluding(IEnumerable? excluded) { @@ -112,7 +113,7 @@ public static DatabaseFilter Excluding(IEnumerable? excluded) public IReadOnlyList ExcludedNames => _excluded is null ? Array.Empty() : Array.AsReadOnly(_excluded); /// - /// The exclusion's text[] parameter for a statement carrying : SQL NULL when nothing is + /// The exclusion's text[] parameter for a statement carrying : SQL NULL when nothing is /// excluded. A new parameter on every call. /// public NpgsqlParameter ExcludeParameter() => @@ -127,8 +128,15 @@ public NpgsqlParameter ExcludeParameter() => /// column: a row with no database name is not a dropped database, so it stays. Independent of the selection, like /// . /// - public static string ExcludeClause(string column, int index) => - $" AND (${index}::text[] IS NULL OR {column} IS NULL OR {column} <> ALL(${index}))"; + public static string ExcludeClause(string column, int index) => ExcludeClause(column, "$" + index.ToString(System.Globalization.CultureInfo.InvariantCulture)); + + /// + /// with the parameter written as (a $n token or a + /// placeholder a shared statement binds later, such as QueryStoreRegressionBaseline.ExcludeToken): the one definition of the + /// predicate, so a NULL-handling change reaches every statement that carries it (#6654 review). + /// + public static string ExcludeClause(string column, string parameter) => + $" AND ({parameter}::text[] IS NULL OR {column} IS NULL OR {column} <> ALL({parameter}))"; /// True when no database is named, so the read covers every database. public bool IsAll => _names is null; diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreRegressionBaseline.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreRegressionBaseline.cs index f658e3ef37..0e49bf0a75 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreRegressionBaseline.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreRegressionBaseline.cs @@ -13,6 +13,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Npgsql; +using PerformanceMonitor.Collectors; namespace PerformanceMonitor.Darling.Storage; @@ -233,10 +234,15 @@ first execution below the first run day's midnight. */ /// The row cap: $5 in the MCP read (a parameter), the literal 50 in the viewer's (#5506). public const string LimitToken = "@@LIMIT"; + /// The databases to leave out (#6654): a text[] parameter, NULL when none, bound by the twin + /// (DatabaseFilter.ExcludeParameter) as $10 in the MCP read and $9 in the viewer's. + public const string ExcludeToken = "@@X"; + /// The statement with its tokens bound to the twin's parameter numbers, for example $6, $7, /// $8, $9. - public static string Bind(string sql, string baselineStart, string runDays, string lowEnd, string highStart, string limit) => + public static string Bind(string sql, string baselineStart, string runDays, string lowEnd, string highStart, string limit, string exclude) => sql.Replace(LimitToken, limit, StringComparison.Ordinal) + .Replace(ExcludeToken, exclude, StringComparison.Ordinal) .Replace(BaselineStartToken, baselineStart, StringComparison.Ordinal) .Replace(RunDaysToken, "(" + runDays + "::date[])", StringComparison.Ordinal) .Replace(LowEndToken, "(" + lowEnd + "::timestamp)", StringComparison.Ordinal) @@ -565,7 +571,13 @@ CASE WHEN NOT f.is_errors THEN 0 ELSE 2 END) AS failure_rank, (f.recent_dur_ms - f.base_dur_ms) * f.recent_execs AS additional_duration_ms FROM flagged AS f - WHERE f.is_cost OR f.is_timeouts OR f.is_errors + WHERE (f.is_cost OR f.is_timeouts OR f.is_errors) + /* #6654: a dropped database's queries are left out here, before the rank and the cap, so they cannot fill the page. */ + {{DatabaseFilter.ExcludeClause("f.database_name", ExcludeToken)}} + /* #6654: the monitor's own statement is left out before the cap too. The collector's drop of it never fires on Darling + (the text arrives separately, so the row it tests is empty), so the stored statement is found by the shared read-side + predicate: the marker against the text in query_store_text, with the MAX(query_text) sample as its fallback. */ + AND {{QueryStoreSelfStatement.NotSelfOfKeyedText("$1", "f.database_name", "f.query_id", "f.query_text_sample")}} ), ranked AS ( SELECT diff --git a/Darling/PerformanceMonitor.Darling.Storage/QueryStoreSelfStatement.cs b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreSelfStatement.cs new file mode 100644 index 0000000000..cb65a55bef --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Storage/QueryStoreSelfStatement.cs @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using PerformanceMonitor.Collectors; + +namespace PerformanceMonitor.Darling.Storage; + +/// +/// The one read-side predicate that hides the monitor's own Query Store statement from every reader that ranks Query Store +/// queries (#6654, #6665). +/// +/// Why a read-side predicate. The collector drops a row whose text carries , +/// but it tests row.QueryText, and on Darling that column is NULL: the Query Store read fetches the text separately +/// (FetchQueryTextSeparately) and lands it once in collect.query_store_text, which has no marker check. So the +/// monitor's statement is stored as an ordinary fact row, and every reader that resolves text from query_store_text can list it +/// (confirmed on a live store: get_query_store_top listed it, 241 executions in 24 hours). The collector is left alone on +/// purpose: a server-side NOT LIKE on the text cost 75% of the read's time (#1565), and the rows already stored stay for the +/// whole retention window, so a read-side filter is needed whatever the collector does. +/// +/// Fail-open. A query with no known text passes: the predicate only removes a query whose text is known and carries the +/// marker. +/// +/// The marker is a plain word, so it is spliced into the SQL as a literal (no quoting needed). +/// +public static class QueryStoreSelfStatement +{ + /// The marker the monitor puts in the text of every statement it sends to a monitored server. + public const string Marker = QueryStoreCollector.SelfQueryMarker; + + private const string Open = "strpos(COALESCE("; + private const string Close = ", ''), '" + Marker + "')"; + + /// The column tests: the text a Query Store top read has already resolved + /// in its LEFT JOIN LATERAL (...) AS t. + public const string LateralTextColumn = "t.query_text"; + + /// The predicate for a read that has already resolved the text as t.query_text, usable inside a constant SQL string. + public const string NotSelfOfLateralText = Open + LateralTextColumn + Close + " = 0"; + + /// The same test, true for the monitor's own statement, over x.query_sql_text of a + /// query_store_text AS x row. For a read that already probes query_store_text by key (a NOT EXISTS), so it adds + /// no second probe, and usable inside a constant SQL string. + public const string SelfOfStoredText = Open + "x.query_sql_text" + Close + " > 0"; + + /// The predicate over a text expression the read has already resolved. + public static string NotSelfOfText(string textExpression) + { + ArgumentException.ThrowIfNullOrWhiteSpace(textExpression); + return Open + textExpression + Close + " = 0"; + } + + /// + /// The predicate for a read that has not resolved the text: the text is looked up in query_store_text by its key + /// (one primary-key probe per row), falling back to when that table has no row for the + /// query. A read whose rows can only carry inline text from before the separate-fetch cutover passes null: the + /// collector's own drop already removed those rows when they were collected. + /// + public static string NotSelfOfKeyedText(string serverIdExpression, string databaseExpression, string queryIdExpression, string? inlineTextExpression = null) + { + var lookup = $"(SELECT x.query_sql_text FROM query_store_text AS x WHERE x.server_id = {serverIdExpression} AND x.database_name = {databaseExpression} AND x.query_id = {queryIdExpression})"; + return NotSelfOfText(inlineTextExpression is null ? lookup : $"COALESCE({lookup}, {inlineTextExpression})"); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs index 2802c9acb3..a9e75ffee5 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStore.cs @@ -235,7 +235,7 @@ FROM query_store_interval_wide /// — both prefixes above produce the same "one row per identity, every /// column deduped's dedupe/the table's own upsert already kept" shape, so this aggregates either one /// identically. References $1, $4 and $6, plus $2/$3 (#5420: the inline-text fallback's window bound, the same $2/$3 both prefixes bind), so both share it unchanged. - private const string QueryStoreTopSuffix = """ + private const string QueryStoreTopSuffix = $$""" ranked AS ( SELECT database_name, @@ -411,7 +411,11 @@ LIMIT 1 ) ) AS query_text ) AS t ON TRUE - WHERE t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%' + /* #6665: the monitor's own statement is left out here too, before the LIMIT, by the shared read-side predicate + (QueryStoreSelfStatement): on Darling its row is stored, because the collector's drop tests an inline text column + that the separate text fetch leaves empty, and the marker is only in the text resolved above. */ + WHERE (t.query_text IS NULL OR t.query_text NOT LIKE 'WAITFOR%') + AND {{QueryStoreSelfStatement.NotSelfOfLateralText}} ORDER BY r.sort_cost DESC, r.database_name, r.query_id, r.plan_id, r.query_hash, r.execution_type_desc, r.replica_role LIMIT $4 ) AS p @@ -681,7 +685,7 @@ reach raw with ZERO extra store round trips versus before this table existed. */ /// a NULL key and an empty-string key stay apart (#5420). Returns the shared . /// $1 server_id, $2/$3 current window, $4/$5 baseline window (naive UTC). /// - public const string QueryStoreComparisonSql = """ + public static readonly string QueryStoreComparisonSql = $$""" WITH deduped_current AS ( /* LOAD-BEARING (correctness, not just perf) — #1841. The rows are CUMULATIVE per-interval snapshots and the collector re-fetches the OPEN interval every cycle, so the SAME interval @@ -840,6 +844,9 @@ ON COALESCE(c.database_name, '') = COALESCE(b.database_name, '') AND COALESCE(c.query_hash, '') = COALESCE(b.query_hash, '') AND (c.database_name IS NULL) = (b.database_name IS NULL) AND (c.query_hash IS NULL) = (b.query_hash IS NULL) + /* #6665: the monitor's own statement is left out of the comparison by the shared read-side predicate. It is applied to the + text the comparison resolved, after the top-100 cut, so a statement that ranks in the top 100 costs the grid one row. */ + WHERE {{QueryStoreSelfStatement.NotSelfOfText("COALESCE(c.query_text, b.query_text)")}} """; /// Query-Store current-vs-baseline comparison rows (shared .Ui item; delta % + NEW/GONE badges). diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs index e4f991b210..dea56584e4 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.QueryStoreRegressions.cs @@ -16,6 +16,10 @@ namespace PerformanceMonitor.Darling.Viewer; +/// The regressions rows with the databases the read left out (#6654 review): the dropped ones, named for the grid's banner. +/// is null when none were left out. +public sealed record ViewerQueryStoreRegressionsRead(List Rows, IReadOnlyList? DatabasesLeftOut); + /// /// One Query-Store regression row — the viewer copy of the Dashboard's /// QueryStoreRegressionItem (Dashboard/Models/QueryStoreRegressionItem.cs), fed by the Postgres @@ -113,13 +117,13 @@ public sealed partial class ViewerDataService /// ), the aggregates, and , /// bound to this read's parameters: $1 server_id, $2 window start (baseline is < $2), $3 window end, $4 database filter /// (text[]), $5 baseline start (window start minus ), $6 the baseline days read from the - /// daily totals (date[]), $7 and $8 the live half's first_execution_time bounds. + /// daily totals (date[]), $7 and $8 the live half's first_execution_time bounds, $9 the databases left out (#6654, text[] or NULL). /// internal static string BuildQueryStoreRegressionsSql(bool wide, bool useDailyTotals) => QueryStoreRegressionBaseline.Bind( (wide ? QueryStoreRegressionBaseline.WideArmsSql : QueryStoreRegressionBaseline.RawArmsSql) + "\n" + QueryStoreRegressionBaseline.AggregatesSql(useDailyTotals) + "\n" + QueryStoreRegressionBaseline.SuffixSql, - "$5", "$6", "$7", "$8", "50"); + "$5", "$6", "$7", "$8", "50", "$9"); /// /// The Query-Store regressions read — the Dashboard's report.query_store_regressions(@start,@end) inline TVF @@ -196,6 +200,16 @@ internal static string BuildQueryStoreRegressionsSql(bool wide, bool useDailyTot /// exactly as the Queries grid passes it (#3953, #5809). /// public async Task> GetQueryStoreRegressionsAsync( + int serverId, DateTime startUtc, DateTime endUtc, IReadOnlyList? databaseNames = null, DateTime? literalEndUtc = null, + CancellationToken cancellationToken = default) => + (await GetQueryStoreRegressionsWithScopeAsync(serverId, startUtc, endUtc, databaseNames, literalEndUtc, cancellationToken)).Rows; + + /// + /// with the databases it left out (#6654 review): the grid has no envelope, so the + /// names come back beside the rows for the banner (ViewerServerTab.DatabasesLeftOutBannerText). Null when none were left + /// out, as answers. + /// + public async Task GetQueryStoreRegressionsWithScopeAsync( int serverId, DateTime startUtc, DateTime endUtc, IReadOnlyList? databaseNames = null, DateTime? literalEndUtc = null, CancellationToken cancellationToken = default) { @@ -206,15 +220,22 @@ public async Task> GetQueryStoreRegressionsA // plan-time chunk exclusion). var baselineStartUtc = QueryStoreRegressionsBaselineStart(startUtc); + /* #6654: with no database chosen the grid leaves out the databases the server DROPPED (named by an earlier database_states + snapshot, absent from the newest), as get_query_store_regressions and the top-procedures ranking do (#5633 item 4): their + queries would otherwise fill the 50-row cap. A database the user picked is read as asked; an unknown set leaves out nothing. */ + var excludedDatabases = databaseNames is { Count: > 0 } + ? null + : (await CurrentDatabaseScope.NarrowAsync(_dataSource, serverId, DatabaseFilter.All, windowStartUtc: baselineStartUtc, asOfUtc: literalEndUtc, logger: ViewerStoreLogger.Instance, cancellationToken: cancellationToken)).ExcludedNames; + /* #5809: the interval table where the Queries grid's own gate allows, raw otherwise. */ var schemaVersion = _cachedStoreSchemaVersion ??= await GetStoreSchemaVersionAsync(cancellationToken); if (schemaVersion is int version && version >= QueryStoreIntervalWideMinSchemaVersion) { var tableRows = await TryGetQueryStoreRegressionsFromTableAsync( - serverId, startUtc, endUtc, literalEndUtc, databaseNames, baselineStartUtc, cancellationToken); + serverId, startUtc, endUtc, literalEndUtc, databaseNames, baselineStartUtc, excludedDatabases, cancellationToken); if (tableRows is not null) { - return tableRows; + return new ViewerQueryStoreRegressionsRead(tableRows, DatabasesLeftOut.Distinct(excludedDatabases)); } } @@ -231,9 +252,9 @@ public async Task> GetQueryStoreRegressionsA connection, transaction, serverId, baselineStartUtc, startUtc, ViewerCommandDeadlines.CurrentInteractiveReadSeconds, null, cancellationToken, clampToRawFloor: true); await using var command = new NpgsqlCommand(plan.BuiltDayCount > 0 ? QueryStoreRegressionsSql : s_rawLiveSql, connection, transaction) { CommandTimeout = ViewerCommandDeadlines.CurrentInteractiveReadSeconds }; - AddQueryStoreRegressionsParameters(command, serverId, startUtc, endUtc, databaseNames, baselineStartUtc, plan); + AddQueryStoreRegressionsParameters(command, serverId, startUtc, endUtc, databaseNames, baselineStartUtc, plan, excludedDatabases); await using var reader = await command.ExecuteReaderAsync(cancellationToken); - return await ReadQueryStoreRegressionRowsAsync(reader, cancellationToken); + return new ViewerQueryStoreRegressionsRead(await ReadQueryStoreRegressionRowsAsync(reader, cancellationToken), DatabasesLeftOut.Distinct(excludedDatabases)); } /// @@ -246,7 +267,7 @@ public async Task> GetQueryStoreRegressionsA /// the span 's notice describes. /// internal async Task?> TryGetQueryStoreRegressionsFromTableAsync( - int serverId, DateTime startUtc, DateTime endUtc, DateTime? literalEndUtc, IReadOnlyList? databaseNames, DateTime baselineStartUtc, CancellationToken cancellationToken) + int serverId, DateTime startUtc, DateTime endUtc, DateTime? literalEndUtc, IReadOnlyList? databaseNames, DateTime baselineStartUtc, IReadOnlyList? excludedDatabases, CancellationToken cancellationToken) { try { @@ -279,7 +300,7 @@ public async Task> GetQueryStoreRegressionsA } await using var command = new NpgsqlCommand(wideSql, connection, transaction) { CommandTimeout = ViewerCommandDeadlines.CurrentInteractiveReadSeconds }; - AddQueryStoreRegressionsParameters(command, serverId, startUtc, endUtc, databaseNames, plan.ClampedStart, baselinePlan); + AddQueryStoreRegressionsParameters(command, serverId, startUtc, endUtc, databaseNames, plan.ClampedStart, baselinePlan, excludedDatabases); await using var reader = await command.ExecuteReaderAsync(cancellationToken); return await ReadQueryStoreRegressionRowsAsync(reader, cancellationToken); } @@ -290,12 +311,12 @@ public async Task> GetQueryStoreRegressionsA } } - /// The eight parameters both forms of the read take: $1 server_id, $2 window start, $3 window end, $4 database + /// The nine parameters both forms of the read take: $1 server_id, $2 window start, $3 window end, $4 database /// filter, $5 baseline start, $6 the baseline days read from the daily totals (#5506), $7 the end of the live half's low - /// first_execution_time range, $8 the start of its high range. + /// first_execution_time range, $8 the start of its high range, $9 the databases left out (#6654). private static void AddQueryStoreRegressionsParameters( NpgsqlCommand command, int serverId, DateTime startUtc, DateTime endUtc, IReadOnlyList? databaseNames, DateTime baselineStartUtc, - QueryStoreRegressionBaseline.Plan plan) + QueryStoreRegressionBaseline.Plan plan, IReadOnlyList? excludedDatabases) { AddServerWindowParameters(command, serverId, startUtc, endUtc); command.Parameters.Add(DatabaseFilterParameter(databaseNames)); @@ -303,6 +324,7 @@ private static void AddQueryStoreRegressionsParameters( command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlTypes.NpgsqlDbType.Array | NpgsqlTypes.NpgsqlDbType.Date, Value = plan.RunDays }); command.Parameters.Add(new NpgsqlParameter { TypedValue = DateTime.SpecifyKind(plan.LowEnd, DateTimeKind.Unspecified) }); command.Parameters.Add(new NpgsqlParameter { TypedValue = DateTime.SpecifyKind(plan.HighStart, DateTimeKind.Unspecified) }); + command.Parameters.Add(DatabaseFilter.Excluding(excludedDatabases).ExcludeParameter()); } private static double? NullableMs(NpgsqlDataReader reader, int ordinal) => diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Queries.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Queries.cs index e0c87f142d..3c393ca2ba 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Queries.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.Queries.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Globalization; using System.Linq; using System.Threading.Tasks; using System.Windows; @@ -466,11 +467,12 @@ private async Task LoadQueryStoreRegressionsAsync(DateTime startUtc, DateTime en step compares its answer with that window's start. The rows are ranked by added duration, not by time, so they name no earlier start and the cap rule does not apply. */ var floorTask = _dataService.GetQueryStoreRegressionsDataStartAsync(_server.ServerId, startUtc, endUtc); - var floorReadTask = _dataService.GetQueryStoreRegressionsAsync(_server.ServerId, startUtc, endUtc, databaseNames: SelectedDatabaseFilter, + var floorReadTask = _dataService.GetQueryStoreRegressionsWithScopeAsync(_server.ServerId, startUtc, endUtc, databaseNames: SelectedDatabaseFilter, literalEndUtc: IsCustomRange ? endUtc : null); /* #5809: the interval table gate's clause 4, as the grid passes it (#3953). */ await AwaitReadWatchingProbeAsync(floorReadTask, floorTask, "Query Store Regressions"); - var rows = floorReadTask.Result; - _queryStoreRegressionsFilterMgr!.UpdateData(rows); + var read = floorReadTask.Result; + _queryStoreRegressionsFilterMgr!.UpdateData(read.Rows); + ShowDatabasesLeftOutBanner(QueryStoreRegressionsLeftOutBanner, read.DatabasesLeftOut); await ShowQueryStoreRegressionsDataStartAsync(QueryStoreRegressionsTruncationBanner, floorTask, startUtc); SetDefaultSortIfNone(QueryStoreRegressionsGrid, "DurationRegressionPercent", ListSortDirection.Descending); } @@ -484,6 +486,37 @@ earlier start and the cap rule does not apply. */ internal const string QueryStoreRegressionsEmptyText = "No regressions to show. Each query is compared with the 7 days before the selected range, so a range that reaches back to the start of the stored Query Store data has nothing earlier to compare with."; + /// + /// The banner text for a grid whose read left databases out (#6654 review), or null when none were left out. The grid has no + /// envelope to carry , and without this a store whose only regressions were in a dropped database + /// would show the empty text that blames the baseline window. The same rule as : at most + /// names, with the full count when there are more, and the same reason and way out, in + /// this grid's words (the database picker, not a tool argument). + /// + internal static string? DatabasesLeftOutBannerText(IReadOnlyList? leftOut) + { + var all = DatabasesLeftOut.Distinct(leftOut); + if (all is null) + { + return null; + } + + var named = DatabasesLeftOut.Named(all)!; + var more = all.Count > named.Count + ? string.Create(CultureInfo.InvariantCulture, $" ({all.Count} in all; the first {DatabasesLeftOut.MaxNamed} are listed)") + : string.Empty; + return "Left out of this grid: " + string.Join(", ", named) + more + ". The server's newest database snapshot no longer lists " + + (all.Count == 1 ? "it" : "them") + " (dropped or detached since an earlier snapshot did). Pick the database in the filter to read its history."; + } + + /// Shows with , or collapses it when no database was left out. + internal static void ShowDatabasesLeftOutBanner(TextBlock banner, IReadOnlyList? leftOut) + { + var text = DatabasesLeftOutBannerText(leftOut); + banner.Text = text ?? string.Empty; + banner.Visibility = text is null ? Visibility.Collapsed : Visibility.Visible; + } + /// /// Raises or hides the "Showing since" banner of the Query Store Regressions grid (#4966). The read compares the range with /// the baseline of the 7 days before it, so the notice keys on that EARLIER window: a server added two days ago covers the diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml index d040a5571f..0f3041eedc 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerServerTab.xaml @@ -1748,6 +1748,7 @@ + @@ -1761,7 +1762,12 @@ Padding="8,4" FontWeight="Bold" Foreground="{DynamicResource ForegroundBrush}" Background="#22FFAA00"/> - + + diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerStoreLogger.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerStoreLogger.cs new file mode 100644 index 0000000000..0e7d29eaf6 --- /dev/null +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerStoreLogger.cs @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using Microsoft.Extensions.Logging; + +namespace PerformanceMonitor.Darling.Viewer; + +/// +/// The the Viewer hands to the storage readers that take one (#6654 review), so a best-effort step that gives up +/// (the dropped-database read, CurrentDatabaseScope) lands in the Viewer's own log as a warning instead of vanishing. +/// Warnings and worse only; the exception text rides the formatter's message. +/// +internal sealed class ViewerStoreLogger : ILogger +{ + public static readonly ViewerStoreLogger Instance = new(); + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Warning; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (IsEnabled(logLevel)) + { + ViewerLogger.Warn("ViewerDataService", formatter(state, exception)); + } + } +} diff --git a/Lite.Tests/QueryStoreRegressionsToolTests.cs b/Lite.Tests/QueryStoreRegressionsToolTests.cs index 884686aba2..e8b2663395 100644 --- a/Lite.Tests/QueryStoreRegressionsToolTests.cs +++ b/Lite.Tests/QueryStoreRegressionsToolTests.cs @@ -174,6 +174,47 @@ the baseline averages would be an avg-of-avgs over two snapshots of one interval Assert.Equal(200, row.GetProperty("recent_exec_count").GetInt64()); } + /// + /// #6654: with no database named, a regression in a database the server dropped (an earlier database_states snapshot lists it, + /// the newest does not) is left out before the cap and named in databases_left_out; naming the database reads it as asked. + /// + [Fact] + public async Task ADroppedDatabasesRegression_IsLeftOut_AndNamed_UntilTheDatabaseIsNamed() + { + var service = new LocalDataService(_duckDb); + var baseNow = Truncate(DateTime.UtcNow); + const string Gone = "GoneDb"; + + await SeedAsync(baseNow.AddDays(-10), 100, 1000, 1000, intervalId: 9, queryId: 99); + foreach (var (queryId, database) in new[] { (20L, Db), (30L, Gone) }) + { + await SeedAsync(baseNow.AddHours(-40), 100, 1000, 1000, intervalId: 1, queryId: queryId, database: database); + await SeedAsync(baseNow.AddMinutes(-30), 100, 3000, 3000, intervalId: 2, queryId: queryId, database: database); + } + + var connection = await SeedConnectionAsync(); + foreach (var (age, names) in new[] { (TimeSpan.FromHours(1), new[] { Db, Gone }), (TimeSpan.FromMinutes(5), new[] { Db }) }) + { + var at = DateTime.UtcNow - age; + var n = 1; + foreach (var name in names) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = "INSERT INTO database_states (collection_id, collection_time, server_id, server_name, database_name, database_id, state_desc, is_in_standby) VALUES " + + $"({_nextId++}, TIMESTAMP '{at:yyyy-MM-dd HH:mm:ss}', {_serverId}, '{ServerName}', '{name}', {n++}, 'ONLINE', false)"; + await cmd.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + } + + var all = Root(await McpQueryTools.GetQueryStoreRegressions(service, _serverManager, ServerName, 24)); + Assert.Equal([20L], all.GetProperty("regressions").EnumerateArray().Select(r => r.GetProperty("query_id").GetInt64()).ToArray()); + Assert.Equal([Gone], all.GetProperty("databases_left_out").EnumerateArray().Select(x => x.GetString()).ToArray()); + + var named = Root(await McpQueryTools.GetQueryStoreRegressions(service, _serverManager, ServerName, 24, Gone)); + Assert.Equal([30L], named.GetProperty("regressions").EnumerateArray().Select(r => r.GetProperty("query_id").GetInt64()).ToArray()); + Assert.True(named.GetProperty("databases_left_out").ValueKind == JsonValueKind.Null); + } + [Fact] public async Task AnOutOfRangeCap_IsRefused_NotSilentlyClamped() { @@ -201,7 +242,7 @@ private async Task SeedConnectionAsync() } private async Task SeedAsync( - DateTime collectionTime, long executions, long avgDurationUs, long avgCpuUs, long intervalId, long queryId = 1) + DateTime collectionTime, long executions, long avgDurationUs, long avgCpuUs, long intervalId, long queryId = 1, string database = Db) { using var readLock = _duckDb.AcquireReadLock(); var connection = await SeedConnectionAsync(); @@ -216,7 +257,7 @@ INSERT INTO query_store_stats cmd.Parameters.Add(new DuckDBParameter { Value = collectionTime }); cmd.Parameters.Add(new DuckDBParameter { Value = _serverId }); cmd.Parameters.Add(new DuckDBParameter { Value = ServerName }); - cmd.Parameters.Add(new DuckDBParameter { Value = Db }); + cmd.Parameters.Add(new DuckDBParameter { Value = database }); cmd.Parameters.Add(new DuckDBParameter { Value = queryId }); cmd.Parameters.Add(new DuckDBParameter { Value = 9L }); cmd.Parameters.Add(new DuckDBParameter { Value = "Regular" }); diff --git a/Lite/Mcp/McpQueryTools.cs b/Lite/Mcp/McpQueryTools.cs index a4ef571163..57d7a7d2b5 100644 --- a/Lite/Mcp/McpQueryTools.cs +++ b/Lite/Mcp/McpQueryTools.cs @@ -624,11 +624,16 @@ public static async Task GetQueryStoreRegressions( the one field whose whole reason for existing is that the cap should not have to be inferred. */ var databases = string.IsNullOrWhiteSpace(database_name) ? null : new[] { database_name }; - var rows = await dataService.GetQueryStoreRegressionsAsync( - resolved.ServerId, hours_back, limit + 1, databases, asOfUtc: windowEnd); - var windowStart = windowEnd.AddHours(-hours_back); var baselineStart = windowStart.AddDays(-LocalDataService.BaselineLookbackDays); + /* #6654: with no database named, the read leaves out the databases the server DROPPED (named by an earlier database_states + snapshot, absent from the newest), as get_top_procedures_by_cpu does (#5633 item 4); the names ride the payload. */ + var droppedDatabases = databases is null + ? await dataService.GetDroppedDatabasesAsync(resolved.ServerId, asOfUtc: string.IsNullOrWhiteSpace(as_of) ? null : windowEnd, windowStartUtc: baselineStart) + : null; + var leftOut = DatabasesLeftOut.Distinct(droppedDatabases); + var rows = await dataService.GetQueryStoreRegressionsAsync( + resolved.ServerId, hours_back, limit + 1, databases, asOfUtc: windowEnd, excludedDatabases: droppedDatabases?.ToList()); /* #4966: where this server's query_store_stats start, checked against the EARLIER of the two windows this tool compares — the baseline's start, not the recent window's: a store that began inside the baseline @@ -643,7 +648,8 @@ public static async Task GetQueryStoreRegressions( emptyAnswer: rows.Count == 0); if (rows.Count == 0) - return await EmptyRegressionsAsync(dataService, resolved.ServerId, resolved.DisplayName, hours_back, windowEnd, notice, databases); + return McpHelpers.WithDatabasesLeftOut( + await EmptyRegressionsAsync(dataService, resolved.ServerId, resolved.DisplayName, hours_back, windowEnd, notice, databases), leftOut)!; var truncated = rows.Count > limit; @@ -669,6 +675,9 @@ public static async Task GetQueryStoreRegressions( returned_count = Math.Min(rows.Count, limit), truncated, regressions = rows.Take(limit).Select(r => RegressionRowJson(r, full_text)), + /* #6654: the dropped databases this read left out, in the words get_top_procedures_by_cpu uses. */ + databases_left_out = DatabasesLeftOut.Named(leftOut), + databases_left_out_note = DatabasesLeftOut.Note(leftOut), }, McpHelpers.JsonOptions); } catch (Exception ex) diff --git a/Lite/Services/LocalDataService.QueryStoreRegressions.cs b/Lite/Services/LocalDataService.QueryStoreRegressions.cs index f0d6a06867..24c2a559cf 100644 --- a/Lite/Services/LocalDataService.QueryStoreRegressions.cs +++ b/Lite/Services/LocalDataService.QueryStoreRegressions.cs @@ -116,7 +116,8 @@ public partial class LocalDataService /// and take at most maxRows / FailureShareDivisor slots (more only when the cost rows leave room). /// public async Task> GetQueryStoreRegressionsAsync( - int serverId, int hoursBack = 24, int maxRows = 50, IReadOnlyList? databaseNames = null, DateTime? asOfUtc = null) + int serverId, int hoursBack = 24, int maxRows = 50, IReadOnlyList? databaseNames = null, DateTime? asOfUtc = null, + IReadOnlyList? excludedDatabases = null) { using var connection = await OpenConnectionAsync(); using var command = connection.CreateCommand(); @@ -126,6 +127,8 @@ public async Task> GetQueryStoreRegressionsAsync( var dbClause = BuildDbInClause(databaseNames, "database_name", 4, out var dbValues); var baselineParamIndex = 4 + dbValues.Count; var limitIndex = baselineParamIndex + 1; + /* #6654: the databases the server dropped, left out of the candidates before the rank and the cap (the exclusion binds after the limit). */ + var excludeClause = BuildDbExcludeClause(excludedDatabases, "f.database_name", limitIndex + 1, out var excludeValues); /* The per-hour rates divide by each window's own length: the baseline is [baseline start, window start), the recent window [window start, window end]. Both are numbers computed here, written as invariant literals. */ @@ -312,7 +315,7 @@ CASE WHEN NOT f.is_errors THEN 0 ELSE 2 END) AS failure_rank, (f.recent_dur_ms - f.base_dur_ms) * f.recent_execs AS additional_duration_ms FROM flagged AS f - WHERE f.is_cost OR f.is_timeouts OR f.is_errors + WHERE (f.is_cost OR f.is_timeouts OR f.is_errors)" + excludeClause + @" ), ranked AS ( SELECT @@ -403,6 +406,8 @@ CASE WHEN r.is_cost THEN COALESCE(r.additional_duration_ms, 0) END DESC, command.Parameters.Add(new DuckDBParameter { Value = db }); command.Parameters.Add(new DuckDBParameter { Value = baselineStartTime }); command.Parameters.Add(new DuckDBParameter { Value = maxRows }); + foreach (var excluded in excludeValues) + command.Parameters.Add(new DuckDBParameter { Value = excluded }); var rows = new List(); var textKeys = new List<(long? Id, DateTime? Time)>(); diff --git a/PerformanceMonitor.Collectors/QueryStoreCollector.cs b/PerformanceMonitor.Collectors/QueryStoreCollector.cs index 2b66ec8992..38788a1dc7 100644 --- a/PerformanceMonitor.Collectors/QueryStoreCollector.cs +++ b/PerformanceMonitor.Collectors/QueryStoreCollector.cs @@ -1788,7 +1788,10 @@ datetimeoffset and need the .UtcDateTime normalization. */ byte budget. This replaced the SQL-side NOT LIKE predicate, which was 75% of the read's elapsed time (full nvarchar(max) scan per row; the field A/B measured 4.3x without it). Reached on BOTH paths: on Azure our own per-database payload runs inside the very - database whose Query Store we are reading, so without this it would collect itself. */ + database whose Query Store we are reading, so without this it would collect itself. + #6665: this tests row.QueryText, which is NULL on the separate-text-fetch path (FetchQueryTextSeparately nulls + query_sql_text, and Darling turns it on), so on that path the drop does not fire and the statement's stats rows are + stored. Darling hides it at READ time instead (QueryStoreSelfStatement); Lite keeps inline text and drops it here. */ if (row.QueryText?.Contains(SelfQueryMarker, StringComparison.Ordinal) == true) { continue;