diff --git a/Darling/Darling.Tests/FindQueriesByTextAllServersSpansLiveTests.cs b/Darling/Darling.Tests/FindQueriesByTextAllServersSpansLiveTests.cs index 6a2b79c47d..ccb8b1f217 100644 --- a/Darling/Darling.Tests/FindQueriesByTextAllServersSpansLiveTests.cs +++ b/Darling/Darling.Tests/FindQueriesByTextAllServersSpansLiveTests.cs @@ -42,9 +42,9 @@ public sealed class FindQueriesByTextAllServersSpansLiveTests private const string CountHash = "0xBB02"; private const string OtherHash = "0xDD04"; - /// Two texts that name dbo.Ledger and one that does not, on two servers. The ledger statement ran on span-a an hour ago, two hours + /// Two texts that name dbo.Ledger and one that does not, on two servers. The ledger statement ran on span-a half an hour ago, two hours /// ago and three days ago, and on span-b, with the same query_hash and database, three days and an hour ago; the count statement ran on - /// span-a three days and a half hour ago; the other statement ran on span-a an hour ago. Only span-a's ledger and other statements sit in + /// span-a three days and a half hour ago; the other statement ran on span-a half an hour ago. Only span-a's ledger and other statements sit in /// the newest span. private static async Task<(ScratchPostgres Scratch, DateTime NewestSample)> OpenScratchAsync(CancellationToken ct) { @@ -68,17 +68,17 @@ await DarlingMcpTestData.ExecAsync(c, ct, VALUES ($1,$2,$3,$4,'Books',$5,sha256($6::text::bytea))", CollectionIdGenerator.Next(), now - age, server, name, hash, text); - await Stat(1, "span-a", LedgerHash, "ledger", TimeSpan.FromHours(1)); + await Stat(1, "span-a", LedgerHash, "ledger", TimeSpan.FromMinutes(30)); await Stat(1, "span-a", LedgerHash, "ledger", TimeSpan.FromHours(2)); await Stat(1, "span-a", LedgerHash, "ledger", TimeSpan.FromDays(3)); await Stat(2, "span-b", LedgerHash, "ledger", TimeSpan.FromDays(3) + TimeSpan.FromHours(1)); await Stat(1, "span-a", CountHash, "count", TimeSpan.FromDays(3) + TimeSpan.FromMinutes(30)); - await Stat(1, "span-a", OtherHash, "other", TimeSpan.FromHours(1)); + await Stat(1, "span-a", OtherHash, "other", TimeSpan.FromMinutes(30)); await DarlingMcpTestData.ExecAsync(c, ct, "CREATE EXTENSION IF NOT EXISTS pg_trgm"); await DarlingMcpTestData.ExecAsync(c, ct, "CREATE INDEX IF NOT EXISTS " + FindQueriesByTextIndexScopeLiveTests.IndexShortName + " ON collect.query_text_dim USING gin (query_text gin_trgm_ops)"); - return (scratch, now - TimeSpan.FromHours(1)); + return (scratch, now - TimeSpan.FromMinutes(30)); } private static QueryTextTrigramIndexTracker ReadyTracker() @@ -103,6 +103,17 @@ private static string[] Keys(JsonElement planCache) => .Select(r => r.GetProperty("server").GetString() + "@" + r.GetProperty("query_hash").GetString() + "@" + r.GetProperty("database_name").GetString()) .ToArray(); + /// How many statements an uncut search runs: one per span of the raw window. + private static int SpanCount() + { + var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + return DarlingMcpFindQueriesByTextTools.RawTierSpans(now - TimescaleSupport.RawRetentionSpan, now).Count; + } + + /// #9219: the newest span a search reads first, a small slice of the window, not a sixteenth of it. + private static double FirstSpanSeconds() => + (TimescaleSupport.RawRetentionSpan / DarlingMcpFindQueriesByTextTools.RawTierSpanParts).TotalSeconds; + private static string Stamp(DateTime naiveUtc) => DateTime.SpecifyKind(naiveUtc, DateTimeKind.Utc).ToString("o", CultureInfo.InvariantCulture); /// The whole budget, counting the statements the source runs. @@ -149,7 +160,7 @@ public async Task ABudgetThatRunsOutAfterTheNewestSpan_KeepsThatSpansRows_AndSay Assert.False(searchedWindow.GetProperty("complete").GetBoolean()); var from = DateTime.Parse(searchedWindow.GetProperty("searched_from").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); var to = DateTime.Parse(searchedWindow.GetProperty("searched_to").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - Assert.InRange((to - from).TotalSeconds, (TimescaleSupport.RawRetentionSpan / DarlingMcpFindQueriesByTextTools.RawTierSpanCount).TotalSeconds - 1, (TimescaleSupport.RawRetentionSpan / DarlingMcpFindQueriesByTextTools.RawTierSpanCount).TotalSeconds + 1); + Assert.InRange((to - from).TotalSeconds, FirstSpanSeconds() - 1, FirstSpanSeconds() + 1); Assert.Equal(searchedWindow.GetProperty("window_to").GetString(), searchedWindow.GetProperty("searched_to").GetString()); } @@ -201,10 +212,10 @@ public async Task TheWholeBudget_ListsEachServerHashAndDatabaseOnce_WithItsNewes string[] expected = ["span-a@" + LedgerHash + "@Books", "span-a@" + CountHash + "@Books", "span-b@" + LedgerHash + "@Books"]; Assert.Equal(expected, Keys(planCache)); Assert.Equal(Stamp(newestSample), planCache.GetProperty("rows")[0].GetProperty("last_collected_at").GetString()); - Assert.Equal(Stamp(newestSample - TimeSpan.FromDays(3)), planCache.GetProperty("rows")[2].GetProperty("last_collected_at").GetString()); + Assert.Equal(Stamp(newestSample - TimeSpan.FromDays(3) - TimeSpan.FromMinutes(30)), planCache.GetProperty("rows")[2].GetProperty("last_collected_at").GetString()); Assert.False(planCache.GetProperty("truncated").GetBoolean()); - Assert.Equal(DarlingMcpFindQueriesByTextTools.RawTierSpanCount, carried.Count); + Assert.Equal(SpanCount(), carried.Count); Assert.Empty(carried[0]); Assert.Equal(new[] { 1 }, carried[1]); Assert.Equal(new[] { 1, 1, 2 }, carried[^1].Order()); @@ -227,12 +238,12 @@ public async Task ALimitBelowTheMatches_IsTruncated_KeepsTheNewestRow_AndStopsAt Assert.Equal("ok", planCache.GetProperty("status").GetString()); Assert.True(planCache.GetProperty("truncated").GetBoolean()); Assert.Equal(["span-a@" + LedgerHash + "@Books"], Keys(planCache)); - Assert.InRange(statements, 2, DarlingMcpFindQueriesByTextTools.RawTierSpanCount - 1); + Assert.InRange(statements, 2, SpanCount() - 1); } /// #9150: a page that fills in the newest span stops the search there (one statement), and the reply says the page is the whole-window /// answer: complete is true and searched_from is the window's start, because spans run newest first and the older ones could not - /// change a full page. truncated says more matches exist. A count statement on span-b half an hour ago makes the newest span hold two rows. + /// change a full page. truncated says more matches exist. A count statement on span-b twenty minutes ago makes the newest span hold two rows. [Fact] public async Task APageThatFillsInTheNewestSpan_IsTheWholeWindowAnswer_AndIsTruncated() { @@ -245,7 +256,7 @@ public async Task APageThatFillsInTheNewestSpan_IsTheWholeWindowAnswer_AndIsTrun await DarlingMcpTestData.ExecAsync(c, ct, @"INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, database_name, query_hash, query_text_digest) VALUES ($1,$2,2,'span-b','Books',$3,sha256('count'::text::bytea))", - CollectionIdGenerator.Next(), DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow) - TimeSpan.FromMinutes(30), CountHash); + CollectionIdGenerator.Next(), DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow) - TimeSpan.FromMinutes(20), CountHash); } var statements = 0; @@ -265,9 +276,9 @@ await DarlingMcpTestData.ExecAsync(c, ct, private const string NullDatabaseHash = "0xC003"; /// A pattern more texts contain than the probe takes (1,100 filler texts name dbo.Common), so the search joins the window in spans. Server - /// span-a ran the first statement an hour ago and two and a half days ago, and the third statement, with no database, an hour ago and three days + /// span-a ran the first statement half an hour ago and two and a half days ago, and the third statement, with no database, half an hour ago and three days /// ago; span-b ran the first statement, with the same query_hash and database, two and a half days ago; span-a ran the second statement three and - /// a half days ago; a statement that does not name dbo.Common ran an hour ago. Only the first and third span-a statements sit in the newest span. + /// a half days ago; a statement that does not name dbo.Common ran half an hour ago. Only the first and third span-a statements sit in the newest span. private static async Task<(ScratchPostgres Scratch, DateTime NewestSample)> OpenCommonScratchAsync(CancellationToken ct) { Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), SkipReason); @@ -295,18 +306,18 @@ await DarlingMcpTestData.ExecAsync(c, ct, VALUES ($1,$2,$3,$4,$5,$6,sha256($7::text::bytea))", CollectionIdGenerator.Next(), now - age, server, name, (object?)database ?? DBNull.Value, hash, text); - await Stat(1, "span-a", "Books", CommonHash, "first", TimeSpan.FromHours(1)); + await Stat(1, "span-a", "Books", CommonHash, "first", TimeSpan.FromMinutes(30)); await Stat(1, "span-a", "Books", CommonHash, "first", TimeSpan.FromDays(2.5)); await Stat(2, "span-b", "Books", CommonHash, "first", TimeSpan.FromDays(2.5)); await Stat(1, "span-a", "Books", SecondCommonHash, "second", TimeSpan.FromDays(3.5)); - await Stat(1, "span-a", null, NullDatabaseHash, "third", TimeSpan.FromHours(1)); + await Stat(1, "span-a", null, NullDatabaseHash, "third", TimeSpan.FromMinutes(30)); await Stat(1, "span-a", null, NullDatabaseHash, "third", TimeSpan.FromDays(3)); - await Stat(1, "span-a", "Books", OtherHash, "other", TimeSpan.FromHours(1)); + await Stat(1, "span-a", "Books", OtherHash, "other", TimeSpan.FromMinutes(30)); await DarlingMcpTestData.ExecAsync(c, ct, "CREATE EXTENSION IF NOT EXISTS pg_trgm"); await DarlingMcpTestData.ExecAsync(c, ct, "CREATE INDEX IF NOT EXISTS " + FindQueriesByTextIndexScopeLiveTests.IndexShortName + " ON collect.query_text_dim USING gin (query_text gin_trgm_ops)"); - return (scratch, now - TimeSpan.FromHours(1)); + return (scratch, now - TimeSpan.FromMinutes(30)); } /// #9150: a pattern more texts contain than the probe takes is joined to the window in spans, newest first. An uncut search runs every @@ -325,7 +336,7 @@ public async Task ACommonPattern_JoinsTheWindowInSpans_AndListsWhatTheWholeWindo Assert.Equal("ok", planCache.GetProperty("status").GetString()); Assert.Equal("join", planCache.GetProperty("sample_read").GetString()); Assert.Equal(JsonValueKind.Null, planCache.GetProperty("matched_texts").ValueKind); - Assert.Equal(DarlingMcpFindQueriesByTextTools.RawTierSpanCount, statements); + Assert.Equal(SpanCount(), statements); string[] expected = [ "span-a@" + CommonHash + "@Books", "span-a@" + NullDatabaseHash + "@", @@ -365,8 +376,7 @@ public async Task ACommonPatternWhoseJoinIsCutAfterTheNewestSpan_KeepsThatSpansR Assert.False(searchedWindow.GetProperty("complete").GetBoolean()); var from = DateTime.Parse(searchedWindow.GetProperty("searched_from").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); var to = DateTime.Parse(searchedWindow.GetProperty("searched_to").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - var span = (TimescaleSupport.RawRetentionSpan / DarlingMcpFindQueriesByTextTools.RawTierSpanCount).TotalSeconds; - Assert.InRange((to - from).TotalSeconds, span - 1, span + 1); + Assert.InRange((to - from).TotalSeconds, FirstSpanSeconds() - 1, FirstSpanSeconds() + 1); Assert.Equal(searchedWindow.GetProperty("window_to").GetString(), searchedWindow.GetProperty("searched_to").GetString()); } diff --git a/Darling/Darling.Tests/FindQueriesByTextBudgetKeepsRowsLiveTests.cs b/Darling/Darling.Tests/FindQueriesByTextBudgetKeepsRowsLiveTests.cs index 85d939b11e..a04a285962 100644 --- a/Darling/Darling.Tests/FindQueriesByTextBudgetKeepsRowsLiveTests.cs +++ b/Darling/Darling.Tests/FindQueriesByTextBudgetKeepsRowsLiveTests.cs @@ -43,9 +43,9 @@ public sealed class FindQueriesByTextBudgetKeepsRowsLiveTests private const string OlderHash = "0xBB02"; private const string UnmatchedHash = "0xDD04"; - /// Two texts that both name dbo.Ledger, and one that does not. The newest statement ran in Books an hour ago, two hours ago + /// Two texts that both name dbo.Ledger, and one that does not. The newest statement ran in Books half an hour ago, two hours ago /// and three days ago, and in Archive, with the same text, three days and an hour ago; the older statement ran in Books three days ago - /// only; the unmatched statement ran an hour ago and three days ago. All sit inside the raw tier's window; only the newest and the + /// only; the unmatched statement ran half an hour ago and three days ago. All sit inside the raw tier's window; only the newest and the /// unmatched statements sit in its newest hours, so a text first judged there is seen again, matched or not, in a later span. private static async Task<(ScratchPostgres Scratch, DateTime NewestSample)> OpenScratchAsync(CancellationToken ct) { @@ -68,14 +68,14 @@ await DarlingMcpTestData.ExecAsync(c, ct, VALUES ($1,$2,1,'ledger',$5,$3,sha256($4::text::bytea))", CollectionIdGenerator.Next(), now - age, hash, text, database); - await Stat(NewestHash, "newest", TimeSpan.FromHours(1)); + await Stat(NewestHash, "newest", TimeSpan.FromMinutes(30)); await Stat(NewestHash, "newest", TimeSpan.FromHours(2)); await Stat(NewestHash, "newest", TimeSpan.FromDays(3)); await Stat(NewestHash, "newest", TimeSpan.FromDays(3) + TimeSpan.FromHours(1), "Archive"); await Stat(OlderHash, "older", TimeSpan.FromDays(3)); - await Stat(UnmatchedHash, "unmatched", TimeSpan.FromHours(1)); + await Stat(UnmatchedHash, "unmatched", TimeSpan.FromMinutes(30)); await Stat(UnmatchedHash, "unmatched", TimeSpan.FromDays(3)); - return (scratch, now - TimeSpan.FromHours(1)); + return (scratch, now - TimeSpan.FromMinutes(30)); } private static async Task PlanCacheAsync(ScratchPostgres scratch, DarlingMcpFindQueriesByTextTools.SourceBudget planCache, int limit, CancellationToken ct) @@ -124,14 +124,14 @@ public async Task ABudgetThatRunsOutAfterTheNewestSpan_ReturnsTheRowsFoundThere_ Assert.False(searchedWindow.GetProperty("complete").GetBoolean()); var from = DateTime.Parse(searchedWindow.GetProperty("searched_from").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); var to = DateTime.Parse(searchedWindow.GetProperty("searched_to").GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - var span = (TimescaleSupport.RawRetentionSpan / DarlingMcpFindQueriesByTextTools.RawTierSpanCount).TotalSeconds; + var span = (TimescaleSupport.RawRetentionSpan / DarlingMcpFindQueriesByTextTools.RawTierSpanParts).TotalSeconds; Assert.InRange((to - from).TotalSeconds, span - 1, span + 1); Assert.Equal(searchedWindow.GetProperty("window_to").GetString(), searchedWindow.GetProperty("searched_to").GetString()); } /// #9150: a page that fills in the newest span stops the search there (one statement), and the reply says the page is the whole-window /// answer: complete is true and searched_from is the window's start, because spans run newest first and the older ones could not - /// change a full page. truncated says more matches exist. A second matched text an hour old makes the newest span hold two rows. + /// change a full page. truncated says more matches exist. A second matched text twenty minutes old makes the newest span hold two rows. [Fact] public async Task APageThatFillsInTheNewestSpan_IsTheWholeWindowAnswer_AndIsTruncated() { @@ -144,7 +144,7 @@ public async Task APageThatFillsInTheNewestSpan_IsTheWholeWindowAnswer_AndIsTrun await DarlingMcpTestData.ExecAsync(c, ct, @"INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, database_name, query_hash, query_text_digest) VALUES ($1,$2,1,'ledger','Books',$3,sha256('older'::text::bytea))", - CollectionIdGenerator.Next(), DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow) - TimeSpan.FromMinutes(30), OlderHash); + CollectionIdGenerator.Next(), DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow) - TimeSpan.FromMinutes(20), OlderHash); } var statements = 0; @@ -198,7 +198,7 @@ public async Task TheWholeBudget_ListsEveryMatchNewestFirst_OncePerHashAndDataba Assert.Equal("ok", planCache.GetProperty("status").GetString()); Assert.Equal([(NewestHash, "Books"), (OlderHash, "Books"), (NewestHash, "Archive")], Keys(planCache)); Assert.Equal(DateTime.SpecifyKind(newestSample, DateTimeKind.Utc).ToString("o"), planCache.GetProperty("rows")[0].GetProperty("last_collected_at").GetString()); - Assert.Equal(DateTime.SpecifyKind(newestSample - TimeSpan.FromDays(3), DateTimeKind.Utc).ToString("o"), planCache.GetProperty("rows")[2].GetProperty("last_collected_at").GetString()); + Assert.Equal(DateTime.SpecifyKind(newestSample - TimeSpan.FromDays(3) - TimeSpan.FromMinutes(30), DateTimeKind.Utc).ToString("o"), planCache.GetProperty("rows")[2].GetProperty("last_collected_at").GetString()); Assert.False(planCache.GetProperty("truncated").GetBoolean()); } @@ -226,7 +226,8 @@ public async Task EveryText_IsJudgedOnce_AndItsVerdictCarriedToTheSpansAfter() var planCache = await PlanCacheAsync(scratch, budget, 20, ct); Assert.Equal("ok", planCache.GetProperty("status").GetString()); - Assert.Equal(DarlingMcpFindQueriesByTextTools.RawTierSpanCount, carried.Count); + var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + Assert.Equal(DarlingMcpFindQueriesByTextTools.RawTierSpans(now - TimescaleSupport.RawRetentionSpan, now).Count, carried.Count); Assert.Empty(carried[0]); Assert.Equal(new[] { Digest("newest"), Digest("unmatched") }.Order(), carried[1].Order()); foreach (var step in carried) diff --git a/Darling/Darling.Tests/FindQueriesByTextFirstSpanLiveTests.cs b/Darling/Darling.Tests/FindQueriesByTextFirstSpanLiveTests.cs new file mode 100644 index 0000000000..ac1553c347 --- /dev/null +++ b/Darling/Darling.Tests/FindQueriesByTextFirstSpanLiveTests.cs @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each test creates its own scratch database through ScratchPostgres and + works entirely inside it, so it cannot race live collection. */ +/// +/// #9219: find_queries_by_text never stops with nothing searched when the newest slice of the raw tier can be read inside the budget. The +/// first span of a search is a small slice of the newest data and the spans widen after it, so a store whose whole window cannot be read in the +/// budget still answers with the newest matches and the span they cover. A slow span is modelled without a sleep: the budget's statement hook +/// takes an advisory lock that a second connection holds, for a span wider than the test allows, so that statement waits on the lock until +/// the source's own timeout cancels it, as a statement that cannot finish in the budget is cancelled. The spans' widths come from the tool's +/// own schedule, so the test needs no knowledge of how wide any span is, only that the window cannot be read whole. +/// +public sealed class FindQueriesByTextFirstSpanLiveTests +{ + private static string? BaseConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private const string SkipReason = "Set DARLING_TEST_PG to a Postgres connection string to run the find_queries_by_text first-span live tests."; + + /// The widest span the modelled store can read inside the budget. The window is days long, so the whole of it never fits. + private static readonly TimeSpan ReadableSpan = TimeSpan.FromHours(2); + + /// The source's budget in the modelled runs: a span the store cannot read waits out all of it, so it is short, and a span it can read + /// takes milliseconds, so it is far from the edge. + private const int ModelledBudgetMilliseconds = 2500; + + private const long LockKey = 921900; + + private const string CommonPattern = "dbo.Common"; + private const string RarePattern = "dbo.Rare"; + + /// One server, "first-a", runs statements that name dbo.Rare 20, 30 and 40 minutes ago, 90 minutes ago, five hours ago and three days + /// ago (R1 to R6), and statements that name dbo.Common (more texts than the probe takes, because 1,100 filler texts name it too) 30 minutes ago + /// on it (C1), 40 minutes ago on "first-b" (C2), 90 minutes ago (C3), five hours ago (C4) and three days ago (C5). Every sample is in the + /// Books database. + private static async Task OpenScratchAsync(CancellationToken ct) + { + Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), SkipReason); + var scratch = await ScratchPostgres.CreateAsync(BaseConnectionString!, ct); + try + { + await using var c = new NpgsqlConnection(scratch.ConnectionString); + await c.OpenAsync(ct); + await PgMigrations.MigrateAsync(c, ct); + await DarlingMcpTestData.RegisterServerAsync(c, 1, "first-a", ct); + await DarlingMcpTestData.RegisterServerAsync(c, 2, "first-b", ct); + await DarlingMcpTestData.ExecAsync(c, ct, + @"INSERT INTO collect.query_text_dim (digest, query_text, last_seen) + SELECT sha256(('filler' || i)::text::bytea), 'SELECT ' || i || ' FROM dbo.Common', timezone('UTC', now()) + FROM generate_series(1, 1100) AS i"); + await DarlingMcpTestData.ExecAsync(c, ct, + @"INSERT INTO collect.query_text_dim (digest, query_text, last_seen) + SELECT sha256(t::bytea), CASE WHEN t LIKE 'r%' THEN 'SELECT * FROM dbo.Rare WHERE k = ' ELSE 'SELECT * FROM dbo.Common WHERE k = ' END || t, timezone('UTC', now()) + FROM unnest(ARRAY['r1','r2','r3','r4','r5','r6','c1','c2','c3','c4','c5']) AS t"); + + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + async Task Stat(int server, string text, TimeSpan age) => + await DarlingMcpTestData.ExecAsync(c, ct, + @"INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, database_name, query_hash, query_text_digest) + VALUES ($1,$2,$3,$4,'Books',$5,sha256($6::text::bytea))", + CollectionIdGenerator.Next(), now - age, server, server == 1 ? "first-a" : "first-b", "0x" + text.ToUpperInvariant(), text); + + await Stat(1, "r1", TimeSpan.FromMinutes(20)); + await Stat(1, "r2", TimeSpan.FromMinutes(30)); + await Stat(1, "r3", TimeSpan.FromMinutes(40)); + await Stat(1, "r4", TimeSpan.FromMinutes(90)); + await Stat(1, "r5", TimeSpan.FromHours(5)); + await Stat(1, "r6", TimeSpan.FromDays(3)); + await Stat(1, "c1", TimeSpan.FromMinutes(30)); + await Stat(2, "c2", TimeSpan.FromMinutes(40)); + await Stat(1, "c3", TimeSpan.FromMinutes(90)); + await Stat(1, "c4", TimeSpan.FromHours(5)); + await Stat(1, "c5", TimeSpan.FromDays(3)); + + await DarlingMcpTestData.ExecAsync(c, ct, "CREATE EXTENSION IF NOT EXISTS pg_trgm"); + await DarlingMcpTestData.ExecAsync(c, ct, + "CREATE INDEX IF NOT EXISTS " + FindQueriesByTextIndexScopeLiveTests.IndexShortName + " ON collect.query_text_dim USING gin (query_text gin_trgm_ops)"); + await DarlingMcpTestData.ExecAsync(c, ct, "ANALYZE collect.query_text_dim"); + return scratch; + } + catch + { + await scratch.DisposeAsync(); + throw; + } + } + + private static QueryTextTrigramIndexTracker ReadyTracker() + { + var tracker = new QueryTextTrigramIndexTracker(); + tracker.Configure(true); + return tracker; + } + + /// A second connection holding the advisory lock a modelled slow span waits on, for as long as the test runs. + private sealed class SlowStore : IAsyncDisposable + { + private readonly NpgsqlConnection _holder; + + private SlowStore(NpgsqlConnection holder) => _holder = holder; + + public static async Task OpenAsync(string connectionString, CancellationToken ct) + { + var holder = new NpgsqlConnection(connectionString); + await holder.OpenAsync(ct); + await DarlingMcpTestData.ExecAsync(holder, ct, "SELECT pg_advisory_lock(" + LockKey.ToString(CultureInfo.InvariantCulture) + ")"); + return new SlowStore(holder); + } + + /// The source's budget for a store that reads a span of up to and cannot read a wider one: before the + /// statement of a wider span the hook waits on the lock the holder has, so the source's timeout cancels it. + public static DarlingMcpFindQueriesByTextTools.SourceBudget Budget(List? widths = null) + { + var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var spans = DarlingMcpFindQueriesByTextTools.RawTierSpans(now - TimescaleSupport.RawRetentionSpan, now); + var step = 0; + return new DarlingMcpFindQueriesByTextTools.SourceBudget(ModelledBudgetMilliseconds) + { + BeforeStatement = async (connection, transaction, token) => + { + var span = spans[Math.Min(step++, spans.Count - 1)]; + var width = (span.Ceiling > now ? now : span.Ceiling) - span.Floor; + widths?.Add(width); + if (width > ReadableSpan) + { + await using var wait = new NpgsqlCommand("SELECT pg_advisory_xact_lock(" + LockKey.ToString(CultureInfo.InvariantCulture) + ")", connection, transaction); + await wait.ExecuteNonQueryAsync(token); + } + }, + }; + } + + public async ValueTask DisposeAsync() => await _holder.DisposeAsync(); + } + + private static async Task AllServersAsync( + string connectionString, string pattern, DarlingMcpFindQueriesByTextTools.SourceBudget planCache, int limit, CancellationToken ct) + { + await using var data = NpgsqlDataSource.Create(connectionString); + var reply = await DarlingMcpFindQueriesByTextTools.FindQueriesByText( + data, pattern, null, null, limit, McpDataDetail.Full, DarlingMcpFindQueriesByTextTools.SourceBudget.Default, planCache, ct, ReadyTracker()); + return JsonDocument.Parse(reply).RootElement.GetProperty("sources").GetProperty("plan_cache"); + } + + private static async Task OneServerAsync( + string connectionString, string pattern, DarlingMcpFindQueriesByTextTools.SourceBudget planCache, int limit, bool indexReady, CancellationToken ct) + { + await using var data = NpgsqlDataSource.Create(connectionString); + var reply = await DarlingMcpFindQueriesByTextTools.FindQueriesByText( + data, pattern, "first-a", null, limit, McpDataDetail.Full, DarlingMcpFindQueriesByTextTools.SourceBudget.Default, planCache, ct, + indexReady ? ReadyTracker() : new QueryTextTrigramIndexTracker()); + return JsonDocument.Parse(reply).RootElement.GetProperty("sources").GetProperty("plan_cache"); + } + + private static string[] AllServersKeys(JsonElement planCache) => + planCache.GetProperty("rows").EnumerateArray().Select(r => r.GetProperty("server").GetString() + "@" + r.GetProperty("query_hash").GetString()).ToArray(); + + private static string[] Hashes(JsonElement planCache) => + planCache.GetProperty("rows").EnumerateArray().Select(r => r.GetProperty("query_hash").GetString()!).ToArray(); + + private static DateTime Instant(JsonElement searched, string name) => + DateTime.Parse(searched.GetProperty(name).GetString()!, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + + /// A stopped search names a span it did search: from a time inside the window to the window's end, and no wider than the spans the + /// store could read. + private static void AssertNamesTheSpanItSearched(JsonElement planCache) + { + var searched = planCache.GetProperty("searched"); + Assert.False(searched.GetProperty("complete").GetBoolean()); + Assert.NotEqual(JsonValueKind.Null, searched.GetProperty("searched_from").ValueKind); + Assert.Equal(searched.GetProperty("window_to").GetString(), searched.GetProperty("searched_to").GetString()); + var covered = Instant(searched, "searched_to") - Instant(searched, "searched_from"); + Assert.InRange(covered.TotalMinutes, 1, TimeSpan.FromHours(5).TotalMinutes); + } + + /// All servers, a pattern more texts contain than the probe takes: the window join. The store reads a span of up to two hours and not + /// a wider one, so the search reads the newest three spans and stops at the fourth. It returns the newest matches and names what it covered, + /// not nothing: a search whose first span was a sixth of a day would have waited on the lock at once. + [Fact] + public async Task AllServers_ACommonPattern_OnAStoreThatCannotReadTheWholeWindow_ReturnsTheNewestMatches_AndSaysWhatItSearched() + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + await using var store = await SlowStore.OpenAsync(scratch.ConnectionString, ct); + + var planCache = await AllServersAsync(scratch.ConnectionString, CommonPattern, SlowStore.Budget(), 20, ct); + + Assert.Equal("stopped", planCache.GetProperty("status").GetString()); + Assert.Equal("join", planCache.GetProperty("sample_read").GetString()); + Assert.Equal(["first-a@0xC1", "first-b@0xC2", "first-a@0xC3"], AllServersKeys(planCache)); + AssertNamesTheSpanItSearched(planCache); + } + + /// The same store, a pattern few texts contain: the digest probe, which reads in the same spans. + [Fact] + public async Task AllServers_ARarePattern_OnAStoreThatCannotReadTheWholeWindow_ReturnsTheNewestMatches_AndSaysWhatItSearched() + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + await using var store = await SlowStore.OpenAsync(scratch.ConnectionString, ct); + + var planCache = await AllServersAsync(scratch.ConnectionString, RarePattern, SlowStore.Budget(), 20, ct); + + Assert.Equal("stopped", planCache.GetProperty("status").GetString()); + Assert.Equal("probe", planCache.GetProperty("sample_read").GetString()); + Assert.Equal(["first-a@0xR1", "first-a@0xR2", "first-a@0xR3", "first-a@0xR4"], AllServersKeys(planCache)); + AssertNamesTheSpanItSearched(planCache); + } + + /// One server, no database, a rare pattern, at each limit. Limits 1, 2 and 3 fill their page in the first two spans, so they answer + /// ok; the default limit does not fill it, so it stops at the span the store cannot read, with the four matches the spans before it + /// held. None of them stops with nothing searched, with the plan-cache index ready or off (an off index reads in spans too). + [Theory] + [InlineData(true, 1)] + [InlineData(true, 2)] + [InlineData(true, 3)] + [InlineData(true, DarlingMcpFindQueriesByTextTools.DefaultLimit)] + [InlineData(false, 1)] + [InlineData(false, 2)] + [InlineData(false, 3)] + [InlineData(false, DarlingMcpFindQueriesByTextTools.DefaultLimit)] + public async Task OneServer_ARarePattern_WithNoDatabase_OnAStoreThatCannotReadTheWholeWindow_NeverStopsWithNothingSearched(bool indexReady, int limit) + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + await using var store = await SlowStore.OpenAsync(scratch.ConnectionString, ct); + + var planCache = await OneServerAsync(scratch.ConnectionString, RarePattern, SlowStore.Budget(), limit, indexReady, ct); + + string[] newest = ["0xR1", "0xR2", "0xR3", "0xR4"]; + Assert.NotEqual(JsonValueKind.Null, planCache.GetProperty("searched").GetProperty("searched_from").ValueKind); + Assert.Equal(newest.Take(limit).ToArray(), Hashes(planCache)); + if (limit < DarlingMcpFindQueriesByTextTools.DefaultLimit) + { + Assert.Equal("ok", planCache.GetProperty("status").GetString()); + Assert.True(planCache.GetProperty("searched").GetProperty("complete").GetBoolean()); + Assert.True(planCache.GetProperty("truncated").GetBoolean()); + } + else + { + Assert.Equal("stopped", planCache.GetProperty("status").GetString()); + AssertNamesTheSpanItSearched(planCache); + Assert.Equal(indexReady ? "probe" : null, planCache.TryGetProperty("sample_read", out var read) ? read.GetString() : null); + } + } + + /// A search that finishes says so: complete and the whole window, in a store that reads every span, for both scopes. + [Fact] + public async Task AFinishedSearch_IsComplete_AndNamesTheWholeWindow() + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + var roomy = new DarlingMcpFindQueriesByTextTools.SourceBudget(60_000); + + foreach (var planCache in new[] + { + await AllServersAsync(scratch.ConnectionString, CommonPattern, roomy, 20, ct), + await OneServerAsync(scratch.ConnectionString, RarePattern, roomy, 20, true, ct), + await OneServerAsync(scratch.ConnectionString, RarePattern, roomy, 20, false, ct), + }) + { + Assert.Equal("ok", planCache.GetProperty("status").GetString()); + var searched = planCache.GetProperty("searched"); + Assert.True(searched.GetProperty("complete").GetBoolean()); + Assert.Equal(searched.GetProperty("window_from").GetString(), searched.GetProperty("searched_from").GetString()); + Assert.Equal(searched.GetProperty("window_to").GetString(), searched.GetProperty("searched_to").GetString()); + } + } + + /// A pattern that is not there answers empty and complete, not stopped, on a store that cannot read the window whole: + /// no text matches, so no sample statement runs and nothing is left to read. + [Fact] + public async Task APatternThatIsNotThere_IsEmptyAndComplete_NotStopped() + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + await using var store = await SlowStore.OpenAsync(scratch.ConnectionString, ct); + + foreach (var planCache in new[] + { + await AllServersAsync(scratch.ConnectionString, "dbo.NotThere", SlowStore.Budget(), 20, ct), + await OneServerAsync(scratch.ConnectionString, "dbo.NotThere", SlowStore.Budget(), 20, true, ct), + }) + { + Assert.Equal("empty", planCache.GetProperty("status").GetString()); + Assert.True(planCache.GetProperty("searched").GetProperty("complete").GetBoolean()); + } + } + + /// The first statement of every search is a small slice of the newest data and no span after it is narrower, so a store that is slow + /// on any part of the window is slow on the newest slice last. + [Fact] + public async Task TheFirstSpanOfASearch_IsTheNewestSlice_AndTheSpansWidenAfterIt() + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + await using var store = await SlowStore.OpenAsync(scratch.ConnectionString, ct); + + var widths = new List(); + _ = await AllServersAsync(scratch.ConnectionString, CommonPattern, SlowStore.Budget(widths), 20, ct); + + Assert.True(widths.Count >= 3, "the store reads the newest spans, so the hook ran for each"); + Assert.True(widths[0] <= TimescaleSupport.RawRetentionSpan / 90, "the first statement reads a small slice of the window"); + for (var i = 1; i < widths.Count; i++) + { + Assert.True(widths[i] >= widths[i - 1], "a span after the first is no narrower than the one before it"); + } + } +} diff --git a/Darling/Darling.Tests/FindQueriesByTextLookupSpanLiveTests.cs b/Darling/Darling.Tests/FindQueriesByTextLookupSpanLiveTests.cs new file mode 100644 index 0000000000..f56ba72c83 --- /dev/null +++ b/Darling/Darling.Tests/FindQueriesByTextLookupSpanLiveTests.cs @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Npgsql; +using NpgsqlTypes; +using PerformanceMonitor.Collectors; +using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Service.Mcp; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Each test creates its own scratch database through ScratchPostgres and + works entirely inside it, so it cannot race live collection. */ +/// +/// #9219: the all-servers window join of find_queries_by_text tests the pattern on the texts of the newest spans' own samples, not on every text +/// in the store. A budget cannot model the difference without a sleep (the cost of a whole-table text match grows with the store, and a test store +/// is small), so the first test counts the rows the statement reads from the text table, which does not depend on timing. +/// +public sealed class FindQueriesByTextLookupSpanLiveTests +{ + private static string? BaseConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private const string SkipReason = "Set DARLING_TEST_PG to a Postgres connection string to run the find_queries_by_text lookup-span live tests."; + + private const int FillerTexts = 6000; + + /// 6,000 texts name dbo.Common and no sample carries them (the pattern is common: past the probe's cap). Server lk-a ran four statements in + /// the newest span (L1 and L2 name it, L3 does not, L4 has no digest), L2 again with another text 90 minutes ago and L6 ninety-five minutes ago (the + /// second span); the newest sighting of the L2 key is the first span's. Server lk-b ran L1's statement three days ago. + private static async Task OpenScratchAsync(CancellationToken ct) + { + Assert.SkipWhen(string.IsNullOrEmpty(BaseConnectionString), SkipReason); + var scratch = await ScratchPostgres.CreateAsync(BaseConnectionString!, ct); + try + { + await using var c = new NpgsqlConnection(scratch.ConnectionString); + await c.OpenAsync(ct); + await PgMigrations.MigrateAsync(c, ct); + await DarlingMcpTestData.RegisterServerAsync(c, 1, "lk-a", ct); + await DarlingMcpTestData.RegisterServerAsync(c, 2, "lk-b", ct); + await DarlingMcpTestData.ExecAsync(c, ct, + @"INSERT INTO collect.query_text_dim (digest, query_text, last_seen) + SELECT sha256(('filler' || i)::text::bytea), 'SELECT ' || i || ' FROM dbo.Common', timezone('UTC', now()) + FROM generate_series(1, " + FillerTexts + ") AS i"); + await DarlingMcpTestData.ExecAsync(c, ct, + @"INSERT INTO collect.query_text_dim (digest, query_text, last_seen) + VALUES (sha256('l1'::text::bytea), 'SELECT * FROM dbo.Common WHERE Id = @p', timezone('UTC', now())), + (sha256('l2'::text::bytea), 'SELECT COUNT(*) FROM dbo.Common', timezone('UTC', now())), + (sha256('l3'::text::bytea), 'SELECT 1 FROM dbo.Other', timezone('UTC', now())), + (sha256('l5'::text::bytea), 'DELETE FROM dbo.Common', timezone('UTC', now())), + (sha256('l6'::text::bytea), 'TRUNCATE TABLE dbo.Common', timezone('UTC', now()))"); + + var now = DarlingMcpTestData.TruncateToSeconds(DateTime.UtcNow); + async Task Stat(int server, string? database, string hash, string? text, TimeSpan age) => + await DarlingMcpTestData.ExecAsync(c, ct, + @"INSERT INTO query_stats (collection_id, collection_time, server_id, server_name, database_name, query_hash, query_text_digest) + VALUES ($1,$2,$3,$4,$5,$6,CASE WHEN $7::text IS NULL THEN NULL ELSE sha256($7::text::bytea) END)", + CollectionIdGenerator.Next(), now - age, server, server == 1 ? "lk-a" : "lk-b", (object?)database ?? DBNull.Value, hash, (object?)text ?? DBNull.Value); + + await Stat(1, "Books", "0xL1", "l1", TimeSpan.FromMinutes(20)); + await Stat(1, "Books", "0xL2", "l2", TimeSpan.FromMinutes(30)); + await Stat(1, "Books", "0xL3", "l3", TimeSpan.FromMinutes(25)); + await Stat(1, "Books", "0xL4", null, TimeSpan.FromMinutes(15)); + await Stat(1, "Books", "0xL2", "l5", TimeSpan.FromMinutes(90)); + await Stat(1, null, "0xL6", "l6", TimeSpan.FromMinutes(95)); + await Stat(2, "Books", "0xL1", "l1", TimeSpan.FromDays(3)); + + await DarlingMcpTestData.ExecAsync(c, ct, "CREATE EXTENSION IF NOT EXISTS pg_trgm"); + await DarlingMcpTestData.ExecAsync(c, ct, + "CREATE INDEX IF NOT EXISTS " + FindQueriesByTextIndexScopeLiveTests.IndexShortName + " ON collect.query_text_dim USING gin (query_text gin_trgm_ops)"); + foreach (var table in new[] { "collect.query_text_dim", "query_stats" }) + { + await DarlingMcpTestData.ExecAsync(c, ct, "ANALYZE " + table); + } + + return scratch; + } + catch + { + await scratch.DisposeAsync(); + throw; + } + } + + private static QueryTextTrigramIndexTracker ReadyTracker() + { + var tracker = new QueryTextTrigramIndexTracker(); + tracker.Configure(true); + return tracker; + } + + /// The text-table rows a plan examined: each scan of query_text_dim counts the rows it returned, the rows it threw away and the + /// rows its index recheck threw away, times its loops. + private static long TextRowsRead(JsonElement node) + { + long total = 0; + if (node.TryGetProperty("Relation Name", out var relation) && relation.GetString() == "query_text_dim") + { + var loops = node.GetProperty("Actual Loops").GetInt64(); + long perLoop = 0; + foreach (var name in new[] { "Actual Rows", "Rows Removed by Filter", "Rows Removed by Index Recheck" }) + { + if (node.TryGetProperty(name, out var value)) + { + perLoop += (long)Math.Ceiling(value.GetDouble()); + } + } + + total += perLoop * Math.Max(loops, 1); + } + + if (node.TryGetProperty("Plans", out var plans)) + { + foreach (var child in plans.EnumerateArray()) + { + total += TextRowsRead(child); + } + } + + return total; + } + + /// A common pattern's first span reads the texts of that span's own samples, not the text table: on a store whose text table holds 6,000 + /// texts that all name the pattern and whose first span carries four of them, the statement the tool runs for the newest span examines a few dozen + /// text rows at most. A statement that matched the whole table in the span would examine all 6,000. + [Fact] + public async Task TheNewestSpansStatement_ExaminesTheTextsOfItsOwnSamples_NotTheTextTable() + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); + var span = DarlingMcpFindQueriesByTextTools.RawTierSpans(now - TimescaleSupport.RawRetentionSpan, now)[0]; + + await using var data = NpgsqlDataSource.Create(scratch.ConnectionString); + await using var c = await data.OpenConnectionAsync(ct); + await using var explain = new NpgsqlCommand("EXPLAIN (ANALYZE, FORMAT JSON) " + DarlingMcpFindQueriesByTextTools.AllServersJoinSql(0), c); + explain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = DarlingMcpFindQueriesByTextTools.EscapeLike("dbo.Common") }); + explain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = 21 }); + explain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = span.Floor }); + explain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Timestamp, Value = span.Ceiling }); + explain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Integer, Value = Array.Empty() }); + explain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Text, Value = Array.Empty() }); + explain.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Text, Value = Array.Empty() }); + var plan = JsonDocument.Parse((string)(await explain.ExecuteScalarAsync(ct))!).RootElement[0].GetProperty("Plan"); + + var read = TextRowsRead(plan); + Assert.True(read < 100, $"the newest span's statement examined {read} text rows on a store with {FillerTexts} texts that name the pattern"); + } + + /// A finished all-servers search over a common pattern lists what the whole-window join lists, in its order, when the newest spans (read by + /// text lookup) and the later spans (read by the table match) both hold rows: a key whose newest sighting is in the first span is not listed again + /// from the second, a key seen only in the second span is, a statement with no digest or a text that does not name the pattern never is, and the + /// same query_hash on the other server is its own row. + [Fact] + public async Task AFinishedSearch_ListsWhatTheWholeWindowJoinLists_WithRowsInTheLookupSpansAndTheLaterOnes() + { + var ct = TestContext.Current.CancellationToken; + await using var scratch = await OpenScratchAsync(ct); + await using var data = NpgsqlDataSource.Create(scratch.ConnectionString); + + var reply = await DarlingMcpFindQueriesByTextTools.FindQueriesByText( + data, "dbo.Common", null, null, 20, McpDataDetail.Full, + DarlingMcpFindQueriesByTextTools.SourceBudget.Default, DarlingMcpFindQueriesByTextTools.SourceBudget.Default, ct, ReadyTracker()); + var planCache = JsonDocument.Parse(reply).RootElement.GetProperty("sources").GetProperty("plan_cache"); + + Assert.Equal("ok", planCache.GetProperty("status").GetString()); + Assert.Equal("join", planCache.GetProperty("sample_read").GetString()); + Assert.True(planCache.GetProperty("searched").GetProperty("complete").GetBoolean()); + var keys = planCache.GetProperty("rows").EnumerateArray() + .Select(r => r.GetProperty("server").GetString() + "@" + r.GetProperty("query_hash").GetString() + "@" + r.GetProperty("database_name").GetString()) + .ToArray(); + Assert.Equal(await WholeWindowJoinKeysAsync(data, "dbo.Common", ct), keys); + Assert.Equal(new[] { "lk-a@0xL1@Books", "lk-a@0xL2@Books", "lk-a@0xL6@", "lk-b@0xL1@Books" }, keys); + } + + /// The whole-window join run directly with the tool's own parameters, its rows as server@query_hash@database in its order. + private static async Task WholeWindowJoinKeysAsync(NpgsqlDataSource data, string pattern, CancellationToken ct) + { + await using var c = await data.OpenConnectionAsync(ct); + await using var join = new NpgsqlCommand( + "SELECT s.server_name, j.query_hash, j.database_name FROM (" + DarlingMcpFindQueriesByTextTools.PlanCacheAllServersSql.TrimEnd() + + ") AS j JOIN collect.servers AS s ON s.server_id = j.server_id ORDER BY j.last_collected DESC, j.server_id, j.query_hash, j.database_name", c); + join.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = DarlingMcpFindQueriesByTextTools.EscapeLike(pattern) }); + join.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = 21 }); + join.Parameters.Add(new NpgsqlParameter + { + NpgsqlDbType = NpgsqlDbType.Timestamp, + Value = DateTime.SpecifyKind(DateTime.UtcNow - TimescaleSupport.RawRetentionSpan, DateTimeKind.Unspecified), + }); + var rows = new List(); + await using var reader = await join.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + rows.Add(reader.GetString(0) + "@" + reader.GetString(1) + "@" + (reader.IsDBNull(2) ? string.Empty : reader.GetString(2))); + } + + return rows.ToArray(); + } +} diff --git a/Darling/Darling.Tests/FindQueriesByTextTests.cs b/Darling/Darling.Tests/FindQueriesByTextTests.cs index 0f6c6fe760..a06061ce9d 100644 --- a/Darling/Darling.Tests/FindQueriesByTextTests.cs +++ b/Darling/Darling.Tests/FindQueriesByTextTests.cs @@ -119,14 +119,32 @@ public void RawTierSpans_TileTheWindowNewestFirst_HalfOpenAndContiguous() } } - /// Four completed spans of the four-day tier are the newest day, a hair over 24 hours once the Query Store source's time is - /// in the window, so the number renders "1" and the word has to follow the rendered number. One span of a sixteen-hour window is the - /// hour the same rule covers. + /// #9219: the first span is a small slice of the newest data and each span after it is no narrower, up to a cap, so a search starts + /// with a statement that is cheap on any store and widens while its budget lasts. The spans still tile the window. + [Fact] + public void RawTierSpans_StartWithANewestSlice_AndWidenWithoutShrinking() + { + var (floor, now) = RawWindow(TimescaleSupport.RawRetentionSpan); + var spans = DarlingMcpFindQueriesByTextTools.RawTierSpans(floor, now); + var lengths = spans.Select((s, i) => (i == 0 ? now - s.Floor : s.Ceiling - s.Floor)).ToList(); + + Assert.True(lengths[0] <= (now - floor) / 90, "the first span must be a small slice of the window"); + for (var i = 1; i < lengths.Count - 1; i++) + { + Assert.True(lengths[i] >= lengths[i - 1], "a span after the first is no narrower than the one before it"); + } + + Assert.True(lengths[^2] <= (now - floor) / 6, "no span but the oldest outgrows a sixth of the window"); + } + + /// Completed spans of a window are the newest part of it, from the oldest completed span's floor. Eight parts of a 288-hour window + /// are a day and a hair once the Query Store source's time is in the window, so the number renders "1" and the word has to follow the rendered + /// number. [Theory] - [InlineData(96, 1, "6 hours")] - [InlineData(96, 4, "1 day")] - [InlineData(96, 16, "4 days")] - [InlineData(16, 1, "1 hour")] + [InlineData(96, 1, "1 hour")] + [InlineData(96, 4, "8 hours")] + [InlineData(96, 6, "1.2 days")] + [InlineData(288, 4, "1 day")] public void DescribeSearched_RendersTheNumber_ThenPicksTheUnitsWordFromIt(int windowHours, int completedSpans, string expected) { var (floor, now) = RawWindow(TimeSpan.FromHours(windowHours)); diff --git a/Darling/Darling.Tests/LiveClockCensusTests.cs b/Darling/Darling.Tests/LiveClockCensusTests.cs index 61bffbd53f..7ad4c4a411 100644 --- a/Darling/Darling.Tests/LiveClockCensusTests.cs +++ b/Darling/Darling.Tests/LiveClockCensusTests.cs @@ -61,6 +61,7 @@ public sealed class LiveClockCensusTests ["DarlingModuleMapGroupedSourceLiveTests.cs"] = "The daily refresh statement reads now() - 2 days with no bound parameter, so the product reads the real clock and a seed placed by an override falls outside its window (the refresh test failed under 12:00Z). The rows sit three hours before now, far inside the window, and the assertions are the newest name per handle, not chunk layout.", ["PerfmonRegroupLiveTests.cs"] = "The only direct clock reads are a poll deadline. Seeds are whole UTC days from the run start date, the policy test seeds three to five days back (the two-days-back version was #5579's 00:00 to 03:00 failure), and the reach test counts chunks from the database clock, so the day-3 chunk cannot drop out of a count at midnight UTC.", ["ForcePlanFailuresAccessPathTests.cs"] = "The shipped read takes its two-hour window from the product's own real clock, so a seed placed by the test clock override falls outside it and the read comes back empty (it failed five of five runs at a 00:30 override). The plan assertions hold across a chunk boundary: at that override they all passed before the read, so the real-clock seed needs no anchor.", + ["FindQueriesByTextLookupSpanLiveTests.cs"] = "find_queries_by_text takes its raw window and spans from the product's own real clock, so the seeds and the spans the plan test binds are placed from that clock, and a seed placed by the test clock override would fall outside them. The plan assertion counts rows read from query_text_dim, which is not chunked, so a chunk boundary inside the newest span does not change it.", ["QueryStoreWatermarkReaderLiveTests.cs"] = "Every batch is stamped with a fixed January date and the floors are offsets from it, so no seeded window moves with the hour. The plan assertions are the access path (an index, no sequential or bitmap scan) and relative buffer counts, which a chunk boundary does not change (the hypertable case compresses a chunk 30 days before the fixed date). The only wall-clock reads are the CollectionTime stamp on a CollectorContext, which no assertion reads (WriteBackfillBatchAsync takes the stamp as an argument).", ["QueryStoreWatermarksWriterLiveTests.cs"] = "Every batch is stamped with a fixed January date, so no seeded window moves with the hour, and the plan assertion is the access path (an index, no sequential scan), which a chunk boundary does not change. The only wall-clock reads are the retention test's ages in hours counted back from now (against a fixed two-day horizon, far from every boundary it checks) and the rung test's bounds around the store's own covered_since.", ["PayloadDimensionLiveTests.cs"] = "Seeds dates 45 and 400 days back, mid-chunk, and asserts on dimension rows, not on chunk or block counts. The clock is only a stamp.", diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFindQueriesByTextTools.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFindQueriesByTextTools.cs index 2505dc7ff4..dcbddcd396 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFindQueriesByTextTools.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingMcpFindQueriesByTextTools.cs @@ -199,7 +199,7 @@ LEFT JOIN query_store_text_seen AS s /// procedure and parameterized text, therefore has its text tested in its newest span only, and a span whose distinct texts number in the /// tens of thousands, where the planner tests the pattern by scanning every text rather than reading each by key, pays that scan once per /// search instead of once per span. The carried arrays grow to the server's distinct digests in the window, 32 bytes and a boolean each. - /// Measured: see . A digest with no text row is judged unmatched. + /// Measured: see . A digest with no text row is judged unmatched. /// #7674: the match is (d.query_text || '') ILIKE, not d.query_text ILIKE, on purpose. With the opt-in trigram index on /// query_text_dim.query_text the planner would otherwise run a bitmap scan of that index over the whole table and join it to this server's /// digests, which the issue measured at 1.7 to 2.0 s cold for one server and one database against 0.2 to 0.7 s without the index. The concatenation is @@ -335,6 +335,68 @@ FROM page AS p JOIN query_text_dim AS d ON d.digest = p.query_text_digest ORDER BY p.last_collected DESC, p.server_id, p.query_hash, p.database_name"; + /// #9219: for the newest spans (): the same parameters and the same + /// rows, but the pattern is tested only on the texts the span's own samples carry, each read by primary key, and not on every text in the store. The + /// window join repeats a whole text match in every span, which on a common pattern reads most of the text table before the span's rows can be + /// listed (about 29,000 heap blocks on the measured store, most of the first span's time), so a budget that fit the span's samples still ran out. The + /// LIMIT 1 keeps the planner from flattening the lookup back into a join with the whole table. Measured on 320,000 texts and 3.0 M samples over + /// four days, a first span of one hour (about 26,000 distinct texts): a pattern every text contains took 2.3 s with the window join and 0.7 s with + /// this statement. It is not used past the newest spans: the lookup costs more per text than the join's single pass over the table, so it stops + /// paying once a span carries a large share of the texts (the four-hour span took 1.3 to 2.7 s here against 2.4 s for the join). + internal const string PlanCacheAllServersLookupSpanSql = @" +WITH seen AS MATERIALIZED ( + SELECT DISTINCT s.query_text_digest AS digest + FROM query_stats AS s + JOIN servers AS v ON v.server_id = s.server_id AND v.is_enabled + WHERE s.collection_time >= $3 + AND s.collection_time < $4 + AND s.query_text_digest IS NOT NULL +), +matched AS MATERIALIZED ( + SELECT f.digest + FROM seen AS f + CROSS JOIN LATERAL (SELECT 1 AS hit + FROM query_text_dim AS d + WHERE d.digest = f.digest + AND d.query_text ILIKE '%' || $1 || '%' ESCAPE '\' + LIMIT 1) AS t +), +hits AS ( + SELECT DISTINCT ON (s.server_id, s.query_hash, s.database_name) + s.server_id, s.query_hash, s.database_name, s.collection_time AS last_collected, s.query_text_digest + FROM query_stats AS s + JOIN matched AS m ON m.digest = s.query_text_digest + JOIN servers AS v ON v.server_id = s.server_id AND v.is_enabled + WHERE s.collection_time >= $3 + AND s.collection_time < $4 + ORDER BY s.server_id, s.query_hash, s.database_name, s.collection_time DESC +), +page AS ( + SELECT h.server_id, h.query_hash, h.database_name, h.last_collected, h.query_text_digest + FROM hits AS h + WHERE NOT EXISTS ( + SELECT 1 + FROM unnest($5::integer[], $6::text[], $7::text[]) AS k(server_id, query_hash, database_name) + WHERE k.server_id = h.server_id + AND k.query_hash IS NOT DISTINCT FROM h.query_hash + AND k.database_name IS NOT DISTINCT FROM h.database_name + ) + ORDER BY h.last_collected DESC, h.server_id, h.query_hash, h.database_name + LIMIT $2 +) +SELECT p.server_id, p.query_hash, p.database_name, p.last_collected, d.query_text +FROM page AS p +JOIN query_text_dim AS d ON d.digest = p.query_text_digest +ORDER BY p.last_collected DESC, p.server_id, p.query_hash, p.database_name"; + + /// #9219: how many of the newest spans the all-servers window join reads with : the first two, an hour + /// each of a four-day tier. The spans after them are wider and carry too large a share of the texts for a lookup per text to beat the join. + internal const int AllServersLookupSpans = 2; + + /// #9219: the window-join statement the all-servers search runs for span (0 is the newest). + internal static string AllServersJoinSql(int step) => + step < AllServersLookupSpans ? PlanCacheAllServersLookupSpanSql : PlanCacheAllServersSpanSql; + /// /// #7849: how many matched texts the all-servers sample read takes as an array () before it falls back to /// the window join (). Measured on a seeded large store (40 servers, 3.0 M sample rows over four days, 300,000 @@ -347,33 +409,47 @@ FROM page AS p public const int ProbeDigestCap = 1000; /// - /// How many spans the one-server plan-cache search cuts the raw window into. Each span is one statement () under - /// what is left of the source's budget, run newest span first, so a budget that runs out leaves the rows the earlier spans found and the - /// reply says how much of the window they cover. Sixteen spans of a four-day window are six hours each: a window whose whole-window - /// statement takes twice the budget still answers from its newest two days. Each text is judged once per search whichever spans it - /// runs in (see ), which keeps the sixteen statements at about the cost of one. Measured warm on a seeded - /// large store (330,000 texts; a server with 30,000 distinct texts in 970,000 samples over four days, the older half compressed): the - /// whole window in one statement takes 0.50 s, and in sixteen spans carrying the judged digests 0.66 s (0.40 s and 0.24 s with a - /// database named; about 36 ms a span while carrying 20,000 to 30,000 digests), where sixteen spans that each judge every text they - /// see take 4.9 s, more than the budget, because a span with tens of thousands of distinct texts judges them by scanning every text in - /// the store and pays that scan once per span; seven coarser spans cut that only to 2.2 s. + /// #9219: how many equal parts the raw window is measured in for the spans of a plan-cache search (): the newest span + /// is one part (an hour of a four-day tier) and the spans after it widen, so the first statement of a search is small whatever the store + /// holds. Each span is one statement ( for one server) under what is left of the source's budget, run newest span + /// first, so a budget that runs out leaves the rows the earlier spans found and the reply says how much of the window they cover. Under + /// sixteen equal six-hour spans the first statement read a sixteenth of the raw tier, and on a large store that statement alone could miss + /// the whole budget (the search then stopped with nothing searched, and which searches did so changed with the limit, because the limit + /// changes the plan). Each text is judged once per search whichever spans it runs in (see ), which keeps the + /// statements of a finished one-server search at about the cost of one; the all-servers join repeats its text match in each span, which is + /// why the spans stop widening at parts and the whole window is twelve spans, not more. /// - public const int RawTierSpanCount = 16; + public const int RawTierSpanParts = 96; + + /// #9219: the widest a span grows, in (twelve hours of a four-day tier). The spans run 1, 1, 2, 4, 8 parts + /// and then this many each, the oldest cut short at the floor. + public const int RawTierSpanMaxParts = 12; /// - /// The spans of the raw window to , newest first: equal lengths, each one's floor the next - /// one's ceiling, the oldest starting exactly at the floor and the newest open a day past , so a sample stamped - /// slightly ahead of this host's clock is still in the first span. + /// The spans of the raw window to , newest first: each one's floor the next one's ceiling, the + /// oldest starting exactly at the floor and the newest open a day past , so a sample stamped slightly ahead of this + /// host's clock is still in the first span. Their lengths are 1, 1, 2, 4, 8 and then parts of + /// (#9219), so the first statement reads the newest slice only and the search widens while time remains. /// internal static IReadOnlyList<(DateTime Floor, DateTime Ceiling)> RawTierSpans(DateTime floor, DateTime now) { - var length = (now - floor) / RawTierSpanCount; - var spans = new (DateTime Floor, DateTime Ceiling)[RawTierSpanCount]; - for (var i = 0; i < RawTierSpanCount; i++) + var part = (now - floor) / RawTierSpanParts; + var spans = new List<(DateTime Floor, DateTime Ceiling)>(); + var ceiling = now.AddDays(1); + var reached = now; + var width = 1; + var next = 1; + while (reached > floor) { - var spanFloor = i == RawTierSpanCount - 1 ? floor : now - length * (i + 1); - var ceiling = i == 0 ? now.AddDays(1) : now - length * i; - spans[i] = (spanFloor, ceiling); + /* The oldest span takes whatever is left, including the few ticks the division by the parts dropped. */ + var spanFloor = reached - floor < part * (width + 1) ? floor : reached - part * width; + + spans.Add((spanFloor, ceiling)); + ceiling = spanFloor; + reached = spanFloor; + + /* 1, 1, 2, 4, 8, 12, 12 ...: the width doubles from the second span on, up to the cap. */ + (width, next) = (Math.Min(next, RawTierSpanMaxParts), next * 2); } return spans; @@ -384,7 +460,7 @@ FROM page AS p /// Query Store source's time is in the window, reads "1 day". internal static string DescribeSearched(int completedSpans, DateTime floor, DateTime now) { - var searched = (now - floor) / RawTierSpanCount * completedSpans; + var searched = now - RawTierSpans(floor, now)[completedSpans - 1].Floor; var (value, unit) = searched.TotalHours < 24 ? (searched.TotalHours, "hour") : (searched.TotalDays, "day"); var rendered = value.ToString("0.#", CultureInfo.InvariantCulture); return rendered + " " + unit + (rendered == "1" ? string.Empty : "s"); @@ -402,7 +478,7 @@ internal static object SearchedWindow(object servers, int? serverCount, string s var stopped = string.Equals(status, "stopped", StringComparison.Ordinal); DateTime? searchedFrom = !stopped ? floor - : completedSpans is > 0 ? now - (now - floor) / RawTierSpanCount * completedSpans.Value : null; + : completedSpans is > 0 ? RawTierSpans(floor, now)[completedSpans.Value - 1].Floor : null; return new { servers, @@ -672,7 +748,7 @@ internal static async Task FindQueriesByText( var (probeWanted, matched) = await matchAhead; - /* The raw window newest span first, one statement per span inside the one transaction and budget (see RawTierSpanCount): the rows + /* The raw window newest span first, one statement per span inside the one transaction and budget (see RawTierSpanParts): the rows a span finds are kept whatever happens to the spans after it, and a span already-found statements ran in again does not list them twice. On the span read the digests each span judged, with their verdicts, are carried to the spans after it, so a text is judged once. */ var now = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Unspecified); @@ -888,7 +964,7 @@ the window join past it. */ } /* The probe runs the raw window newest span first, one statement per span inside the one transaction and budget (see - RawTierSpanCount): the rows a span finds are kept whatever happens to the spans after it, and a server, query_hash and + RawTierSpanParts): the rows a span finds are kept whatever happens to the spans after it, and a server, query_hash and database an earlier span found is not listed again. #9150: the window join runs in the same spans, matching the pattern in each (the matched texts are too many to bind as an array). */ if (step >= spans.Count || collected.Count > limit) @@ -900,7 +976,7 @@ the window join past it. */ var wanted = limit + 1 - collected.Count; var probed = digests; return probed is null - ? new SourceStatement(PlanCacheAllServersSpanSql, p => BindAllServersSpan( + ? new SourceStatement(AllServersJoinSql(step), p => BindAllServersSpan( p, new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Text, Value = escaped }, wanted, span, collected)) : new SourceStatement(PlanCacheAllServersProbeSql, p => BindAllServersSpan( p, new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Bytea, Value = probed }, wanted, span, collected));