From 8636733cbd85edf64caa9aa4181e949918f31972 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:28:34 -0400 Subject: [PATCH 1/4] A parallel query waiting on itself is not blocking (#6645) --- .../Darling.Tests/BlockingWaitCountTests.cs | 51 +++++++++++++++++++ .../PgAnomalyDetector.cs | 9 +++- .../DarlingAlertReadAdapter.cs | 9 +++- Lite.Tests/BlockingChainReconstructorTests.cs | 25 +++++++++ Lite.Tests/BlockingWaitDuckDbTests.cs | 23 +++++++++ Lite/Analysis/AnomalyDetector.cs | 7 +-- Lite/Services/LocalDataService.Blocking.cs | 7 ++- .../BlockingChainReconstructor.cs | 5 ++ PerformanceMonitor.Common/BlockingWaits.cs | 16 +++++- 9 files changed, 142 insertions(+), 10 deletions(-) diff --git a/Darling/Darling.Tests/BlockingWaitCountTests.cs b/Darling/Darling.Tests/BlockingWaitCountTests.cs index d6f03944c8..0b9e51e6b1 100644 --- a/Darling/Darling.Tests/BlockingWaitCountTests.cs +++ b/Darling/Darling.Tests/BlockingWaitCountTests.cs @@ -89,6 +89,30 @@ public void IssueShape_FortyThreeReportsOfFourWaits_AreFourEvents() Assert.Equal(new[] { 55900L, 45900L, 45900L, 30900L }.OrderBy(x => x), waits.Select(w => w.WaitTimeMs).OrderBy(x => x)); } + /// + /// #6645: the threads of one parallel query write reports with blocked spid = blocking spid (different ecid). + /// That is the query waiting on itself, so it is no wait: the rule drops it and the alert rows follow. + /// + [Fact] + public void ParallelQuerySelfWaits_AreNotWaits() + { + var rows = OneWait(59, 59, T0, 6); + rows.AddRange(OneWait(59, 59, T0.AddMinutes(1), 3)); + var real = OneWait(102, 100, T0.AddMinutes(2), 5); + rows.AddRange(real); + + var wait = Assert.Single(Collapse(rows)); + Assert.Equal(real.Max(r => r.WaitTimeMs), wait.WaitTimeMs); + Assert.Equal(100, wait.BlockingSpid); + + var counted = AlertContextBuilders.BlockingWaitRows(rows, Array.Empty()); + Assert.Equal(100, Assert.Single(counted).BlockingSpid); + Assert.Empty(AlertContextBuilders.BlockingWaitRows(OneWait(59, 59, T0, 6), Array.Empty())); + + /* A report that named no session (stored with NULL spids, read as 0) is still a blocking event. */ + Assert.Single(AlertContextBuilders.BlockingWaitRows(OneWait(0, 0, T0, 1), Array.Empty())); + } + [Fact] public void ReportOrder_DoesNotChangeTheCount() { @@ -297,6 +321,33 @@ public async Task PostgresSql_AgreesWithTheCSharpRule_OnTheSameReports() Assert.Equal(expected.Max(w => w.WaitTimeMs), Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); } + [Fact] + public async Task PostgresSql_LeavesOutParallelQuerySelfWaits_AndCountsTheRealOne() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), "Set DARLING_TEST_PG to run the live wait-count SQL test."); + + var rows = new List(); + rows.AddRange(OneWait(59, 59, T0, 6)); + rows.AddRange(OneWait(59, 59, T0.AddMinutes(10), 4)); + rows.AddRange(OneWait(102, 100, T0.AddMinutes(20), 5)); + + /* #6645: reportCount adds the per-wait report tally, which must count only the real wait's reports. */ + var sql = "WITH " + BlockingWaits.WaitsCteSql( + BlockingWaits.SqlDialect.Postgres, "waits", ValuesSource(rows, 1), new[] { "server_id" }, reportCount: true) + + " SELECT COUNT(*), COUNT(DISTINCT blocking_spid), MAX(wait_reports) FROM waits"; + + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(); + Assert.True(await reader.ReadAsync()); + + Assert.Equal(1, Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture)); + Assert.Equal(1, Convert.ToInt32(reader.GetValue(1), CultureInfo.InvariantCulture)); + Assert.Equal(5, Convert.ToInt32(reader.GetValue(2), CultureInfo.InvariantCulture)); + } + /* ── #5644 wave-1 review: sub-millisecond starts and NULL times, in C# and in the store's SQL ── */ internal sealed record NullableReport(int Blocked, int Blocker, DateTime? Time, long? Wait); diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs index d2f3bba7b9..3839b184d4 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs @@ -307,12 +307,15 @@ the CURRENT window counts report and deadlock events by when they HAPPENED (even snapshot's event_time IS its collection time. The BASELINE this count is judged against comes from the continuous aggregates, which bucket the hypertable's own column (collection_time) by design. */ public const string BlockingWindowSql = @" +/* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking. */ SELECT COALESCE(NULLIF( (SELECT COUNT(*) FROM v_blocked_process_reports - WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4), 0), + WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4 + AND NOT COALESCE(blocked_spid = blocking_spid, FALSE)), 0), (SELECT COUNT(*) FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3)) AS current_blocking, + WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 + AND NOT COALESCE(blocked_spid = blocking_spid, FALSE))) AS current_blocking, (SELECT COUNT(*) FROM v_deadlocks WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time < $3 AND collection_time >= $4) AS current_deadlocks"; @@ -325,9 +328,11 @@ the CURRENT window counts report and deadlock events by when they HAPPENED (even COALESCE(NULLIF( (SELECT COUNT(*) FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $5 + AND NOT COALESCE(blocked_spid = blocking_spid, FALSE) AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x)))), 0), (SELECT COUNT(*) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 + AND NOT COALESCE(blocked_spid = blocking_spid, FALSE) AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x))))) AS current_blocking, 0::bigint AS current_deadlocks"; diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs index 1d820bf4b0..8d32d986f3 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs @@ -576,6 +576,11 @@ private async Task> GetRecentBlockedProcessReportsC /// ), its wait_time_ms summed and its distinct /// blocked SPIDs counted. $1 server_id, used twice. /// + /// #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not + /// blocking. The two aggregates leave those rows out with FILTER rather than the WHERE, so a newest snapshot + /// that holds only such rows still reads as a snapshot (its time and freshness) with nothing blocked. + /// + /// /// ONE statement, deliberately: Npgsql fails SILENTLY on multi-statement commands with positional /// parameters, so the freshness probe cannot be batched onto this — the snapshot time comes back as /// a column of this same aggregate instead, which is one round trip rather than two anyway. @@ -584,8 +589,8 @@ private async Task> GetRecentBlockedProcessReportsC public const string CurrentBlockingWaitSql = @" SELECT collection_time, - CAST(COALESCE(SUM(wait_time_ms), 0) AS bigint) AS total_wait_ms, - CAST(COUNT(DISTINCT blocked_spid) AS integer) AS blocked_sessions + CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)), 0) AS bigint) AS total_wait_ms, + CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)) AS integer) AS blocked_sessions FROM dmv_blocking_snapshots WHERE server_id = $1 AND collection_time = ( diff --git a/Lite.Tests/BlockingChainReconstructorTests.cs b/Lite.Tests/BlockingChainReconstructorTests.cs index f4768b1f55..8fab6c2cc6 100644 --- a/Lite.Tests/BlockingChainReconstructorTests.cs +++ b/Lite.Tests/BlockingChainReconstructorTests.cs @@ -54,6 +54,31 @@ public void Empty_ProducesNoChains() Assert.Empty(result.Chains); } + /// + /// #6645: a parallel query's threads write reports with blocked spid = blocking spid (different ecid). That + /// is the query waiting on itself: no chain, no cycle, and a real chain beside it is unchanged. + /// + [Fact] + public void ParallelQuerySelfWaits_MakeNoChainAndNoCycle() + { + var self = new[] + { + Pair(59, 59, blockedEcid: 2, blockingEcid: 1), Pair(59, 59, blockedEcid: 5, blockingEcid: 1), + Pair(59, 59, blockedEcid: 6, blockingEcid: 1), Pair(59, 59, blockedEcid: 1, blockingEcid: 2), + }; + + var onlySelf = Run(self); + Assert.Empty(onlySelf.Chains); + Assert.False(onlySelf.CycleDetected); + + var withReal = Run(self.Append(Pair(52, 51))); + var chain = Assert.Single(withReal.Chains); + Assert.Equal(51, chain.ApexSpid); + Assert.Equal(1, chain.Depth); + Assert.Equal(1, chain.VictimCount); + Assert.False(withReal.CycleDetected); + } + [Fact] public void DepthFourLine_ReportsApexDepthAndVictims() { diff --git a/Lite.Tests/BlockingWaitDuckDbTests.cs b/Lite.Tests/BlockingWaitDuckDbTests.cs index 573e91bb30..f4a4259599 100644 --- a/Lite.Tests/BlockingWaitDuckDbTests.cs +++ b/Lite.Tests/BlockingWaitDuckDbTests.cs @@ -134,6 +134,29 @@ public async Task SeveralReportsOfOneWait_CountOnce() Assert.Equal(55_000.0, (await BlockingFactAsync()).Metadata["max_wait_time_ms"]); } + /// + /// #6645: the threads of one parallel query write reports with blocked_spid = blocking_spid (a different ecid + /// each). That is the query waiting on itself, not blocking, so it is no wait and no head blocker: the fact, + /// the overview card, the C# rule and the current-wait snapshot all leave it out. + /// + [Fact] + public async Task ParallelQuerySelfWaits_AreNotBlocking() + { + await SeedWaitAsync(59, 59, Ago(40), reports: 4); + await SeedWaitAsync(59, 59, Ago(30), reports: 3, dmv: true); + await SeedWaitAsync(52, 51, Ago(20), reports: 5); + + await AssertAllThreeAgreeAsync(1); + + /* The real lock wait's blocker is the only head blocker; the parallel query's own spid is not one. */ + Assert.Equal(1.0, (await BlockingFactAsync()).Metadata["distinct_head_blockers"]); + + var snapshot = await new LocalDataService(_duckDb).GetCurrentBlockingWaitAsync(ServerId); + Assert.NotNull(snapshot); + Assert.Equal(0, snapshot!.Value.BlockedSessionCount); + Assert.Equal(0L, snapshot.Value.TotalWaitMs); + } + [Fact] public async Task TheAuditsElevenNineNineAndSixReports_AreFourWaits() { diff --git a/Lite/Analysis/AnomalyDetector.cs b/Lite/Analysis/AnomalyDetector.cs index 4189c32b5f..9b4ca653ff 100644 --- a/Lite/Analysis/AnomalyDetector.cs +++ b/Lite/Analysis/AnomalyDetector.cs @@ -795,15 +795,16 @@ The DMV arm below stays on collection_time because a snapshot's event_time IS it var scopeList = context.SeparatelyMonitoredDatabases; var scoped = scopeList is { Count: > 0 }; var rowScope = SeparatelyMonitoredScope.BprFilter(scopeList, 4); - /* current_blocking: prefer the blocked-process-report; fall back to the always-on DMV + /* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking. + current_blocking: prefer the blocked-process-report; fall back to the always-on DMV snapshot so RDS (where the BPR session is empty) still counts blocking. Mirrors the overview/alert path (LocalDataService.Overview.cs / LocalDataService.Blocking.cs). */ cmd.CommandText = (@" SELECT COALESCE(NULLIF( - (SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3{SCOPE}") + @" AS ev), 0), + (SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND NOT COALESCE(blocked_spid = blocking_spid, FALSE){SCOPE}") + @" AS ev), 0), (SELECT COUNT(*) FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3{SCOPE})) AS current_blocking, + WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 AND NOT COALESCE(blocked_spid = blocking_spid, FALSE){SCOPE})) AS current_blocking, (SELECT " + StoredEventCopies.DeadlockDistinctCount + @" FROM v_deadlocks AS dl WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time < $3) AS current_deadlocks") .Replace("{SCOPE}", rowScope); diff --git a/Lite/Services/LocalDataService.Blocking.cs b/Lite/Services/LocalDataService.Blocking.cs index 6f8b3824a7..8bf7eb61b4 100644 --- a/Lite/Services/LocalDataService.Blocking.cs +++ b/Lite/Services/LocalDataService.Blocking.cs @@ -45,11 +45,14 @@ public partial class LocalDataService DuckDB.NET materializes as System.Numerics.BigInteger — a type Convert.ToInt64 cannot convert (BigInteger does not implement IConvertible), so the read threw at alert-check time without them. COUNT is cast for the same reason its result is read as an int. */ + /* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not + blocking. FILTER (not WHERE) so a newest snapshot of only such rows still reads as a snapshot with + nothing blocked, its time and freshness intact. */ command.CommandText = @" SELECT collection_time, - CAST(COALESCE(SUM(wait_time_ms), 0) AS BIGINT) AS total_wait_ms, - CAST(COUNT(DISTINCT blocked_spid) AS INTEGER) AS blocked_sessions + CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)), 0) AS BIGINT) AS total_wait_ms, + CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)) AS INTEGER) AS blocked_sessions FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND collection_time = ( diff --git a/PerformanceMonitor.Analysis/BlockingChainReconstructor.cs b/PerformanceMonitor.Analysis/BlockingChainReconstructor.cs index 2a1fc936ed..30057675ee 100644 --- a/PerformanceMonitor.Analysis/BlockingChainReconstructor.cs +++ b/PerformanceMonitor.Analysis/BlockingChainReconstructor.cs @@ -212,6 +212,11 @@ public static BlockingReconstruction Reconstruct( foreach (var row in pairs) { + // #6645: a parallel query's threads write reports with blocked spid = blocking spid (different ecid). + // That is the query waiting on itself, not blocking: make no node, so no edge or cycle either. + if (row.BlockingSpid == row.BlockedSpid) + continue; + // The collector reconstructs cumulatively (loop = null) so an episode's per-scan re-fires merge into // one chain (preserves window-level depth/victims for the severity fact); the viewer scopes per scan. var loop = scopeByMonitorLoop ? row.MonitorLoop : null; diff --git a/PerformanceMonitor.Common/BlockingWaits.cs b/PerformanceMonitor.Common/BlockingWaits.cs index 2106c814ff..897062bf9c 100644 --- a/PerformanceMonitor.Common/BlockingWaits.cs +++ b/PerformanceMonitor.Common/BlockingWaits.cs @@ -71,7 +71,8 @@ public static double WaitStartEpochMs(DateTime eventTime, long waitTimeMs) => (eventTime - DateTime.UnixEpoch).TotalMilliseconds - waitTimeMs; /// - /// Collapses reports to one row per wait: the report with the longest wait time (the latest one on a tie). + /// Collapses reports to one row per wait (a report whose blocked and blocking spid are the same, and not 0, is a + /// parallel query waiting on itself, #6645, and is dropped): the report with the longest wait time (the latest one on a tie). /// keeps rows that must never merge apart (the report source, for one). A row with /// no event time or no wait time has no derived start and cannot be placed in a wait, so it stays a wait of /// its own (the SQL form does the same). The result is ordered by the @@ -89,6 +90,15 @@ public static List Collapse( var placeable = new List<(T Row, double Start, DateTime Time, long Wait)>(); foreach (var row in reports) { + /* #6645: a parallel query's threads report waits on their own session (same spid, different ecid). + That is the query waiting on itself, not blocking. Spid 0 is a report that named no session (a + withheld-whole marker, stored with NULL spids), which is still a blocking event: the SQL form keeps + it too, as NULL = NULL is not true. */ + if (blockedSpid(row) == blockingSpid(row) && blockedSpid(row) != 0) + { + continue; + } + var time = eventTime(row); var wait = waitTimeMs(row); if (time is null || wait is null) @@ -187,6 +197,10 @@ public static string WaitsCteSql( {name}_marked AS ( SELECT s.*, {epochMs} - s.wait_time_ms AS wait_start_ms FROM ({source}) AS s + /* #6645: a parallel query's threads write reports with blocked_spid = blocking_spid (different ecid). That is + the query waiting on itself, not blocking: it is no wait, no head blocker and no report. A report that + names no session (NULL spids) is kept: the comparison is not true for it. */ + WHERE NOT COALESCE(s.blocked_spid = s.blocking_spid, FALSE) ), {name}_gapped AS ( SELECT m.*, From 6d29768add71b06bed8cfa3d49023d8af3e1ae66 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:43:50 -0400 Subject: [PATCH 2/4] Keep spid 0/0 reports as waits in the SQL, and filter the Lite blocking baseline the same way (#6645) The collector stores 0, never NULL, for a report that names no session. The C# rule keeps those rows, but the SQL predicate dropped them (0 = 0 is true). One BlockingWaits.NotSelfWaitSql now carries the rule for every SQL surface. The Lite blocking baseline counts the same reports as the current window. --- .../Darling.Tests/BlockingWaitCountTests.cs | 8 ++++-- .../PgAnomalyDetector.cs | 19 +++++++++----- .../DarlingAlertReadAdapter.cs | 6 ++--- Lite.Tests/BlockingWaitDuckDbTests.cs | 26 +++++++++++++++++++ Lite/Analysis/AnomalyDetector.cs | 4 +-- Lite/Analysis/BaselineProvider.cs | 4 ++- Lite/Services/LocalDataService.Blocking.cs | 9 ++++--- PerformanceMonitor.Common/BlockingWaits.cs | 22 +++++++++++----- 8 files changed, 73 insertions(+), 25 deletions(-) diff --git a/Darling/Darling.Tests/BlockingWaitCountTests.cs b/Darling/Darling.Tests/BlockingWaitCountTests.cs index 0b9e51e6b1..8269735c64 100644 --- a/Darling/Darling.Tests/BlockingWaitCountTests.cs +++ b/Darling/Darling.Tests/BlockingWaitCountTests.cs @@ -109,7 +109,7 @@ public void ParallelQuerySelfWaits_AreNotWaits() Assert.Equal(100, Assert.Single(counted).BlockingSpid); Assert.Empty(AlertContextBuilders.BlockingWaitRows(OneWait(59, 59, T0, 6), Array.Empty())); - /* A report that named no session (stored with NULL spids, read as 0) is still a blocking event. */ + /* A report that named no session (the collector stores 0, never NULL) is still a blocking event. */ Assert.Single(AlertContextBuilders.BlockingWaitRows(OneWait(0, 0, T0, 1), Array.Empty())); } @@ -304,6 +304,10 @@ public async Task PostgresSql_AgreesWithTheCSharpRule_OnTheSameReports() rows.AddRange(OneWait(102, 100, T0.AddMinutes(30), 6)); rows.AddRange(OneWait(103, 100, T0, 4)); rows.AddRange(OneWait(102, 101, T0.AddMinutes(1), 7)); + /* #6645 round 1: a report that named no session is stored as (0, 0) and is a wait in the C# rule, so the SQL + must keep it; a parallel query's equal non-zero spids are dropped by both. */ + rows.AddRange(OneWait(0, 0, T0.AddMinutes(40), 3)); + rows.AddRange(OneWait(59, 59, T0.AddMinutes(50), 4)); var sql = "WITH " + BlockingWaits.WaitsCteSql( BlockingWaits.SqlDialect.Postgres, "waits", ValuesSource(rows, 1), new[] { "server_id" }) @@ -317,7 +321,7 @@ public async Task PostgresSql_AgreesWithTheCSharpRule_OnTheSameReports() var expected = Collapse(rows); Assert.Equal(expected.Count, Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture)); - Assert.Equal(6, expected.Count); + Assert.Equal(7, expected.Count); Assert.Equal(expected.Max(w => w.WaitTimeMs), Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture)); } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs index 3839b184d4..953759d6d6 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs @@ -305,17 +305,22 @@ the CURRENT window counts report and deadlock events by when they HAPPENED (even deadlock_time), so the spike counts the events the grids show; $4 is the EventWindowFloor for $2, the partition-column bound with no upper side. The DMV snapshot arm stays on collection_time because a snapshot's event_time IS its collection time. The BASELINE this count is judged against comes from the - continuous aggregates, which bucket the hypertable's own column (collection_time) by design. */ - public const string BlockingWindowSql = @" + continuous aggregates, which bucket the hypertable's own column (collection_time) by design. + + #6645: this CURRENT count leaves out a parallel query's self-waits, but the BASELINE aggregate + (collect.blocked_process_baseline, count(*) over every report row) still counts them until a store + migration rebuilds it with the same predicate, so a server with many self-waits has a high baseline and + its real blocking is under-flagged in the meantime. */ + public const string BlockingWindowSql = $@" /* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking. */ SELECT COALESCE(NULLIF( (SELECT COUNT(*) FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4 - AND NOT COALESCE(blocked_spid = blocking_spid, FALSE)), 0), + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}), 0), (SELECT COUNT(*) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 - AND NOT COALESCE(blocked_spid = blocking_spid, FALSE))) AS current_blocking, + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql})) AS current_blocking, (SELECT COUNT(*) FROM v_deadlocks WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time < $3 AND collection_time >= $4) AS current_deadlocks"; @@ -323,16 +328,16 @@ AND NOT COALESCE(blocked_spid = blocking_spid, FALSE))) AS current_blocking, own targets, skipped on both arms, a NULL database still counting; $5 = the event-window floor, which takes the number after the list so the scoped read keeps the list where it was). The deadlock count comes from DeadlockGraphsCountSql through the every-process rule, so this read carries no deadlock column. */ - public const string BlockingSkippingSeparateCountSql = @" + public const string BlockingSkippingSeparateCountSql = $@" SELECT COALESCE(NULLIF( (SELECT COUNT(*) FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $5 - AND NOT COALESCE(blocked_spid = blocking_spid, FALSE) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x)))), 0), (SELECT COUNT(*) FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 - AND NOT COALESCE(blocked_spid = blocking_spid, FALSE) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x))))) AS current_blocking, 0::bigint AS current_deadlocks"; diff --git a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs index 8d32d986f3..297e480a7d 100644 --- a/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs +++ b/Darling/PerformanceMonitor.Darling.Service/DarlingAlertReadAdapter.cs @@ -586,11 +586,11 @@ private async Task> GetRecentBlockedProcessReportsC /// a column of this same aggregate instead, which is one round trip rather than two anyway. /// /// - public const string CurrentBlockingWaitSql = @" + public const string CurrentBlockingWaitSql = $@" SELECT collection_time, - CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)), 0) AS bigint) AS total_wait_ms, - CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)) AS integer) AS blocked_sessions + CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE {BlockingWaits.NotSelfWaitSql}), 0) AS bigint) AS total_wait_ms, + CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE {BlockingWaits.NotSelfWaitSql}) AS integer) AS blocked_sessions FROM dmv_blocking_snapshots WHERE server_id = $1 AND collection_time = ( diff --git a/Lite.Tests/BlockingWaitDuckDbTests.cs b/Lite.Tests/BlockingWaitDuckDbTests.cs index f4a4259599..00f8974db5 100644 --- a/Lite.Tests/BlockingWaitDuckDbTests.cs +++ b/Lite.Tests/BlockingWaitDuckDbTests.cs @@ -157,6 +157,32 @@ public async Task ParallelQuerySelfWaits_AreNotBlocking() Assert.Equal(0L, snapshot.Value.TotalWaitMs); } + /// + /// #6645 round 1: the collector stores 0, never NULL, for a report that names no session, and the C# rule keeps + /// a (0, 0) row (0 = 0 is true in SQL, so the SQL predicate must carry the same exception). The fact, the overview + /// card and the C# rule count it as a wait; the equal non-zero spids beside it are still dropped. + /// + [Fact] + public async Task AReportThatNamedNoSession_StoredAsSpidZeroZero_IsStillAWait() + { + await SeedWaitAsync(0, 0, Ago(40), reports: 3); + await SeedWaitAsync(59, 59, Ago(30), reports: 4); + await SeedWaitAsync(52, 51, Ago(20), reports: 5); + + await AssertAllThreeAgreeAsync(2); + } + + [Fact] + public async Task ADmvSnapshotOfSpidZeroZero_IsStillBlockedInTheCurrentWaitRead() + { + await SeedWaitAsync(0, 0, Ago(10), reports: 2, dmv: true); + + var snapshot = await new LocalDataService(_duckDb).GetCurrentBlockingWaitAsync(ServerId); + Assert.NotNull(snapshot); + Assert.Equal(1, snapshot!.Value.BlockedSessionCount); + Assert.True(snapshot.Value.TotalWaitMs > 0); + } + [Fact] public async Task TheAuditsElevenNineNineAndSixReports_AreFourWaits() { diff --git a/Lite/Analysis/AnomalyDetector.cs b/Lite/Analysis/AnomalyDetector.cs index 9b4ca653ff..9bc6b628a3 100644 --- a/Lite/Analysis/AnomalyDetector.cs +++ b/Lite/Analysis/AnomalyDetector.cs @@ -802,9 +802,9 @@ snapshot so RDS (where the BPR session is empty) still counts blocking. Mirrors cmd.CommandText = (@" SELECT COALESCE(NULLIF( - (SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND NOT COALESCE(blocked_spid = blocking_spid, FALSE){SCOPE}") + @" AS ev), 0), + (SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql + "{SCOPE}") + @" AS ev), 0), (SELECT COUNT(*) FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 AND NOT COALESCE(blocked_spid = blocking_spid, FALSE){SCOPE})) AS current_blocking, + WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql + @"{SCOPE})) AS current_blocking, (SELECT " + StoredEventCopies.DeadlockDistinctCount + @" FROM v_deadlocks AS dl WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time < $3) AS current_deadlocks") .Replace("{SCOPE}", rowScope); diff --git a/Lite/Analysis/BaselineProvider.cs b/Lite/Analysis/BaselineProvider.cs index 769a25614f..6afc16516f 100644 --- a/Lite/Analysis/BaselineProvider.cs +++ b/Lite/Analysis/BaselineProvider.cs @@ -711,10 +711,12 @@ FROM v_file_io_stats AND (delta_reads > 0 OR delta_writes > 0) )," + RobustTierScaffold, + // #6645: the baseline counts the same reports the current window does (BlockingWaits.NotSelfWaitSql), so the 3x ratio + // compares like with like. The Darling twin reads a stored aggregate that still counts them. // Event-based — mean = events per covered hour for this bucket, sample_count = covered days. // No restart exclusion needed (event counts, not cumulative). See EventBaselineSql (#4731). MetricNames.Blocking => OnEventTime(EventBaselineSql("blocked_process_report", "v_collection_log", - StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3") + " AS ev", "COUNT(*)"), "event_time"), + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql) + " AS ev", "COUNT(*)"), "event_time"), // Event-based — same approach as blocking MetricNames.Deadlock => OnEventTime(EventBaselineSql("deadlocks", "v_collection_log", diff --git a/Lite/Services/LocalDataService.Blocking.cs b/Lite/Services/LocalDataService.Blocking.cs index 8bf7eb61b4..bbeea0ad21 100644 --- a/Lite/Services/LocalDataService.Blocking.cs +++ b/Lite/Services/LocalDataService.Blocking.cs @@ -46,13 +46,14 @@ DuckDB.NET materializes as System.Numerics.BigInteger — a type Convert.ToInt64 (BigInteger does not implement IConvertible), so the read threw at alert-check time without them. COUNT is cast for the same reason its result is read as an int. */ /* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not - blocking. FILTER (not WHERE) so a newest snapshot of only such rows still reads as a snapshot with - nothing blocked, its time and freshness intact. */ + blocking. FILTER (not WHERE) so a stored snapshot of only such rows still reads as a snapshot with + nothing blocked, its time and freshness intact. The DMV collector no longer writes such rows + (DmvBlockingSnapshotCollector filters them), so this bites only on rows already stored. */ command.CommandText = @" SELECT collection_time, - CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)), 0) AS BIGINT) AS total_wait_ms, - CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE NOT COALESCE(blocked_spid = blocking_spid, FALSE)) AS INTEGER) AS blocked_sessions + CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE " + BlockingWaits.NotSelfWaitSql + @"), 0) AS BIGINT) AS total_wait_ms, + CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE " + BlockingWaits.NotSelfWaitSql + @") AS INTEGER) AS blocked_sessions FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND collection_time = ( diff --git a/PerformanceMonitor.Common/BlockingWaits.cs b/PerformanceMonitor.Common/BlockingWaits.cs index 897062bf9c..913e490521 100644 --- a/PerformanceMonitor.Common/BlockingWaits.cs +++ b/PerformanceMonitor.Common/BlockingWaits.cs @@ -45,6 +45,16 @@ namespace PerformanceMonitor.Common; /// public static class BlockingWaits { + /// + /// #6645: the SQL predicate that is true for a row that is a real wait and false for a parallel query's thread + /// waiting on its own session (the same non-zero spid on both sides). The one definition every read surface + /// aliases, in DuckDB and PostgreSQL. It is 's rule: a (0, 0) row is a report that + /// named no session (the collector stores 0, never NULL), which stays a blocking event, so the equal-spid test + /// carries blocked_spid <> 0; a NULL spid makes the test NULL, which COALESCE turns to FALSE and keeps + /// the row. Column names are unqualified: every use has one source relation in scope. + /// + public const string NotSelfWaitSql = "NOT COALESCE(blocked_spid = blocking_spid AND blocked_spid <> 0, FALSE)"; + /// How far apart (milliseconds) two reports' derived wait starts may be and still be one wait. The /// threshold is at least 5 s, so 2 s cannot merge two blocked-process-report waits and is far above the few /// milliseconds of timestamp jitter between reports of one wait. It alone is not enough for DMV snapshots: see . @@ -91,9 +101,9 @@ public static List Collapse( foreach (var row in reports) { /* #6645: a parallel query's threads report waits on their own session (same spid, different ecid). - That is the query waiting on itself, not blocking. Spid 0 is a report that named no session (a - withheld-whole marker, stored with NULL spids), which is still a blocking event: the SQL form keeps - it too, as NULL = NULL is not true. */ + That is the query waiting on itself, not blocking. Spid 0 is a report that named no session (the + collector stores 0, not NULL, when a report has no spid: BlockedProcessReportCollector), which is + still a blocking event: NotSelfWaitSql keeps (0, 0) rows too, by the same `<> 0` test. */ if (blockedSpid(row) == blockingSpid(row) && blockedSpid(row) != 0) { continue; @@ -198,9 +208,9 @@ public static string WaitsCteSql( SELECT s.*, {epochMs} - s.wait_time_ms AS wait_start_ms FROM ({source}) AS s /* #6645: a parallel query's threads write reports with blocked_spid = blocking_spid (different ecid). That is - the query waiting on itself, not blocking: it is no wait, no head blocker and no report. A report that - names no session (NULL spids) is kept: the comparison is not true for it. */ - WHERE NOT COALESCE(s.blocked_spid = s.blocking_spid, FALSE) + the query waiting on itself, not blocking: it is no wait, no head blocker and no report. NotSelfWaitSql + keeps a report that names no session (spids 0/0, or NULL). */ + WHERE {NotSelfWaitSql} ), {name}_gapped AS ( SELECT m.*, From 09b386ee5104276d1e6559dcd61d05cf96dddd05 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:22:56 -0400 Subject: [PATCH 3/4] Decide XE-or-DMV on XE reports that are not self-waits (#6645) --- Darling/Darling.Tests/BlockingWaitCountTests.cs | 13 +++++++++++++ PerformanceMonitor.Alerting/AlertContextBuilders.cs | 7 ++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/BlockingWaitCountTests.cs b/Darling/Darling.Tests/BlockingWaitCountTests.cs index 8269735c64..b6f52ef391 100644 --- a/Darling/Darling.Tests/BlockingWaitCountTests.cs +++ b/Darling/Darling.Tests/BlockingWaitCountTests.cs @@ -238,6 +238,19 @@ public void XeReportsInANonExcludedDatabase_AreTheSource_AndDmvWaitsAreIgnored() Assert.Equal(102, wait.BlockedSpid); } + /// #6645 round 1: XE reports that are all a parallel query's self-waits are no XE reports, so the DMV + /// waits beside them are the source, as in the SQL read surfaces. + [Fact] + public void XeReportsThatAreAllSelfWaits_FallBackToTheDmvWaits() + { + var rows = OneWait(59, 59, T0, 6); + rows.AddRange(OneWait(202, 200, T0.AddMinutes(5), 3, source: BlockedProcessAlertRow.DmvSnapshotSource)); + + var wait = Assert.Single(AlertContextBuilders.BlockingWaitRows(rows, Array.Empty())); + Assert.Equal(BlockedProcessAlertRow.DmvSnapshotSource, wait.Source); + Assert.Equal(200, wait.BlockingSpid); + } + [Fact] public void NoXeReportsAtAll_CountsTheDmvWaits_OutsideTheExcludedDatabases() { diff --git a/PerformanceMonitor.Alerting/AlertContextBuilders.cs b/PerformanceMonitor.Alerting/AlertContextBuilders.cs index 5cd2faaa28..4d531102d6 100644 --- a/PerformanceMonitor.Alerting/AlertContextBuilders.cs +++ b/PerformanceMonitor.Alerting/AlertContextBuilders.cs @@ -499,7 +499,12 @@ public static IReadOnlyList BlockingWaitRows( { if (events == null || events.Count == 0) return Array.Empty(); - var hasXeReports = events.Any(r => r.Source == BlockedProcessAlertRow.XeReportSource); + /* #6645: a parallel query's self-wait report (the same non-zero spid on both sides) is not a blocking + report, so a window whose XE reports are all self-waits has no XE reports and falls back to the DMV + waits, as the SQL read surfaces do (BlockingWaits.NotSelfWaitSql). Spid 0 is a report that named no + session and still counts. */ + var hasXeReports = events.Any(r => r.Source == BlockedProcessAlertRow.XeReportSource + && !(r.BlockedSpid == r.BlockingSpid && r.BlockedSpid != 0)); var waits = BlockingWaits.Collapse( FilterBlocking(events, excludedDatabases), From b85ad464fe28eec9bd34060b38dc41aa84589814 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:35:45 -0400 Subject: [PATCH 4/4] Pin Lite's self-wait filter in the twin test's blocking arm (#6645) Lite's blocking baseline arm now filters self-waits with BlockingWaits.NotSelfWaitSql (6d29768ad), so the expected event source at line 228 of DarlingEventBaselineCoveredDaysTests.cs follows it. Test expectation only; no behavior change. --- Darling/Darling.Tests/DarlingEventBaselineCoveredDaysTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Darling/Darling.Tests/DarlingEventBaselineCoveredDaysTests.cs b/Darling/Darling.Tests/DarlingEventBaselineCoveredDaysTests.cs index 4f9333a0f3..0d1250f27d 100644 --- a/Darling/Darling.Tests/DarlingEventBaselineCoveredDaysTests.cs +++ b/Darling/Darling.Tests/DarlingEventBaselineCoveredDaysTests.cs @@ -225,7 +225,7 @@ public void TheTwoProducts_CarryTheSameEventBaselineBody_AndTheSameCollectors() foreach (var (metric, eventTable, aggregate, liteEvents) in new[] { ("Blocking", "blocked_process_reports", TimescaleSupport.BlockedProcessBaselineView, - "StoredEventCopies.BlockedProcessReports(\"server_id = $1 AND event_time >= $2 AND event_time < $3\") + \" AS ev\""), + "StoredEventCopies.BlockedProcessReports(\"server_id = $1 AND event_time >= $2 AND event_time < $3 AND \" + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql) + \" AS ev\""), ("Deadlock", "deadlocks", TimescaleSupport.DeadlockBaselineView, "v_deadlocks"), }) {