From 9784da31849926605e79bd94df821a8437bd3763 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 24 Oct 2026 04:27:00 -0400
Subject: [PATCH 1/4] Darling blocking baseline leaves out self-waits, like the
window it is compared with (#6690)
---
Darling/Darling.Tests/BaselineSupplyTests.cs | 6 +-
.../FleetPartialDatabaseFailureBandTests.cs | 125 ++++++++++++++++++
.../PgAnomalyDetector.cs | 7 +-
.../TimescaleSupport.cs | 56 +++++++-
4 files changed, 183 insertions(+), 11 deletions(-)
diff --git a/Darling/Darling.Tests/BaselineSupplyTests.cs b/Darling/Darling.Tests/BaselineSupplyTests.cs
index 1ee0375f70..d2a5628f42 100644
--- a/Darling/Darling.Tests/BaselineSupplyTests.cs
+++ b/Darling/Darling.Tests/BaselineSupplyTests.cs
@@ -107,13 +107,13 @@ and the legacy one they still fall back to are shared the same way. */
Assert.Contains("delta_wait_time_ms >= 0", TimescaleSupport.LegacyCreateWaitStatsBaselineSql, StringComparison.Ordinal);
Assert.DoesNotContain("sample_interval_seconds", TimescaleSupport.LegacyCreateWaitStatsBaselineSql, StringComparison.Ordinal);
- /* blocked_process_reports: neither family filters, so the shared aggregate carries no WHERE at all.
- A WHERE appearing here later means one family narrowed the other's supply. */
+ /* blocked_process_reports: both families leave a parallel query's self-waits out of their current window
+ (#6663), so the shared aggregate carries the same predicate (#6690), or its baseline reads high. */
var blocking = PgBaselineProvider.GetBaselineQuery(MetricNames.Blocking)!;
var blockingPerMinute = PgBaselineProvider.GetBaselineQuery(MetricNames.BlockingPerMinute)!;
Assert.Contains("FROM blocked_process_baseline", blocking, StringComparison.Ordinal);
Assert.Contains("FROM blocked_process_baseline", blockingPerMinute, StringComparison.Ordinal);
- Assert.DoesNotContain("WHERE", TimescaleSupport.CreateBlockedProcessBaselineSql, StringComparison.Ordinal);
+ Assert.Contains(PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql, TimescaleSupport.CreateBlockedProcessBaselineSql, StringComparison.Ordinal);
}
///
diff --git a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
index 77d180ea1e..020022600b 100644
--- a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
+++ b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
@@ -600,4 +600,129 @@ public async Task StoreWhoseRollupPredatesTheNoteColumn_IsRebuiltAndRefreshedOnc
Assert.Equal("HEALTHY", warmedBands[collectors[1]]);
Assert.Equal("HEALTHY", warmedBands[collectors[2]]);
}
+
+ /* ───────────── #6690: the blocking baseline aggregate leaves self-waits out ───────────── */
+
+ private const string BlockedReportInsertSql = @"
+INSERT INTO collect.blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocked_ecid, blocking_spid, blocking_ecid, wait_time_ms, wait_resource, lock_mode, blocked_status, blocked_isolation_level, blocked_log_used, blocked_transaction_count, blocked_client_app, blocked_host_name, blocked_login_name, blocked_sql_text, blocking_status, blocking_isolation_level, blocking_client_app, blocking_host_name, blocking_login_name, blocking_sql_text, blocked_transaction_name, blocking_transaction_name, blocked_last_tran_started, blocking_last_tran_started, blocked_last_batch_started, blocking_last_batch_started, blocked_last_batch_completed, blocking_last_batch_completed, blocked_priority, blocking_priority, blocked_process_report_xml, object_id, database_id, contentious_object, monitor_loop, blocked_query_plan_xml, blocking_query_plan_xml)
+VALUES ($1, $2, -6690, 'srv-6690', $2, 'testdb', $3, 0, $4, 0, 100, 'KEY', 'X', 'suspended', 'READ COMMITTED', 0, 1, 'app', 'host', 'login', 'select 1', 'suspended', 'READ COMMITTED', 'app', 'host', 'login', 'select 2', NULL, NULL, $2, $2, $2, $2, $2, $2, 0, 0, '', 1, 1, 'obj', 1, NULL, NULL)";
+
+ private static string PreSelfWaitFilterBaselineSql()
+ => Regex.Replace(TimescaleSupport.CreateBlockedProcessBaselineSql, @"WHERE [^\r\n]*\r?\n", string.Empty);
+
+ private static async Task InsertBlockedReportAsync(NpgsqlConnection connection, long id, DateTime t, int blocked, int blocking, CancellationToken ct)
+ {
+ await using var insert = new NpgsqlCommand(BlockedReportInsertSql, connection);
+ insert.Parameters.AddWithValue(id);
+ insert.Parameters.AddWithValue(t);
+ insert.Parameters.AddWithValue(blocked);
+ insert.Parameters.AddWithValue(blocking);
+ await insert.ExecuteNonQueryAsync(ct);
+ }
+
+ private static async Task BaselineEventsAsync(NpgsqlConnection connection, DateTime from, DateTime to, CancellationToken ct)
+ {
+ await using var sum = new NpgsqlCommand("SELECT COALESCE(sum(event_count), 0) FROM collect.blocked_process_baseline WHERE server_id = -6690 AND collection_time >= $1 AND collection_time < $2", connection);
+ sum.Parameters.AddWithValue(from);
+ sum.Parameters.AddWithValue(to);
+ return Convert.ToInt64(await sum.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture);
+ }
+
+ [Fact]
+ public async Task BlockingBaselineBuiltWithoutTheSelfWaitFilter_IsRebuiltFilteredAndBackfilledOverTheWholeWindow_AgainstDevPostgres()
+ {
+ var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #6690 reshape test.");
+ var ct = TestContext.Current.CancellationToken;
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+ Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "the dev fixture is expected to have TimescaleDB installed");
+
+ /* No scheduler: the refresh policy's own run must not move the watermark under the test. */
+ await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection))
+ {
+ await stop.ExecuteNonQueryAsync(ct);
+ }
+
+ await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct);
+
+ /* The pre-#6690 aggregate: the same select with no WHERE. */
+ var oldShape = PreSelfWaitFilterBaselineSql();
+ Assert.DoesNotContain("blocking_spid", oldShape, StringComparison.Ordinal);
+ await using (var create = new NpgsqlCommand(oldShape.Replace("WITH NO DATA", "WITH DATA", StringComparison.Ordinal), connection))
+ {
+ await create.ExecuteNonQueryAsync(ct);
+ }
+
+ /* A recent day and a day older than any refresh policy's window, each holding one real report (spid 1
+ blocked by 2) and one self-wait (spid 5 waiting on itself). */
+ var now = LiveClock.Now();
+ long id = 66900;
+ foreach (var t in new[] { now.AddHours(-3), now.AddDays(-10) })
+ {
+ await InsertBlockedReportAsync(connection, id++, t, 1, 2, ct);
+ await InsertBlockedReportAsync(connection, id++, t, 5, 5, ct);
+ }
+
+ await using (var refreshAll = new NpgsqlCommand("CALL refresh_continuous_aggregate('collect.blocked_process_baseline', NULL, NULL)", connection))
+ {
+ await refreshAll.ExecuteNonQueryAsync(ct);
+ }
+
+ Assert.Equal(2L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct));
+
+ /* The sweep drops it (only it), the ensure sweep recreates it filtered. */
+ Assert.Equal(1, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct));
+ await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct);
+
+ /* The trap: the policy's recent window is all a refresh would cover, so the old day reads EMPTY. */
+ await using (var recentOnly = new NpgsqlCommand("CALL refresh_continuous_aggregate('collect.blocked_process_baseline', $1, $2)", connection))
+ {
+ recentOnly.Parameters.AddWithValue(now.AddDays(-1));
+ recentOnly.Parameters.AddWithValue(now.AddHours(1));
+ await recentOnly.ExecuteNonQueryAsync(ct);
+ }
+
+ Assert.Equal(0L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct));
+
+ /* The start path's coverage-gated backfill closes it over the whole baseline window, counting only the real report. */
+ Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1);
+ Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct));
+ Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-1), now.AddHours(1), ct));
+
+ /* A converged store: nothing left to drop. */
+ Assert.Equal(0, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct));
+ Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct));
+ }
+
+ [Fact]
+ public async Task PlainBlockingBaselineViewWithoutTheSelfWaitFilter_IsReplacedInPlace_AgainstDevPostgres()
+ {
+ var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the live #6690 plain-view test.");
+ var ct = TestContext.Current.CancellationToken;
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+
+ /* A store without TimescaleDB that built the plain fallback view before #6690. */
+ await using (var create = new NpgsqlCommand(TimescaleSupport.CreateBaselineFallbackViewSql(TimescaleSupport.BlockedProcessBaselineView, PreSelfWaitFilterBaselineSql()), connection))
+ {
+ await create.ExecuteNonQueryAsync(ct);
+ }
+
+ var now = LiveClock.Now();
+ await InsertBlockedReportAsync(connection, 66950, now.AddHours(-3), 1, 2, ct);
+ await InsertBlockedReportAsync(connection, 66951, now.AddHours(-3), 5, 5, ct);
+
+ Assert.Equal(2L, await BaselineEventsAsync(connection, now.AddDays(-1), now, ct));
+ await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct);
+ Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-1), now, ct));
+
+ /* Converged: the next pass fills and replaces nothing. */
+ Assert.Equal(0, await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct));
+ }
}
diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs
index 953759d6d6..b2da181b43 100644
--- a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs
+++ b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs
@@ -307,10 +307,9 @@ the CURRENT window counts report and deadlock events by when they HAPPENED (even
snapshot's event_time IS its collection time. The BASELINE this count is judged against comes from the
continuous aggregates, which bucket the hypertable's own column (collection_time) by design.
- #6645: this CURRENT count leaves out a parallel query's self-waits, but the BASELINE aggregate
- (collect.blocked_process_baseline, count(*) over every report row) still counts them until a store
- migration rebuilds it with the same predicate, so a server with many self-waits has a high baseline and
- its real blocking is under-flagged in the meantime. */
+ #6645: this CURRENT count leaves out a parallel query's self-waits, and so does the BASELINE aggregate
+ (collect.blocked_process_baseline, #6690: its select carries the same BlockingWaits.NotSelfWaitSql), so the
+ ratio compares like with like. A store that built the aggregate earlier is rebuilt on start. */
public const string BlockingWindowSql = $@"
/* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking. */
SELECT
diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
index 6339e209ae..d7b86a5aff 100644
--- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
+++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
@@ -791,9 +791,13 @@ WHERE delta_execution_count > 0
WITH NO DATA";
- /// Blocking AND BlockingPerMinute baseline supply -- both families share this source and both
- /// have NO row-level filter, so one aggregate serves both. Event counts per collection re-aggregate to
- /// either shape: Blocking sums them per hour/dow bucket, BlockingPerMinute re-buckets by minute.
+ /// Blocking AND BlockingPerMinute baseline supply -- both families share this source and the same
+ /// row-level filter, so one aggregate serves both. Since #6663 both leave a parallel query's self-waits out of
+ /// their current window (), so the baseline they are compared with
+ /// leaves them out too (#6690): a count over every report row would read a server's baseline high and under-flag
+ /// its real blocking. Event counts per collection re-aggregate to either shape: Blocking sums them per
+ /// hour/dow bucket, BlockingPerMinute re-buckets by minute. A store that built this aggregate before #6690 is
+ /// rebuilt by .
public const string CreateBlockedProcessBaselineSql = @"CREATE MATERIALIZED VIEW IF NOT EXISTS collect.blocked_process_baseline
WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS
SELECT
@@ -802,6 +806,7 @@ WHERE delta_execution_count > 0
collection_time,
count(*) AS event_count
FROM collect.blocked_process_reports
+WHERE " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql + @"
GROUP BY server_id, bucket, collection_time
WITH NO DATA";
@@ -1068,6 +1073,23 @@ public static async Task EnsureBaselineFallbackViewsAsync(
{
if (await probe.ExecuteScalarAsync(cancellationToken) is true)
{
+ /* #6690: the one relation that is "present" and still wrong: a plain view built before the
+ blocking baseline left self-waits out. A plain view has no data to lose, so it is replaced
+ in place; a continuous aggregate is never touched here (the reshape sweep rebuilds it). */
+ if (view == BlockedProcessBaselineView && await IsStalePlainBlockedBaselineViewAsync(connection, cancellationToken))
+ {
+ using var replace = new NpgsqlCommand(CreateBaselineFallbackViewSql(view, createSql), connection)
+ {
+ CommandTimeout = SetupTimeoutSeconds,
+ };
+ await replace.ExecuteNonQueryAsync(cancellationToken);
+ filled++;
+
+ logger?.LogInformation(
+ "Baseline relation {View} was a plain view that still counted blocking self-waits (#6690) -- replaced it with the filtered select.",
+ view);
+ }
+
continue;
}
}
@@ -1094,6 +1116,25 @@ public static async Task EnsureBaselineFallbackViewsAsync(
return filled;
}
+ ///
+ /// #6690: is collect.blocked_process_baseline a PLAIN view (not a continuous aggregate) whose select does
+ /// not carry the self-wait filter? is null for a plain view and for a
+ /// store without the extension, which is what separates it from an aggregate (also a relkind='v' view);
+ /// the filter is recognised by its blocking_spid column, which the pre-#6690 select never named.
+ ///
+ private static async Task IsStalePlainBlockedBaselineViewAsync(NpgsqlConnection connection, CancellationToken cancellationToken)
+ {
+ if (await ResolveMaterializationAsync(connection, BlockedProcessBaselineView, cancellationToken) is not null)
+ {
+ return false;
+ }
+
+ using var probe = new NpgsqlCommand(
+ $"SELECT COALESCE(position('blocking_spid' IN pg_get_viewdef('collect.{BlockedProcessBaselineView}'::regclass)) = 0, false) FROM pg_class WHERE oid = 'collect.{BlockedProcessBaselineView}'::regclass AND relkind = 'v'",
+ connection) { CommandTimeout = SetupTimeoutSeconds };
+ return await probe.ExecuteScalarAsync(cancellationToken) is true;
+ }
+
/// Does this baseline relation exist under any implementation — continuous aggregate or plain view?
public static string BaselineRelationExistsSql(string view)
=> $"SELECT to_regclass('collect.{view}') IS NOT NULL";
@@ -6202,6 +6243,13 @@ The ensure sweep recreates it WITH NO DATA and the eight-day refresh window re-m
answer from raw, exactly; WarmCollectionHealthHourlyAsync closes that window with one refresh. */
(View: CollectionHealthHourlyView,
StaleCheck: "SELECT (EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'collect' AND table_name = 'collection_health_hourly') AND NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'collect' AND table_name = 'collection_health_hourly' AND column_name = 'latest_run_note'))"),
+ /* blocked_process_baseline gained the self-wait filter (#6690) -> stale iff the aggregate EXISTS but its
+ definition does not mention blocking_spid (the filter's column; the old text had no WHERE at all).
+ Nothing is built on it, so the CASCADE drops only the view; the ensure sweep recreates it WITH NO DATA
+ and the start path's coverage-gated BackfillBaselineAggregatesAsync (its materialization is empty, so
+ its gate fires on this very start) re-materializes it from raw. NULL view_definition is not stale. */
+ (View: BlockedProcessBaselineView,
+ StaleCheck: "SELECT EXISTS (SELECT 1 FROM timescaledb_information.continuous_aggregates WHERE view_schema = 'collect' AND view_name = 'blocked_process_baseline' AND position('blocking_spid' IN view_definition) = 0)"),
};
var dropped = 0;
@@ -6237,7 +6285,7 @@ The ensure sweep recreates it WITH NO DATA and the eight-day refresh window re-m
hold = AggregateJobHold.None;
dropped++;
logger?.LogInformation(
- "TimescaleDB: dropped stale continuous aggregate {View} (composer-dimension reshape) — recreated in the new shape this cycle.",
+ "TimescaleDB: dropped stale continuous aggregate {View} (reshape) — recreated in the new shape this cycle.",
view);
}
catch (OperationCanceledException) when (hold.Unscheduled.Count > 0)
From 9047271f4a2476f80792e52ed53a969ba4cd8d18 Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 24 Oct 2026 05:08:32 -0400
Subject: [PATCH 2/4] Leave self-waits out of Lite's per-minute blocking series
and baseline (#6705)
---
Lite.Tests/BlockingStatsReaderTests.cs | 63 +++++++++++++++++--
Lite.Tests/EventBaselineCoveredDaysTests.cs | 29 +++++++++
Lite/Analysis/BaselineProvider.cs | 4 +-
Lite/Services/LocalDataService.Blocking.cs | 18 +++---
.../LocalDataService.BlockingStats.cs | 4 +-
5 files changed, 100 insertions(+), 18 deletions(-)
diff --git a/Lite.Tests/BlockingStatsReaderTests.cs b/Lite.Tests/BlockingStatsReaderTests.cs
index 4a3306518f..99a7e765a2 100644
--- a/Lite.Tests/BlockingStatsReaderTests.cs
+++ b/Lite.Tests/BlockingStatsReaderTests.cs
@@ -151,6 +151,57 @@ public async Task BlockingDurationStats_FallsBackToDmvSnapshot_OnlyWhenNoXeInWin
Assert.Equal(1000, bucket.TotalDurationMs);
}
+ // ── #6645 / #6705: a self-wait (same non-zero blocked and blocking spid) is not blocking, on both arms ──
+
+ [Fact]
+ public async Task BlockingSeries_LeaveSelfWaitsOut_OnTheXeArm()
+ {
+ var service = new LocalDataService(_duckDb);
+
+ /* One minute: a parallel query waiting on itself (spid 77 blocking 77), a real block and a (0, 0) report. A (0, 0) report
+ is a system spid pair the predicate keeps; the self-wait goes, with its 9000 ms. */
+ await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(5), "DB1", 9000, blockedSpid: 77, blockingSpid: 77);
+ await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(15), "DB1", 1000);
+ await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(25), "DB1", 3000, blockedSpid: 0, blockingSpid: 0);
+
+ var trend = Assert.Single(await service.GetBlockingTrendAsync(ServerId));
+ Assert.Equal(2, trend.Count);
+
+ var duration = Assert.Single(await service.GetBlockingDurationStatsAsync(ServerId));
+ Assert.Equal(2, duration.EventCount);
+ Assert.Equal(4000, duration.TotalDurationMs);
+ Assert.Equal(3000, duration.MaxDurationMs);
+
+ Assert.Equal(2, (await service.GetBlockingSlicerDataAsync(ServerId, 24)).Sum(b => b.SessionCount));
+ Assert.True(await service.HasBlockedProcessReportsInWindowAsync(ServerId, MinuteA.AddMinutes(-1), MinuteA.AddMinutes(1)));
+ }
+
+ [Fact]
+ public async Task BlockingSeries_OnlySelfWaitReportsInXe_DrawTheDmvArm_AndDmvSelfWaitsGo()
+ {
+ var service = new LocalDataService(_duckDb);
+
+ /* The XE arm holds only a self-wait, so once the predicate filters it the arm is empty and the DMV arm is drawn. The DMV
+ arm leaves its own self-wait out. */
+ await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(5), "DB1", 9000, blockedSpid: 77, blockingSpid: 77);
+ await SeedDmvBlockingSnapshotAsync(MinuteB.AddSeconds(5), "DB1", 2000);
+ await SeedDmvBlockingSnapshotAsync(MinuteB.AddSeconds(10), "DB1", 8000, blockedSpid: 88, blockingSpid: 88);
+
+ var trend = Assert.Single(await service.GetBlockingTrendAsync(ServerId));
+ Assert.Equal(MinuteB, trend.Time);
+ Assert.Equal("DMV snapshot", trend.Source);
+ Assert.Equal(1, trend.Count);
+
+ var duration = Assert.Single(await service.GetBlockingDurationStatsAsync(ServerId));
+ Assert.Equal(MinuteB, duration.Time);
+ Assert.Equal("DMV snapshot", duration.Source);
+ Assert.Equal(2000, duration.TotalDurationMs);
+
+ var slicer = await service.GetBlockingSlicerDataAsync(ServerId, 24);
+ Assert.Equal(1, slicer.Sum(b => b.SessionCount));
+ Assert.False(await service.HasBlockedProcessReportsInWindowAsync(ServerId, MinuteA.AddMinutes(-1), MinuteB.AddMinutes(1)));
+ }
+
// ── Deadlock severity aggregate: victim_count + total / max / avg wait, reconciling with the count ──
[Fact]
@@ -217,7 +268,7 @@ await SeedDeadlockAsync(MinuteFloor(DateTime.UtcNow.AddHours(-48)).AddSeconds(5)
// ── Seeding helpers (raw INSERT via the shared initializer's connection) ──
- private async Task SeedBlockedProcessReportAsync(DateTime eventTime, string databaseName, long waitTimeMs)
+ private async Task SeedBlockedProcessReportAsync(DateTime eventTime, string databaseName, long waitTimeMs, int blockedSpid = 55, int blockingSpid = 66)
{
using var readLock = _duckDb.AcquireReadLock();
var connection = await SeedConnectionAsync();
@@ -234,9 +285,9 @@ INSERT INTO blocked_process_reports
cmd.Parameters.Add(new DuckDBParameter { Value = "TestSrv" });
cmd.Parameters.Add(new DuckDBParameter { Value = eventTime });
cmd.Parameters.Add(new DuckDBParameter { Value = databaseName });
- cmd.Parameters.Add(new DuckDBParameter { Value = 55 });
+ cmd.Parameters.Add(new DuckDBParameter { Value = blockedSpid });
cmd.Parameters.Add(new DuckDBParameter { Value = eventTime });
- cmd.Parameters.Add(new DuckDBParameter { Value = 66 });
+ cmd.Parameters.Add(new DuckDBParameter { Value = blockingSpid });
cmd.Parameters.Add(new DuckDBParameter { Value = eventTime });
cmd.Parameters.Add(new DuckDBParameter { Value = waitTimeMs });
cmd.Parameters.Add(new DuckDBParameter { Value = "X" });
@@ -245,7 +296,7 @@ INSERT INTO blocked_process_reports
await cmd.ExecuteNonQueryAsync();
}
- private async Task SeedDmvBlockingSnapshotAsync(DateTime eventTime, string databaseName, long waitTimeMs)
+ private async Task SeedDmvBlockingSnapshotAsync(DateTime eventTime, string databaseName, long waitTimeMs, int blockedSpid = 55, int blockingSpid = 66)
{
using var readLock = _duckDb.AcquireReadLock();
var connection = await SeedConnectionAsync();
@@ -261,8 +312,8 @@ INSERT INTO dmv_blocking_snapshots
cmd.Parameters.Add(new DuckDBParameter { Value = "TestSrv" });
cmd.Parameters.Add(new DuckDBParameter { Value = eventTime });
cmd.Parameters.Add(new DuckDBParameter { Value = databaseName });
- cmd.Parameters.Add(new DuckDBParameter { Value = 55 });
- cmd.Parameters.Add(new DuckDBParameter { Value = 66 });
+ cmd.Parameters.Add(new DuckDBParameter { Value = blockedSpid });
+ cmd.Parameters.Add(new DuckDBParameter { Value = blockingSpid });
cmd.Parameters.Add(new DuckDBParameter { Value = waitTimeMs });
cmd.Parameters.Add(new DuckDBParameter { Value = "X" });
cmd.Parameters.Add(new DuckDBParameter { Value = 1 });
diff --git a/Lite.Tests/EventBaselineCoveredDaysTests.cs b/Lite.Tests/EventBaselineCoveredDaysTests.cs
index 71c83e155e..506c10d143 100644
--- a/Lite.Tests/EventBaselineCoveredDaysTests.cs
+++ b/Lite.Tests/EventBaselineCoveredDaysTests.cs
@@ -573,4 +573,33 @@ await ExecAsync(
Assert.NotNull(row);
Assert.Equal(1.0 / 5.0, row!.Mean, 9);
}
+
+ ///
+ /// #6705: the blocking-per-minute baseline counts the same reports the current window does, so a self-wait (the same
+ /// non-zero blocked and blocking spid) does not make a minute heavier. Each Tuesday has one minute holding a real report
+ /// and a self-wait, and another holding only a self-wait: the first counts 1, the second is not a minute at all, and a
+ /// (0, 0) report still counts.
+ ///
+ [Fact]
+ public async Task TheBlockingPerMinuteBaseline_LeavesSelfWaitsOut()
+ {
+ async Task Report(DateTime at, int blockedSpid, int blockingSpid) => await ExecAsync(
+ "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, wait_time_ms, blocked_spid, blocking_spid) "
+ + "VALUES ($1, $2, $2, $3, 'TestServer', 1000, $4, $5)",
+ _nextId--, at, ServerId, blockedSpid, blockingSpid);
+
+ foreach (var tuesday in Tuesdays)
+ {
+ await Report(tuesday.AddMinutes(10), 55, 66);
+ await Report(tuesday.AddMinutes(10).AddSeconds(20), 77, 77);
+ await Report(tuesday.AddMinutes(20), 88, 88);
+ await Report(tuesday.AddMinutes(30), 0, 0);
+ }
+
+ var row = await RowAsync(MetricNames.BlockingPerMinute, Hour, Tuesday);
+
+ Assert.NotNull(row);
+ Assert.Equal(10L, row!.SampleCount);
+ Assert.Equal(1.0, row.Mean, 9);
+ }
}
diff --git a/Lite/Analysis/BaselineProvider.cs b/Lite/Analysis/BaselineProvider.cs
index 6afc16516f..ed33deba4e 100644
--- a/Lite/Analysis/BaselineProvider.cs
+++ b/Lite/Analysis/BaselineProvider.cs
@@ -712,7 +712,7 @@ FROM v_file_io_stats
)," + RobustTierScaffold,
// #6645: the baseline counts the same reports the current window does (BlockingWaits.NotSelfWaitSql), so the 3x ratio
- // compares like with like. The Darling twin reads a stored aggregate that still counts them.
+ // compares like with like. The Darling twin's stored aggregate leaves them out too (#6705).
// Event-based — mean = events per covered hour for this bucket, sample_count = covered days.
// No restart exclusion needed (event counts, not cumulative). See EventBaselineSql (#4731).
MetricNames.Blocking => OnEventTime(EventBaselineSql("blocked_process_report", "v_collection_log",
@@ -775,7 +775,7 @@ FROM with_rate
WITH per_minute AS (
SELECT DATE_TRUNC('minute', event_time) AS minute_bucket,
COUNT(*)::DOUBLE PRECISION AS event_count
- FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3") + @" AS ev
+ FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql) + @" AS ev
GROUP BY minute_bucket
),
clean AS (
diff --git a/Lite/Services/LocalDataService.Blocking.cs b/Lite/Services/LocalDataService.Blocking.cs
index bbeea0ad21..ded10e45ab 100644
--- a/Lite/Services/LocalDataService.Blocking.cs
+++ b/Lite/Services/LocalDataService.Blocking.cs
@@ -1001,7 +1001,7 @@ WITH bpr AS (
COUNT(DISTINCT blocking_spid) AS distinct_blockers,
COUNT(DISTINCT blocked_spid) AS distinct_blocked,
COUNT(DISTINCT database_name) AS distinct_databases
- FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + @" AS ev
+ FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + @" AS ev
GROUP BY date_trunc('hour', event_time)
),
dmv AS (
@@ -1013,7 +1013,7 @@ dmv AS (
COUNT(DISTINCT blocked_spid) AS distinct_blocked,
COUNT(DISTINCT database_name) AS distinct_databases
FROM v_dmv_blocking_snapshots
- WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3" + dbClause + @"
+ WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause + @"
GROUP BY date_trunc('hour', collection_time)
)
SELECT bucket, event_count, total_wait_sec, distinct_blockers, distinct_blocked, distinct_databases FROM bpr
@@ -1090,7 +1090,8 @@ GROUP BY date_trunc('hour', deadlock_time)
///
/// #5098: whether the XE blocked process reports have a row in the window, with the SAME predicate (and database filter) as the
/// bpr arm of and . When true those reads drew
- /// the XE rows alone (the DMV arm is skipped), so the note must name the XE start, not a DMV-covered one.
+ /// the XE rows alone (the DMV arm is skipped), so the note must name the XE start, not a DMV-covered one. #6705: the predicate
+ /// leaves out self-waits (), as those reads do on both arms.
///
public async Task HasBlockedProcessReportsInWindowAsync(int serverId, DateTime startUtc, DateTime endUtc, IReadOnlyList? databaseNames = null)
{
@@ -1098,7 +1099,7 @@ public async Task HasBlockedProcessReportsInWindowAsync(int serverId, Date
using var command = connection.CreateCommand();
var dbClause = BuildDbInClause(databaseNames, "database_name", 4, out var dbValues);
- command.CommandText = "SELECT 1 FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + " AS ev LIMIT 1";
+ command.CommandText = "SELECT 1 FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + " AS ev LIMIT 1";
command.Parameters.Add(new DuckDBParameter { Value = serverId });
command.Parameters.Add(new DuckDBParameter { Value = startUtc });
command.Parameters.Add(new DuckDBParameter { Value = endUtc });
@@ -1110,8 +1111,9 @@ public async Task HasBlockedProcessReportsInWindowAsync(int serverId, Date
}
///
- /// #5098: the earliest XE blocked process report in the window, with the SAME predicate and database filter as
- /// . Null when there is none. Opens its own connection.
+ /// #5098: the earliest XE blocked process report in the window, with the same database filter as
+ /// . It names when the XE session started capturing, so a self-wait report
+ /// counts here (#6705). Null when there is none. Opens its own connection.
///
public async Task GetEarliestBlockedProcessReportInWindowAsync(int serverId, DateTime startUtc, DateTime endUtc, IReadOnlyList? databaseNames = null)
{
@@ -1212,13 +1214,13 @@ Group by event_time (when blocking actually occurred) rather than collection_tim
command.CommandText = @"
WITH bpr AS (
SELECT DATE_TRUNC('minute', event_time) AS bucket, COUNT(*) AS incident_count, 'blocked-process-report' AS source
- FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + @" AS ev
+ FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + @" AS ev
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
SELECT DATE_TRUNC('minute', event_time) AS bucket, COUNT(*) AS incident_count, 'DMV snapshot' AS source
FROM v_dmv_blocking_snapshots
- WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause + @"
+ WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause + @"
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, incident_count, source FROM bpr
diff --git a/Lite/Services/LocalDataService.BlockingStats.cs b/Lite/Services/LocalDataService.BlockingStats.cs
index dd9e31c3d2..8c9ae586fa 100644
--- a/Lite/Services/LocalDataService.BlockingStats.cs
+++ b/Lite/Services/LocalDataService.BlockingStats.cs
@@ -96,7 +96,7 @@ WITH bpr AS (
CAST(COALESCE(MAX(wait_time_ms), 0) AS BIGINT) AS max_duration_ms,
CAST(COALESCE(AVG(wait_time_ms), 0) AS DOUBLE) AS avg_duration_ms,
'blocked-process-report' AS source
- FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + @" AS ev
+ FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + @" AS ev
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
@@ -108,7 +108,7 @@ dmv AS (
CAST(COALESCE(AVG(wait_time_ms), 0) AS DOUBLE) AS avg_duration_ms,
'DMV snapshot' AS source
FROM v_dmv_blocking_snapshots
- WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause + @"
+ WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause + @"
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, event_count, total_duration_ms, max_duration_ms, avg_duration_ms, source FROM bpr
From 19e2ca831c091762407ac0990f5c69ab51953fea Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 24 Oct 2026 05:18:58 -0400
Subject: [PATCH 3/4] Blocking trend bars leave self-waits out like the band,
and the rebuilt blocking baseline records where its supply begins (#6705)
The Viewer and MCP blocking trend and duration reads, and the existence check that picks their arm, take BlockingWaits.NotSelfWaitSql on both arms. A rebuilt blocked_process_baseline records a supply floor (view comment) on the backfill pass, and the Blocking baseline starts its window no earlier, so days the aggregate cannot back are not counted as quiet.
---
.../FleetPartialDatabaseFailureBandTests.cs | 105 ++++++++++++++++++
.../Darling.Tests/ViewerBlockingStatsTests.cs | 78 ++++++++++++-
.../PgBaselineProvider.cs | 13 +++
.../Mcp/DarlingBlockingTrendReader.cs | 7 ++
.../Mcp/DarlingDataReader.cs | 5 +
.../TimescaleSupport.cs | 92 ++++++++++++++-
.../ViewerDataService.Blocking.cs | 8 +-
.../ViewerDataService.BlockingStats.cs | 6 +-
.../ViewerDataService.BlockingTrends.cs | 10 +-
9 files changed, 312 insertions(+), 12 deletions(-)
diff --git a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
index 020022600b..ac81fc5c76 100644
--- a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
+++ b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
@@ -18,8 +18,10 @@
using System.Threading;
using System.Threading.Tasks;
using Npgsql;
+using PerformanceMonitor.Analysis.Baselines;
using PerformanceMonitor.Collectors;
using PerformanceMonitor.Common;
+using PerformanceMonitor.Darling.Analysis;
using PerformanceMonitor.Darling.Service.Mcp;
using PerformanceMonitor.Darling.Storage;
using PerformanceMonitor.Darling.Viewer;
@@ -725,4 +727,107 @@ public async Task PlainBlockingBaselineViewWithoutTheSelfWaitFilter_IsReplacedIn
/* Converged: the next pass fills and replaces nothing. */
Assert.Equal(0, await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct));
}
+
+ /* ───────────── #6705: the rebuilt aggregate's supply floor (the #3590 trap) ───────────── */
+
+ [Theory]
+ [InlineData(null, null)]
+ [InlineData("", null)]
+ [InlineData("not a floor", null)]
+ [InlineData("supply_floor_utc=garbage", null)]
+ [InlineData("supply_floor_utc=2026-09-24", null)]
+ [InlineData("supply_floor_utc=2026-09-24T12:34:56", "2026-09-24T12:34:56")]
+ public void TheSupplyFloorComment_ParsesOnlyAValidFloor(string? comment, string? expected)
+ => Assert.Equal(
+ expected is null ? null : DateTime.ParseExact(expected, "yyyy-MM-dd'T'HH:mm:ss", CultureInfo.InvariantCulture),
+ TimescaleSupport.ParseBlockedBaselineSupplyFloor(comment));
+
+ private static async Task<(long Covered, double Events)> BlockingBaselineTotalsAsync(NpgsqlDataSource postgres, DateTime analysisTime, CancellationToken ct)
+ {
+ var map = await new PgBaselineProvider(postgres).GetBucketMapAsync(-6690, MetricNames.Blocking, analysisTime, analysisTime.AddHours(1), ct);
+ return (map.Buckets.Values.Sum(b => b.SampleCount), map.Buckets.Values.Sum(b => b.Mean * b.SampleCount));
+ }
+
+ [Fact]
+ public async Task BlockingBaseline_OfARebuiltAggregate_CountsNoDayBeforeTheFloor_AndAStoreWithNoFloorReadsAsBefore_AgainstDevPostgres()
+ {
+ var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #6705 supply-floor test.");
+ var ct = TestContext.Current.CancellationToken;
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+ Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "the dev fixture is expected to have TimescaleDB installed");
+ await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection))
+ {
+ await stop.ExecuteNonQueryAsync(ct);
+ }
+
+ await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct);
+
+ /* The operator keeps blocked process reports 10 days, not the 30 of the default. */
+ await using (var retention = new NpgsqlCommand("INSERT INTO config.config_collector_schedules (server_id, collector_name, frequency_minutes, retention_days, enabled) VALUES (NULL, 'blocked_process_report', 1, 10, TRUE)", connection))
+ {
+ await retention.ExecuteNonQueryAsync(ct);
+ }
+
+ await using (var create = new NpgsqlCommand(PreSelfWaitFilterBaselineSql().Replace("WITH NO DATA", "WITH DATA", StringComparison.Ordinal), connection))
+ {
+ await create.ExecuteNonQueryAsync(ct);
+ }
+
+ /* Raw holds one real report 2 days back and one 8 days back (inside the 10), each at 14:00 UTC. The collector logged a
+ SUCCESS run at 14:00 on each of the last 30 days, which the log keeps twice as long as the reports. */
+ var now = LiveClock.Now();
+ var dayStart = now.Date;
+ long id = 67050;
+ foreach (var back in new[] { 2, 8 })
+ {
+ await InsertBlockedReportAsync(connection, id++, dayStart.AddDays(-back).AddHours(14), 1, 2, ct);
+ }
+
+ await using (var log = new NpgsqlCommand(@"
+INSERT INTO collect.collection_log (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, error_message, rows_collected, sql_duration_ms, duckdb_duration_ms)
+SELECT -670500 - g, -6690, 'srv-6690', 'blocked_process_report', $1 - make_interval(days => g) + interval '14 hours', 100, 'SUCCESS', NULL, 0, 80, 20
+FROM generate_series(0, 29) AS g", connection))
+ {
+ log.Parameters.AddWithValue(DateTime.SpecifyKind(dayStart, DateTimeKind.Unspecified));
+ await log.ExecuteNonQueryAsync(ct);
+ }
+
+ /* The reshape drops the old aggregate, the ensure sweep recreates it, the start path's backfill records the floor. */
+ Assert.Equal(1, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct));
+ await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct);
+ Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct));
+ var before = DateTime.UtcNow;
+ Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1);
+ var floor = await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct);
+ Assert.NotNull(floor);
+ Assert.InRange(floor!.Value, before.AddDays(-10).AddMinutes(-5), DateTime.UtcNow.AddDays(-10).AddSeconds(1));
+
+ /* A later start does not move it. */
+ await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct);
+ Assert.Equal(floor, await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct));
+
+ await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString);
+ var runs = Enumerable.Range(0, 30).Select(g => dayStart.AddDays(-g).AddHours(14)).ToList();
+ var coveredAfterFloor = runs.Count(t => t >= floor.Value && t < dayStart);
+ Assert.InRange(coveredAfterFloor, 8, 10);
+
+ var (covered, events) = await BlockingBaselineTotalsAsync(postgres, now, ct);
+ Assert.Equal(coveredAfterFloor, covered);
+ Assert.Equal(2.0, events, precision: 6);
+
+ /* No floor: every logged day of the window is a covered day, as before. */
+ await using (var clear = new NpgsqlCommand($"COMMENT ON VIEW collect.{TimescaleSupport.BlockedProcessBaselineView} IS NULL", connection))
+ {
+ await clear.ExecuteNonQueryAsync(ct);
+ }
+
+ Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct));
+ var (coveredWithoutFloor, eventsWithoutFloor) = await BlockingBaselineTotalsAsync(postgres, now, ct);
+ Assert.Equal(runs.Count(t => t < dayStart), coveredWithoutFloor);
+ Assert.Equal(2.0, eventsWithoutFloor, precision: 6);
+ }
}
diff --git a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs
index c36d5eb701..b2849a358b 100644
--- a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs
+++ b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs
@@ -176,8 +176,80 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup
}
}
+ ///
+ /// #6705: a parallel query waiting on itself (the same non-zero spid on both sides) is not blocking, on the desktop trend,
+ /// the duration stats, the existence check that picks the arm, and the MCP trend and stats reads. A minute with one self-wait
+ /// report and one real report counts 1; a window whose only XE reports are self-waits draws the DMV arm, as the detector does.
+ ///
+ [Fact]
+ public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWaitsDrawTheDmvArm_AgainstDevPostgres()
+ {
+ var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+ Assert.SkipWhen(string.IsNullOrEmpty(connectionString),
+ "Set DARLING_TEST_PG to a Postgres connection string to run the live self-wait trend test.");
+ var ct = TestContext.Current.CancellationToken;
+
+ using var connection = new NpgsqlConnection(connectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+ await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct);
+ await DeleteRowsAsync(connection, "dmv_blocking_snapshots", StatsServerId, ct);
+
+ await using var viewer = new ViewerDataService(connectionString!);
+ await using var postgres = NpgsqlDataSource.Create(connectionString!);
+
+ var bodySucceeded = false;
+ try
+ {
+ var start = TruncateToMinute(DateTime.UtcNow.AddMinutes(-10));
+ var end = start.AddMinutes(11);
+
+ /* One real report (1000 ms) and one self-wait (7000 ms, spid 7 on itself) in the same minute. */
+ await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(5), 1000);
+ await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7);
+ await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(20), 9999);
+
+ var trend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end);
+ Assert.Equal(1, trend.Sum(p => p.Count));
+ var mcpTrend = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
+ Assert.Equal(1, mcpTrend.Sum(p => p.Count));
+
+ var stats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end);
+ Assert.Equal(1, stats.Sum(p => p.EventCount));
+ Assert.Equal(1000, stats.Sum(p => p.TotalDurationMs));
+ Assert.Equal(1000, stats.Max(p => p.MaxDurationMs));
+ var mcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
+ Assert.Equal(1, mcpStats.Sum(p => p.EventCount));
+ Assert.Equal(1000, mcpStats.Sum(p => p.TotalDurationMs));
+ Assert.True(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end));
+
+ /* Only the self-wait remains in XE: the window draws the DMV snapshot, and the existence check says so. */
+ await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct);
+ await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7);
+
+ var dmvTrend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end);
+ Assert.Equal(1, dmvTrend.Sum(p => p.Count));
+ Assert.All(dmvTrend, p => Assert.Equal("DMV snapshot", p.Source));
+ var dmvMcp = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
+ Assert.All(dmvMcp, p => Assert.Equal("DMV snapshot", p.Source));
+ var dmvStats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end);
+ Assert.Equal(9999, dmvStats.Sum(p => p.TotalDurationMs));
+ Assert.False(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end));
+
+ bodySucceeded = true;
+ }
+ finally
+ {
+ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) =>
+ {
+ await DeleteRowsAsync(cleanup, "blocked_process_reports", StatsServerId, cleanupCt);
+ await DeleteRowsAsync(cleanup, "dmv_blocking_snapshots", StatsServerId, cleanupCt);
+ });
+ }
+ }
+
private static async Task InsertBlockedProcessReportAsync(
- NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs)
+ NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs, int blockedSpid = 55, int blockingSpid = 60)
{
using var command = new NpgsqlCommand(@"
INSERT INTO blocked_process_reports
@@ -191,8 +263,8 @@ INSERT INTO blocked_process_reports
command.Parameters.AddWithValue(ServerName);
command.Parameters.AddWithValue(DateTime.SpecifyKind(eventTimeUtc, DateTimeKind.Unspecified));
command.Parameters.AddWithValue("AppDb");
- command.Parameters.AddWithValue(55);
- command.Parameters.AddWithValue(60);
+ command.Parameters.AddWithValue(blockedSpid);
+ command.Parameters.AddWithValue(blockingSpid);
command.Parameters.AddWithValue(waitTimeMs);
command.Parameters.AddWithValue("X");
command.Parameters.AddWithValue("SELECT 1");
diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs
index f0497c7661..434f311f72 100644
--- a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs
+++ b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs
@@ -896,6 +896,19 @@ analyze_server and compare_analysis (BaselineCache) without changing anyone's an
query = await ChooseSupplyAsync(connection, serverId, metricName, query, windowStart, cancellationToken);
}
+ /* #6705 (the #3590 trap): the Blocking arm counts a collection_log run with no event as a measured zero, and the
+ log is kept twice as long as the events behind blocked_process_baseline. After that aggregate was rebuilt it
+ holds only what raw still held, so a day before that would read as a quiet day, not a missing one. The
+ start path recorded where its supply begins; the window starts no earlier. Absent or unreadable is no floor,
+ and it is never derived from the aggregate's own oldest bucket (a server that rarely blocks has real zero days).
+ The text of the arm is unchanged: it still reads $2. */
+ if (keys is null && metricName == MetricNames.Blocking
+ && await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, cancellationToken) is DateTime supplyFloor
+ && supplyFloor > windowStart)
+ {
+ windowStart = supplyFloor;
+ }
+
/* #3653 Q6: the target's clock over this window, read on the same connection and INSIDE this try on
purpose — a store that cannot answer a one-row indexed read of server_properties cannot answer the
aggregate scan either, and one classified catch (AnalysisShutdownResidueTests pins exactly one) is
diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs
index f459be87ce..8b5e1d2be0 100644
--- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs
@@ -50,6 +50,11 @@ public sealed record BlockingTrendReadPoint(DateTime Time, int Count, string? So
/// The database predicate is the list form (#5244), on BOTH arms, as the desktop's BlockingTrendSql has it.
/// The DMV arm's NOT EXISTS (SELECT 1 FROM bpr) therefore asks whether the XE source has rows for the CHOSEN databases:
/// a database with only DMV-captured blocking still charts, whatever other databases' XE reports hold.
+ ///
+ /// A parallel query waiting on itself is not blocking (#6645, #6705): both arms carry
+ /// , as PgAnomalyDetector.BlockingWindowSql does, so the bars (and the web
+ /// Overview strip drawn from this read) agree with the baseline band. The DMV arm is therefore drawn when the FILTERED XE
+ /// arm is empty, a window whose only XE reports are self-waits included.
///
public static readonly string BlockingTrendSql = $$"""
WITH bpr AS (
@@ -58,6 +63,7 @@ FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
+ AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
@@ -66,6 +72,7 @@ FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
+ AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, incident_count, source FROM bpr
diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs
index 3c24870107..3704fa45ae 100644
--- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs
@@ -4822,6 +4822,9 @@ public static async Task> GetBlockedSessionTrendAsy
/// is taken only when the XE source has no row for the CHOSEN databases (NOT EXISTS (SELECT 1 FROM bpr) over the
/// filtered bpr), exactly as the desktop's blocking reads and get_blocking_trend choose, so the tools answer
/// from one source for one filter and the two sources are never mixed. The unfiltered series is exactly what it was.
+ /// Both arms leave out a parallel query waiting on itself (, #6645, #6705),
+ /// as get_blocking_trend does, so the count and the severity reconcile; the DMV fallback is taken when the FILTERED XE
+ /// arm is empty.
///
public static readonly string BlockingDurationStatsSql = $$"""
WITH bpr AS (
@@ -4836,6 +4839,7 @@ FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
+ AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
@@ -4850,6 +4854,7 @@ FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
+ AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, event_count, total_duration_ms, max_duration_ms, avg_duration_ms, source FROM bpr
diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
index d8ddb66ab0..3aaf4660c2 100644
--- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
+++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
@@ -792,10 +792,11 @@ WHERE delta_execution_count > 0
/// Blocking AND BlockingPerMinute baseline supply -- both families share this source and the same
- /// row-level filter, so one aggregate serves both. Since #6663 both leave a parallel query's self-waits out of
- /// their current window (), so the baseline they are compared with
- /// leaves them out too (#6690): a count over every report row would read a server's baseline high and under-flag
- /// its real blocking. Event counts per collection re-aggregate to either shape: Blocking sums them per
+ /// row-level filter, so one aggregate serves both. Both are compared with series that leave a parallel query's
+ /// self-waits out (): Blocking with its current window
+ /// (PgAnomalyDetector.BlockingWindowSql, #6663), BlockingPerMinute with the bars drawn over its band (the Viewer's
+ /// BlockingTrendSql and get_blocking_trend, #6705). So the baseline leaves them out too (#6690): a count over
+ /// every report row would read a server's baseline high and under-flag its real blocking. Event counts per collection re-aggregate to either shape: Blocking sums them per
/// hour/dow bucket, BlockingPerMinute re-buckets by minute. A store that built this aggregate before #6690 is
/// rebuilt by .
public const string CreateBlockedProcessBaselineSql = @"CREATE MATERIALIZED VIEW IF NOT EXISTS collect.blocked_process_baseline
@@ -908,6 +909,13 @@ that created it — see BaselineBackfillProbeSql. */
continue;
}
+ /* #6705: an EMPTY materialization over a raw table that holds rows is a rebuilt (or first-built)
+ aggregate, and it is about to hold only what raw still holds. Say where that begins BEFORE the refresh. */
+ if (view == BlockedProcessBaselineView && coverageOldest is null)
+ {
+ await RecordBlockedBaselineSupplyFloorAsync(connection, logger, DateTime.UtcNow, cancellationToken);
+ }
+
var after = await RefreshFromAsync(connection, view, needFrom.Value, force: false, cancellationToken);
/* VERIFY, DO NOT ASSUME -- the failure this guards is SILENT. On a CAGG that has just been
@@ -1135,6 +1143,82 @@ private static async Task IsStalePlainBlockedBaselineViewAsync(NpgsqlConne
return await probe.ExecuteScalarAsync(cancellationToken) is true;
}
+ ///
+ /// #6705 (the #3590 trap): the prefix of the view comment that records where the Blocking baseline's EVENT supply begins.
+ /// 's event arm counts a collection_log run with no event as a measured zero, and that
+ /// log is kept twice the base, so a day the rebuilt blocked_process_baseline cannot back would read as a quiet day, not a
+ /// missing one. The comment is the floor: the instant the rebuild's raw supply begins (see
+ /// ). No migration: the service owns the view, and a rebuild
+ /// (DROP ... CASCADE) clears the comment with the view.
+ ///
+ public const string BlockedBaselineSupplyFloorPrefix = "supply_floor_utc=";
+
+ private const string SupplyFloorFormat = "yyyy-MM-dd'T'HH:mm:ss";
+
+ /// The view comment and the fleet blocked_process_report retention in force, in one row. A view that does not exist reads NULL.
+ private static readonly string BlockedBaselineSupplyFloorProbeSql =
+ $"SELECT obj_description(to_regclass('collect.{BlockedProcessBaselineView}'), 'pg_class'), "
+ + DarlingRetentionHorizons.FleetRetentionDaysSql("blocked_process_report");
+
+ /// The floor a view comment records; null for an absent comment or one that is not a floor.
+ public static DateTime? ParseBlockedBaselineSupplyFloor(string? comment)
+ => comment is not null
+ && comment.StartsWith(BlockedBaselineSupplyFloorPrefix, StringComparison.Ordinal)
+ && DateTime.TryParseExact(comment.AsSpan(BlockedBaselineSupplyFloorPrefix.Length), SupplyFloorFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out var floor)
+ ? floor
+ : null;
+
+ private static async Task<(DateTime? Floor, int RetentionDays)> ReadBlockedBaselineSupplyAsync(NpgsqlConnection connection, CancellationToken cancellationToken)
+ {
+ using var probe = new NpgsqlCommand(BlockedBaselineSupplyFloorProbeSql, connection) { CommandTimeout = SetupTimeoutSeconds };
+ await using var reader = await probe.ExecuteReaderAsync(cancellationToken);
+ await reader.ReadAsync(cancellationToken);
+ return (ParseBlockedBaselineSupplyFloor(reader.IsDBNull(0) ? null : reader.GetString(0)), Convert.ToInt32(reader.GetValue(1), CultureInfo.InvariantCulture));
+ }
+
+ ///
+ /// The instant before which the rebuilt blocked_process_baseline cannot back a day, or null (no floor recorded: the
+ /// aggregate was never rebuilt by this build, or the comment is not a floor). The provider raises the Blocking
+ /// baseline's window start to it, so the days the aggregate does not hold are left out of the covered days rather than
+ /// counted as quiet. Never the aggregate's own oldest bucket: a server that rarely blocks has real zero days.
+ ///
+ public static async Task ReadBlockedBaselineSupplyFloorAsync(NpgsqlConnection connection, CancellationToken cancellationToken = default)
+ {
+ if (connection is null)
+ {
+ throw new ArgumentNullException(nameof(connection));
+ }
+
+ return (await ReadBlockedBaselineSupplyAsync(connection, cancellationToken)).Floor;
+ }
+
+ ///
+ /// Records the floor on the pass that runs the backfill of a rebuilt blocked_process_baseline:
+ /// minus the blocked_process_report retention the daily purge applies right now
+ /// (, the fleet override or the default). Raw drops its chunks at
+ /// that horizon and the backfill clamps to what raw holds, so the rebuilt aggregate begins there; the aggregate's own 35 days
+ /// never remove a materialized hour that raw has since dropped (the purge is drop_chunks, which records no invalidation,
+ /// and the refresh policy reaches back one day). Written once: a floor already recorded for this view is left alone, so a
+ /// store whose aggregate stays empty (a server with only self-waits) does not move its floor forward on every start.
+ ///
+ private static async Task RecordBlockedBaselineSupplyFloorAsync(NpgsqlConnection connection, ILogger? logger, DateTime utcNow, CancellationToken cancellationToken)
+ {
+ var (recorded, retentionDays) = await ReadBlockedBaselineSupplyAsync(connection, cancellationToken);
+ if (recorded is not null)
+ {
+ return;
+ }
+
+ var floor = utcNow - TimeSpan.FromDays(retentionDays);
+ using var comment = new NpgsqlCommand(
+ $"COMMENT ON VIEW collect.{BlockedProcessBaselineView} IS '{BlockedBaselineSupplyFloorPrefix}{floor.ToString(SupplyFloorFormat, CultureInfo.InvariantCulture)}'",
+ connection) { CommandTimeout = SetupTimeoutSeconds };
+ await comment.ExecuteNonQueryAsync(cancellationToken);
+ logger?.LogInformation(
+ "TimescaleDB: {View} was rebuilt over the {RetentionDays} days of blocked process reports the store keeps -- the Blocking baseline counts no day before {Floor:u}.",
+ BlockedProcessBaselineView, retentionDays, floor);
+ }
+
/// Does this baseline relation exist under any implementation — continuous aggregate or plain view?
public static string BaselineRelationExistsSql(string view)
=> $"SELECT to_regclass('collect.{view}') IS NOT NULL";
diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs
index 6b9073f241..120d2925fb 100644
--- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs
+++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs
@@ -315,16 +315,18 @@ public async Task ReadRecentBlockedProcessRepor
///
/// Whether the XE blocked_process_reports source holds a report in the window: the same predicate the Blocking charts' reads
- /// use for their bpr arm (event time in the window, the event-window floor on collection time, the database filter),
- /// so true here means the charts draw from XE rows and not from the DMV fallback beside them.
+ /// use for their bpr arm (event time in the window, the event-window floor on collection time, the database filter,
+ /// and : a parallel query waiting on itself is not blocking, #6705), so true here
+ /// means the charts draw from XE rows and not from the DMV fallback beside them.
///
- public const string BlockedProcessReportsInWindowSql = """
+ public const string BlockedProcessReportsInWindowSql = $"""
SELECT EXISTS (
SELECT 1
FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $5
AND ($4::text[] IS NULL OR database_name = ANY($4))
+ AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
)
""";
diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs
index ca3525bf2a..f764c86554 100644
--- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs
+++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs
@@ -50,8 +50,10 @@ public sealed partial class ViewerDataService
/// drop a row (an event is collected after it happens). $5 is the database filter (#5244), the same
/// text[] (NULL for all) the count trend binds, on BOTH arms so the severity reconciles with it.
/// The last column names the arm that answered, the tag Darling's MCP answers carry.
+ /// Both arms leave out a parallel query waiting on itself (, #6645, #6705), as the
+ /// count trend does, so the two reconcile and the severity is the blocking's, not the query's own.
///
- public const string BlockingDurationStatsSql = """
+ public const string BlockingDurationStatsSql = $"""
WITH bpr AS (
SELECT
DATE_TRUNC('minute', event_time) AS bucket,
@@ -64,6 +66,7 @@ FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
AND ($5::text[] IS NULL OR database_name = ANY($5))
+ AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
@@ -78,6 +81,7 @@ FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
AND ($5::text[] IS NULL OR database_name = ANY($5))
+ AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, event_count, total_duration_ms, max_duration_ms, avg_duration_ms, source FROM bpr
diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs
index b4298aa91b..eb50e3ef7a 100644
--- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs
+++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs
@@ -68,14 +68,21 @@ public sealed partial class ViewerDataService
/// collection_time, which this event-time window alone gives the planner nothing to exclude a chunk
/// on (#4229); the floor lets it skip every chunk older than the window, without being able to drop a row
/// (an event is collected after it happens).
+ ///
+ /// A parallel query waiting on itself is not blocking (#6645, #6705): both arms carry
+ /// , so the bars agree with the baseline band drawn over them
+ /// (blocked_process_baseline leaves self-waits out, #6690) and with PgAnomalyDetector.BlockingWindowSql.
+ /// The DMV arm is drawn when the FILTERED XE arm is empty, so a window whose only XE reports are self-waits falls back to
+ /// the DMV snapshots, as the anomaly detector does; takes the same predicate.
///
- public const string BlockingTrendSql = """
+ public const string BlockingTrendSql = $"""
WITH bpr AS (
SELECT DATE_TRUNC('minute', event_time) AS bucket, COUNT(*) AS incident_count, 'blocked-process-report' AS source
FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $5
AND ($4::text[] IS NULL OR database_name = ANY($4))
+ AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
@@ -84,6 +91,7 @@ FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $5
AND ($4::text[] IS NULL OR database_name = ANY($4))
+ AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, incident_count, source FROM bpr
From 44b8d014fd2db4d6b6d4b526174dc1939ae6200e Mon Sep 17 00:00:00 2001
From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com>
Date: Sat, 24 Oct 2026 05:40:59 -0400
Subject: [PATCH 4/4] Leave self-waits out of the Darling blocking slicer and
the fleet band's report count, record the baseline supply floor when the view
is created (#6705)
---
.../FleetPartialDatabaseFailureBandTests.cs | 40 ++++++++++++++--
.../Darling.Tests/ViewerBlockingStatsTests.cs | 48 ++++++++++++++-----
.../Mcp/DarlingFleetReader.cs | 7 ++-
.../TimescaleSupport.cs | 45 ++++++++++++++++-
.../ViewerDataService.Blocking.cs | 4 +-
.../ViewerDataService.BlockingSlicers.cs | 9 +++-
6 files changed, 133 insertions(+), 20 deletions(-)
diff --git a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
index ac81fc5c76..2912023223 100644
--- a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
+++ b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs
@@ -748,6 +748,28 @@ public void TheSupplyFloorComment_ParsesOnlyAValidFloor(string? comment, string?
return (map.Buckets.Values.Sum(b => b.SampleCount), map.Buckets.Values.Sum(b => b.Mean * b.SampleCount));
}
+ /// #6705: the ensure sweep records no floor on a fresh store, where raw holds nothing and the aggregate will hold everything raw ever gets.
+ [Fact]
+ public async Task EnsureOnAFreshStore_RecordsNoSupplyFloor_AgainstDevPostgres()
+ {
+ var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
+ Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #6705 fresh-store test.");
+ var ct = TestContext.Current.CancellationToken;
+ await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct);
+ await using var connection = new NpgsqlConnection(scratch.ConnectionString);
+ await connection.OpenAsync(ct);
+ await PgMigrations.MigrateAsync(connection, ct);
+ Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "the dev fixture is expected to have TimescaleDB installed");
+ await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection))
+ {
+ await stop.ExecuteNonQueryAsync(ct);
+ }
+
+ await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct);
+ await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct);
+ Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct));
+ }
+
[Fact]
public async Task BlockingBaseline_OfARebuiltAggregate_CountsNoDayBeforeTheFloor_AndAStoreWithNoFloorReadsAsBefore_AgainstDevPostgres()
{
@@ -796,16 +818,26 @@ FROM generate_series(0, 29) AS g", connection))
await log.ExecuteNonQueryAsync(ct);
}
- /* The reshape drops the old aggregate, the ensure sweep recreates it, the start path's backfill records the floor. */
+ /* The reshape drops the old aggregate and the ensure sweep recreates it over raw rows, which records the floor on that
+ pass: a service that stops before the backfill, with the refresh policy then materializing the last day, still has it. */
Assert.Equal(1, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct));
- await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct);
- Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct));
var before = DateTime.UtcNow;
- Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1);
+ await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct);
var floor = await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct);
Assert.NotNull(floor);
Assert.InRange(floor!.Value, before.AddDays(-10).AddMinutes(-5), DateTime.UtcNow.AddDays(-10).AddSeconds(1));
+ /* The policy's run materializes the last day before the backfill; the backfill then sees a non-empty materialization. */
+ await using (var recentOnly = new NpgsqlCommand("CALL refresh_continuous_aggregate('collect.blocked_process_baseline', $1, $2)", connection))
+ {
+ recentOnly.Parameters.AddWithValue(now.AddDays(-1));
+ recentOnly.Parameters.AddWithValue(now.AddHours(1));
+ await recentOnly.ExecuteNonQueryAsync(ct);
+ }
+
+ Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1);
+ Assert.Equal(floor, await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct));
+
/* A later start does not move it. */
await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct);
Assert.Equal(floor, await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct));
diff --git a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs
index b2849a358b..9dab43e4dc 100644
--- a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs
+++ b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs
@@ -180,6 +180,8 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup
/// #6705: a parallel query waiting on itself (the same non-zero spid on both sides) is not blocking, on the desktop trend,
/// the duration stats, the existence check that picks the arm, and the MCP trend and stats reads. A minute with one self-wait
/// report and one real report counts 1; a window whose only XE reports are self-waits draws the DMV arm, as the detector does.
+ /// The Blocking-tab slicer reads the same pair by the hour and agrees (Lite's BlockingSeries_LeaveSelfWaitsOut_OnTheXeArm).
+ /// A (0, 0) report is a system spid pair the predicate keeps, so it counts.
///
[Fact]
public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWaitsDrawTheDmvArm_AgainstDevPostgres()
@@ -204,25 +206,36 @@ public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWai
var start = TruncateToMinute(DateTime.UtcNow.AddMinutes(-10));
var end = start.AddMinutes(11);
- /* One real report (1000 ms) and one self-wait (7000 ms, spid 7 on itself) in the same minute. */
+ /* One real report (1000 ms), one self-wait (7000 ms, spid 7 on itself) and one (0, 0) report (3000 ms) in the same minute. */
await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(5), 1000);
+ await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(25), 3000, blockedSpid: 0, blockingSpid: 0);
await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7);
await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(20), 9999);
+ /* A DMV self-wait (spid 166 on itself, 5555 ms) in the same minute: every DMV-arm read below must leave it out. */
+ await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(25), 5555, blockedSpid: 166, blockingSpid: 166, collectionId: 2);
var trend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end);
- Assert.Equal(1, trend.Sum(p => p.Count));
+ Assert.Equal(2, trend.Sum(p => p.Count));
var mcpTrend = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
- Assert.Equal(1, mcpTrend.Sum(p => p.Count));
+ Assert.Equal(2, mcpTrend.Sum(p => p.Count));
var stats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end);
- Assert.Equal(1, stats.Sum(p => p.EventCount));
- Assert.Equal(1000, stats.Sum(p => p.TotalDurationMs));
- Assert.Equal(1000, stats.Max(p => p.MaxDurationMs));
+ Assert.Equal(2, stats.Sum(p => p.EventCount));
+ Assert.Equal(4000, stats.Sum(p => p.TotalDurationMs));
+ Assert.Equal(3000, stats.Max(p => p.MaxDurationMs));
var mcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
- Assert.Equal(1, mcpStats.Sum(p => p.EventCount));
- Assert.Equal(1000, mcpStats.Sum(p => p.TotalDurationMs));
+ Assert.Equal(2, mcpStats.Sum(p => p.EventCount));
+ Assert.Equal(4000, mcpStats.Sum(p => p.TotalDurationMs));
Assert.True(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end));
+ /* The slicer: the real and (0, 0) reports count, the 7000 ms self-wait does not (4.0 s, not 11.0 s), and it
+ names two blockers (60 and 0) and two blocked spids (55 and 0), not the self-wait's spid 7. */
+ var slicer = Assert.Single(await viewer.GetBlockingSlicerDataAsync(StatsServerId, start.AddMinutes(-1), end));
+ Assert.Equal(2, slicer.SessionCount);
+ Assert.Equal(4.0, slicer.TotalCpu);
+ Assert.Equal(2, slicer.TotalElapsed);
+ Assert.Equal(2, slicer.TotalReads);
+
/* Only the self-wait remains in XE: the window draws the DMV snapshot, and the existence check says so. */
await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct);
await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7);
@@ -232,10 +245,20 @@ public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWai
Assert.All(dmvTrend, p => Assert.Equal("DMV snapshot", p.Source));
var dmvMcp = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
Assert.All(dmvMcp, p => Assert.Equal("DMV snapshot", p.Source));
+ Assert.Equal(1, dmvMcp.Sum(p => p.Count));
var dmvStats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end);
Assert.Equal(9999, dmvStats.Sum(p => p.TotalDurationMs));
+ Assert.Equal(1, dmvStats.Sum(p => p.EventCount));
+ var dmvMcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
+ Assert.Equal(1, dmvMcpStats.Sum(p => p.EventCount));
+ Assert.Equal(9999, dmvMcpStats.Sum(p => p.TotalDurationMs));
Assert.False(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end));
+ /* The slicer draws the DMV arm too: its one real snapshot (9999 ms), none of the XE self-wait's 7000 ms or the DMV self-wait's 5555 ms. */
+ var dmvSlicer = Assert.Single(await viewer.GetBlockingSlicerDataAsync(StatsServerId, start.AddMinutes(-1), end));
+ Assert.Equal(1, dmvSlicer.SessionCount);
+ Assert.Equal(9.999, dmvSlicer.TotalCpu, 3);
+
bodySucceeded = true;
}
finally
@@ -275,21 +298,22 @@ INSERT INTO blocked_process_reports
}
private static async Task InsertDmvBlockingSnapshotAsync(
- NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs)
+ NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs,
+ int blockedSpid = 155, int blockingSpid = 160, long collectionId = 1)
{
using var command = new NpgsqlCommand(@"
INSERT INTO dmv_blocking_snapshots
(collection_id, collection_time, server_id, server_name, event_time, database_name,
blocked_spid, blocking_spid, wait_time_ms, lock_mode, blocked_sql_text, blocking_sql_text, contentious_object)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", connection);
- command.Parameters.AddWithValue(1L);
+ command.Parameters.AddWithValue(collectionId);
command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTimeUtc, DateTimeKind.Unspecified));
command.Parameters.AddWithValue(serverId);
command.Parameters.AddWithValue(ServerName);
command.Parameters.AddWithValue(DateTime.SpecifyKind(eventTimeUtc, DateTimeKind.Unspecified));
command.Parameters.AddWithValue("AppDb");
- command.Parameters.AddWithValue(155);
- command.Parameters.AddWithValue(160);
+ command.Parameters.AddWithValue(blockedSpid);
+ command.Parameters.AddWithValue(blockingSpid);
command.Parameters.AddWithValue(waitTimeMs);
command.Parameters.AddWithValue("S");
command.Parameters.AddWithValue("SELECT 2");
diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs
index 921b5b7b00..a2431c1ac5 100644
--- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs
+++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs
@@ -355,7 +355,10 @@ the alert engine and the analysis fact share. The DMV side applies the same rule
band's count tiers (5 / 20 per hour) were measured in blocked-process REPORTS per server-hour, so the band
reads those; the tile, the alert value and the MCP counts stay waits. Each source is read once (the
*_reports CTE, shared by the report count and the waits chain), so the window and the floor are still
- applied once per source. */
+ applied once per source.
+
+ #6705: both reads leave a parallel query waiting on itself out (BlockingWaits.NotSelfWaitSql), so the report count
+ the band reads agrees with the waits count, which WaitsCteSql already filters. */
public static readonly string FleetBlockingSql = @"
WITH xe_reports AS (
SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms
@@ -363,6 +366,7 @@ FROM v_blocked_process_reports
WHERE event_time >= $1
AND event_time <= $2
AND collection_time >= $3
+ AND " + BlockingWaits.NotSelfWaitSql + @"
),
dmv_reports AS (
SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms
@@ -370,6 +374,7 @@ FROM v_dmv_blocking_snapshots
WHERE event_time >= $1
AND event_time <= $2
AND collection_time >= $3
+ AND " + BlockingWaits.NotSelfWaitSql + @"
)," + BlockingWaits.WaitsCteSql(
BlockingWaits.SqlDialect.Postgres, "xe_waits",
"SELECT * FROM xe_reports", new[] { "server_id" }) + "," + BlockingWaits.WaitsCteSql(
diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
index 3aaf4660c2..022c7bc4b2 100644
--- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
+++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs
@@ -913,7 +913,7 @@ that created it — see BaselineBackfillProbeSql. */
aggregate, and it is about to hold only what raw still holds. Say where that begins BEFORE the refresh. */
if (view == BlockedProcessBaselineView && coverageOldest is null)
{
- await RecordBlockedBaselineSupplyFloorAsync(connection, logger, DateTime.UtcNow, cancellationToken);
+ await TryRecordBlockedBaselineSupplyFloorAsync(connection, logger, cancellationToken);
}
var after = await RefreshFromAsync(connection, view, needFrom.Value, force: false, cancellationToken);
@@ -1219,6 +1219,31 @@ private static async Task RecordBlockedBaselineSupplyFloorAsync(NpgsqlConnection
BlockedProcessBaselineView, retentionDays, floor);
}
+ private static async Task RawHoldsRowsAsync(NpgsqlConnection connection, string sourceTable, CancellationToken cancellationToken)
+ {
+ using var probe = new NpgsqlCommand($"SELECT EXISTS (SELECT 1 FROM collect.{sourceTable})", connection) { CommandTimeout = SetupTimeoutSeconds };
+ return await probe.ExecuteScalarAsync(cancellationToken) is true;
+ }
+
+ ///
+ /// #6705: , failure-isolated. A failed comment write must not skip the
+ /// backfill that follows it (it would fail again on every start and the baseline would never be backfilled); it logs and the
+ /// pass goes on, and the next start tries the write again.
+ ///
+ private static async Task TryRecordBlockedBaselineSupplyFloorAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken)
+ {
+ try
+ {
+ await RecordBlockedBaselineSupplyFloorAsync(connection, logger, DateTime.UtcNow, cancellationToken);
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ logger?.LogWarning(
+ "Could not record where the {View} supply begins -- the Blocking baseline counts quiet days it cannot back until this succeeds: {Message}",
+ BlockedProcessBaselineView, ex.Message);
+ }
+ }
+
/// Does this baseline relation exist under any implementation — continuous aggregate or plain view?
public static string BaselineRelationExistsSql(string view)
=> $"SELECT to_regclass('collect.{view}') IS NOT NULL";
@@ -6501,11 +6526,29 @@ under the exact names the baseline aggregates need (#1757). CREATE MATERIALIZED
{
try
{
+ /* #6705: was the blocking baseline absent before this CREATE? The plain fallback view was dropped above, so
+ a relation that exists here is a real aggregate. Read before the CREATE, as IF NOT EXISTS says nothing. */
+ var createsBlockedBaseline = false;
+ if (view == BlockedProcessBaselineView)
+ {
+ using var exists = new NpgsqlCommand(BaselineRelationExistsSql(view), connection) { CommandTimeout = SetupTimeoutSeconds };
+ createsBlockedBaseline = await exists.ExecuteScalarAsync(cancellationToken) is false;
+ }
+
using (var create = new NpgsqlCommand(createSql, connection) { CommandTimeout = SetupTimeoutSeconds })
{
await create.ExecuteNonQueryAsync(cancellationToken);
}
+ /* #6705: the floor is written on the pass that CREATES the baseline, before its refresh policy can materialize
+ anything. Written only at backfill time, a service that stopped between this CREATE and the backfill left the
+ policy to materialize the last day first, so the backfill saw a non-empty materialization and recorded
+ nothing. A fresh store (raw empty) records nothing: its aggregate will hold everything raw ever gets. */
+ if (createsBlockedBaseline && await RawHoldsRowsAsync(connection, SourceTableFor(view), cancellationToken))
+ {
+ await TryRecordBlockedBaselineSupplyFloorAsync(connection, logger, cancellationToken);
+ }
+
/* #3893: an off-grid aggregate's materialization width is set HERE, after its CREATE and BEFORE
its policy exists, because nothing later would be early enough. #3620's width ensure runs in
the aggregate-compression step, a later convergence stage, and walks the compression targets
diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs
index 120d2925fb..c869cf38aa 100644
--- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs
+++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs
@@ -437,7 +437,9 @@ OR EXISTS (SELECT 1 FROM inside AS i CROSS JOIN first_on AS f
///
/// Where the XE blocked-process-report data starts for one server, as two columns: the XE collector's first logged run
/// in the window (or the window's start when a run exists before it, which is covered), and the earliest XE report in the
- /// window (the same predicate as ). The caller takes the earlier of the two,
+ /// window, a self-wait report included (it still proves the XE session was capturing, so this read leaves out the
+ /// filter that takes, #6705).
+ /// The caller takes the earlier of the two,
/// through , which also folds in the threshold's history.
/// $1 server, $2 window start, $3 window end, $4 database filter, $5 the event-window floor on collection time. Both run
/// reads are bounded, the first run in [$2, $3] and the newest run before $2, and ride
diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs
index d61241d174..d50e1ffa33 100644
--- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs
+++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs
@@ -37,8 +37,13 @@ public sealed partial class ViewerDataService
/// $1 server_id, $2 window start, $3 window end (naive UTC), $4 database filter. The XE arm windows and
/// buckets on the report's own event_time; the DMV arm stays on collection_time, which IS its
/// event time. $5 is the for $2 (XE arm only; no upper bound).
+ ///
+ /// A parallel query waiting on itself is not blocking (#6645, #6705): both arms carry
+ /// , as Lite's slicer does, so the bars agree with
+ /// and the baseline band. The DMV arm is drawn when the FILTERED XE arm has no buckets, so a
+ /// window whose only XE reports are self-waits falls back to the DMV snapshots.
///
- public const string BlockingSlicerSql = """
+ public const string BlockingSlicerSql = $"""
WITH bpr AS (
SELECT
date_trunc('hour', event_time) AS bucket,
@@ -51,6 +56,7 @@ FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $5
AND ($4::text[] IS NULL OR database_name = ANY($4))
+ AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
GROUP BY date_trunc('hour', event_time)
),
dmv AS (
@@ -64,6 +70,7 @@ dmv AS (
FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3
AND ($4::text[] IS NULL OR database_name = ANY($4))
+ AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
GROUP BY date_trunc('hour', collection_time)
)
SELECT bucket, event_count, total_wait_sec, distinct_blockers, distinct_blocked, distinct_databases FROM bpr