From 9784da31849926605e79bd94df821a8437bd3763 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 24 Oct 2026 04:27:00 -0400 Subject: [PATCH 1/4] Darling blocking baseline leaves out self-waits, like the window it is compared with (#6690) --- Darling/Darling.Tests/BaselineSupplyTests.cs | 6 +- .../FleetPartialDatabaseFailureBandTests.cs | 125 ++++++++++++++++++ .../PgAnomalyDetector.cs | 7 +- .../TimescaleSupport.cs | 56 +++++++- 4 files changed, 183 insertions(+), 11 deletions(-) diff --git a/Darling/Darling.Tests/BaselineSupplyTests.cs b/Darling/Darling.Tests/BaselineSupplyTests.cs index 1ee0375f70..d2a5628f42 100644 --- a/Darling/Darling.Tests/BaselineSupplyTests.cs +++ b/Darling/Darling.Tests/BaselineSupplyTests.cs @@ -107,13 +107,13 @@ and the legacy one they still fall back to are shared the same way. */ Assert.Contains("delta_wait_time_ms >= 0", TimescaleSupport.LegacyCreateWaitStatsBaselineSql, StringComparison.Ordinal); Assert.DoesNotContain("sample_interval_seconds", TimescaleSupport.LegacyCreateWaitStatsBaselineSql, StringComparison.Ordinal); - /* blocked_process_reports: neither family filters, so the shared aggregate carries no WHERE at all. - A WHERE appearing here later means one family narrowed the other's supply. */ + /* blocked_process_reports: both families leave a parallel query's self-waits out of their current window + (#6663), so the shared aggregate carries the same predicate (#6690), or its baseline reads high. */ var blocking = PgBaselineProvider.GetBaselineQuery(MetricNames.Blocking)!; var blockingPerMinute = PgBaselineProvider.GetBaselineQuery(MetricNames.BlockingPerMinute)!; Assert.Contains("FROM blocked_process_baseline", blocking, StringComparison.Ordinal); Assert.Contains("FROM blocked_process_baseline", blockingPerMinute, StringComparison.Ordinal); - Assert.DoesNotContain("WHERE", TimescaleSupport.CreateBlockedProcessBaselineSql, StringComparison.Ordinal); + Assert.Contains(PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql, TimescaleSupport.CreateBlockedProcessBaselineSql, StringComparison.Ordinal); } /// diff --git a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs index 77d180ea1e..020022600b 100644 --- a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs +++ b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs @@ -600,4 +600,129 @@ public async Task StoreWhoseRollupPredatesTheNoteColumn_IsRebuiltAndRefreshedOnc Assert.Equal("HEALTHY", warmedBands[collectors[1]]); Assert.Equal("HEALTHY", warmedBands[collectors[2]]); } + + /* ───────────── #6690: the blocking baseline aggregate leaves self-waits out ───────────── */ + + private const string BlockedReportInsertSql = @" +INSERT INTO collect.blocked_process_reports (blocked_report_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocked_ecid, blocking_spid, blocking_ecid, wait_time_ms, wait_resource, lock_mode, blocked_status, blocked_isolation_level, blocked_log_used, blocked_transaction_count, blocked_client_app, blocked_host_name, blocked_login_name, blocked_sql_text, blocking_status, blocking_isolation_level, blocking_client_app, blocking_host_name, blocking_login_name, blocking_sql_text, blocked_transaction_name, blocking_transaction_name, blocked_last_tran_started, blocking_last_tran_started, blocked_last_batch_started, blocking_last_batch_started, blocked_last_batch_completed, blocking_last_batch_completed, blocked_priority, blocking_priority, blocked_process_report_xml, object_id, database_id, contentious_object, monitor_loop, blocked_query_plan_xml, blocking_query_plan_xml) +VALUES ($1, $2, -6690, 'srv-6690', $2, 'testdb', $3, 0, $4, 0, 100, 'KEY', 'X', 'suspended', 'READ COMMITTED', 0, 1, 'app', 'host', 'login', 'select 1', 'suspended', 'READ COMMITTED', 'app', 'host', 'login', 'select 2', NULL, NULL, $2, $2, $2, $2, $2, $2, 0, 0, '', 1, 1, 'obj', 1, NULL, NULL)"; + + private static string PreSelfWaitFilterBaselineSql() + => Regex.Replace(TimescaleSupport.CreateBlockedProcessBaselineSql, @"WHERE [^\r\n]*\r?\n", string.Empty); + + private static async Task InsertBlockedReportAsync(NpgsqlConnection connection, long id, DateTime t, int blocked, int blocking, CancellationToken ct) + { + await using var insert = new NpgsqlCommand(BlockedReportInsertSql, connection); + insert.Parameters.AddWithValue(id); + insert.Parameters.AddWithValue(t); + insert.Parameters.AddWithValue(blocked); + insert.Parameters.AddWithValue(blocking); + await insert.ExecuteNonQueryAsync(ct); + } + + private static async Task BaselineEventsAsync(NpgsqlConnection connection, DateTime from, DateTime to, CancellationToken ct) + { + await using var sum = new NpgsqlCommand("SELECT COALESCE(sum(event_count), 0) FROM collect.blocked_process_baseline WHERE server_id = -6690 AND collection_time >= $1 AND collection_time < $2", connection); + sum.Parameters.AddWithValue(from); + sum.Parameters.AddWithValue(to); + return Convert.ToInt64(await sum.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + } + + [Fact] + public async Task BlockingBaselineBuiltWithoutTheSelfWaitFilter_IsRebuiltFilteredAndBackfilledOverTheWholeWindow_AgainstDevPostgres() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #6690 reshape test."); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "the dev fixture is expected to have TimescaleDB installed"); + + /* No scheduler: the refresh policy's own run must not move the watermark under the test. */ + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + + /* The pre-#6690 aggregate: the same select with no WHERE. */ + var oldShape = PreSelfWaitFilterBaselineSql(); + Assert.DoesNotContain("blocking_spid", oldShape, StringComparison.Ordinal); + await using (var create = new NpgsqlCommand(oldShape.Replace("WITH NO DATA", "WITH DATA", StringComparison.Ordinal), connection)) + { + await create.ExecuteNonQueryAsync(ct); + } + + /* A recent day and a day older than any refresh policy's window, each holding one real report (spid 1 + blocked by 2) and one self-wait (spid 5 waiting on itself). */ + var now = LiveClock.Now(); + long id = 66900; + foreach (var t in new[] { now.AddHours(-3), now.AddDays(-10) }) + { + await InsertBlockedReportAsync(connection, id++, t, 1, 2, ct); + await InsertBlockedReportAsync(connection, id++, t, 5, 5, ct); + } + + await using (var refreshAll = new NpgsqlCommand("CALL refresh_continuous_aggregate('collect.blocked_process_baseline', NULL, NULL)", connection)) + { + await refreshAll.ExecuteNonQueryAsync(ct); + } + + Assert.Equal(2L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct)); + + /* The sweep drops it (only it), the ensure sweep recreates it filtered. */ + Assert.Equal(1, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct)); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + + /* The trap: the policy's recent window is all a refresh would cover, so the old day reads EMPTY. */ + await using (var recentOnly = new NpgsqlCommand("CALL refresh_continuous_aggregate('collect.blocked_process_baseline', $1, $2)", connection)) + { + recentOnly.Parameters.AddWithValue(now.AddDays(-1)); + recentOnly.Parameters.AddWithValue(now.AddHours(1)); + await recentOnly.ExecuteNonQueryAsync(ct); + } + + Assert.Equal(0L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct)); + + /* The start path's coverage-gated backfill closes it over the whole baseline window, counting only the real report. */ + Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1); + Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct)); + Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-1), now.AddHours(1), ct)); + + /* A converged store: nothing left to drop. */ + Assert.Equal(0, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct)); + Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-11), now.AddDays(-9), ct)); + } + + [Fact] + public async Task PlainBlockingBaselineViewWithoutTheSelfWaitFilter_IsReplacedInPlace_AgainstDevPostgres() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string to run the live #6690 plain-view test."); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + + /* A store without TimescaleDB that built the plain fallback view before #6690. */ + await using (var create = new NpgsqlCommand(TimescaleSupport.CreateBaselineFallbackViewSql(TimescaleSupport.BlockedProcessBaselineView, PreSelfWaitFilterBaselineSql()), connection)) + { + await create.ExecuteNonQueryAsync(ct); + } + + var now = LiveClock.Now(); + await InsertBlockedReportAsync(connection, 66950, now.AddHours(-3), 1, 2, ct); + await InsertBlockedReportAsync(connection, 66951, now.AddHours(-3), 5, 5, ct); + + Assert.Equal(2L, await BaselineEventsAsync(connection, now.AddDays(-1), now, ct)); + await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct); + Assert.Equal(1L, await BaselineEventsAsync(connection, now.AddDays(-1), now, ct)); + + /* Converged: the next pass fills and replaces nothing. */ + Assert.Equal(0, await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct)); + } } diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs index 953759d6d6..b2da181b43 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs @@ -307,10 +307,9 @@ the CURRENT window counts report and deadlock events by when they HAPPENED (even snapshot's event_time IS its collection time. The BASELINE this count is judged against comes from the continuous aggregates, which bucket the hypertable's own column (collection_time) by design. - #6645: this CURRENT count leaves out a parallel query's self-waits, but the BASELINE aggregate - (collect.blocked_process_baseline, count(*) over every report row) still counts them until a store - migration rebuilds it with the same predicate, so a server with many self-waits has a high baseline and - its real blocking is under-flagged in the meantime. */ + #6645: this CURRENT count leaves out a parallel query's self-waits, and so does the BASELINE aggregate + (collect.blocked_process_baseline, #6690: its select carries the same BlockingWaits.NotSelfWaitSql), so the + ratio compares like with like. A store that built the aggregate earlier is rebuilt on start. */ public const string BlockingWindowSql = $@" /* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking. */ SELECT diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index 6339e209ae..d7b86a5aff 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -791,9 +791,13 @@ WHERE delta_execution_count > 0 WITH NO DATA"; - /// Blocking AND BlockingPerMinute baseline supply -- both families share this source and both - /// have NO row-level filter, so one aggregate serves both. Event counts per collection re-aggregate to - /// either shape: Blocking sums them per hour/dow bucket, BlockingPerMinute re-buckets by minute. + /// Blocking AND BlockingPerMinute baseline supply -- both families share this source and the same + /// row-level filter, so one aggregate serves both. Since #6663 both leave a parallel query's self-waits out of + /// their current window (), so the baseline they are compared with + /// leaves them out too (#6690): a count over every report row would read a server's baseline high and under-flag + /// its real blocking. Event counts per collection re-aggregate to either shape: Blocking sums them per + /// hour/dow bucket, BlockingPerMinute re-buckets by minute. A store that built this aggregate before #6690 is + /// rebuilt by . public const string CreateBlockedProcessBaselineSql = @"CREATE MATERIALIZED VIEW IF NOT EXISTS collect.blocked_process_baseline WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT @@ -802,6 +806,7 @@ WHERE delta_execution_count > 0 collection_time, count(*) AS event_count FROM collect.blocked_process_reports +WHERE " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql + @" GROUP BY server_id, bucket, collection_time WITH NO DATA"; @@ -1068,6 +1073,23 @@ public static async Task EnsureBaselineFallbackViewsAsync( { if (await probe.ExecuteScalarAsync(cancellationToken) is true) { + /* #6690: the one relation that is "present" and still wrong: a plain view built before the + blocking baseline left self-waits out. A plain view has no data to lose, so it is replaced + in place; a continuous aggregate is never touched here (the reshape sweep rebuilds it). */ + if (view == BlockedProcessBaselineView && await IsStalePlainBlockedBaselineViewAsync(connection, cancellationToken)) + { + using var replace = new NpgsqlCommand(CreateBaselineFallbackViewSql(view, createSql), connection) + { + CommandTimeout = SetupTimeoutSeconds, + }; + await replace.ExecuteNonQueryAsync(cancellationToken); + filled++; + + logger?.LogInformation( + "Baseline relation {View} was a plain view that still counted blocking self-waits (#6690) -- replaced it with the filtered select.", + view); + } + continue; } } @@ -1094,6 +1116,25 @@ public static async Task EnsureBaselineFallbackViewsAsync( return filled; } + /// + /// #6690: is collect.blocked_process_baseline a PLAIN view (not a continuous aggregate) whose select does + /// not carry the self-wait filter? is null for a plain view and for a + /// store without the extension, which is what separates it from an aggregate (also a relkind='v' view); + /// the filter is recognised by its blocking_spid column, which the pre-#6690 select never named. + /// + private static async Task IsStalePlainBlockedBaselineViewAsync(NpgsqlConnection connection, CancellationToken cancellationToken) + { + if (await ResolveMaterializationAsync(connection, BlockedProcessBaselineView, cancellationToken) is not null) + { + return false; + } + + using var probe = new NpgsqlCommand( + $"SELECT COALESCE(position('blocking_spid' IN pg_get_viewdef('collect.{BlockedProcessBaselineView}'::regclass)) = 0, false) FROM pg_class WHERE oid = 'collect.{BlockedProcessBaselineView}'::regclass AND relkind = 'v'", + connection) { CommandTimeout = SetupTimeoutSeconds }; + return await probe.ExecuteScalarAsync(cancellationToken) is true; + } + /// Does this baseline relation exist under any implementation — continuous aggregate or plain view? public static string BaselineRelationExistsSql(string view) => $"SELECT to_regclass('collect.{view}') IS NOT NULL"; @@ -6202,6 +6243,13 @@ The ensure sweep recreates it WITH NO DATA and the eight-day refresh window re-m answer from raw, exactly; WarmCollectionHealthHourlyAsync closes that window with one refresh. */ (View: CollectionHealthHourlyView, StaleCheck: "SELECT (EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'collect' AND table_name = 'collection_health_hourly') AND NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = 'collect' AND table_name = 'collection_health_hourly' AND column_name = 'latest_run_note'))"), + /* blocked_process_baseline gained the self-wait filter (#6690) -> stale iff the aggregate EXISTS but its + definition does not mention blocking_spid (the filter's column; the old text had no WHERE at all). + Nothing is built on it, so the CASCADE drops only the view; the ensure sweep recreates it WITH NO DATA + and the start path's coverage-gated BackfillBaselineAggregatesAsync (its materialization is empty, so + its gate fires on this very start) re-materializes it from raw. NULL view_definition is not stale. */ + (View: BlockedProcessBaselineView, + StaleCheck: "SELECT EXISTS (SELECT 1 FROM timescaledb_information.continuous_aggregates WHERE view_schema = 'collect' AND view_name = 'blocked_process_baseline' AND position('blocking_spid' IN view_definition) = 0)"), }; var dropped = 0; @@ -6237,7 +6285,7 @@ The ensure sweep recreates it WITH NO DATA and the eight-day refresh window re-m hold = AggregateJobHold.None; dropped++; logger?.LogInformation( - "TimescaleDB: dropped stale continuous aggregate {View} (composer-dimension reshape) — recreated in the new shape this cycle.", + "TimescaleDB: dropped stale continuous aggregate {View} (reshape) — recreated in the new shape this cycle.", view); } catch (OperationCanceledException) when (hold.Unscheduled.Count > 0) From 9047271f4a2476f80792e52ed53a969ba4cd8d18 Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:08:32 -0400 Subject: [PATCH 2/4] Leave self-waits out of Lite's per-minute blocking series and baseline (#6705) --- Lite.Tests/BlockingStatsReaderTests.cs | 63 +++++++++++++++++-- Lite.Tests/EventBaselineCoveredDaysTests.cs | 29 +++++++++ Lite/Analysis/BaselineProvider.cs | 4 +- Lite/Services/LocalDataService.Blocking.cs | 18 +++--- .../LocalDataService.BlockingStats.cs | 4 +- 5 files changed, 100 insertions(+), 18 deletions(-) diff --git a/Lite.Tests/BlockingStatsReaderTests.cs b/Lite.Tests/BlockingStatsReaderTests.cs index 4a3306518f..99a7e765a2 100644 --- a/Lite.Tests/BlockingStatsReaderTests.cs +++ b/Lite.Tests/BlockingStatsReaderTests.cs @@ -151,6 +151,57 @@ public async Task BlockingDurationStats_FallsBackToDmvSnapshot_OnlyWhenNoXeInWin Assert.Equal(1000, bucket.TotalDurationMs); } + // ── #6645 / #6705: a self-wait (same non-zero blocked and blocking spid) is not blocking, on both arms ── + + [Fact] + public async Task BlockingSeries_LeaveSelfWaitsOut_OnTheXeArm() + { + var service = new LocalDataService(_duckDb); + + /* One minute: a parallel query waiting on itself (spid 77 blocking 77), a real block and a (0, 0) report. A (0, 0) report + is a system spid pair the predicate keeps; the self-wait goes, with its 9000 ms. */ + await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(5), "DB1", 9000, blockedSpid: 77, blockingSpid: 77); + await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(15), "DB1", 1000); + await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(25), "DB1", 3000, blockedSpid: 0, blockingSpid: 0); + + var trend = Assert.Single(await service.GetBlockingTrendAsync(ServerId)); + Assert.Equal(2, trend.Count); + + var duration = Assert.Single(await service.GetBlockingDurationStatsAsync(ServerId)); + Assert.Equal(2, duration.EventCount); + Assert.Equal(4000, duration.TotalDurationMs); + Assert.Equal(3000, duration.MaxDurationMs); + + Assert.Equal(2, (await service.GetBlockingSlicerDataAsync(ServerId, 24)).Sum(b => b.SessionCount)); + Assert.True(await service.HasBlockedProcessReportsInWindowAsync(ServerId, MinuteA.AddMinutes(-1), MinuteA.AddMinutes(1))); + } + + [Fact] + public async Task BlockingSeries_OnlySelfWaitReportsInXe_DrawTheDmvArm_AndDmvSelfWaitsGo() + { + var service = new LocalDataService(_duckDb); + + /* The XE arm holds only a self-wait, so once the predicate filters it the arm is empty and the DMV arm is drawn. The DMV + arm leaves its own self-wait out. */ + await SeedBlockedProcessReportAsync(MinuteA.AddSeconds(5), "DB1", 9000, blockedSpid: 77, blockingSpid: 77); + await SeedDmvBlockingSnapshotAsync(MinuteB.AddSeconds(5), "DB1", 2000); + await SeedDmvBlockingSnapshotAsync(MinuteB.AddSeconds(10), "DB1", 8000, blockedSpid: 88, blockingSpid: 88); + + var trend = Assert.Single(await service.GetBlockingTrendAsync(ServerId)); + Assert.Equal(MinuteB, trend.Time); + Assert.Equal("DMV snapshot", trend.Source); + Assert.Equal(1, trend.Count); + + var duration = Assert.Single(await service.GetBlockingDurationStatsAsync(ServerId)); + Assert.Equal(MinuteB, duration.Time); + Assert.Equal("DMV snapshot", duration.Source); + Assert.Equal(2000, duration.TotalDurationMs); + + var slicer = await service.GetBlockingSlicerDataAsync(ServerId, 24); + Assert.Equal(1, slicer.Sum(b => b.SessionCount)); + Assert.False(await service.HasBlockedProcessReportsInWindowAsync(ServerId, MinuteA.AddMinutes(-1), MinuteB.AddMinutes(1))); + } + // ── Deadlock severity aggregate: victim_count + total / max / avg wait, reconciling with the count ── [Fact] @@ -217,7 +268,7 @@ await SeedDeadlockAsync(MinuteFloor(DateTime.UtcNow.AddHours(-48)).AddSeconds(5) // ── Seeding helpers (raw INSERT via the shared initializer's connection) ── - private async Task SeedBlockedProcessReportAsync(DateTime eventTime, string databaseName, long waitTimeMs) + private async Task SeedBlockedProcessReportAsync(DateTime eventTime, string databaseName, long waitTimeMs, int blockedSpid = 55, int blockingSpid = 66) { using var readLock = _duckDb.AcquireReadLock(); var connection = await SeedConnectionAsync(); @@ -234,9 +285,9 @@ INSERT INTO blocked_process_reports cmd.Parameters.Add(new DuckDBParameter { Value = "TestSrv" }); cmd.Parameters.Add(new DuckDBParameter { Value = eventTime }); cmd.Parameters.Add(new DuckDBParameter { Value = databaseName }); - cmd.Parameters.Add(new DuckDBParameter { Value = 55 }); + cmd.Parameters.Add(new DuckDBParameter { Value = blockedSpid }); cmd.Parameters.Add(new DuckDBParameter { Value = eventTime }); - cmd.Parameters.Add(new DuckDBParameter { Value = 66 }); + cmd.Parameters.Add(new DuckDBParameter { Value = blockingSpid }); cmd.Parameters.Add(new DuckDBParameter { Value = eventTime }); cmd.Parameters.Add(new DuckDBParameter { Value = waitTimeMs }); cmd.Parameters.Add(new DuckDBParameter { Value = "X" }); @@ -245,7 +296,7 @@ INSERT INTO blocked_process_reports await cmd.ExecuteNonQueryAsync(); } - private async Task SeedDmvBlockingSnapshotAsync(DateTime eventTime, string databaseName, long waitTimeMs) + private async Task SeedDmvBlockingSnapshotAsync(DateTime eventTime, string databaseName, long waitTimeMs, int blockedSpid = 55, int blockingSpid = 66) { using var readLock = _duckDb.AcquireReadLock(); var connection = await SeedConnectionAsync(); @@ -261,8 +312,8 @@ INSERT INTO dmv_blocking_snapshots cmd.Parameters.Add(new DuckDBParameter { Value = "TestSrv" }); cmd.Parameters.Add(new DuckDBParameter { Value = eventTime }); cmd.Parameters.Add(new DuckDBParameter { Value = databaseName }); - cmd.Parameters.Add(new DuckDBParameter { Value = 55 }); - cmd.Parameters.Add(new DuckDBParameter { Value = 66 }); + cmd.Parameters.Add(new DuckDBParameter { Value = blockedSpid }); + cmd.Parameters.Add(new DuckDBParameter { Value = blockingSpid }); cmd.Parameters.Add(new DuckDBParameter { Value = waitTimeMs }); cmd.Parameters.Add(new DuckDBParameter { Value = "X" }); cmd.Parameters.Add(new DuckDBParameter { Value = 1 }); diff --git a/Lite.Tests/EventBaselineCoveredDaysTests.cs b/Lite.Tests/EventBaselineCoveredDaysTests.cs index 71c83e155e..506c10d143 100644 --- a/Lite.Tests/EventBaselineCoveredDaysTests.cs +++ b/Lite.Tests/EventBaselineCoveredDaysTests.cs @@ -573,4 +573,33 @@ await ExecAsync( Assert.NotNull(row); Assert.Equal(1.0 / 5.0, row!.Mean, 9); } + + /// + /// #6705: the blocking-per-minute baseline counts the same reports the current window does, so a self-wait (the same + /// non-zero blocked and blocking spid) does not make a minute heavier. Each Tuesday has one minute holding a real report + /// and a self-wait, and another holding only a self-wait: the first counts 1, the second is not a minute at all, and a + /// (0, 0) report still counts. + /// + [Fact] + public async Task TheBlockingPerMinuteBaseline_LeavesSelfWaitsOut() + { + async Task Report(DateTime at, int blockedSpid, int blockingSpid) => await ExecAsync( + "INSERT INTO blocked_process_reports (blocked_report_id, collection_time, event_time, server_id, server_name, wait_time_ms, blocked_spid, blocking_spid) " + + "VALUES ($1, $2, $2, $3, 'TestServer', 1000, $4, $5)", + _nextId--, at, ServerId, blockedSpid, blockingSpid); + + foreach (var tuesday in Tuesdays) + { + await Report(tuesday.AddMinutes(10), 55, 66); + await Report(tuesday.AddMinutes(10).AddSeconds(20), 77, 77); + await Report(tuesday.AddMinutes(20), 88, 88); + await Report(tuesday.AddMinutes(30), 0, 0); + } + + var row = await RowAsync(MetricNames.BlockingPerMinute, Hour, Tuesday); + + Assert.NotNull(row); + Assert.Equal(10L, row!.SampleCount); + Assert.Equal(1.0, row.Mean, 9); + } } diff --git a/Lite/Analysis/BaselineProvider.cs b/Lite/Analysis/BaselineProvider.cs index 6afc16516f..ed33deba4e 100644 --- a/Lite/Analysis/BaselineProvider.cs +++ b/Lite/Analysis/BaselineProvider.cs @@ -712,7 +712,7 @@ FROM v_file_io_stats )," + RobustTierScaffold, // #6645: the baseline counts the same reports the current window does (BlockingWaits.NotSelfWaitSql), so the 3x ratio - // compares like with like. The Darling twin reads a stored aggregate that still counts them. + // compares like with like. The Darling twin's stored aggregate leaves them out too (#6705). // Event-based — mean = events per covered hour for this bucket, sample_count = covered days. // No restart exclusion needed (event counts, not cumulative). See EventBaselineSql (#4731). MetricNames.Blocking => OnEventTime(EventBaselineSql("blocked_process_report", "v_collection_log", @@ -775,7 +775,7 @@ FROM with_rate WITH per_minute AS ( SELECT DATE_TRUNC('minute', event_time) AS minute_bucket, COUNT(*)::DOUBLE PRECISION AS event_count - FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3") + @" AS ev + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql) + @" AS ev GROUP BY minute_bucket ), clean AS ( diff --git a/Lite/Services/LocalDataService.Blocking.cs b/Lite/Services/LocalDataService.Blocking.cs index bbeea0ad21..ded10e45ab 100644 --- a/Lite/Services/LocalDataService.Blocking.cs +++ b/Lite/Services/LocalDataService.Blocking.cs @@ -1001,7 +1001,7 @@ WITH bpr AS ( COUNT(DISTINCT blocking_spid) AS distinct_blockers, COUNT(DISTINCT blocked_spid) AS distinct_blocked, COUNT(DISTINCT database_name) AS distinct_databases - FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + @" AS ev + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + @" AS ev GROUP BY date_trunc('hour', event_time) ), dmv AS ( @@ -1013,7 +1013,7 @@ dmv AS ( COUNT(DISTINCT blocked_spid) AS distinct_blocked, COUNT(DISTINCT database_name) AS distinct_databases FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3" + dbClause + @" + WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause + @" GROUP BY date_trunc('hour', collection_time) ) SELECT bucket, event_count, total_wait_sec, distinct_blockers, distinct_blocked, distinct_databases FROM bpr @@ -1090,7 +1090,8 @@ GROUP BY date_trunc('hour', deadlock_time) /// /// #5098: whether the XE blocked process reports have a row in the window, with the SAME predicate (and database filter) as the /// bpr arm of and . When true those reads drew - /// the XE rows alone (the DMV arm is skipped), so the note must name the XE start, not a DMV-covered one. + /// the XE rows alone (the DMV arm is skipped), so the note must name the XE start, not a DMV-covered one. #6705: the predicate + /// leaves out self-waits (), as those reads do on both arms. /// public async Task HasBlockedProcessReportsInWindowAsync(int serverId, DateTime startUtc, DateTime endUtc, IReadOnlyList? databaseNames = null) { @@ -1098,7 +1099,7 @@ public async Task HasBlockedProcessReportsInWindowAsync(int serverId, Date using var command = connection.CreateCommand(); var dbClause = BuildDbInClause(databaseNames, "database_name", 4, out var dbValues); - command.CommandText = "SELECT 1 FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + " AS ev LIMIT 1"; + command.CommandText = "SELECT 1 FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + " AS ev LIMIT 1"; command.Parameters.Add(new DuckDBParameter { Value = serverId }); command.Parameters.Add(new DuckDBParameter { Value = startUtc }); command.Parameters.Add(new DuckDBParameter { Value = endUtc }); @@ -1110,8 +1111,9 @@ public async Task HasBlockedProcessReportsInWindowAsync(int serverId, Date } /// - /// #5098: the earliest XE blocked process report in the window, with the SAME predicate and database filter as - /// . Null when there is none. Opens its own connection. + /// #5098: the earliest XE blocked process report in the window, with the same database filter as + /// . It names when the XE session started capturing, so a self-wait report + /// counts here (#6705). Null when there is none. Opens its own connection. /// public async Task GetEarliestBlockedProcessReportInWindowAsync(int serverId, DateTime startUtc, DateTime endUtc, IReadOnlyList? databaseNames = null) { @@ -1212,13 +1214,13 @@ Group by event_time (when blocking actually occurred) rather than collection_tim command.CommandText = @" WITH bpr AS ( SELECT DATE_TRUNC('minute', event_time) AS bucket, COUNT(*) AS incident_count, 'blocked-process-report' AS source - FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + @" AS ev + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + @" AS ev GROUP BY DATE_TRUNC('minute', event_time) ), dmv AS ( SELECT DATE_TRUNC('minute', event_time) AS bucket, COUNT(*) AS incident_count, 'DMV snapshot' AS source FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause + @" + WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause + @" GROUP BY DATE_TRUNC('minute', event_time) ) SELECT bucket, incident_count, source FROM bpr diff --git a/Lite/Services/LocalDataService.BlockingStats.cs b/Lite/Services/LocalDataService.BlockingStats.cs index dd9e31c3d2..8c9ae586fa 100644 --- a/Lite/Services/LocalDataService.BlockingStats.cs +++ b/Lite/Services/LocalDataService.BlockingStats.cs @@ -96,7 +96,7 @@ WITH bpr AS ( CAST(COALESCE(MAX(wait_time_ms), 0) AS BIGINT) AS max_duration_ms, CAST(COALESCE(AVG(wait_time_ms), 0) AS DOUBLE) AS avg_duration_ms, 'blocked-process-report' AS source - FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause) + @" AS ev + FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause) + @" AS ev GROUP BY DATE_TRUNC('minute', event_time) ), dmv AS ( @@ -108,7 +108,7 @@ dmv AS ( CAST(COALESCE(AVG(wait_time_ms), 0) AS DOUBLE) AS avg_duration_ms, 'DMV snapshot' AS source FROM v_dmv_blocking_snapshots - WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3" + dbClause + @" + WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND " + BlockingWaits.NotSelfWaitSql + dbClause + @" GROUP BY DATE_TRUNC('minute', event_time) ) SELECT bucket, event_count, total_duration_ms, max_duration_ms, avg_duration_ms, source FROM bpr From 19e2ca831c091762407ac0990f5c69ab51953fea Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:18:58 -0400 Subject: [PATCH 3/4] Blocking trend bars leave self-waits out like the band, and the rebuilt blocking baseline records where its supply begins (#6705) The Viewer and MCP blocking trend and duration reads, and the existence check that picks their arm, take BlockingWaits.NotSelfWaitSql on both arms. A rebuilt blocked_process_baseline records a supply floor (view comment) on the backfill pass, and the Blocking baseline starts its window no earlier, so days the aggregate cannot back are not counted as quiet. --- .../FleetPartialDatabaseFailureBandTests.cs | 105 ++++++++++++++++++ .../Darling.Tests/ViewerBlockingStatsTests.cs | 78 ++++++++++++- .../PgBaselineProvider.cs | 13 +++ .../Mcp/DarlingBlockingTrendReader.cs | 7 ++ .../Mcp/DarlingDataReader.cs | 5 + .../TimescaleSupport.cs | 92 ++++++++++++++- .../ViewerDataService.Blocking.cs | 8 +- .../ViewerDataService.BlockingStats.cs | 6 +- .../ViewerDataService.BlockingTrends.cs | 10 +- 9 files changed, 312 insertions(+), 12 deletions(-) diff --git a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs index 020022600b..ac81fc5c76 100644 --- a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs +++ b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs @@ -18,8 +18,10 @@ using System.Threading; using System.Threading.Tasks; using Npgsql; +using PerformanceMonitor.Analysis.Baselines; using PerformanceMonitor.Collectors; using PerformanceMonitor.Common; +using PerformanceMonitor.Darling.Analysis; using PerformanceMonitor.Darling.Service.Mcp; using PerformanceMonitor.Darling.Storage; using PerformanceMonitor.Darling.Viewer; @@ -725,4 +727,107 @@ public async Task PlainBlockingBaselineViewWithoutTheSelfWaitFilter_IsReplacedIn /* Converged: the next pass fills and replaces nothing. */ Assert.Equal(0, await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct)); } + + /* ───────────── #6705: the rebuilt aggregate's supply floor (the #3590 trap) ───────────── */ + + [Theory] + [InlineData(null, null)] + [InlineData("", null)] + [InlineData("not a floor", null)] + [InlineData("supply_floor_utc=garbage", null)] + [InlineData("supply_floor_utc=2026-09-24", null)] + [InlineData("supply_floor_utc=2026-09-24T12:34:56", "2026-09-24T12:34:56")] + public void TheSupplyFloorComment_ParsesOnlyAValidFloor(string? comment, string? expected) + => Assert.Equal( + expected is null ? null : DateTime.ParseExact(expected, "yyyy-MM-dd'T'HH:mm:ss", CultureInfo.InvariantCulture), + TimescaleSupport.ParseBlockedBaselineSupplyFloor(comment)); + + private static async Task<(long Covered, double Events)> BlockingBaselineTotalsAsync(NpgsqlDataSource postgres, DateTime analysisTime, CancellationToken ct) + { + var map = await new PgBaselineProvider(postgres).GetBucketMapAsync(-6690, MetricNames.Blocking, analysisTime, analysisTime.AddHours(1), ct); + return (map.Buckets.Values.Sum(b => b.SampleCount), map.Buckets.Values.Sum(b => b.Mean * b.SampleCount)); + } + + [Fact] + public async Task BlockingBaseline_OfARebuiltAggregate_CountsNoDayBeforeTheFloor_AndAStoreWithNoFloorReadsAsBefore_AgainstDevPostgres() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #6705 supply-floor test."); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "the dev fixture is expected to have TimescaleDB installed"); + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + + /* The operator keeps blocked process reports 10 days, not the 30 of the default. */ + await using (var retention = new NpgsqlCommand("INSERT INTO config.config_collector_schedules (server_id, collector_name, frequency_minutes, retention_days, enabled) VALUES (NULL, 'blocked_process_report', 1, 10, TRUE)", connection)) + { + await retention.ExecuteNonQueryAsync(ct); + } + + await using (var create = new NpgsqlCommand(PreSelfWaitFilterBaselineSql().Replace("WITH NO DATA", "WITH DATA", StringComparison.Ordinal), connection)) + { + await create.ExecuteNonQueryAsync(ct); + } + + /* Raw holds one real report 2 days back and one 8 days back (inside the 10), each at 14:00 UTC. The collector logged a + SUCCESS run at 14:00 on each of the last 30 days, which the log keeps twice as long as the reports. */ + var now = LiveClock.Now(); + var dayStart = now.Date; + long id = 67050; + foreach (var back in new[] { 2, 8 }) + { + await InsertBlockedReportAsync(connection, id++, dayStart.AddDays(-back).AddHours(14), 1, 2, ct); + } + + await using (var log = new NpgsqlCommand(@" +INSERT INTO collect.collection_log (log_id, server_id, server_name, collector_name, collection_time, duration_ms, status, error_message, rows_collected, sql_duration_ms, duckdb_duration_ms) +SELECT -670500 - g, -6690, 'srv-6690', 'blocked_process_report', $1 - make_interval(days => g) + interval '14 hours', 100, 'SUCCESS', NULL, 0, 80, 20 +FROM generate_series(0, 29) AS g", connection)) + { + log.Parameters.AddWithValue(DateTime.SpecifyKind(dayStart, DateTimeKind.Unspecified)); + await log.ExecuteNonQueryAsync(ct); + } + + /* The reshape drops the old aggregate, the ensure sweep recreates it, the start path's backfill records the floor. */ + Assert.Equal(1, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct)); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct)); + var before = DateTime.UtcNow; + Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1); + var floor = await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct); + Assert.NotNull(floor); + Assert.InRange(floor!.Value, before.AddDays(-10).AddMinutes(-5), DateTime.UtcNow.AddDays(-10).AddSeconds(1)); + + /* A later start does not move it. */ + await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct); + Assert.Equal(floor, await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct)); + + await using var postgres = NpgsqlDataSource.Create(scratch.ConnectionString); + var runs = Enumerable.Range(0, 30).Select(g => dayStart.AddDays(-g).AddHours(14)).ToList(); + var coveredAfterFloor = runs.Count(t => t >= floor.Value && t < dayStart); + Assert.InRange(coveredAfterFloor, 8, 10); + + var (covered, events) = await BlockingBaselineTotalsAsync(postgres, now, ct); + Assert.Equal(coveredAfterFloor, covered); + Assert.Equal(2.0, events, precision: 6); + + /* No floor: every logged day of the window is a covered day, as before. */ + await using (var clear = new NpgsqlCommand($"COMMENT ON VIEW collect.{TimescaleSupport.BlockedProcessBaselineView} IS NULL", connection)) + { + await clear.ExecuteNonQueryAsync(ct); + } + + Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct)); + var (coveredWithoutFloor, eventsWithoutFloor) = await BlockingBaselineTotalsAsync(postgres, now, ct); + Assert.Equal(runs.Count(t => t < dayStart), coveredWithoutFloor); + Assert.Equal(2.0, eventsWithoutFloor, precision: 6); + } } diff --git a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs index c36d5eb701..b2849a358b 100644 --- a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs +++ b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs @@ -176,8 +176,80 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup } } + /// + /// #6705: a parallel query waiting on itself (the same non-zero spid on both sides) is not blocking, on the desktop trend, + /// the duration stats, the existence check that picks the arm, and the MCP trend and stats reads. A minute with one self-wait + /// report and one real report counts 1; a window whose only XE reports are self-waits draws the DMV arm, as the detector does. + /// + [Fact] + public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWaitsDrawTheDmvArm_AgainstDevPostgres() + { + var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(connectionString), + "Set DARLING_TEST_PG to a Postgres connection string to run the live self-wait trend test."); + var ct = TestContext.Current.CancellationToken; + + using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct); + await DeleteRowsAsync(connection, "dmv_blocking_snapshots", StatsServerId, ct); + + await using var viewer = new ViewerDataService(connectionString!); + await using var postgres = NpgsqlDataSource.Create(connectionString!); + + var bodySucceeded = false; + try + { + var start = TruncateToMinute(DateTime.UtcNow.AddMinutes(-10)); + var end = start.AddMinutes(11); + + /* One real report (1000 ms) and one self-wait (7000 ms, spid 7 on itself) in the same minute. */ + await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(5), 1000); + await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7); + await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(20), 9999); + + var trend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end); + Assert.Equal(1, trend.Sum(p => p.Count)); + var mcpTrend = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct); + Assert.Equal(1, mcpTrend.Sum(p => p.Count)); + + var stats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end); + Assert.Equal(1, stats.Sum(p => p.EventCount)); + Assert.Equal(1000, stats.Sum(p => p.TotalDurationMs)); + Assert.Equal(1000, stats.Max(p => p.MaxDurationMs)); + var mcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct); + Assert.Equal(1, mcpStats.Sum(p => p.EventCount)); + Assert.Equal(1000, mcpStats.Sum(p => p.TotalDurationMs)); + Assert.True(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end)); + + /* Only the self-wait remains in XE: the window draws the DMV snapshot, and the existence check says so. */ + await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct); + await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7); + + var dmvTrend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end); + Assert.Equal(1, dmvTrend.Sum(p => p.Count)); + Assert.All(dmvTrend, p => Assert.Equal("DMV snapshot", p.Source)); + var dmvMcp = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct); + Assert.All(dmvMcp, p => Assert.Equal("DMV snapshot", p.Source)); + var dmvStats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end); + Assert.Equal(9999, dmvStats.Sum(p => p.TotalDurationMs)); + Assert.False(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end)); + + bodySucceeded = true; + } + finally + { + await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) => + { + await DeleteRowsAsync(cleanup, "blocked_process_reports", StatsServerId, cleanupCt); + await DeleteRowsAsync(cleanup, "dmv_blocking_snapshots", StatsServerId, cleanupCt); + }); + } + } + private static async Task InsertBlockedProcessReportAsync( - NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs) + NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs, int blockedSpid = 55, int blockingSpid = 60) { using var command = new NpgsqlCommand(@" INSERT INTO blocked_process_reports @@ -191,8 +263,8 @@ INSERT INTO blocked_process_reports command.Parameters.AddWithValue(ServerName); command.Parameters.AddWithValue(DateTime.SpecifyKind(eventTimeUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue("AppDb"); - command.Parameters.AddWithValue(55); - command.Parameters.AddWithValue(60); + command.Parameters.AddWithValue(blockedSpid); + command.Parameters.AddWithValue(blockingSpid); command.Parameters.AddWithValue(waitTimeMs); command.Parameters.AddWithValue("X"); command.Parameters.AddWithValue("SELECT 1"); diff --git a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs index f0497c7661..434f311f72 100644 --- a/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs +++ b/Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs @@ -896,6 +896,19 @@ analyze_server and compare_analysis (BaselineCache) without changing anyone's an query = await ChooseSupplyAsync(connection, serverId, metricName, query, windowStart, cancellationToken); } + /* #6705 (the #3590 trap): the Blocking arm counts a collection_log run with no event as a measured zero, and the + log is kept twice as long as the events behind blocked_process_baseline. After that aggregate was rebuilt it + holds only what raw still held, so a day before that would read as a quiet day, not a missing one. The + start path recorded where its supply begins; the window starts no earlier. Absent or unreadable is no floor, + and it is never derived from the aggregate's own oldest bucket (a server that rarely blocks has real zero days). + The text of the arm is unchanged: it still reads $2. */ + if (keys is null && metricName == MetricNames.Blocking + && await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, cancellationToken) is DateTime supplyFloor + && supplyFloor > windowStart) + { + windowStart = supplyFloor; + } + /* #3653 Q6: the target's clock over this window, read on the same connection and INSIDE this try on purpose — a store that cannot answer a one-row indexed read of server_properties cannot answer the aggregate scan either, and one classified catch (AnalysisShutdownResidueTests pins exactly one) is diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs index f459be87ce..8b5e1d2be0 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingBlockingTrendReader.cs @@ -50,6 +50,11 @@ public sealed record BlockingTrendReadPoint(DateTime Time, int Count, string? So /// The database predicate is the list form (#5244), on BOTH arms, as the desktop's BlockingTrendSql has it. /// The DMV arm's NOT EXISTS (SELECT 1 FROM bpr) therefore asks whether the XE source has rows for the CHOSEN databases: /// a database with only DMV-captured blocking still charts, whatever other databases' XE reports hold. + /// + /// A parallel query waiting on itself is not blocking (#6645, #6705): both arms carry + /// , as PgAnomalyDetector.BlockingWindowSql does, so the bars (and the web + /// Overview strip drawn from this read) agree with the baseline band. The DMV arm is therefore drawn when the FILTERED XE + /// arm is empty, a window whose only XE reports are self-waits included. /// public static readonly string BlockingTrendSql = $$""" WITH bpr AS ( @@ -58,6 +63,7 @@ FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4 {{DatabaseFilter.All.Clause("database_name", 5)}} + AND {{BlockingWaits.NotSelfWaitSql}} GROUP BY DATE_TRUNC('minute', event_time) ), dmv AS ( @@ -66,6 +72,7 @@ FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4 {{DatabaseFilter.All.Clause("database_name", 5)}} + AND {{BlockingWaits.NotSelfWaitSql}} GROUP BY DATE_TRUNC('minute', event_time) ) SELECT bucket, incident_count, source FROM bpr diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs index 3c24870107..3704fa45ae 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingDataReader.cs @@ -4822,6 +4822,9 @@ public static async Task> GetBlockedSessionTrendAsy /// is taken only when the XE source has no row for the CHOSEN databases (NOT EXISTS (SELECT 1 FROM bpr) over the /// filtered bpr), exactly as the desktop's blocking reads and get_blocking_trend choose, so the tools answer /// from one source for one filter and the two sources are never mixed. The unfiltered series is exactly what it was. + /// Both arms leave out a parallel query waiting on itself (, #6645, #6705), + /// as get_blocking_trend does, so the count and the severity reconcile; the DMV fallback is taken when the FILTERED XE + /// arm is empty. /// public static readonly string BlockingDurationStatsSql = $$""" WITH bpr AS ( @@ -4836,6 +4839,7 @@ FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4 {{DatabaseFilter.All.Clause("database_name", 5)}} + AND {{BlockingWaits.NotSelfWaitSql}} GROUP BY DATE_TRUNC('minute', event_time) ), dmv AS ( @@ -4850,6 +4854,7 @@ FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4 {{DatabaseFilter.All.Clause("database_name", 5)}} + AND {{BlockingWaits.NotSelfWaitSql}} GROUP BY DATE_TRUNC('minute', event_time) ) SELECT bucket, event_count, total_duration_ms, max_duration_ms, avg_duration_ms, source FROM bpr diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index d8ddb66ab0..3aaf4660c2 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -792,10 +792,11 @@ WHERE delta_execution_count > 0 /// Blocking AND BlockingPerMinute baseline supply -- both families share this source and the same - /// row-level filter, so one aggregate serves both. Since #6663 both leave a parallel query's self-waits out of - /// their current window (), so the baseline they are compared with - /// leaves them out too (#6690): a count over every report row would read a server's baseline high and under-flag - /// its real blocking. Event counts per collection re-aggregate to either shape: Blocking sums them per + /// row-level filter, so one aggregate serves both. Both are compared with series that leave a parallel query's + /// self-waits out (): Blocking with its current window + /// (PgAnomalyDetector.BlockingWindowSql, #6663), BlockingPerMinute with the bars drawn over its band (the Viewer's + /// BlockingTrendSql and get_blocking_trend, #6705). So the baseline leaves them out too (#6690): a count over + /// every report row would read a server's baseline high and under-flag its real blocking. Event counts per collection re-aggregate to either shape: Blocking sums them per /// hour/dow bucket, BlockingPerMinute re-buckets by minute. A store that built this aggregate before #6690 is /// rebuilt by . public const string CreateBlockedProcessBaselineSql = @"CREATE MATERIALIZED VIEW IF NOT EXISTS collect.blocked_process_baseline @@ -908,6 +909,13 @@ that created it — see BaselineBackfillProbeSql. */ continue; } + /* #6705: an EMPTY materialization over a raw table that holds rows is a rebuilt (or first-built) + aggregate, and it is about to hold only what raw still holds. Say where that begins BEFORE the refresh. */ + if (view == BlockedProcessBaselineView && coverageOldest is null) + { + await RecordBlockedBaselineSupplyFloorAsync(connection, logger, DateTime.UtcNow, cancellationToken); + } + var after = await RefreshFromAsync(connection, view, needFrom.Value, force: false, cancellationToken); /* VERIFY, DO NOT ASSUME -- the failure this guards is SILENT. On a CAGG that has just been @@ -1135,6 +1143,82 @@ private static async Task IsStalePlainBlockedBaselineViewAsync(NpgsqlConne return await probe.ExecuteScalarAsync(cancellationToken) is true; } + /// + /// #6705 (the #3590 trap): the prefix of the view comment that records where the Blocking baseline's EVENT supply begins. + /// 's event arm counts a collection_log run with no event as a measured zero, and that + /// log is kept twice the base, so a day the rebuilt blocked_process_baseline cannot back would read as a quiet day, not a + /// missing one. The comment is the floor: the instant the rebuild's raw supply begins (see + /// ). No migration: the service owns the view, and a rebuild + /// (DROP ... CASCADE) clears the comment with the view. + /// + public const string BlockedBaselineSupplyFloorPrefix = "supply_floor_utc="; + + private const string SupplyFloorFormat = "yyyy-MM-dd'T'HH:mm:ss"; + + /// The view comment and the fleet blocked_process_report retention in force, in one row. A view that does not exist reads NULL. + private static readonly string BlockedBaselineSupplyFloorProbeSql = + $"SELECT obj_description(to_regclass('collect.{BlockedProcessBaselineView}'), 'pg_class'), " + + DarlingRetentionHorizons.FleetRetentionDaysSql("blocked_process_report"); + + /// The floor a view comment records; null for an absent comment or one that is not a floor. + public static DateTime? ParseBlockedBaselineSupplyFloor(string? comment) + => comment is not null + && comment.StartsWith(BlockedBaselineSupplyFloorPrefix, StringComparison.Ordinal) + && DateTime.TryParseExact(comment.AsSpan(BlockedBaselineSupplyFloorPrefix.Length), SupplyFloorFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out var floor) + ? floor + : null; + + private static async Task<(DateTime? Floor, int RetentionDays)> ReadBlockedBaselineSupplyAsync(NpgsqlConnection connection, CancellationToken cancellationToken) + { + using var probe = new NpgsqlCommand(BlockedBaselineSupplyFloorProbeSql, connection) { CommandTimeout = SetupTimeoutSeconds }; + await using var reader = await probe.ExecuteReaderAsync(cancellationToken); + await reader.ReadAsync(cancellationToken); + return (ParseBlockedBaselineSupplyFloor(reader.IsDBNull(0) ? null : reader.GetString(0)), Convert.ToInt32(reader.GetValue(1), CultureInfo.InvariantCulture)); + } + + /// + /// The instant before which the rebuilt blocked_process_baseline cannot back a day, or null (no floor recorded: the + /// aggregate was never rebuilt by this build, or the comment is not a floor). The provider raises the Blocking + /// baseline's window start to it, so the days the aggregate does not hold are left out of the covered days rather than + /// counted as quiet. Never the aggregate's own oldest bucket: a server that rarely blocks has real zero days. + /// + public static async Task ReadBlockedBaselineSupplyFloorAsync(NpgsqlConnection connection, CancellationToken cancellationToken = default) + { + if (connection is null) + { + throw new ArgumentNullException(nameof(connection)); + } + + return (await ReadBlockedBaselineSupplyAsync(connection, cancellationToken)).Floor; + } + + /// + /// Records the floor on the pass that runs the backfill of a rebuilt blocked_process_baseline: + /// minus the blocked_process_report retention the daily purge applies right now + /// (, the fleet override or the default). Raw drops its chunks at + /// that horizon and the backfill clamps to what raw holds, so the rebuilt aggregate begins there; the aggregate's own 35 days + /// never remove a materialized hour that raw has since dropped (the purge is drop_chunks, which records no invalidation, + /// and the refresh policy reaches back one day). Written once: a floor already recorded for this view is left alone, so a + /// store whose aggregate stays empty (a server with only self-waits) does not move its floor forward on every start. + /// + private static async Task RecordBlockedBaselineSupplyFloorAsync(NpgsqlConnection connection, ILogger? logger, DateTime utcNow, CancellationToken cancellationToken) + { + var (recorded, retentionDays) = await ReadBlockedBaselineSupplyAsync(connection, cancellationToken); + if (recorded is not null) + { + return; + } + + var floor = utcNow - TimeSpan.FromDays(retentionDays); + using var comment = new NpgsqlCommand( + $"COMMENT ON VIEW collect.{BlockedProcessBaselineView} IS '{BlockedBaselineSupplyFloorPrefix}{floor.ToString(SupplyFloorFormat, CultureInfo.InvariantCulture)}'", + connection) { CommandTimeout = SetupTimeoutSeconds }; + await comment.ExecuteNonQueryAsync(cancellationToken); + logger?.LogInformation( + "TimescaleDB: {View} was rebuilt over the {RetentionDays} days of blocked process reports the store keeps -- the Blocking baseline counts no day before {Floor:u}.", + BlockedProcessBaselineView, retentionDays, floor); + } + /// Does this baseline relation exist under any implementation — continuous aggregate or plain view? public static string BaselineRelationExistsSql(string view) => $"SELECT to_regclass('collect.{view}') IS NOT NULL"; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs index 6b9073f241..120d2925fb 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs @@ -315,16 +315,18 @@ public async Task ReadRecentBlockedProcessRepor /// /// Whether the XE blocked_process_reports source holds a report in the window: the same predicate the Blocking charts' reads - /// use for their bpr arm (event time in the window, the event-window floor on collection time, the database filter), - /// so true here means the charts draw from XE rows and not from the DMV fallback beside them. + /// use for their bpr arm (event time in the window, the event-window floor on collection time, the database filter, + /// and : a parallel query waiting on itself is not blocking, #6705), so true here + /// means the charts draw from XE rows and not from the DMV fallback beside them. /// - public const string BlockedProcessReportsInWindowSql = """ + public const string BlockedProcessReportsInWindowSql = $""" SELECT EXISTS ( SELECT 1 FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $5 AND ($4::text[] IS NULL OR database_name = ANY($4)) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} ) """; diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs index ca3525bf2a..f764c86554 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingStats.cs @@ -50,8 +50,10 @@ public sealed partial class ViewerDataService /// drop a row (an event is collected after it happens). $5 is the database filter (#5244), the same /// text[] (NULL for all) the count trend binds, on BOTH arms so the severity reconciles with it. /// The last column names the arm that answered, the tag Darling's MCP answers carry. + /// Both arms leave out a parallel query waiting on itself (, #6645, #6705), as the + /// count trend does, so the two reconcile and the severity is the blocking's, not the query's own. /// - public const string BlockingDurationStatsSql = """ + public const string BlockingDurationStatsSql = $""" WITH bpr AS ( SELECT DATE_TRUNC('minute', event_time) AS bucket, @@ -64,6 +66,7 @@ FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4 AND ($5::text[] IS NULL OR database_name = ANY($5)) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} GROUP BY DATE_TRUNC('minute', event_time) ), dmv AS ( @@ -78,6 +81,7 @@ FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $4 AND ($5::text[] IS NULL OR database_name = ANY($5)) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} GROUP BY DATE_TRUNC('minute', event_time) ) SELECT bucket, event_count, total_duration_ms, max_duration_ms, avg_duration_ms, source FROM bpr diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs index b4298aa91b..eb50e3ef7a 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingTrends.cs @@ -68,14 +68,21 @@ public sealed partial class ViewerDataService /// collection_time, which this event-time window alone gives the planner nothing to exclude a chunk /// on (#4229); the floor lets it skip every chunk older than the window, without being able to drop a row /// (an event is collected after it happens). + /// + /// A parallel query waiting on itself is not blocking (#6645, #6705): both arms carry + /// , so the bars agree with the baseline band drawn over them + /// (blocked_process_baseline leaves self-waits out, #6690) and with PgAnomalyDetector.BlockingWindowSql. + /// The DMV arm is drawn when the FILTERED XE arm is empty, so a window whose only XE reports are self-waits falls back to + /// the DMV snapshots, as the anomaly detector does; takes the same predicate. /// - public const string BlockingTrendSql = """ + public const string BlockingTrendSql = $""" WITH bpr AS ( SELECT DATE_TRUNC('minute', event_time) AS bucket, COUNT(*) AS incident_count, 'blocked-process-report' AS source FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $5 AND ($4::text[] IS NULL OR database_name = ANY($4)) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} GROUP BY DATE_TRUNC('minute', event_time) ), dmv AS ( @@ -84,6 +91,7 @@ FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $5 AND ($4::text[] IS NULL OR database_name = ANY($4)) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} GROUP BY DATE_TRUNC('minute', event_time) ) SELECT bucket, incident_count, source FROM bpr From 44b8d014fd2db4d6b6d4b526174dc1939ae6200e Mon Sep 17 00:00:00 2001 From: Erik Darling <2136037+erikdarlingdata@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:40:59 -0400 Subject: [PATCH 4/4] Leave self-waits out of the Darling blocking slicer and the fleet band's report count, record the baseline supply floor when the view is created (#6705) --- .../FleetPartialDatabaseFailureBandTests.cs | 40 ++++++++++++++-- .../Darling.Tests/ViewerBlockingStatsTests.cs | 48 ++++++++++++++----- .../Mcp/DarlingFleetReader.cs | 7 ++- .../TimescaleSupport.cs | 45 ++++++++++++++++- .../ViewerDataService.Blocking.cs | 4 +- .../ViewerDataService.BlockingSlicers.cs | 9 +++- 6 files changed, 133 insertions(+), 20 deletions(-) diff --git a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs index ac81fc5c76..2912023223 100644 --- a/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs +++ b/Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs @@ -748,6 +748,28 @@ public void TheSupplyFloorComment_ParsesOnlyAValidFloor(string? comment, string? return (map.Buckets.Values.Sum(b => b.SampleCount), map.Buckets.Values.Sum(b => b.Mean * b.SampleCount)); } + /// #6705: the ensure sweep records no floor on a fresh store, where raw holds nothing and the aggregate will hold everything raw ever gets. + [Fact] + public async Task EnsureOnAFreshStore_RecordsNoSupplyFloor_AgainstDevPostgres() + { + var baseConnectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + Assert.SkipWhen(string.IsNullOrEmpty(baseConnectionString), "Set DARLING_TEST_PG to a Postgres connection string (with TimescaleDB installed) to run the live #6705 fresh-store test."); + var ct = TestContext.Current.CancellationToken; + await using var scratch = await ScratchPostgres.CreateAsync(baseConnectionString!, ct); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await PgMigrations.MigrateAsync(connection, ct); + Assert.True(await TimescaleSupport.TryEnableAsync(connection, null, ct), "the dev fixture is expected to have TimescaleDB installed"); + await using (var stop = new NpgsqlCommand("SELECT _timescaledb_functions.stop_background_workers()", connection)) + { + await stop.ExecuteNonQueryAsync(ct); + } + + await TimescaleSupport.ConvertToHypertablesAsync(connection, null, ct); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); + Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct)); + } + [Fact] public async Task BlockingBaseline_OfARebuiltAggregate_CountsNoDayBeforeTheFloor_AndAStoreWithNoFloorReadsAsBefore_AgainstDevPostgres() { @@ -796,16 +818,26 @@ FROM generate_series(0, 29) AS g", connection)) await log.ExecuteNonQueryAsync(ct); } - /* The reshape drops the old aggregate, the ensure sweep recreates it, the start path's backfill records the floor. */ + /* The reshape drops the old aggregate and the ensure sweep recreates it over raw rows, which records the floor on that + pass: a service that stops before the backfill, with the refresh policy then materializing the last day, still has it. */ Assert.Equal(1, await TimescaleSupport.DropStaleContinuousAggregatesAsync(connection, null, ct)); - await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); - Assert.Null(await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct)); var before = DateTime.UtcNow; - Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1); + await TimescaleSupport.EnsureContinuousAggregatesAsync(connection, null, ct); var floor = await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct); Assert.NotNull(floor); Assert.InRange(floor!.Value, before.AddDays(-10).AddMinutes(-5), DateTime.UtcNow.AddDays(-10).AddSeconds(1)); + /* The policy's run materializes the last day before the backfill; the backfill then sees a non-empty materialization. */ + await using (var recentOnly = new NpgsqlCommand("CALL refresh_continuous_aggregate('collect.blocked_process_baseline', $1, $2)", connection)) + { + recentOnly.Parameters.AddWithValue(now.AddDays(-1)); + recentOnly.Parameters.AddWithValue(now.AddHours(1)); + await recentOnly.ExecuteNonQueryAsync(ct); + } + + Assert.True(await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct) >= 1); + Assert.Equal(floor, await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct)); + /* A later start does not move it. */ await TimescaleSupport.BackfillBaselineAggregatesAsync(connection, null, ct); Assert.Equal(floor, await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, ct)); diff --git a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs index b2849a358b..9dab43e4dc 100644 --- a/Darling/Darling.Tests/ViewerBlockingStatsTests.cs +++ b/Darling/Darling.Tests/ViewerBlockingStatsTests.cs @@ -180,6 +180,8 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup /// #6705: a parallel query waiting on itself (the same non-zero spid on both sides) is not blocking, on the desktop trend, /// the duration stats, the existence check that picks the arm, and the MCP trend and stats reads. A minute with one self-wait /// report and one real report counts 1; a window whose only XE reports are self-waits draws the DMV arm, as the detector does. + /// The Blocking-tab slicer reads the same pair by the hour and agrees (Lite's BlockingSeries_LeaveSelfWaitsOut_OnTheXeArm). + /// A (0, 0) report is a system spid pair the predicate keeps, so it counts. /// [Fact] public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWaitsDrawTheDmvArm_AgainstDevPostgres() @@ -204,25 +206,36 @@ public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWai var start = TruncateToMinute(DateTime.UtcNow.AddMinutes(-10)); var end = start.AddMinutes(11); - /* One real report (1000 ms) and one self-wait (7000 ms, spid 7 on itself) in the same minute. */ + /* One real report (1000 ms), one self-wait (7000 ms, spid 7 on itself) and one (0, 0) report (3000 ms) in the same minute. */ await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(5), 1000); + await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(25), 3000, blockedSpid: 0, blockingSpid: 0); await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7); await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(20), 9999); + /* A DMV self-wait (spid 166 on itself, 5555 ms) in the same minute: every DMV-arm read below must leave it out. */ + await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(25), 5555, blockedSpid: 166, blockingSpid: 166, collectionId: 2); var trend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end); - Assert.Equal(1, trend.Sum(p => p.Count)); + Assert.Equal(2, trend.Sum(p => p.Count)); var mcpTrend = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct); - Assert.Equal(1, mcpTrend.Sum(p => p.Count)); + Assert.Equal(2, mcpTrend.Sum(p => p.Count)); var stats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end); - Assert.Equal(1, stats.Sum(p => p.EventCount)); - Assert.Equal(1000, stats.Sum(p => p.TotalDurationMs)); - Assert.Equal(1000, stats.Max(p => p.MaxDurationMs)); + Assert.Equal(2, stats.Sum(p => p.EventCount)); + Assert.Equal(4000, stats.Sum(p => p.TotalDurationMs)); + Assert.Equal(3000, stats.Max(p => p.MaxDurationMs)); var mcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct); - Assert.Equal(1, mcpStats.Sum(p => p.EventCount)); - Assert.Equal(1000, mcpStats.Sum(p => p.TotalDurationMs)); + Assert.Equal(2, mcpStats.Sum(p => p.EventCount)); + Assert.Equal(4000, mcpStats.Sum(p => p.TotalDurationMs)); Assert.True(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end)); + /* The slicer: the real and (0, 0) reports count, the 7000 ms self-wait does not (4.0 s, not 11.0 s), and it + names two blockers (60 and 0) and two blocked spids (55 and 0), not the self-wait's spid 7. */ + var slicer = Assert.Single(await viewer.GetBlockingSlicerDataAsync(StatsServerId, start.AddMinutes(-1), end)); + Assert.Equal(2, slicer.SessionCount); + Assert.Equal(4.0, slicer.TotalCpu); + Assert.Equal(2, slicer.TotalElapsed); + Assert.Equal(2, slicer.TotalReads); + /* Only the self-wait remains in XE: the window draws the DMV snapshot, and the existence check says so. */ await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct); await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7); @@ -232,10 +245,20 @@ public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWai Assert.All(dmvTrend, p => Assert.Equal("DMV snapshot", p.Source)); var dmvMcp = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct); Assert.All(dmvMcp, p => Assert.Equal("DMV snapshot", p.Source)); + Assert.Equal(1, dmvMcp.Sum(p => p.Count)); var dmvStats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end); Assert.Equal(9999, dmvStats.Sum(p => p.TotalDurationMs)); + Assert.Equal(1, dmvStats.Sum(p => p.EventCount)); + var dmvMcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct); + Assert.Equal(1, dmvMcpStats.Sum(p => p.EventCount)); + Assert.Equal(9999, dmvMcpStats.Sum(p => p.TotalDurationMs)); Assert.False(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end)); + /* The slicer draws the DMV arm too: its one real snapshot (9999 ms), none of the XE self-wait's 7000 ms or the DMV self-wait's 5555 ms. */ + var dmvSlicer = Assert.Single(await viewer.GetBlockingSlicerDataAsync(StatsServerId, start.AddMinutes(-1), end)); + Assert.Equal(1, dmvSlicer.SessionCount); + Assert.Equal(9.999, dmvSlicer.TotalCpu, 3); + bodySucceeded = true; } finally @@ -275,21 +298,22 @@ INSERT INTO blocked_process_reports } private static async Task InsertDmvBlockingSnapshotAsync( - NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs) + NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs, + int blockedSpid = 155, int blockingSpid = 160, long collectionId = 1) { using var command = new NpgsqlCommand(@" INSERT INTO dmv_blocking_snapshots (collection_id, collection_time, server_id, server_name, event_time, database_name, blocked_spid, blocking_spid, wait_time_ms, lock_mode, blocked_sql_text, blocking_sql_text, contentious_object) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", connection); - command.Parameters.AddWithValue(1L); + command.Parameters.AddWithValue(collectionId); command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTimeUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue(serverId); command.Parameters.AddWithValue(ServerName); command.Parameters.AddWithValue(DateTime.SpecifyKind(eventTimeUtc, DateTimeKind.Unspecified)); command.Parameters.AddWithValue("AppDb"); - command.Parameters.AddWithValue(155); - command.Parameters.AddWithValue(160); + command.Parameters.AddWithValue(blockedSpid); + command.Parameters.AddWithValue(blockingSpid); command.Parameters.AddWithValue(waitTimeMs); command.Parameters.AddWithValue("S"); command.Parameters.AddWithValue("SELECT 2"); diff --git a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs index 921b5b7b00..a2431c1ac5 100644 --- a/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs +++ b/Darling/PerformanceMonitor.Darling.Service/Mcp/DarlingFleetReader.cs @@ -355,7 +355,10 @@ the alert engine and the analysis fact share. The DMV side applies the same rule band's count tiers (5 / 20 per hour) were measured in blocked-process REPORTS per server-hour, so the band reads those; the tile, the alert value and the MCP counts stay waits. Each source is read once (the *_reports CTE, shared by the report count and the waits chain), so the window and the floor are still - applied once per source. */ + applied once per source. + + #6705: both reads leave a parallel query waiting on itself out (BlockingWaits.NotSelfWaitSql), so the report count + the band reads agrees with the waits count, which WaitsCteSql already filters. */ public static readonly string FleetBlockingSql = @" WITH xe_reports AS ( SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms @@ -363,6 +366,7 @@ FROM v_blocked_process_reports WHERE event_time >= $1 AND event_time <= $2 AND collection_time >= $3 + AND " + BlockingWaits.NotSelfWaitSql + @" ), dmv_reports AS ( SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms @@ -370,6 +374,7 @@ FROM v_dmv_blocking_snapshots WHERE event_time >= $1 AND event_time <= $2 AND collection_time >= $3 + AND " + BlockingWaits.NotSelfWaitSql + @" )," + BlockingWaits.WaitsCteSql( BlockingWaits.SqlDialect.Postgres, "xe_waits", "SELECT * FROM xe_reports", new[] { "server_id" }) + "," + BlockingWaits.WaitsCteSql( diff --git a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs index 3aaf4660c2..022c7bc4b2 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/TimescaleSupport.cs @@ -913,7 +913,7 @@ that created it — see BaselineBackfillProbeSql. */ aggregate, and it is about to hold only what raw still holds. Say where that begins BEFORE the refresh. */ if (view == BlockedProcessBaselineView && coverageOldest is null) { - await RecordBlockedBaselineSupplyFloorAsync(connection, logger, DateTime.UtcNow, cancellationToken); + await TryRecordBlockedBaselineSupplyFloorAsync(connection, logger, cancellationToken); } var after = await RefreshFromAsync(connection, view, needFrom.Value, force: false, cancellationToken); @@ -1219,6 +1219,31 @@ private static async Task RecordBlockedBaselineSupplyFloorAsync(NpgsqlConnection BlockedProcessBaselineView, retentionDays, floor); } + private static async Task RawHoldsRowsAsync(NpgsqlConnection connection, string sourceTable, CancellationToken cancellationToken) + { + using var probe = new NpgsqlCommand($"SELECT EXISTS (SELECT 1 FROM collect.{sourceTable})", connection) { CommandTimeout = SetupTimeoutSeconds }; + return await probe.ExecuteScalarAsync(cancellationToken) is true; + } + + /// + /// #6705: , failure-isolated. A failed comment write must not skip the + /// backfill that follows it (it would fail again on every start and the baseline would never be backfilled); it logs and the + /// pass goes on, and the next start tries the write again. + /// + private static async Task TryRecordBlockedBaselineSupplyFloorAsync(NpgsqlConnection connection, ILogger? logger, CancellationToken cancellationToken) + { + try + { + await RecordBlockedBaselineSupplyFloorAsync(connection, logger, DateTime.UtcNow, cancellationToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogWarning( + "Could not record where the {View} supply begins -- the Blocking baseline counts quiet days it cannot back until this succeeds: {Message}", + BlockedProcessBaselineView, ex.Message); + } + } + /// Does this baseline relation exist under any implementation — continuous aggregate or plain view? public static string BaselineRelationExistsSql(string view) => $"SELECT to_regclass('collect.{view}') IS NOT NULL"; @@ -6501,11 +6526,29 @@ under the exact names the baseline aggregates need (#1757). CREATE MATERIALIZED { try { + /* #6705: was the blocking baseline absent before this CREATE? The plain fallback view was dropped above, so + a relation that exists here is a real aggregate. Read before the CREATE, as IF NOT EXISTS says nothing. */ + var createsBlockedBaseline = false; + if (view == BlockedProcessBaselineView) + { + using var exists = new NpgsqlCommand(BaselineRelationExistsSql(view), connection) { CommandTimeout = SetupTimeoutSeconds }; + createsBlockedBaseline = await exists.ExecuteScalarAsync(cancellationToken) is false; + } + using (var create = new NpgsqlCommand(createSql, connection) { CommandTimeout = SetupTimeoutSeconds }) { await create.ExecuteNonQueryAsync(cancellationToken); } + /* #6705: the floor is written on the pass that CREATES the baseline, before its refresh policy can materialize + anything. Written only at backfill time, a service that stopped between this CREATE and the backfill left the + policy to materialize the last day first, so the backfill saw a non-empty materialization and recorded + nothing. A fresh store (raw empty) records nothing: its aggregate will hold everything raw ever gets. */ + if (createsBlockedBaseline && await RawHoldsRowsAsync(connection, SourceTableFor(view), cancellationToken)) + { + await TryRecordBlockedBaselineSupplyFloorAsync(connection, logger, cancellationToken); + } + /* #3893: an off-grid aggregate's materialization width is set HERE, after its CREATE and BEFORE its policy exists, because nothing later would be early enough. #3620's width ensure runs in the aggregate-compression step, a later convergence stage, and walks the compression targets diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs index 120d2925fb..c869cf38aa 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.Blocking.cs @@ -437,7 +437,9 @@ OR EXISTS (SELECT 1 FROM inside AS i CROSS JOIN first_on AS f /// /// Where the XE blocked-process-report data starts for one server, as two columns: the XE collector's first logged run /// in the window (or the window's start when a run exists before it, which is covered), and the earliest XE report in the - /// window (the same predicate as ). The caller takes the earlier of the two, + /// window, a self-wait report included (it still proves the XE session was capturing, so this read leaves out the + /// filter that takes, #6705). + /// The caller takes the earlier of the two, /// through , which also folds in the threshold's history. /// $1 server, $2 window start, $3 window end, $4 database filter, $5 the event-window floor on collection time. Both run /// reads are bounded, the first run in [$2, $3] and the newest run before $2, and ride diff --git a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs index d61241d174..d50e1ffa33 100644 --- a/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs +++ b/Darling/PerformanceMonitor.Darling.Viewer/ViewerDataService.BlockingSlicers.cs @@ -37,8 +37,13 @@ public sealed partial class ViewerDataService /// $1 server_id, $2 window start, $3 window end (naive UTC), $4 database filter. The XE arm windows and /// buckets on the report's own event_time; the DMV arm stays on collection_time, which IS its /// event time. $5 is the for $2 (XE arm only; no upper bound). + /// + /// A parallel query waiting on itself is not blocking (#6645, #6705): both arms carry + /// , as Lite's slicer does, so the bars agree with + /// and the baseline band. The DMV arm is drawn when the FILTERED XE arm has no buckets, so a + /// window whose only XE reports are self-waits falls back to the DMV snapshots. /// - public const string BlockingSlicerSql = """ + public const string BlockingSlicerSql = $""" WITH bpr AS ( SELECT date_trunc('hour', event_time) AS bucket, @@ -51,6 +56,7 @@ FROM v_blocked_process_reports WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3 AND collection_time >= $5 AND ($4::text[] IS NULL OR database_name = ANY($4)) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} GROUP BY date_trunc('hour', event_time) ), dmv AS ( @@ -64,6 +70,7 @@ dmv AS ( FROM v_dmv_blocking_snapshots WHERE server_id = $1 AND collection_time >= $2 AND collection_time <= $3 AND ($4::text[] IS NULL OR database_name = ANY($4)) + AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql} GROUP BY date_trunc('hour', collection_time) ) SELECT bucket, event_count, total_wait_sec, distinct_blockers, distinct_blocked, distinct_databases FROM bpr