Skip to content
Merged
6 changes: 3 additions & 3 deletions Darling/Darling.Tests/BaselineSupplyTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,13 @@ and the legacy one they still fall back to are shared the same way. */
Assert.Contains("delta_wait_time_ms >= 0", TimescaleSupport.LegacyCreateWaitStatsBaselineSql, StringComparison.Ordinal);
Assert.DoesNotContain("sample_interval_seconds", TimescaleSupport.LegacyCreateWaitStatsBaselineSql, StringComparison.Ordinal);

/* blocked_process_reports: neither family filters, so the shared aggregate carries no WHERE at all.
A WHERE appearing here later means one family narrowed the other's supply. */
/* blocked_process_reports: both families leave a parallel query's self-waits out of their current window
(#6663), so the shared aggregate carries the same predicate (#6690), or its baseline reads high. */
var blocking = PgBaselineProvider.GetBaselineQuery(MetricNames.Blocking)!;
var blockingPerMinute = PgBaselineProvider.GetBaselineQuery(MetricNames.BlockingPerMinute)!;
Assert.Contains("FROM blocked_process_baseline", blocking, StringComparison.Ordinal);
Assert.Contains("FROM blocked_process_baseline", blockingPerMinute, StringComparison.Ordinal);
Assert.DoesNotContain("WHERE", TimescaleSupport.CreateBlockedProcessBaselineSql, StringComparison.Ordinal);
Assert.Contains(PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql, TimescaleSupport.CreateBlockedProcessBaselineSql, StringComparison.Ordinal);
}

/// <summary>
Expand Down
262 changes: 262 additions & 0 deletions Darling/Darling.Tests/FleetPartialDatabaseFailureBandTests.cs

Large diffs are not rendered by default.

110 changes: 103 additions & 7 deletions Darling/Darling.Tests/ViewerBlockingStatsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,103 @@ await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup
}
}

/// <summary>
/// #6705: a parallel query waiting on itself (the same non-zero spid on both sides) is not blocking, on the desktop trend,
/// the duration stats, the existence check that picks the arm, and the MCP trend and stats reads. A minute with one self-wait
/// report and one real report counts 1; a window whose only XE reports are self-waits draws the DMV arm, as the detector does.
/// The Blocking-tab slicer reads the same pair by the hour and agrees (Lite's <c>BlockingSeries_LeaveSelfWaitsOut_OnTheXeArm</c>).
/// A (0, 0) report is a system spid pair the predicate keeps, so it counts.
/// </summary>
[Fact]
public async Task SelfWaitReports_AreNotBlocking_OnEveryTrendRead_AndOnlySelfWaitsDrawTheDmvArm_AgainstDevPostgres()
{
var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
Assert.SkipWhen(string.IsNullOrEmpty(connectionString),
"Set DARLING_TEST_PG to a Postgres connection string to run the live self-wait trend test.");
var ct = TestContext.Current.CancellationToken;

using var connection = new NpgsqlConnection(connectionString);
await connection.OpenAsync(ct);
await PgMigrations.MigrateAsync(connection, ct);
await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct);
await DeleteRowsAsync(connection, "dmv_blocking_snapshots", StatsServerId, ct);

await using var viewer = new ViewerDataService(connectionString!);
await using var postgres = NpgsqlDataSource.Create(connectionString!);

var bodySucceeded = false;
try
{
var start = TruncateToMinute(DateTime.UtcNow.AddMinutes(-10));
var end = start.AddMinutes(11);

/* One real report (1000 ms), one self-wait (7000 ms, spid 7 on itself) and one (0, 0) report (3000 ms) in the same minute. */
await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(5), 1000);
await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(25), 3000, blockedSpid: 0, blockingSpid: 0);
await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7);
await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(20), 9999);
/* A DMV self-wait (spid 166 on itself, 5555 ms) in the same minute: every DMV-arm read below must leave it out. */
await InsertDmvBlockingSnapshotAsync(connection, StatsServerId, start, start.AddSeconds(25), 5555, blockedSpid: 166, blockingSpid: 166, collectionId: 2);

var trend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end);
Assert.Equal(2, trend.Sum(p => p.Count));
var mcpTrend = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
Assert.Equal(2, mcpTrend.Sum(p => p.Count));

var stats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end);
Assert.Equal(2, stats.Sum(p => p.EventCount));
Assert.Equal(4000, stats.Sum(p => p.TotalDurationMs));
Assert.Equal(3000, stats.Max(p => p.MaxDurationMs));
var mcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
Assert.Equal(2, mcpStats.Sum(p => p.EventCount));
Assert.Equal(4000, mcpStats.Sum(p => p.TotalDurationMs));
Assert.True(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end));

/* The slicer: the real and (0, 0) reports count, the 7000 ms self-wait does not (4.0 s, not 11.0 s), and it
names two blockers (60 and 0) and two blocked spids (55 and 0), not the self-wait's spid 7. */
var slicer = Assert.Single(await viewer.GetBlockingSlicerDataAsync(StatsServerId, start.AddMinutes(-1), end));
Assert.Equal(2, slicer.SessionCount);
Assert.Equal(4.0, slicer.TotalCpu);
Assert.Equal(2, slicer.TotalElapsed);
Assert.Equal(2, slicer.TotalReads);

/* Only the self-wait remains in XE: the window draws the DMV snapshot, and the existence check says so. */
await DeleteRowsAsync(connection, "blocked_process_reports", StatsServerId, ct);
await InsertBlockedProcessReportAsync(connection, StatsServerId, start, start.AddSeconds(30), 7000, blockedSpid: 7, blockingSpid: 7);

var dmvTrend = await viewer.GetBlockingTrendAsync(StatsServerId, start.AddMinutes(-1), end);
Assert.Equal(1, dmvTrend.Sum(p => p.Count));
Assert.All(dmvTrend, p => Assert.Equal("DMV snapshot", p.Source));
var dmvMcp = await PerformanceMonitor.Darling.Service.Mcp.DarlingBlockingTrendReader.GetBlockingTrendAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
Assert.All(dmvMcp, p => Assert.Equal("DMV snapshot", p.Source));
Assert.Equal(1, dmvMcp.Sum(p => p.Count));
var dmvStats = await viewer.GetBlockingDurationStatsAsync(StatsServerId, start.AddMinutes(-1), end);
Assert.Equal(9999, dmvStats.Sum(p => p.TotalDurationMs));
Assert.Equal(1, dmvStats.Sum(p => p.EventCount));
var dmvMcpStats = await PerformanceMonitor.Darling.Service.Mcp.DarlingDataReader.GetBlockingDurationStatsAsync(postgres, StatsServerId, start.AddMinutes(-1), end, ct);
Assert.Equal(1, dmvMcpStats.Sum(p => p.EventCount));
Assert.Equal(9999, dmvMcpStats.Sum(p => p.TotalDurationMs));
Assert.False(await viewer.HasBlockedProcessReportsInWindowAsync(StatsServerId, start.AddMinutes(-1), end));

/* The slicer draws the DMV arm too: its one real snapshot (9999 ms), none of the XE self-wait's 7000 ms or the DMV self-wait's 5555 ms. */
var dmvSlicer = Assert.Single(await viewer.GetBlockingSlicerDataAsync(StatsServerId, start.AddMinutes(-1), end));
Assert.Equal(1, dmvSlicer.SessionCount);
Assert.Equal(9.999, dmvSlicer.TotalCpu, 3);

bodySucceeded = true;
}
finally
{
await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) =>
{
await DeleteRowsAsync(cleanup, "blocked_process_reports", StatsServerId, cleanupCt);
await DeleteRowsAsync(cleanup, "dmv_blocking_snapshots", StatsServerId, cleanupCt);
});
}
}

private static async Task InsertBlockedProcessReportAsync(
NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs)
NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs, int blockedSpid = 55, int blockingSpid = 60)
{
using var command = new NpgsqlCommand(@"
INSERT INTO blocked_process_reports
Expand All @@ -191,8 +286,8 @@ INSERT INTO blocked_process_reports
command.Parameters.AddWithValue(ServerName);
command.Parameters.AddWithValue(DateTime.SpecifyKind(eventTimeUtc, DateTimeKind.Unspecified));
command.Parameters.AddWithValue("AppDb");
command.Parameters.AddWithValue(55);
command.Parameters.AddWithValue(60);
command.Parameters.AddWithValue(blockedSpid);
command.Parameters.AddWithValue(blockingSpid);
command.Parameters.AddWithValue(waitTimeMs);
command.Parameters.AddWithValue("X");
command.Parameters.AddWithValue("SELECT 1");
Expand All @@ -203,21 +298,22 @@ INSERT INTO blocked_process_reports
}

private static async Task InsertDmvBlockingSnapshotAsync(
NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs)
NpgsqlConnection connection, int serverId, DateTime collectionTimeUtc, DateTime eventTimeUtc, long waitTimeMs,
int blockedSpid = 155, int blockingSpid = 160, long collectionId = 1)
{
using var command = new NpgsqlCommand(@"
INSERT INTO dmv_blocking_snapshots
(collection_id, collection_time, server_id, server_name, event_time, database_name,
blocked_spid, blocking_spid, wait_time_ms, lock_mode, blocked_sql_text, blocking_sql_text, contentious_object)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)", connection);
command.Parameters.AddWithValue(1L);
command.Parameters.AddWithValue(collectionId);
command.Parameters.AddWithValue(DateTime.SpecifyKind(collectionTimeUtc, DateTimeKind.Unspecified));
command.Parameters.AddWithValue(serverId);
command.Parameters.AddWithValue(ServerName);
command.Parameters.AddWithValue(DateTime.SpecifyKind(eventTimeUtc, DateTimeKind.Unspecified));
command.Parameters.AddWithValue("AppDb");
command.Parameters.AddWithValue(155);
command.Parameters.AddWithValue(160);
command.Parameters.AddWithValue(blockedSpid);
command.Parameters.AddWithValue(blockingSpid);
command.Parameters.AddWithValue(waitTimeMs);
command.Parameters.AddWithValue("S");
command.Parameters.AddWithValue("SELECT 2");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,10 +307,9 @@ the CURRENT window counts report and deadlock events by when they HAPPENED (even
snapshot's event_time IS its collection time. The BASELINE this count is judged against comes from the
continuous aggregates, which bucket the hypertable's own column (collection_time) by design.

#6645: this CURRENT count leaves out a parallel query's self-waits, but the BASELINE aggregate
(collect.blocked_process_baseline, count(*) over every report row) still counts them until a store
migration rebuilds it with the same predicate, so a server with many self-waits has a high baseline and
its real blocking is under-flagged in the meantime. */
#6645: this CURRENT count leaves out a parallel query's self-waits, and so does the BASELINE aggregate
(collect.blocked_process_baseline, #6690: its select carries the same BlockingWaits.NotSelfWaitSql), so the
ratio compares like with like. A store that built the aggregate earlier is rebuilt on start. */
public const string BlockingWindowSql = $@"
/* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking. */
SELECT
Expand Down
13 changes: 13 additions & 0 deletions Darling/PerformanceMonitor.Darling.Analysis/PgBaselineProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -896,6 +896,19 @@ analyze_server and compare_analysis (BaselineCache) without changing anyone's an
query = await ChooseSupplyAsync(connection, serverId, metricName, query, windowStart, cancellationToken);
}

/* #6705 (the #3590 trap): the Blocking arm counts a collection_log run with no event as a measured zero, and the
log is kept twice as long as the events behind blocked_process_baseline. After that aggregate was rebuilt it
holds only what raw still held, so a day before that would read as a quiet day, not a missing one. The
start path recorded where its supply begins; the window starts no earlier. Absent or unreadable is no floor,
and it is never derived from the aggregate's own oldest bucket (a server that rarely blocks has real zero days).
The text of the arm is unchanged: it still reads $2. */
if (keys is null && metricName == MetricNames.Blocking
&& await TimescaleSupport.ReadBlockedBaselineSupplyFloorAsync(connection, cancellationToken) is DateTime supplyFloor
&& supplyFloor > windowStart)
{
windowStart = supplyFloor;
}

/* #3653 Q6: the target's clock over this window, read on the same connection and INSIDE this try on
purpose — a store that cannot answer a one-row indexed read of server_properties cannot answer the
aggregate scan either, and one classified catch (AnalysisShutdownResidueTests pins exactly one) is
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ public sealed record BlockingTrendReadPoint(DateTime Time, int Count, string? So
/// <para><b>The database predicate is the list form (#5244), on BOTH arms</b>, as the desktop's <c>BlockingTrendSql</c> has it.
/// The DMV arm's <c>NOT EXISTS (SELECT 1 FROM bpr)</c> therefore asks whether the XE source has rows for the CHOSEN databases:
/// a database with only DMV-captured blocking still charts, whatever other databases' XE reports hold.</para>
///
/// <para><b>A parallel query waiting on itself is not blocking (#6645, #6705)</b>: both arms carry
/// <see cref="BlockingWaits.NotSelfWaitSql"/>, as <c>PgAnomalyDetector.BlockingWindowSql</c> does, so the bars (and the web
/// Overview strip drawn from this read) agree with the baseline band. The DMV arm is therefore drawn when the FILTERED XE
/// arm is empty, a window whose only XE reports are self-waits included.</para>
/// </summary>
public static readonly string BlockingTrendSql = $$"""
WITH bpr AS (
Expand All @@ -58,6 +63,7 @@ FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
Expand All @@ -66,6 +72,7 @@ FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, incident_count, source FROM bpr
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4822,6 +4822,9 @@ public static async Task<List<BlockedSessionTrendRow>> GetBlockedSessionTrendAsy
/// is taken only when the XE source has no row for the CHOSEN databases (<c>NOT EXISTS (SELECT 1 FROM bpr)</c> over the
/// filtered <c>bpr</c>), exactly as the desktop's blocking reads and <c>get_blocking_trend</c> choose, so the tools answer
/// from one source for one filter and the two sources are never mixed. The unfiltered series is exactly what it was.</para>
/// <para>Both arms leave out a parallel query waiting on itself (<see cref="BlockingWaits.NotSelfWaitSql"/>, #6645, #6705),
/// as <c>get_blocking_trend</c> does, so the count and the severity reconcile; the DMV fallback is taken when the FILTERED XE
/// arm is empty.</para>
/// </summary>
public static readonly string BlockingDurationStatsSql = $$"""
WITH bpr AS (
Expand All @@ -4836,6 +4839,7 @@ FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
),
dmv AS (
Expand All @@ -4850,6 +4854,7 @@ FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND event_time >= $2 AND event_time <= $3
AND collection_time >= $4
{{DatabaseFilter.All.Clause("database_name", 5)}}
AND {{BlockingWaits.NotSelfWaitSql}}
GROUP BY DATE_TRUNC('minute', event_time)
)
SELECT bucket, event_count, total_duration_ms, max_duration_ms, avg_duration_ms, source FROM bpr
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -355,21 +355,26 @@ the alert engine and the analysis fact share. The DMV side applies the same rule
band's count tiers (5 / 20 per hour) were measured in blocked-process REPORTS per server-hour, so the band
reads those; the tile, the alert value and the MCP counts stay waits. Each source is read once (the
*_reports CTE, shared by the report count and the waits chain), so the window and the floor are still
applied once per source. */
applied once per source.

#6705: both reads leave a parallel query waiting on itself out (BlockingWaits.NotSelfWaitSql), so the report count
the band reads agrees with the waits count, which WaitsCteSql already filters. */
public static readonly string FleetBlockingSql = @"
WITH xe_reports AS (
SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms
FROM v_blocked_process_reports
WHERE event_time >= $1
AND event_time <= $2
AND collection_time >= $3
AND " + BlockingWaits.NotSelfWaitSql + @"
),
dmv_reports AS (
SELECT server_id, blocked_spid, blocking_spid, event_time, wait_time_ms
FROM v_dmv_blocking_snapshots
WHERE event_time >= $1
AND event_time <= $2
AND collection_time >= $3
AND " + BlockingWaits.NotSelfWaitSql + @"
)," + BlockingWaits.WaitsCteSql(
BlockingWaits.SqlDialect.Postgres, "xe_waits",
"SELECT * FROM xe_reports", new[] { "server_id" }) + "," + BlockingWaits.WaitsCteSql(
Expand Down
Loading
Loading