Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 69 additions & 1 deletion Darling/Darling.Tests/BlockingWaitCountTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,30 @@ public void IssueShape_FortyThreeReportsOfFourWaits_AreFourEvents()
Assert.Equal(new[] { 55900L, 45900L, 45900L, 30900L }.OrderBy(x => x), waits.Select(w => w.WaitTimeMs).OrderBy(x => x));
}

/// <summary>
/// #6645: the threads of one parallel query write reports with blocked spid = blocking spid (different ecid).
/// That is the query waiting on itself, so it is no wait: the rule drops it and the alert rows follow.
/// </summary>
[Fact]
public void ParallelQuerySelfWaits_AreNotWaits()
{
var rows = OneWait(59, 59, T0, 6);
rows.AddRange(OneWait(59, 59, T0.AddMinutes(1), 3));
var real = OneWait(102, 100, T0.AddMinutes(2), 5);
rows.AddRange(real);

var wait = Assert.Single(Collapse(rows));
Assert.Equal(real.Max(r => r.WaitTimeMs), wait.WaitTimeMs);
Assert.Equal(100, wait.BlockingSpid);

var counted = AlertContextBuilders.BlockingWaitRows(rows, Array.Empty<string>());
Assert.Equal(100, Assert.Single(counted).BlockingSpid);
Assert.Empty(AlertContextBuilders.BlockingWaitRows(OneWait(59, 59, T0, 6), Array.Empty<string>()));

/* A report that named no session (the collector stores 0, never NULL) is still a blocking event. */
Assert.Single(AlertContextBuilders.BlockingWaitRows(OneWait(0, 0, T0, 1), Array.Empty<string>()));
}

[Fact]
public void ReportOrder_DoesNotChangeTheCount()
{
Expand Down Expand Up @@ -214,6 +238,19 @@ public void XeReportsInANonExcludedDatabase_AreTheSource_AndDmvWaitsAreIgnored()
Assert.Equal(102, wait.BlockedSpid);
}

/// <summary>#6645 round 1: XE reports that are all a parallel query's self-waits are no XE reports, so the DMV
/// waits beside them are the source, as in the SQL read surfaces.</summary>
[Fact]
public void XeReportsThatAreAllSelfWaits_FallBackToTheDmvWaits()
{
var rows = OneWait(59, 59, T0, 6);
rows.AddRange(OneWait(202, 200, T0.AddMinutes(5), 3, source: BlockedProcessAlertRow.DmvSnapshotSource));

var wait = Assert.Single(AlertContextBuilders.BlockingWaitRows(rows, Array.Empty<string>()));
Assert.Equal(BlockedProcessAlertRow.DmvSnapshotSource, wait.Source);
Assert.Equal(200, wait.BlockingSpid);
}

[Fact]
public void NoXeReportsAtAll_CountsTheDmvWaits_OutsideTheExcludedDatabases()
{
Expand Down Expand Up @@ -280,6 +317,10 @@ public async Task PostgresSql_AgreesWithTheCSharpRule_OnTheSameReports()
rows.AddRange(OneWait(102, 100, T0.AddMinutes(30), 6));
rows.AddRange(OneWait(103, 100, T0, 4));
rows.AddRange(OneWait(102, 101, T0.AddMinutes(1), 7));
/* #6645 round 1: a report that named no session is stored as (0, 0) and is a wait in the C# rule, so the SQL
must keep it; a parallel query's equal non-zero spids are dropped by both. */
rows.AddRange(OneWait(0, 0, T0.AddMinutes(40), 3));
rows.AddRange(OneWait(59, 59, T0.AddMinutes(50), 4));

var sql = "WITH " + BlockingWaits.WaitsCteSql(
BlockingWaits.SqlDialect.Postgres, "waits", ValuesSource(rows, 1), new[] { "server_id" })
Expand All @@ -293,10 +334,37 @@ public async Task PostgresSql_AgreesWithTheCSharpRule_OnTheSameReports()

var expected = Collapse(rows);
Assert.Equal(expected.Count, Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture));
Assert.Equal(6, expected.Count);
Assert.Equal(7, expected.Count);
Assert.Equal(expected.Max(w => w.WaitTimeMs), Convert.ToInt64(reader.GetValue(1), CultureInfo.InvariantCulture));
}

[Fact]
public async Task PostgresSql_LeavesOutParallelQuerySelfWaits_AndCountsTheRealOne()
{
var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
Assert.SkipWhen(string.IsNullOrWhiteSpace(connectionString), "Set DARLING_TEST_PG to run the live wait-count SQL test.");

var rows = new List<BlockedProcessAlertRow>();
rows.AddRange(OneWait(59, 59, T0, 6));
rows.AddRange(OneWait(59, 59, T0.AddMinutes(10), 4));
rows.AddRange(OneWait(102, 100, T0.AddMinutes(20), 5));

/* #6645: reportCount adds the per-wait report tally, which must count only the real wait's reports. */
var sql = "WITH " + BlockingWaits.WaitsCteSql(
BlockingWaits.SqlDialect.Postgres, "waits", ValuesSource(rows, 1), new[] { "server_id" }, reportCount: true)
+ " SELECT COUNT(*), COUNT(DISTINCT blocking_spid), MAX(wait_reports) FROM waits";

await using var connection = new NpgsqlConnection(connectionString);
await connection.OpenAsync();
await using var command = new NpgsqlCommand(sql, connection);
await using var reader = await command.ExecuteReaderAsync();
Assert.True(await reader.ReadAsync());

Assert.Equal(1, Convert.ToInt32(reader.GetValue(0), CultureInfo.InvariantCulture));
Assert.Equal(1, Convert.ToInt32(reader.GetValue(1), CultureInfo.InvariantCulture));
Assert.Equal(5, Convert.ToInt32(reader.GetValue(2), CultureInfo.InvariantCulture));
}

/* ── #5644 wave-1 review: sub-millisecond starts and NULL times, in C# and in the store's SQL ── */

internal sealed record NullableReport(int Blocked, int Blocker, DateTime? Time, long? Wait);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ public void TheTwoProducts_CarryTheSameEventBaselineBody_AndTheSameCollectors()
foreach (var (metric, eventTable, aggregate, liteEvents) in new[]
{
("Blocking", "blocked_process_reports", TimescaleSupport.BlockedProcessBaselineView,
"StoredEventCopies.BlockedProcessReports(\"server_id = $1 AND event_time >= $2 AND event_time < $3\") + \" AS ev\""),
"StoredEventCopies.BlockedProcessReports(\"server_id = $1 AND event_time >= $2 AND event_time < $3 AND \" + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql) + \" AS ev\""),
("Deadlock", "deadlocks", TimescaleSupport.DeadlockBaselineView, "v_deadlocks"),
})
{
Expand Down
20 changes: 15 additions & 5 deletions Darling/PerformanceMonitor.Darling.Analysis/PgAnomalyDetector.cs
Original file line number Diff line number Diff line change
Expand Up @@ -305,29 +305,39 @@ the CURRENT window counts report and deadlock events by when they HAPPENED (even
deadlock_time), so the spike counts the events the grids show; $4 is the EventWindowFloor for $2, the
partition-column bound with no upper side. The DMV snapshot arm stays on collection_time because a
snapshot's event_time IS its collection time. The BASELINE this count is judged against comes from the
continuous aggregates, which bucket the hypertable's own column (collection_time) by design. */
public const string BlockingWindowSql = @"
continuous aggregates, which bucket the hypertable's own column (collection_time) by design.

#6645: this CURRENT count leaves out a parallel query's self-waits, but the BASELINE aggregate
(collect.blocked_process_baseline, count(*) over every report row) still counts them until a store
migration rebuilds it with the same predicate, so a server with many self-waits has a high baseline and
its real blocking is under-flagged in the meantime. */
public const string BlockingWindowSql = $@"
/* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking. */
SELECT
COALESCE(NULLIF(
(SELECT COUNT(*) FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4), 0),
WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $4
AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}), 0),
(SELECT COUNT(*) FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3)) AS current_blocking,
WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3
AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql})) AS current_blocking,
(SELECT COUNT(*) FROM v_deadlocks
WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time < $3 AND collection_time >= $4) AS current_deadlocks";

/* The same read for an Azure SQL Database master target ($4 = the names of the databases monitored as their
own targets, skipped on both arms, a NULL database still counting; $5 = the event-window floor, which takes
the number after the list so the scoped read keeps the list where it was). The deadlock count comes from DeadlockGraphsCountSql through the every-process
rule, so this read carries no deadlock column. */
public const string BlockingSkippingSeparateCountSql = @"
public const string BlockingSkippingSeparateCountSql = $@"
SELECT
COALESCE(NULLIF(
(SELECT COUNT(*) FROM v_blocked_process_reports
WHERE server_id = $1 AND event_time >= $2 AND event_time < $3 AND collection_time >= $5
AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x)))), 0),
(SELECT COUNT(*) FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3
AND {PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql}
AND (database_name IS NULL OR NOT (lower(database_name) = ANY(SELECT lower(x) FROM unnest($4::text[]) x))))) AS current_blocking,
0::bigint AS current_deadlocks";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -576,16 +576,21 @@ private async Task<List<BlockedProcessAlertRow>> GetRecentBlockedProcessReportsC
/// <see cref="CurrentBlockingWaitResult"/>), its <c>wait_time_ms</c> summed and its distinct
/// blocked SPIDs counted. $1 server_id, used twice.
/// <para>
/// #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not
/// blocking. The two aggregates leave those rows out with FILTER rather than the WHERE, so a newest snapshot
/// that holds only such rows still reads as a snapshot (its time and freshness) with nothing blocked.
/// </para>
/// <para>
/// ONE statement, deliberately: Npgsql fails SILENTLY on multi-statement commands with positional
/// parameters, so the freshness probe cannot be batched onto this — the snapshot time comes back as
/// a column of this same aggregate instead, which is one round trip rather than two anyway.
/// </para>
/// </summary>
public const string CurrentBlockingWaitSql = @"
public const string CurrentBlockingWaitSql = $@"
SELECT
collection_time,
CAST(COALESCE(SUM(wait_time_ms), 0) AS bigint) AS total_wait_ms,
CAST(COUNT(DISTINCT blocked_spid) AS integer) AS blocked_sessions
CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE {BlockingWaits.NotSelfWaitSql}), 0) AS bigint) AS total_wait_ms,
CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE {BlockingWaits.NotSelfWaitSql}) AS integer) AS blocked_sessions
FROM dmv_blocking_snapshots
WHERE server_id = $1
AND collection_time = (
Expand Down
25 changes: 25 additions & 0 deletions Lite.Tests/BlockingChainReconstructorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,31 @@ public void Empty_ProducesNoChains()
Assert.Empty(result.Chains);
}

/// <summary>
/// #6645: a parallel query's threads write reports with blocked spid = blocking spid (different ecid). That
/// is the query waiting on itself: no chain, no cycle, and a real chain beside it is unchanged.
/// </summary>
[Fact]
public void ParallelQuerySelfWaits_MakeNoChainAndNoCycle()
{
var self = new[]
{
Pair(59, 59, blockedEcid: 2, blockingEcid: 1), Pair(59, 59, blockedEcid: 5, blockingEcid: 1),
Pair(59, 59, blockedEcid: 6, blockingEcid: 1), Pair(59, 59, blockedEcid: 1, blockingEcid: 2),
};

var onlySelf = Run(self);
Assert.Empty(onlySelf.Chains);
Assert.False(onlySelf.CycleDetected);

var withReal = Run(self.Append(Pair(52, 51)));
var chain = Assert.Single(withReal.Chains);
Assert.Equal(51, chain.ApexSpid);
Assert.Equal(1, chain.Depth);
Assert.Equal(1, chain.VictimCount);
Assert.False(withReal.CycleDetected);
}

[Fact]
public void DepthFourLine_ReportsApexDepthAndVictims()
{
Expand Down
49 changes: 49 additions & 0 deletions Lite.Tests/BlockingWaitDuckDbTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,55 @@ public async Task SeveralReportsOfOneWait_CountOnce()
Assert.Equal(55_000.0, (await BlockingFactAsync()).Metadata["max_wait_time_ms"]);
}

/// <summary>
/// #6645: the threads of one parallel query write reports with blocked_spid = blocking_spid (a different ecid
/// each). That is the query waiting on itself, not blocking, so it is no wait and no head blocker: the fact,
/// the overview card, the C# rule and the current-wait snapshot all leave it out.
/// </summary>
[Fact]
public async Task ParallelQuerySelfWaits_AreNotBlocking()
{
await SeedWaitAsync(59, 59, Ago(40), reports: 4);
await SeedWaitAsync(59, 59, Ago(30), reports: 3, dmv: true);
await SeedWaitAsync(52, 51, Ago(20), reports: 5);

await AssertAllThreeAgreeAsync(1);

/* The real lock wait's blocker is the only head blocker; the parallel query's own spid is not one. */
Assert.Equal(1.0, (await BlockingFactAsync()).Metadata["distinct_head_blockers"]);

var snapshot = await new LocalDataService(_duckDb).GetCurrentBlockingWaitAsync(ServerId);
Assert.NotNull(snapshot);
Assert.Equal(0, snapshot!.Value.BlockedSessionCount);
Assert.Equal(0L, snapshot.Value.TotalWaitMs);
}

/// <summary>
/// #6645 round 1: the collector stores 0, never NULL, for a report that names no session, and the C# rule keeps
/// a (0, 0) row (0 = 0 is true in SQL, so the SQL predicate must carry the same exception). The fact, the overview
/// card and the C# rule count it as a wait; the equal non-zero spids beside it are still dropped.
/// </summary>
[Fact]
public async Task AReportThatNamedNoSession_StoredAsSpidZeroZero_IsStillAWait()
{
await SeedWaitAsync(0, 0, Ago(40), reports: 3);
await SeedWaitAsync(59, 59, Ago(30), reports: 4);
await SeedWaitAsync(52, 51, Ago(20), reports: 5);

await AssertAllThreeAgreeAsync(2);
}

[Fact]
public async Task ADmvSnapshotOfSpidZeroZero_IsStillBlockedInTheCurrentWaitRead()
{
await SeedWaitAsync(0, 0, Ago(10), reports: 2, dmv: true);

var snapshot = await new LocalDataService(_duckDb).GetCurrentBlockingWaitAsync(ServerId);
Assert.NotNull(snapshot);
Assert.Equal(1, snapshot!.Value.BlockedSessionCount);
Assert.True(snapshot.Value.TotalWaitMs > 0);
}

[Fact]
public async Task TheAuditsElevenNineNineAndSixReports_AreFourWaits()
{
Expand Down
7 changes: 4 additions & 3 deletions Lite/Analysis/AnomalyDetector.cs
Original file line number Diff line number Diff line change
Expand Up @@ -795,15 +795,16 @@ The DMV arm below stays on collection_time because a snapshot's event_time IS it
var scopeList = context.SeparatelyMonitoredDatabases;
var scoped = scopeList is { Count: > 0 };
var rowScope = SeparatelyMonitoredScope.BprFilter(scopeList, 4);
/* current_blocking: prefer the blocked-process-report; fall back to the always-on DMV
/* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not blocking.
current_blocking: prefer the blocked-process-report; fall back to the always-on DMV
snapshot so RDS (where the BPR session is empty) still counts blocking. Mirrors the
overview/alert path (LocalDataService.Overview.cs / LocalDataService.Blocking.cs). */
cmd.CommandText = (@"
SELECT
COALESCE(NULLIF(
(SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3{SCOPE}") + @" AS ev), 0),
(SELECT COUNT(*) FROM " + StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql + "{SCOPE}") + @" AS ev), 0),
(SELECT COUNT(*) FROM v_dmv_blocking_snapshots
WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3{SCOPE})) AS current_blocking,
WHERE server_id = $1 AND collection_time >= $2 AND collection_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql + @"{SCOPE})) AS current_blocking,
(SELECT " + StoredEventCopies.DeadlockDistinctCount + @" FROM v_deadlocks AS dl WHERE server_id = $1 AND deadlock_time >= $2 AND deadlock_time < $3) AS current_deadlocks")
.Replace("{SCOPE}", rowScope);

Expand Down
4 changes: 3 additions & 1 deletion Lite/Analysis/BaselineProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -711,10 +711,12 @@ FROM v_file_io_stats
AND (delta_reads > 0 OR delta_writes > 0)
)," + RobustTierScaffold,

// #6645: the baseline counts the same reports the current window does (BlockingWaits.NotSelfWaitSql), so the 3x ratio
// compares like with like. The Darling twin reads a stored aggregate that still counts them.
// Event-based — mean = events per covered hour for this bucket, sample_count = covered days.
// No restart exclusion needed (event counts, not cumulative). See EventBaselineSql (#4731).
MetricNames.Blocking => OnEventTime(EventBaselineSql("blocked_process_report", "v_collection_log",
StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3") + " AS ev", "COUNT(*)"), "event_time"),
StoredEventCopies.BlockedProcessReports("server_id = $1 AND event_time >= $2 AND event_time < $3 AND " + PerformanceMonitor.Common.BlockingWaits.NotSelfWaitSql) + " AS ev", "COUNT(*)"), "event_time"),

// Event-based — same approach as blocking
MetricNames.Deadlock => OnEventTime(EventBaselineSql("deadlocks", "v_collection_log",
Expand Down
8 changes: 6 additions & 2 deletions Lite/Services/LocalDataService.Blocking.cs
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,15 @@ public partial class LocalDataService
DuckDB.NET materializes as System.Numerics.BigInteger — a type Convert.ToInt64 cannot convert
(BigInteger does not implement IConvertible), so the read threw at alert-check time without
them. COUNT is cast for the same reason its result is read as an int. */
/* #6645: a row whose blocked and blocking spid are the same is a parallel query waiting on itself, not
blocking. FILTER (not WHERE) so a stored snapshot of only such rows still reads as a snapshot with
nothing blocked, its time and freshness intact. The DMV collector no longer writes such rows
(DmvBlockingSnapshotCollector filters them), so this bites only on rows already stored. */
command.CommandText = @"
SELECT
collection_time,
CAST(COALESCE(SUM(wait_time_ms), 0) AS BIGINT) AS total_wait_ms,
CAST(COUNT(DISTINCT blocked_spid) AS INTEGER) AS blocked_sessions
CAST(COALESCE(SUM(wait_time_ms) FILTER (WHERE " + BlockingWaits.NotSelfWaitSql + @"), 0) AS BIGINT) AS total_wait_ms,
CAST(COUNT(DISTINCT blocked_spid) FILTER (WHERE " + BlockingWaits.NotSelfWaitSql + @") AS INTEGER) AS blocked_sessions
FROM v_dmv_blocking_snapshots
WHERE server_id = $1
AND collection_time = (
Expand Down
7 changes: 6 additions & 1 deletion PerformanceMonitor.Alerting/AlertContextBuilders.cs
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,12 @@ public static IReadOnlyList<BlockedProcessAlertRow> BlockingWaitRows(
{
if (events == null || events.Count == 0) return Array.Empty<BlockedProcessAlertRow>();

var hasXeReports = events.Any(r => r.Source == BlockedProcessAlertRow.XeReportSource);
/* #6645: a parallel query's self-wait report (the same non-zero spid on both sides) is not a blocking
report, so a window whose XE reports are all self-waits has no XE reports and falls back to the DMV
waits, as the SQL read surfaces do (BlockingWaits.NotSelfWaitSql). Spid 0 is a report that named no
session and still counts. */
var hasXeReports = events.Any(r => r.Source == BlockedProcessAlertRow.XeReportSource
&& !(r.BlockedSpid == r.BlockingSpid && r.BlockedSpid != 0));

var waits = BlockingWaits.Collapse(
FilterBlocking(events, excludedDatabases),
Expand Down
Loading
Loading