Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 44 additions & 15 deletions Darling/Darling.Tests/AlertEngineTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1366,6 +1366,9 @@ public async Task AzureMaster_EventsForADatabaseNotInTheList_StillAlert()

Assert.Contains(h.Deliverer.Outcomes, o => o.MetricName == "Blocking Detected");
Assert.Contains(h.Deliverer.Outcomes, o => o.MetricName == "Deadlocks Detected");
/* #6703: a count of one reads in the singular. */
Assert.Equal("1 blocking session", Assert.Single(h.Deliverer.Outcomes, o => o.MetricName == "Blocking Detected").ShortMessage);
Assert.Equal("1 deadlock in the last hour", Assert.Single(h.Deliverer.Outcomes, o => o.MetricName == "Deadlocks Detected").ShortMessage);
}

[Fact]
Expand All @@ -1382,7 +1385,7 @@ public async Task AzureMaster_Blocking_CountAndWatermarkCoverOnlyTheUnlistedData

var outcome = Assert.Single(h.Deliverer.Outcomes, o => o.MetricName == "Blocking Detected");
Assert.Equal("2", outcome.CurrentValue);
Assert.Equal("2 blocking session(s)", outcome.ShortMessage);
Assert.Equal("2 blocking sessions", outcome.ShortMessage);
Assert.Equal(2, h.StateStore.EdgeWatermarks[(Key, AlertEngine.BlockingWatermarkMetric)]);
}

Expand All @@ -1400,7 +1403,7 @@ public async Task AzureMaster_Deadlocks_CountAndWatermarkCoverOnlyTheUnlistedDat

var outcome = Assert.Single(h.Deliverer.Outcomes, o => o.MetricName == "Deadlocks Detected");
Assert.Equal("2", outcome.CurrentValue);
Assert.Equal("2 deadlock(s) in the last hour", outcome.ShortMessage);
Assert.Equal("2 deadlocks in the last hour", outcome.ShortMessage);
Assert.Equal(2, h.StateStore.EdgeWatermarks[(Key, AlertEngine.DeadlockWatermarkMetric)]);
}

Expand Down Expand Up @@ -1498,7 +1501,7 @@ public async Task BlockingWait_FiresAtThresholdInclusive_WithRealNumericsAndCont
await DriveBlockingWaitAsync(engine, h, 600_000, samples: AlertEngine.BlockingWaitBreachSamples, from: Harness.SampleBase);

var fired = Assert.Single(h.Deliverer.Outcomes, o => o.MetricName == "Blocking Wait Time");
Assert.Equal("600s across 3 blocked session(s)", fired.CurrentValue);
Assert.Equal("600s across 3 blocked sessions", fired.CurrentValue);
Assert.Equal("600s", fired.ThresholdValue);
/* #1830: the numerics must carry the real values — the display text is prose no parser recovers. */
Assert.Equal(600d, fired.NumericCurrentValue);
Expand Down Expand Up @@ -1544,7 +1547,7 @@ public async Task BlockingWait_ASingleSnapshotAtThreeTimesTheBar_FiresAtOnce_And

var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("Blocking Wait Time", fired.MetricName);
Assert.Equal("180s across 145 blocked session(s)", fired.CurrentValue);
Assert.Equal("180s across 145 blocked sessions", fired.CurrentValue);
Assert.Equal(AlertEngine.BlockingWaitFiredBySingleSnapshot, FiredBy(fired));
var gateItem = fired.Context!.Details[0];
Assert.Equal("180s", Assert.Single(gateItem.Fields, f => f.Label == "Total Blocked Wait").Value);
Expand Down Expand Up @@ -2756,7 +2759,7 @@ public async Task LongRunningQuery_ForwardsEverySettingsKnobToTheAdapter_AndFire

var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("Long-Running Query", fired.MetricName);
Assert.Equal("1 query(s), longest 35m", fired.CurrentValue); /* :385 */
Assert.Equal("1 query, longest 35m", fired.CurrentValue); /* :385 */
Assert.Equal("30m", fired.ThresholdValue);
Assert.Equal(35d, fired.NumericCurrentValue); /* :389 */

Expand Down Expand Up @@ -2843,7 +2846,7 @@ service logins (multi-day CDC-shaped background), the application's admin login

var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("Long-Running Query", fired.MetricName);
Assert.Equal("2 query(s), longest 66m", fired.CurrentValue);
Assert.Equal("2 queries, longest 66m", fired.CurrentValue);
Assert.StartsWith("Session #65 running 66m", fired.ShortMessage, StringComparison.Ordinal);
/* The excluded sessions are not on the card as SESSIONS (the knob item lists the prefix itself, by design). */
Assert.DoesNotContain("Job 0x", fired.DetailText, StringComparison.Ordinal);
Expand All @@ -2866,7 +2869,7 @@ public async Task LongRunningQuery_AnExcludedSessionIsNotEvaluated_AndTheCardCou
{
/* Measured on one production store class: 191 distinct sessions over the 30-minute bar in 7 days, the p90
seen in 6,192 snapshots — permanent background requests no gate can separate from a runaway query. With
the knob set, the permanent session is not read into the decision at all (not counted in "N query(s)",
the knob set, the permanent session is not read into the decision at all (not counted in "N queries",
not the longest, not fingerprinted), and the card's knob item says how many the knob removed, by arm, so
the operator can see it working. */
var h = new Harness();
Expand All @@ -2883,7 +2886,7 @@ the operator can see it working. */
var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("Long-Running Query", fired.MetricName);
/* One query, and the longest is the 35-minute one — the two permanent sessions never entered the count. */
Assert.Equal("1 query(s), longest 35m", fired.CurrentValue);
Assert.Equal("1 query, longest 35m", fired.CurrentValue);
Assert.Equal(35d, fired.NumericCurrentValue);
Assert.StartsWith("Session #73 running 35m", fired.ShortMessage, StringComparison.Ordinal);

Expand Down Expand Up @@ -2923,7 +2926,7 @@ matches nothing real — the entry is listed on the card so the misspelling is v
await h.Build().EvaluateServerAsync(Harness.Snapshot());

var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("2 query(s), longest 150m", fired.CurrentValue);
Assert.Equal("2 queries, longest 150m", fired.CurrentValue);
var knobItem = fired.Context!.Details[^1];
Assert.Equal("0", Assert.Single(knobItem.Fields, f => f.Label == AlertContextBuilders.LongRunningQueryExcludedCountLabel).Value);
Assert.Equal("SQLAgent - TSQL JobStep*", Assert.Single(knobItem.Fields, f => f.Label == "Excluded Program Prefixes").Value);
Expand Down Expand Up @@ -3006,7 +3009,7 @@ public async Task LongRunningQuery_ExcludedDatabasesAreRemovedAheadOfTheCap_AndT
runs long; on a server where that ETL holds the SIX longest sessions and the cap is five, the old
post-read filter returned an EMPTY page — five excluded rows read, five dropped — and the alert was off
for every other database without a word. With the list applied in the read, the human's query is on
the page, the fire says "1 query(s)", and the card's new item says six sessions were in excluded
the page, the fire says "1 query", and the card's new item says six sessions were in excluded
databases — so a short page is a page an operator can read. */
var h = new Harness();
h.Settings.LongRunningQueryEnabled = true;
Expand All @@ -3023,7 +3026,7 @@ public async Task LongRunningQuery_ExcludedDatabasesAreRemovedAheadOfTheCap_AndT

var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("Long-Running Query", fired.MetricName);
Assert.Equal("1 query(s), longest 35m", fired.CurrentValue);
Assert.Equal("1 query, longest 35m", fired.CurrentValue);
Assert.StartsWith("Session #73 running 35m", fired.ShortMessage, StringComparison.Ordinal);
/* None of the six ETL sessions is on the card as a session — not their text, not their ids. */
Assert.DoesNotContain("FactSales", fired.DetailText, StringComparison.Ordinal);
Expand Down Expand Up @@ -3085,7 +3088,7 @@ public async Task LongRunningQuery_TheKnobAndTheDatabaseListCompose_ASessionBoth
await h.Build().EvaluateServerAsync(Harness.Snapshot());

var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("1 query(s), longest 35m", fired.CurrentValue);
Assert.Equal("1 query, longest 35m", fired.CurrentValue);

/* Knob item first (61 and 65 by prefix; 62 by login), database item last (63 and 64) — 2 + 1 + 2 = 5, the
five sessions the page does not show, each counted exactly once. */
Expand Down Expand Up @@ -3279,7 +3282,7 @@ public async Task Cpu_ConcurrentMaintenanceSessions_OneLineEach_CappedWithAState
var detail = Assert.Single(h.Deliverer.Outcomes).DetailText!;
Assert.Equal(AlertContextBuilders.ActiveMaintenanceMaxLines,
detail.Split('\n').Count(line => line.Contains("Active maintenance: BACKUP DATABASE", StringComparison.Ordinal)));
Assert.EndsWith("Active maintenance: 2 more maintenance session(s) not shown", detail, StringComparison.Ordinal);
Assert.EndsWith("Active maintenance: 2 more maintenance sessions not shown", detail, StringComparison.Ordinal);
Assert.Contains("10m 0s elapsed", detail, StringComparison.Ordinal);
}

Expand Down Expand Up @@ -4408,7 +4411,7 @@ NEW run of the same job alerts without waiting out the old run's cooldown. */
await engine.EvaluateServerAsync(Harness.Snapshot());
var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("Long-Running Job", fired.MetricName);
Assert.Equal("1 job(s) exceeding 3x average", fired.CurrentValue); /* :606 */
Assert.Equal("1 job exceeding 3x average", fired.CurrentValue); /* :606 */
Assert.Equal(400d, fired.NumericCurrentValue); /* :610 */
Assert.Equal(300d, fired.NumericThresholdValue); /* :611 multiplier*100 */

Expand Down Expand Up @@ -4460,6 +4463,29 @@ fresh evidence resumes real evaluation. */
Assert.Equal("Long-Running Jobs Cleared", Assert.Single(h.Resolutions).Title);
}

[Fact]
public async Task AnomalousJob_TwoJobsReadInThePlural()
{
/* #6703: "2 jobs exceeding", where one job reads "1 job exceeding". */
var h = new Harness();
h.Settings.LongRunningJobEnabled = true;
var start = new DateTime(2026, 7, 1, 11, 0, 0);
h.Adapter.AnomalousJobs.Add(new AnomalousJobInfo
{
JobName = "Nightly ETL", JobId = "job-1", StartTime = start,
CurrentDurationSeconds = 3600, AvgDurationSeconds = 900, PercentOfAverage = 400
});
h.Adapter.AnomalousJobs.Add(new AnomalousJobInfo
{
JobName = "Index Rebuild", JobId = "job-2", StartTime = start,
CurrentDurationSeconds = 3600, AvgDurationSeconds = 900, PercentOfAverage = 400
});

await h.Build().EvaluateServerAsync(Harness.Snapshot());

Assert.Equal("2 jobs exceeding 3x average", Assert.Single(h.Deliverer.Outcomes).CurrentValue);
}

/* ---------------- failed jobs ---------------- */

[Fact]
Expand All @@ -4477,7 +4503,7 @@ public async Task FailedJobs_WatermarkDedups_AndPersistsTheServerLocalRunTime()
await engine.EvaluateServerAsync(Harness.Snapshot());
var fired = Assert.Single(h.Deliverer.Outcomes);
Assert.Equal("Failed Agent Job", fired.MetricName);
Assert.Equal("1 job failure(s) in last 60m — Backup.Full", fired.CurrentValue); /* :701 */
Assert.Equal("1 job failure in last 60m — Backup.Full", fired.CurrentValue); /* :701 */
Assert.Equal((Key, firstFailure), Assert.Single(h.StateStore.SavedFailedJob));

/* The same failure lingering in the lookback window never re-fires (:667). */
Expand All @@ -4491,6 +4517,9 @@ public async Task FailedJobs_WatermarkDedups_AndPersistsTheServerLocalRunTime()
await engine.EvaluateServerAsync(Harness.Snapshot());
Assert.Equal(2, h.Deliverer.Outcomes.Count);
Assert.Equal(firstFailure.AddMinutes(30), h.StateStore.FailedJobWatermarks[Key]);
/* #6703: two failures in the window read "2 job failures", the one-failure fire above "1 job failure". */
Assert.StartsWith("2 job failures in last 60m", h.Deliverer.Outcomes[1].CurrentValue, StringComparison.Ordinal);
Assert.StartsWith("2 job failures", h.Deliverer.Outcomes[1].ShortMessage, StringComparison.Ordinal);
}

/// <summary>
Expand Down
23 changes: 18 additions & 5 deletions Darling/Darling.Tests/DarlingPgOperationalAlertTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,20 @@ public void BuildPgDeadlockIncident_FallsBackToPidAndParticipantCount_WhenVictim
var incident = DarlingWorker.BuildPgDeadlockIncident(
DeadlockRow("hash-2", victimStatement: null, victimPid: 555, participantCount: 3));

Assert.Equal("victim pid 555, 3 participant(s)", incident.InvolvedObjects[0]);
Assert.Equal("victim pid 555, 3 participants", incident.InvolvedObjects[0]);
}

[Fact]
public void BuildPgIncidents_WithACountOfOne_ReadInTheSingular()
{
/* #6703: "1 participant" and "blocking 1 session", where the counts above read in the plural. */
var deadlock = DarlingWorker.BuildPgDeadlockIncident(
DeadlockRow("hash-2", victimStatement: null, victimPid: 555, participantCount: 1));
Assert.Equal("victim pid 555, 1 participant", deadlock.InvolvedObjects[0]);

var blocking = DarlingWorker.BuildPgBlockingIncident(
BlockingRow(rootBackendId: 1, rootPid: 42, totalVictims: 1, databases: Array.Empty<string>(), rootQuery: null));
Assert.Equal("root pid 42 blocking 1 session", blocking.InvolvedObjects[0]);
}

[Fact]
Expand Down Expand Up @@ -229,7 +242,7 @@ public void BuildPgBlockingIncident_IncludesDatabaseRootPidVictimCountAndQuery()
var incident = DarlingWorker.BuildPgBlockingIncident(
BlockingRow(rootBackendId: 1, rootPid: 42, totalVictims: 5, databases: new[] { "sales" }, rootQuery: "SELECT * FROM x"));

Assert.Equal("root pid 42 blocking 5 session(s) in [sales]: SELECT * FROM x", incident.InvolvedObjects[0]);
Assert.Equal("root pid 42 blocking 5 sessions in [sales]: SELECT * FROM x", incident.InvolvedObjects[0]);
Assert.Equal("sales", incident.Database);
}

Expand All @@ -239,7 +252,7 @@ public void BuildPgBlockingIncident_OmitsTrailingQueryClause_WhenRootQueryIsMiss
var incident = DarlingWorker.BuildPgBlockingIncident(
BlockingRow(rootBackendId: 1, rootPid: 42, totalVictims: 2, databases: new[] { "sales" }, rootQuery: null));

Assert.Equal("root pid 42 blocking 2 session(s) in [sales]", incident.InvolvedObjects[0]);
Assert.Equal("root pid 42 blocking 2 sessions in [sales]", incident.InvolvedObjects[0]);
}

[Fact]
Expand Down Expand Up @@ -272,7 +285,7 @@ public void BuildPgBlockingIncident_OmitsDatabaseClause_WhenNoDatabasesResolved(
var incident = DarlingWorker.BuildPgBlockingIncident(
BlockingRow(rootBackendId: 1, rootPid: 42, totalVictims: 2, databases: Array.Empty<string>(), rootQuery: null));

Assert.Equal("root pid 42 blocking 2 session(s)", incident.InvolvedObjects[0]);
Assert.Equal("root pid 42 blocking 2 sessions", incident.InvolvedObjects[0]);
Assert.Null(incident.Database);
}

Expand Down Expand Up @@ -491,7 +504,7 @@ await deliverer.DeliverAsync(
},
DetailText: null, NumericCurrentValue: 3, NumericThresholdValue: 1, Muted: false,
Severity: AlertEngine.GradeDeadlockFire(3, settings.DeadlockRateThresholds),
ShortMessage: "3 deadlock(s) in the last hour"),
ShortMessage: "3 deadlocks in the last hour"),
TestContext.Current.CancellationToken);

/* The CPU arm's shape: no context at all, the grade on the outcome. */
Expand Down
Loading
Loading