Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 84 additions & 5 deletions Darling/Darling.Tests/DarlingEventBaselineCoveredDaysTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,18 @@ public sealed class DarlingEventBaselineCoveredDaysLiveTests

private static readonly DateTime AnalysisTime = new(2026, 3, 4, 14, 0, 0, DateTimeKind.Unspecified);

/// <summary>
/// #6706: the provider measures the raw tables' purge horizon from its clock. The seeded data is March 2026, so the clock is
/// set at the window's end (Mar 4 00:00 UTC): at the default 30-day retention the horizon is then the window start (Feb 2)
/// and every assertion below stands as it did before the clamp existed.
/// </summary>
private static readonly TimeProvider Clock = new FixedClock(new DateTimeOffset(2026, 3, 4, 0, 0, 0, TimeSpan.Zero));

private sealed class FixedClock(DateTimeOffset now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => now;
}

private static readonly DateTime[] Tuesdays =
[
new(2026, 2, 3, 14, 0, 0, DateTimeKind.Unspecified), new(2026, 2, 10, 14, 0, 0, DateTimeKind.Unspecified),
Expand Down Expand Up @@ -316,7 +328,7 @@ public async Task Live_AQuietHour_OnAServerThatHasCapturedEvents_IsARowWithMeanZ
await SeedEventAsync(connection, family, serverId, ElsewhereEvent, ct);
await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct);

var map = await new PgBaselineProvider(postgres).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);
var map = await new PgBaselineProvider(postgres, timeProvider: Clock).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);

Assert.Equal(24 * 7, map.Buckets.Count);
var tuesday = map.Buckets[(Hour, Tuesday)];
Expand Down Expand Up @@ -364,7 +376,7 @@ public async Task Live_ACollectorThatLoggedSuccessEveryHour_ForAServerWhoseSourc
await SeedQuietMonthAsync(connection, serverId, collector, ct);
await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct);

var map = await new PgBaselineProvider(postgres).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);
var map = await new PgBaselineProvider(postgres, timeProvider: Clock).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);

Assert.Empty(map.Buckets);

Expand Down Expand Up @@ -412,7 +424,7 @@ public async Task Live_EventsOnTwoOfFiveCoveredDays_DivideByFive_NotByTwo(string

await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct);

var map = await new PgBaselineProvider(postgres).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);
var map = await new PgBaselineProvider(postgres, timeProvider: Clock).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);

var tuesday = map.Buckets[(Hour, Tuesday)];
Assert.Equal(6.0 / 5.0, tuesday.Mean, 9);
Expand Down Expand Up @@ -463,7 +475,7 @@ await SeedRunsAsync(connection, serverId, collector, Tuesdays[4], Tuesdays[4].Ad
await SeedEventAsync(connection, family, serverId, ElsewhereEvent, ct);
await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct);

var map = await new PgBaselineProvider(postgres).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);
var map = await new PgBaselineProvider(postgres, timeProvider: Clock).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct);

var tuesday = map.Buckets[(Hour, Tuesday)];
Assert.Equal(4L, tuesday.SampleCount);
Expand Down Expand Up @@ -507,7 +519,7 @@ public async Task Live_TwoCallsHoursApartOnOneUtcDay_ShareOneCompute_NextDayReco
await SeedEventAsync(connection, family, serverId, ElsewhereEvent, ct);
await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct);

var provider = new PgBaselineProvider(postgres);
var provider = new PgBaselineProvider(postgres, timeProvider: Clock);
var day = AnalysisTime.Date;

var (morning, firstReads) = await CommandCapture.CountBaselineReadsAsync(
Expand Down Expand Up @@ -535,6 +547,73 @@ public async Task Live_TwoCallsHoursApartOnOneUtcDay_ShareOneCompute_NextDayReco
}
}

/// <summary>
/// #6706: on a plain-view store, raw events are purged at the collector's retention while <c>collection_log</c> is kept twice
/// as long. With retention 7 days and runs logged back 35 days, the days before the horizon (Feb 25 00:00 for a clock of
/// Mar 4 00:00) have logged runs and no events, and used to count as measured zeros. Events sit only on the last seven days
/// (three on Tuesday Mar 3 at 14:10, one on Sunday Mar 1 at 09:10). The Tuesday 14:00 bucket is now Mar 3 alone, so its mean
/// is the three events over one day, not over five. Then the same data under the default 30-day retention reads as it did
/// before: five Tuesdays, mean 3/5.
/// </summary>
[Theory]
[InlineData("blocking")]
[InlineData("deadlock")]
public async Task Live_RetentionBelowTheWindow_CountsOnlyTheDaysTheRawTableStillHolds(string family)
{
var connectionString = Environment.GetEnvironmentVariable("DARLING_TEST_PG");
Assert.SkipWhen(string.IsNullOrEmpty(connectionString),
"Set DARLING_TEST_PG to a Postgres connection string to run the live event-baseline coverage test.");

var ct = TestContext.Current.CancellationToken;
var (metric, collector, serverId) = Family(family);

using var connection = new NpgsqlConnection(connectionString);
await connection.OpenAsync(ct);
await PgMigrations.MigrateAsync(connection, ct);
await CleanupAsync(connection, serverId, ct);
await ExecAsync(connection, ct, "DELETE FROM config.config_collector_schedules WHERE server_id IS NULL AND collector_name = $1", collector);

await using var postgres = NpgsqlDataSource.Create(connectionString!);
var bodySucceeded = false;
try
{
await SeedQuietMonthAsync(connection, serverId, collector, ct);
for (var i = 0; i < 3; i++)
{
await SeedEventAsync(connection, family, serverId, Tuesdays[4].AddMinutes(10 + i), ct);
}

await SeedEventAsync(connection, family, serverId, new DateTime(2026, 3, 1, 9, 10, 0, DateTimeKind.Unspecified), ct);
await TimescaleSupport.EnsureBaselineFallbackViewsAsync(connection, null, ct);

await ExecAsync(connection, ct,
"INSERT INTO config.config_collector_schedules (server_id, collector_name, retention_days) VALUES (NULL, $1, 7)", collector);

var short7 = (await new PgBaselineProvider(postgres, timeProvider: Clock).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct))
.Buckets[(Hour, Tuesday)];
Assert.Equal(3.0, short7.Mean, 9);
Assert.Equal(1L, short7.SampleCount);
Assert.Equal(1L, short7.DistinctDays);

await ExecAsync(connection, ct, "DELETE FROM config.config_collector_schedules WHERE server_id IS NULL AND collector_name = $1", collector);

var thirty = (await new PgBaselineProvider(postgres, timeProvider: Clock).GetBucketMapAsync(serverId, metric, AnalysisTime, AnalysisTime.AddHours(1), ct))
.Buckets[(Hour, Tuesday)];
Assert.Equal(3.0 / 5.0, thirty.Mean, 9);
Assert.Equal(5L, thirty.SampleCount);

bodySucceeded = true;
}
finally
{
await LiveStoreCleanup.RunAsync(connectionString!, bodySucceeded, async (cleanup, cleanupCt) =>
{
await ExecAsync(cleanup, cleanupCt, "DELETE FROM config.config_collector_schedules WHERE server_id IS NULL AND collector_name = $1", collector);
await TearDownAsync(cleanup, serverId, cleanupCt);
});
}
}

/// <summary>SUCCESS runs every 15 minutes from <paramref name="from"/> to <paramref name="to"/> with no rows, carrying <paramref name="errorMessage"/> when given.</summary>
private static Task SeedRunsAsync(
NpgsqlConnection connection, int serverId, string collector, DateTime from, DateTime to, string? errorMessage, int batch, CancellationToken ct) =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ public class PgBaselineProvider
private readonly ILogger? _logger;
private readonly BaselineLocalClock _localClock;

/* #6706: the clock the plain-view event arms measure the purge horizon from. Tests set it; the service never does. */
private readonly TimeProvider _time;

/// <summary>Cache TTL — baselines are recomputed after this interval.</summary>
public static TimeSpan CacheTtl { get; set; } = TimeSpan.FromHours(1);

Expand All @@ -147,11 +150,14 @@ public class PgBaselineProvider
/// <param name="sharedCache">#3941: the process's <see cref="BaselineCache"/>, consulted on a miss in this
/// provider's own cache and handed every successful compute. Null (every caller before #3941) keeps a private
/// cache and nothing else.</param>
public PgBaselineProvider(NpgsqlDataSource postgres, ILogger? logger = null, BaselineCache? sharedCache = null)
/// <param name="timeProvider">#6706: the clock the Blocking and Deadlock arms measure the raw tables' purge horizon
/// from. Null is <see cref="TimeProvider.System"/>.</param>
public PgBaselineProvider(NpgsqlDataSource postgres, ILogger? logger = null, BaselineCache? sharedCache = null, TimeProvider? timeProvider = null)
{
_postgres = postgres ?? throw new ArgumentNullException(nameof(postgres));
_logger = logger;
_shared = sharedCache;
_time = timeProvider ?? TimeProvider.System;
/* Information, not Warning: "this host cannot resolve that zone id" is a statement about the host's
configuration, made once per zone per process by the resolver itself — see BaselineLocalClock. */
_localClock = new BaselineLocalClock(message => _logger?.LogInformation("{Message}", message));
Expand Down Expand Up @@ -851,6 +857,66 @@ private async Task<string> ChooseSupplyAsync(
return prefersSuccessor ? query : legacyQuery;
}

/// <summary>The collector whose raw rows back each event arm, and the baseline relation the arm reads them through (#6706).</summary>
internal static (string Collector, string View)? EventSupplyFor(string metricName) => metricName switch
{
MetricNames.Blocking => ("blocked_process_report", TimescaleSupport.BlockedProcessBaselineView),
MetricNames.Deadlock => ("deadlocks", TimescaleSupport.DeadlockBaselineView),
_ => null,
};

/// <summary>
/// #6706: is this baseline relation an ordinary view over the raw table, rather than a continuous aggregate or absent?
/// A continuous aggregate is itself a <c>relkind='v'</c> view, so the catalog check alone is not enough; the
/// aggregate half is <see cref="TimescaleSupport.ResolveMaterializationAsync"/> (#3684: one definition, not a copy).
/// </summary>
internal static string IsPlainBaselineViewSql(string view)
=> $"SELECT EXISTS (SELECT 1 FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'collect' AND c.relname = '{view}' AND c.relkind = 'v')";

/// <summary>
/// #6706: on a store where the Blocking or Deadlock arm's event relation is a plain view over the raw table (no
/// TimescaleDB, or the fallback view of a partly built one), the events reach back only as far as the raw table's purge
/// horizon, while the <c>logged</c> side reads <c>collection_log</c>, kept twice as long. With the collector's retention
/// below the 30-day window, every day between the horizon and the window start has logged runs and no events, and each
/// would count as a measured zero. The window starts no earlier than now minus the retention in force (the fleet override
/// or the default, <see cref="DarlingRetentionHorizons.FleetRetentionDaysSql"/>, the rule the purge applies; the purge never
/// removes a row newer than that). A continuous aggregate keeps 35 days whatever raw retention is, so it is left alone (the
/// aggregate check is <see cref="TimescaleSupport.ResolveMaterializationAsync"/>: null for a plain view or a store without
/// the extension), as is every other metric. At the default 30 days the floor sits within a day of the window start (the window ends at the UTC
/// midnight), so a default store changes only in the hours of its oldest day that the purge may already have taken. The arm's
/// text is unchanged: it still reads <c>$2</c>.
/// </summary>
private async Task<DateTime> ClampToPlainViewPurgeHorizonAsync(
NpgsqlConnection connection, string metricName, DateTime windowStart, CancellationToken cancellationToken)
{
if (EventSupplyFor(metricName) is not var (collector, view))
{
return windowStart;
}

using (var plain = new NpgsqlCommand(IsPlainBaselineViewSql(view), connection) { CommandTimeout = DarlingAnalysisService.AnalysisCommandTimeoutSeconds })
{
if (await plain.ExecuteScalarAsync(cancellationToken) is not true)
{
return windowStart;
}
}

if (await TimescaleSupport.ResolveMaterializationAsync(connection, view, cancellationToken) is not null)
{
return windowStart;
}

long retentionDays;
using (var retention = new NpgsqlCommand("SELECT " + DarlingRetentionHorizons.FleetRetentionDaysSql(collector), connection) { CommandTimeout = DarlingAnalysisService.AnalysisCommandTimeoutSeconds })
{
retentionDays = Convert.ToInt64(await retention.ExecuteScalarAsync(cancellationToken), System.Globalization.CultureInfo.InvariantCulture);
}

var horizon = DateTime.SpecifyKind(_time.GetUtcNow().UtcDateTime.AddDays(-retentionDays), DateTimeKind.Unspecified);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Non-blocking, and the body already names half of it: the horizon is anchored on the wall clock, while every other input of this compute is anchored on windowEnd — the cache key's own instant.

The PR body says a cached entry "is valid for the UTC day like every other input of the compute". It isn't quite like the others. windowEnd = RoundedKeyTime(...) is the key; windowStart, the clock resolve and the $2/$3 binds all derive from it, which is what lets this class's own doc claim (lines 38–42, restated at 951–956) that "the window ends on the analysis HOUR the cache keys on, so a shared entry is the answer a fresh compute would give." With the horizon on _time.GetUtcNow(), two computes inside one cached UTC day — the worker's pass at 01:00 and an MCP analyze_server at 23:00 on a cache miss, sharing BaselineCache — ask for windows 22 hours apart at the start. Whichever lands first wins the day, so a shared entry is no longer the answer a fresh compute would give. Bounded (at most the oldest day's hours) and in the safe direction, so not a correctness complaint.

windowEnd would restore it, and it is the instant the purge horizon is actually being compared against — the window the arm reads ends there, so "what did raw still hold over this window" is a question about windowEnd, not about now:

var horizon = windowEnd.AddDays(-retentionDays);

That also makes the whole TimeProvider seam unnecessary: the fixed clock the six existing live tests now have to be handed (DarlingEventBaselineCoveredDaysTests.cs:278) exists only because the horizon reads a clock the test cannot set otherwise, and a windowEnd-anchored floor is already deterministic from AnalysisTime. On a default-retention store it is also exactly the "rounded down to the UTC day" variant the body offers, for free, since windowEnd for these two daily arms is a UTC midnight (RoundedDay, line 479) — windowEnd - 30d == windowStart, so the default store stops changing at all rather than losing the oldest day's early hours.

The one behavioural difference to weigh before taking it: for an analysis whose analysisTime is historical (context.TimeRangeStart is now - hoursBack, so windowEnd trails now by up to a day at the default and further on a wide hours_back), a windowEnd anchor floors the window where raw stood then rather than now, which admits days the purge has since taken. Both readings are defensible; the clock-anchored one is the more conservative, the key-anchored one is the one the cache contract is written for.

return horizon > windowStart ? horizon : windowStart;
}

private static async Task<DateTime?> OldestBucketAsync(NpgsqlConnection connection, string relation, int serverId, CancellationToken cancellationToken)
{
using var probe = new NpgsqlCommand(SupplyOldestBucketSql(relation), connection) { CommandTimeout = DarlingAnalysisService.AnalysisCommandTimeoutSeconds };
Expand Down Expand Up @@ -894,6 +960,7 @@ analyze_server and compare_analysis (BaselineCache) without changing anyone's an
if (keys is null)
{
query = await ChooseSupplyAsync(connection, serverId, metricName, query, windowStart, cancellationToken);
windowStart = await ClampToPlainViewPurgeHorizonAsync(connection, metricName, windowStart, cancellationToken);
}

/* #6705 (the #3590 trap): the Blocking arm counts a collection_log run with no event as a measured zero, and the
Expand Down
Loading