Skip to content
Merged
176 changes: 176 additions & 0 deletions Darling/Darling.Tests/DarlingStoreTableHealthCatalogReadsLiveTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
/*
* Copyright (c) 2026 Erik Darling, Darling Data LLC
*
* This file is part of the SQL Server Performance Monitor.
*
* Licensed under the MIT License. See LICENSE file in the project root for full license information.
*/

using System;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using Npgsql;
using NpgsqlTypes;
using PerformanceMonitor.Darling.Storage;
using Xunit;

namespace Darling.Tests;

/// <summary>
/// The relation list the per-index read sizes from (#9224). On a store with thousands of chunks the planner
/// estimated millions of rows for them, and every join and sort above them was planned for that many rows. The estimate is a
/// count the plan gives, so the test compares it with the rows the statement returns.
/// </summary>
/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Every test reaches DARLING_TEST_PG only to
CREATE and DROP its own database through ScratchPostgres and works entirely inside it. */
public sealed class DarlingStoreTableHealthCatalogReadsLiveTests
{
private static string? BaseConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG");

private static async Task ExecAsync(NpgsqlConnection connection, string sql, System.Threading.CancellationToken ct)
{
await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
await command.ExecuteNonQueryAsync(ct);
}

private static async Task<(long Planned, long Actual)> PlannedAndActualRowsAsync(
NpgsqlConnection connection, string sql, System.Threading.CancellationToken ct)
{
await using var explain = new NpgsqlCommand("EXPLAIN (FORMAT JSON) " + sql, connection) { CommandTimeout = 300 };
var json = (string)(await explain.ExecuteScalarAsync(ct))!;
using var doc = JsonDocument.Parse(json);
var planned = (long)doc.RootElement[0].GetProperty("Plan").GetProperty("Plan Rows").GetDouble();

long actual = 0;
await using var run = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 };
await using var reader = await run.ExecuteReaderAsync(ct);
while (await reader.ReadAsync(ct))
{
actual++;
}

return (planned, actual);
}

private const string SeedSql = @"
CREATE SCHEMA zz9224;
DO $$
BEGIN
FOR i IN 1..150 LOOP
EXECUTE format('CREATE TABLE zz9224.t%s (id integer PRIMARY KEY, v text)', i);
EXECUTE format('CREATE INDEX t%s_v ON zz9224.t%s (v)', i, i);
END LOOP;
END $$;
CREATE TABLE zz9224.ht (ts timestamptz NOT NULL, dev integer NOT NULL, v double precision);
SELECT create_hypertable('zz9224.ht', 'ts', chunk_time_interval => interval '1 day');
CREATE INDEX ht_dev ON zz9224.ht (dev, ts DESC);
INSERT INTO zz9224.ht SELECT now() - (d || ' days')::interval, d, 1 FROM generate_series(0, 11) AS d;
CREATE TABLE zz9224.pt (d date NOT NULL, v text) PARTITION BY RANGE (d);
CREATE TABLE zz9224.pt_1 PARTITION OF zz9224.pt FOR VALUES FROM ('2026-01-01') TO ('2026-02-01');
CREATE TABLE zz9224.pt_2 PARTITION OF zz9224.pt FOR VALUES FROM ('2026-02-01') TO ('2026-03-01');
CREATE INDEX pt_v ON zz9224.pt (v);
ANALYZE;";

/// <summary>The rows the <c>est</c> CTE (the per-index estimate) produced, found by its subplan name in an EXPLAIN ANALYZE plan.</summary>
private static long? EstimateRowsProduced(JsonElement node)
{
if (node.TryGetProperty("Subplan Name", out var name) && name.GetString() == "CTE est")
{
return (long)node.GetProperty("Actual Rows").GetDouble();
}

if (node.TryGetProperty("Plans", out var children))
{
foreach (var child in children.EnumerateArray())
{
var found = EstimateRowsProduced(child);
if (found is not null)
{
return found;
}
}
}

return null;
}

[Fact]
public async Task TheIndexList_IsNotPlannedForMillionsOfRows_WhenTheStoreHasManyTables()
{
var baseCs = BaseConnectionString;
Assert.SkipWhen(string.IsNullOrEmpty(baseCs), "Set DARLING_TEST_PG to a Postgres connection string to run the #9224 live tests.");
var ct = TestContext.Current.CancellationToken;

await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct);
Assert.SkipUnless(scratch.TimescaleEnabled, "TimescaleDB is not enabled on this cluster.");
await using var connection = new NpgsqlConnection(scratch.ConnectionString);
await connection.OpenAsync(ct);

await ExecAsync(connection, SeedSql, ct);

var indexes = await PlannedAndActualRowsAsync(connection, DarlingStoreTableHealthReader.IndexRelationsSql(timescale: true), ct);

// The seeded store has well over a hundred indexes, so the list is not trivial.
Assert.True(indexes.Actual >= 300, $"indexes listed {indexes.Actual}");

// One set-returning function per partitioned root is estimated at 1000 rows, so the bound is generous. The old plan
// was a hundred times the real count and more.
Assert.True(indexes.Planned <= indexes.Actual * 20 + 2000, $"indexes: planned {indexes.Planned} rows, listed {indexes.Actual}");
}

[Fact]
public async Task ThePerIndexEstimate_IsKeptToTheIndexesTheReadWasHandedIn()
{
var baseCs = BaseConnectionString;
Assert.SkipWhen(string.IsNullOrEmpty(baseCs), "Set DARLING_TEST_PG to a Postgres connection string to run the #9224 live tests.");
var ct = TestContext.Current.CancellationToken;

await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct);
Assert.SkipUnless(scratch.TimescaleEnabled, "TimescaleDB is not enabled on this cluster.");
await using var connection = new NpgsqlConnection(scratch.ConnectionString);
await connection.OpenAsync(ct);
await ExecAsync(connection, SeedSql, ct);

// Indexes the read does not list: a temporary table is not a store relation, so the index list leaves it out, but its btree
// indexes are in the database for the estimate to read.
await ExecAsync(connection, @"
CREATE TEMPORARY TABLE zz9224_tmp (c1 int, c2 int, c3 int, c4 int, c5 int, c6 int, c7 int, c8 int, c9 int, c10 int);
DO $$
BEGIN
FOR i IN 1..600 LOOP
EXECUTE format('CREATE INDEX zz9224_tmp_%s ON zz9224_tmp (c%s)', i, 1 + i % 10);
END LOOP;
END $$;
ANALYZE;", ct);

// What the reader hands the statement: the relations the list names, and the size of each.
long[] oids;
long[] bytes;
long indexesHandedIn;
await using (var sizes = new NpgsqlCommand(
"SELECT array_agg(l.oid ORDER BY l.oid), array_agg(pg_catalog.pg_relation_size(l.oid::oid) ORDER BY l.oid), "
+ "count(*) FILTER (WHERE c.relkind IN ('i', 'I')) FROM ("
+ DarlingStoreTableHealthReader.IndexRelationsSql(timescale: true) + ") AS l(oid) JOIN pg_catalog.pg_class AS c ON c.oid = l.oid::oid", connection))
await using (var sizeReader = await sizes.ExecuteReaderAsync(ct))
{
Assert.True(await sizeReader.ReadAsync(ct));
oids = (long[])sizeReader.GetValue(0);
bytes = (long[])sizeReader.GetValue(1);
indexesHandedIn = sizeReader.GetInt64(2);
}

await using var explain = new NpgsqlCommand(
"EXPLAIN (ANALYZE, FORMAT JSON) " + DarlingStoreTableHealthReader.BuildIndexSql(timescale: true, presized: true), connection) { CommandTimeout = 300 };
explain.Parameters.Add(new NpgsqlParameter { Value = oids, NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Bigint });
explain.Parameters.Add(new NpgsqlParameter { Value = bytes, NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Bigint });
using var doc = JsonDocument.Parse((string)(await explain.ExecuteScalarAsync(ct))!);
var estimated = EstimateRowsProduced(doc.RootElement[0].GetProperty("Plan"))
?? throw new InvalidOperationException("The per-index plan has no estimate node.");

// The per-index estimate used to run over every btree index in the database, the temporary table's and TimescaleDB's own
// among them, and size each one a second time. Kept to the indexes the read was handed, it produces no more rows than
// the read has indexes.
Assert.True(estimated <= indexesHandedIn, $"per-index estimate: {estimated} rows for {indexesHandedIn} indexes handed in");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -556,12 +556,16 @@ chunk_rels AS (
SELECT t.obj_schema, t.obj_name, t.kind, pc.oid AS rel_oid,
EXISTS (SELECT 1 FROM _timescaledb_catalog.compression_settings AS cs
WHERE cs.relid::oid = pc.oid AND cs.compress_relid IS NOT NULL) AS is_compressed
FROM targets AS t
JOIN timescaledb_information.chunks AS ch
ON ch.hypertable_schema = t.root_schema
AND ch.hypertable_name = t.root_name
JOIN pg_catalog.pg_namespace AS pn ON pn.nspname = ch.chunk_schema
JOIN pg_catalog.pg_class AS pc ON pc.relnamespace = pn.oid AND pc.relname = ch.chunk_name
/* Chunk membership from core inheritance, not timescaledb_information.chunks: that view joins every chunk to its dimension
slices and to all of pg_class, and its row count is estimated far too low, which let the planner pick the wrong join order
(#9224). No column of _timescaledb_catalog.chunk is named: TimescaleDB 2.29 rebuilt that catalog, so a column of it exists on
one side of that version only. The compression_settings reads (relid, compress_relid), here and in TimescaleMemberArms, are
the catalog columns still named, and both exist on 2.28.1 and on 2.29 and later. A chunk dropped by retention has no relation
and no pg_inherits row, as in the view. The
tiered-storage (OSM) chunk is a foreign table, which the view skipped.*/
FROM root_rels AS t
JOIN pg_catalog.pg_inherits AS inh ON inh.inhparent = t.rel_oid
JOIN pg_catalog.pg_class AS pc ON pc.oid = inh.inhrelid AND pc.relkind <> 'f'
),
";

Expand Down Expand Up @@ -612,13 +616,16 @@ AND c.relpersistence <> 't'
AND NOT {StoreSelfMetrics.SystemSchemaPredicateSql}{(timescale ? "\n AND " + StoreSelfMetrics.TimescaleInventoriedPredicateSql : string.Empty)}
),
members AS (
{(timescale ? TimescaleMemberArms : string.Empty)} SELECT pr.nspname AS obj_schema, pr.relname AS obj_name, CASE WHEN pr.relkind = 'p' THEN 'partitioned' ELSE 'table' END AS kind, x.rel_oid AS rel_oid, 'heap'::text AS member_role, (pr.relkind = 'p') AS grouped, false AS is_compressed
{(timescale ? TimescaleMemberArms : string.Empty)} SELECT pr.nspname AS obj_schema, pr.relname AS obj_name, 'table'::text AS kind, pr.oid AS rel_oid, 'heap'::text AS member_role, false AS grouped, false AS is_compressed
FROM plain_roots AS pr
WHERE pr.relkind <> 'p'
UNION ALL
/* Only the partitioned roots are walked: a set-returning function in a lateral join over every root is estimated at 1000
rows per root, which made this CTE millions of estimated rows on a store with thousands of tables (#9224). */
SELECT pr.nspname, pr.relname, 'partitioned'::text, t.relid, 'heap'::text, true, false
FROM plain_roots AS pr
CROSS JOIN LATERAL (
SELECT pr.oid AS rel_oid WHERE pr.relkind <> 'p'
UNION ALL
SELECT t.relid FROM pg_catalog.pg_partition_tree(pr.oid::regclass) AS t WHERE pr.relkind = 'p' AND t.isleaf
) AS x
CROSS JOIN LATERAL pg_catalog.pg_partition_tree(pr.oid::regclass) AS t
WHERE pr.relkind = 'p' AND t.isleaf
)";

/// <summary>
Expand Down Expand Up @@ -701,7 +708,7 @@ of chunks the size functions (a file stat or more per relation) are most of the
FROM members AS m
JOIN pg_catalog.pg_class AS c ON c.oid = m.rel_oid{(presized ? @"
LEFT JOIN unnest($1::bigint[], $2::bigint[], $3::bigint[], $4::bigint[]) AS z(rel_oid, heap_bytes, index_bytes, toast_bytes)
ON z.rel_oid = m.rel_oid::bigint" : string.Empty)}
ON m.rel_oid = z.rel_oid::oid" : string.Empty)}
),
est AS MATERIALIZED (
{PgTableBloatStatsCollector.BuildQueryText(EstimateRelationFilterSql)}
Expand Down Expand Up @@ -941,10 +948,17 @@ CROSS JOIN LATERAL pg_catalog.pg_partition_tree(p.indexrelid::regclass) AS t
WHERE t.isleaf
)";

/// <summary>
/// The condition that keeps the estimate in the presized per-index read to the indexes it sized (#9224), in place of every
/// btree index in the database. The size column the estimate carries is not read there: the read takes it from its own list.
/// </summary>
private const string EstimateIndexFilterSql = @"
AND c.oid::bigint = ANY($1::bigint[])";

/// <summary>The statement for the per-index read: see <see cref="StoreIndexHealthRow"/>.</summary>
public static string BuildIndexSql(bool timescale, bool presized = false) => $@"
WITH est AS MATERIALIZED (
{PgIndexBloatCollector.QueryText}
{(presized ? PgIndexBloatCollector.BuildQueryText("0::bigint", EstimateIndexFilterSql) : PgIndexBloatCollector.QueryText)}
),
{BuildMembersCtes(timescale)},
{PartitionIndexCtes()},
Expand All @@ -970,7 +984,7 @@ THEN pg_catalog.substr(ic.relname, pg_catalog.length(c.relname) + 2) END,
{(presized ? "COALESCE(zh.bytes, 0)" : "pg_catalog.pg_relation_size(c.oid)")} AS heap_bytes,
COALESCE(s.idx_scan, 0) AS idx_scan,
e.est_reclaimable_bytes,
e.index_bytes AS est_index_bytes,
{(presized ? "CASE WHEN e.index_oid IS NOT NULL THEN COALESCE(zi.bytes, 0) END" : "e.index_bytes")} AS est_index_bytes,
e.skipped_reason
FROM members AS m
JOIN pg_catalog.pg_class AS c ON c.oid = m.rel_oid
Expand All @@ -980,8 +994,8 @@ FROM members AS m
LEFT JOIN pg_catalog.pg_stat_all_indexes AS s ON s.indexrelid = x.indexrelid
LEFT JOIN ptree AS pt ON pt.leaf_index_oid = x.indexrelid
LEFT JOIN est AS e ON e.index_oid = x.indexrelid{(presized ? @"
LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zi(rel_oid, bytes) ON zi.rel_oid = x.indexrelid::bigint
LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zh(rel_oid, bytes) ON zh.rel_oid = c.oid::bigint" : string.Empty)}
LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zi(rel_oid, bytes) ON x.indexrelid = zi.rel_oid::oid
LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zh(rel_oid, bytes) ON c.oid = zh.rel_oid::oid" : string.Empty)}
),
grp AS (
SELECT
Expand Down Expand Up @@ -1474,14 +1488,14 @@ public static async Task<List<StoreTableHealthRow>> GetAsync(
/// The relations the per-table read sizes (#9170): every member relation, once each. The same <c>members</c> the read
/// itself groups, so the sizes cover exactly the relations it sums.
/// </summary>
private static string MemberRelationsSql(bool timescale) => $@"
internal static string MemberRelationsSql(bool timescale) => $@"
WITH {BuildMembersCtes(timescale)}
SELECT DISTINCT m.rel_oid::bigint
FROM members AS m
JOIN pg_catalog.pg_class AS c ON c.oid = m.rel_oid";

/// <summary>The relations the per-index read sizes (#9170): every member relation (the heap each index hangs under) and every index on one.</summary>
private static string IndexRelationsSql(bool timescale) => $@"
internal static string IndexRelationsSql(bool timescale) => $@"
WITH {BuildMembersCtes(timescale)}
SELECT m.rel_oid::bigint
FROM members AS m
Expand Down
15 changes: 12 additions & 3 deletions PerformanceMonitor.Collectors/PgIndexBloatCollector.cs
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,16 @@ public readonly record struct Row(
/// other two names after, which on a store with thousands of chunks in one schema compared every index column with every
/// statistics row of the schema (1.7 billion pairs, over three minutes, #8806). <c>OFFSET 0</c> keeps the subquery from
/// being flattened back into that join.</para></summary>
public const string QueryText = @"WITH params AS (
public static readonly string QueryText = BuildQueryText("pg_catalog.pg_relation_size(c.oid)", string.Empty);

/// <summary>
/// The estimate statement, with the two things the store health read (#9224) changes about it. <paramref name="indexBytesSql"/>
/// is the expression for an index's size (the collector's own is <c>pg_catalog.pg_relation_size(c.oid)</c>, a file stat per index).
/// <paramref name="indexFilterSql"/> (empty for the collector) is one more condition on the index set, before the column and
/// statistics joins: the store read keeps the estimate to the indexes it reads and takes their sizes from its own list, so no
/// size function runs a second time.
/// </summary>
public static string BuildQueryText(string indexBytesSql, string indexFilterSql) => $@"WITH params AS (
SELECT
pg_catalog.current_setting('block_size')::int AS bs,
CASE WHEN pg_catalog.version() ~ '64-bit|x86_64|ppc64|ia64|amd64|aarch64|arm64'
Expand All @@ -192,7 +201,7 @@ idx AS (
c.relname AS index_name,
t.oid AS table_oid,
c.relpages::bigint AS index_pages,
pg_catalog.pg_relation_size(c.oid) AS index_bytes,
{indexBytesSql} AS index_bytes,
t.reltuples AS table_rows,
x.indnatts,
x.indkey,
Expand All @@ -208,7 +217,7 @@ FROM pg_catalog.pg_class AS c
AND am.amname = 'btree'
AND x.indisvalid
AND x.indisready
AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast'){indexFilterSql}
OFFSET 0
),
keycols AS (
Expand Down
Loading