diff --git a/Darling/Darling.Tests/DarlingStoreTableHealthCatalogReadsLiveTests.cs b/Darling/Darling.Tests/DarlingStoreTableHealthCatalogReadsLiveTests.cs new file mode 100644 index 0000000000..be80d847c7 --- /dev/null +++ b/Darling/Darling.Tests/DarlingStoreTableHealthCatalogReadsLiveTests.cs @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2026 Erik Darling, Darling Data LLC + * + * This file is part of the SQL Server Performance Monitor. + * + * Licensed under the MIT License. See LICENSE file in the project root for full license information. + */ + +using System; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using Npgsql; +using NpgsqlTypes; +using PerformanceMonitor.Darling.Storage; +using Xunit; + +namespace Darling.Tests; + +/// +/// The relation list the per-index read sizes from (#9224). On a store with thousands of chunks the planner +/// estimated millions of rows for them, and every join and sort above them was planned for that many rows. The estimate is a +/// count the plan gives, so the test compares it with the rows the statement returns. +/// +/* #1776 own-store: deliberately NOT [Collection("live-postgres")]. Every test reaches DARLING_TEST_PG only to + CREATE and DROP its own database through ScratchPostgres and works entirely inside it. */ +public sealed class DarlingStoreTableHealthCatalogReadsLiveTests +{ + private static string? BaseConnectionString => Environment.GetEnvironmentVariable("DARLING_TEST_PG"); + + private static async Task ExecAsync(NpgsqlConnection connection, string sql, System.Threading.CancellationToken ct) + { + await using var command = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 }; + await command.ExecuteNonQueryAsync(ct); + } + + private static async Task<(long Planned, long Actual)> PlannedAndActualRowsAsync( + NpgsqlConnection connection, string sql, System.Threading.CancellationToken ct) + { + await using var explain = new NpgsqlCommand("EXPLAIN (FORMAT JSON) " + sql, connection) { CommandTimeout = 300 }; + var json = (string)(await explain.ExecuteScalarAsync(ct))!; + using var doc = JsonDocument.Parse(json); + var planned = (long)doc.RootElement[0].GetProperty("Plan").GetProperty("Plan Rows").GetDouble(); + + long actual = 0; + await using var run = new NpgsqlCommand(sql, connection) { CommandTimeout = 300 }; + await using var reader = await run.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + actual++; + } + + return (planned, actual); + } + + private const string SeedSql = @" +CREATE SCHEMA zz9224; +DO $$ +BEGIN + FOR i IN 1..150 LOOP + EXECUTE format('CREATE TABLE zz9224.t%s (id integer PRIMARY KEY, v text)', i); + EXECUTE format('CREATE INDEX t%s_v ON zz9224.t%s (v)', i, i); + END LOOP; +END $$; +CREATE TABLE zz9224.ht (ts timestamptz NOT NULL, dev integer NOT NULL, v double precision); +SELECT create_hypertable('zz9224.ht', 'ts', chunk_time_interval => interval '1 day'); +CREATE INDEX ht_dev ON zz9224.ht (dev, ts DESC); +INSERT INTO zz9224.ht SELECT now() - (d || ' days')::interval, d, 1 FROM generate_series(0, 11) AS d; +CREATE TABLE zz9224.pt (d date NOT NULL, v text) PARTITION BY RANGE (d); +CREATE TABLE zz9224.pt_1 PARTITION OF zz9224.pt FOR VALUES FROM ('2026-01-01') TO ('2026-02-01'); +CREATE TABLE zz9224.pt_2 PARTITION OF zz9224.pt FOR VALUES FROM ('2026-02-01') TO ('2026-03-01'); +CREATE INDEX pt_v ON zz9224.pt (v); +ANALYZE;"; + + /// The rows the est CTE (the per-index estimate) produced, found by its subplan name in an EXPLAIN ANALYZE plan. + private static long? EstimateRowsProduced(JsonElement node) + { + if (node.TryGetProperty("Subplan Name", out var name) && name.GetString() == "CTE est") + { + return (long)node.GetProperty("Actual Rows").GetDouble(); + } + + if (node.TryGetProperty("Plans", out var children)) + { + foreach (var child in children.EnumerateArray()) + { + var found = EstimateRowsProduced(child); + if (found is not null) + { + return found; + } + } + } + + return null; + } + + [Fact] + public async Task TheIndexList_IsNotPlannedForMillionsOfRows_WhenTheStoreHasManyTables() + { + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), "Set DARLING_TEST_PG to a Postgres connection string to run the #9224 live tests."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + Assert.SkipUnless(scratch.TimescaleEnabled, "TimescaleDB is not enabled on this cluster."); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + + await ExecAsync(connection, SeedSql, ct); + + var indexes = await PlannedAndActualRowsAsync(connection, DarlingStoreTableHealthReader.IndexRelationsSql(timescale: true), ct); + + // The seeded store has well over a hundred indexes, so the list is not trivial. + Assert.True(indexes.Actual >= 300, $"indexes listed {indexes.Actual}"); + + // One set-returning function per partitioned root is estimated at 1000 rows, so the bound is generous. The old plan + // was a hundred times the real count and more. + Assert.True(indexes.Planned <= indexes.Actual * 20 + 2000, $"indexes: planned {indexes.Planned} rows, listed {indexes.Actual}"); + } + + [Fact] + public async Task ThePerIndexEstimate_IsKeptToTheIndexesTheReadWasHandedIn() + { + var baseCs = BaseConnectionString; + Assert.SkipWhen(string.IsNullOrEmpty(baseCs), "Set DARLING_TEST_PG to a Postgres connection string to run the #9224 live tests."); + var ct = TestContext.Current.CancellationToken; + + await using var scratch = await ScratchPostgres.CreateAsync(baseCs!, ct); + Assert.SkipUnless(scratch.TimescaleEnabled, "TimescaleDB is not enabled on this cluster."); + await using var connection = new NpgsqlConnection(scratch.ConnectionString); + await connection.OpenAsync(ct); + await ExecAsync(connection, SeedSql, ct); + + // Indexes the read does not list: a temporary table is not a store relation, so the index list leaves it out, but its btree + // indexes are in the database for the estimate to read. + await ExecAsync(connection, @" +CREATE TEMPORARY TABLE zz9224_tmp (c1 int, c2 int, c3 int, c4 int, c5 int, c6 int, c7 int, c8 int, c9 int, c10 int); +DO $$ +BEGIN + FOR i IN 1..600 LOOP + EXECUTE format('CREATE INDEX zz9224_tmp_%s ON zz9224_tmp (c%s)', i, 1 + i % 10); + END LOOP; +END $$; +ANALYZE;", ct); + + // What the reader hands the statement: the relations the list names, and the size of each. + long[] oids; + long[] bytes; + long indexesHandedIn; + await using (var sizes = new NpgsqlCommand( + "SELECT array_agg(l.oid ORDER BY l.oid), array_agg(pg_catalog.pg_relation_size(l.oid::oid) ORDER BY l.oid), " + + "count(*) FILTER (WHERE c.relkind IN ('i', 'I')) FROM (" + + DarlingStoreTableHealthReader.IndexRelationsSql(timescale: true) + ") AS l(oid) JOIN pg_catalog.pg_class AS c ON c.oid = l.oid::oid", connection)) + await using (var sizeReader = await sizes.ExecuteReaderAsync(ct)) + { + Assert.True(await sizeReader.ReadAsync(ct)); + oids = (long[])sizeReader.GetValue(0); + bytes = (long[])sizeReader.GetValue(1); + indexesHandedIn = sizeReader.GetInt64(2); + } + + await using var explain = new NpgsqlCommand( + "EXPLAIN (ANALYZE, FORMAT JSON) " + DarlingStoreTableHealthReader.BuildIndexSql(timescale: true, presized: true), connection) { CommandTimeout = 300 }; + explain.Parameters.Add(new NpgsqlParameter { Value = oids, NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Bigint }); + explain.Parameters.Add(new NpgsqlParameter { Value = bytes, NpgsqlDbType = NpgsqlDbType.Array | NpgsqlDbType.Bigint }); + using var doc = JsonDocument.Parse((string)(await explain.ExecuteScalarAsync(ct))!); + var estimated = EstimateRowsProduced(doc.RootElement[0].GetProperty("Plan")) + ?? throw new InvalidOperationException("The per-index plan has no estimate node."); + + // The per-index estimate used to run over every btree index in the database, the temporary table's and TimescaleDB's own + // among them, and size each one a second time. Kept to the indexes the read was handed, it produces no more rows than + // the read has indexes. + Assert.True(estimated <= indexesHandedIn, $"per-index estimate: {estimated} rows for {indexesHandedIn} indexes handed in"); + } +} diff --git a/Darling/PerformanceMonitor.Darling.Storage/DarlingStoreTableHealthReader.cs b/Darling/PerformanceMonitor.Darling.Storage/DarlingStoreTableHealthReader.cs index 13d4167372..d87f4d6edd 100644 --- a/Darling/PerformanceMonitor.Darling.Storage/DarlingStoreTableHealthReader.cs +++ b/Darling/PerformanceMonitor.Darling.Storage/DarlingStoreTableHealthReader.cs @@ -556,12 +556,16 @@ chunk_rels AS ( SELECT t.obj_schema, t.obj_name, t.kind, pc.oid AS rel_oid, EXISTS (SELECT 1 FROM _timescaledb_catalog.compression_settings AS cs WHERE cs.relid::oid = pc.oid AND cs.compress_relid IS NOT NULL) AS is_compressed - FROM targets AS t - JOIN timescaledb_information.chunks AS ch - ON ch.hypertable_schema = t.root_schema - AND ch.hypertable_name = t.root_name - JOIN pg_catalog.pg_namespace AS pn ON pn.nspname = ch.chunk_schema - JOIN pg_catalog.pg_class AS pc ON pc.relnamespace = pn.oid AND pc.relname = ch.chunk_name + /* Chunk membership from core inheritance, not timescaledb_information.chunks: that view joins every chunk to its dimension + slices and to all of pg_class, and its row count is estimated far too low, which let the planner pick the wrong join order + (#9224). No column of _timescaledb_catalog.chunk is named: TimescaleDB 2.29 rebuilt that catalog, so a column of it exists on + one side of that version only. The compression_settings reads (relid, compress_relid), here and in TimescaleMemberArms, are + the catalog columns still named, and both exist on 2.28.1 and on 2.29 and later. A chunk dropped by retention has no relation + and no pg_inherits row, as in the view. The + tiered-storage (OSM) chunk is a foreign table, which the view skipped.*/ + FROM root_rels AS t + JOIN pg_catalog.pg_inherits AS inh ON inh.inhparent = t.rel_oid + JOIN pg_catalog.pg_class AS pc ON pc.oid = inh.inhrelid AND pc.relkind <> 'f' ), "; @@ -612,13 +616,16 @@ AND c.relpersistence <> 't' AND NOT {StoreSelfMetrics.SystemSchemaPredicateSql}{(timescale ? "\n AND " + StoreSelfMetrics.TimescaleInventoriedPredicateSql : string.Empty)} ), members AS ( -{(timescale ? TimescaleMemberArms : string.Empty)} SELECT pr.nspname AS obj_schema, pr.relname AS obj_name, CASE WHEN pr.relkind = 'p' THEN 'partitioned' ELSE 'table' END AS kind, x.rel_oid AS rel_oid, 'heap'::text AS member_role, (pr.relkind = 'p') AS grouped, false AS is_compressed +{(timescale ? TimescaleMemberArms : string.Empty)} SELECT pr.nspname AS obj_schema, pr.relname AS obj_name, 'table'::text AS kind, pr.oid AS rel_oid, 'heap'::text AS member_role, false AS grouped, false AS is_compressed + FROM plain_roots AS pr + WHERE pr.relkind <> 'p' + UNION ALL + /* Only the partitioned roots are walked: a set-returning function in a lateral join over every root is estimated at 1000 + rows per root, which made this CTE millions of estimated rows on a store with thousands of tables (#9224). */ + SELECT pr.nspname, pr.relname, 'partitioned'::text, t.relid, 'heap'::text, true, false FROM plain_roots AS pr - CROSS JOIN LATERAL ( - SELECT pr.oid AS rel_oid WHERE pr.relkind <> 'p' - UNION ALL - SELECT t.relid FROM pg_catalog.pg_partition_tree(pr.oid::regclass) AS t WHERE pr.relkind = 'p' AND t.isleaf - ) AS x + CROSS JOIN LATERAL pg_catalog.pg_partition_tree(pr.oid::regclass) AS t + WHERE pr.relkind = 'p' AND t.isleaf )"; /// @@ -701,7 +708,7 @@ of chunks the size functions (a file stat or more per relation) are most of the FROM members AS m JOIN pg_catalog.pg_class AS c ON c.oid = m.rel_oid{(presized ? @" LEFT JOIN unnest($1::bigint[], $2::bigint[], $3::bigint[], $4::bigint[]) AS z(rel_oid, heap_bytes, index_bytes, toast_bytes) - ON z.rel_oid = m.rel_oid::bigint" : string.Empty)} + ON m.rel_oid = z.rel_oid::oid" : string.Empty)} ), est AS MATERIALIZED ( {PgTableBloatStatsCollector.BuildQueryText(EstimateRelationFilterSql)} @@ -941,10 +948,17 @@ CROSS JOIN LATERAL pg_catalog.pg_partition_tree(p.indexrelid::regclass) AS t WHERE t.isleaf )"; + /// + /// The condition that keeps the estimate in the presized per-index read to the indexes it sized (#9224), in place of every + /// btree index in the database. The size column the estimate carries is not read there: the read takes it from its own list. + /// + private const string EstimateIndexFilterSql = @" + AND c.oid::bigint = ANY($1::bigint[])"; + /// The statement for the per-index read: see . public static string BuildIndexSql(bool timescale, bool presized = false) => $@" WITH est AS MATERIALIZED ( -{PgIndexBloatCollector.QueryText} +{(presized ? PgIndexBloatCollector.BuildQueryText("0::bigint", EstimateIndexFilterSql) : PgIndexBloatCollector.QueryText)} ), {BuildMembersCtes(timescale)}, {PartitionIndexCtes()}, @@ -970,7 +984,7 @@ THEN pg_catalog.substr(ic.relname, pg_catalog.length(c.relname) + 2) END, {(presized ? "COALESCE(zh.bytes, 0)" : "pg_catalog.pg_relation_size(c.oid)")} AS heap_bytes, COALESCE(s.idx_scan, 0) AS idx_scan, e.est_reclaimable_bytes, - e.index_bytes AS est_index_bytes, + {(presized ? "CASE WHEN e.index_oid IS NOT NULL THEN COALESCE(zi.bytes, 0) END" : "e.index_bytes")} AS est_index_bytes, e.skipped_reason FROM members AS m JOIN pg_catalog.pg_class AS c ON c.oid = m.rel_oid @@ -980,8 +994,8 @@ FROM members AS m LEFT JOIN pg_catalog.pg_stat_all_indexes AS s ON s.indexrelid = x.indexrelid LEFT JOIN ptree AS pt ON pt.leaf_index_oid = x.indexrelid LEFT JOIN est AS e ON e.index_oid = x.indexrelid{(presized ? @" - LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zi(rel_oid, bytes) ON zi.rel_oid = x.indexrelid::bigint - LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zh(rel_oid, bytes) ON zh.rel_oid = c.oid::bigint" : string.Empty)} + LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zi(rel_oid, bytes) ON x.indexrelid = zi.rel_oid::oid + LEFT JOIN unnest($1::bigint[], $2::bigint[]) AS zh(rel_oid, bytes) ON c.oid = zh.rel_oid::oid" : string.Empty)} ), grp AS ( SELECT @@ -1474,14 +1488,14 @@ public static async Task> GetAsync( /// The relations the per-table read sizes (#9170): every member relation, once each. The same members the read /// itself groups, so the sizes cover exactly the relations it sums. /// - private static string MemberRelationsSql(bool timescale) => $@" + internal static string MemberRelationsSql(bool timescale) => $@" WITH {BuildMembersCtes(timescale)} SELECT DISTINCT m.rel_oid::bigint FROM members AS m JOIN pg_catalog.pg_class AS c ON c.oid = m.rel_oid"; /// The relations the per-index read sizes (#9170): every member relation (the heap each index hangs under) and every index on one. - private static string IndexRelationsSql(bool timescale) => $@" + internal static string IndexRelationsSql(bool timescale) => $@" WITH {BuildMembersCtes(timescale)} SELECT m.rel_oid::bigint FROM members AS m diff --git a/PerformanceMonitor.Collectors/PgIndexBloatCollector.cs b/PerformanceMonitor.Collectors/PgIndexBloatCollector.cs index ea94880662..ed9d7191f3 100644 --- a/PerformanceMonitor.Collectors/PgIndexBloatCollector.cs +++ b/PerformanceMonitor.Collectors/PgIndexBloatCollector.cs @@ -178,7 +178,16 @@ public readonly record struct Row( /// other two names after, which on a store with thousands of chunks in one schema compared every index column with every /// statistics row of the schema (1.7 billion pairs, over three minutes, #8806). OFFSET 0 keeps the subquery from /// being flattened back into that join. - public const string QueryText = @"WITH params AS ( + public static readonly string QueryText = BuildQueryText("pg_catalog.pg_relation_size(c.oid)", string.Empty); + + /// + /// The estimate statement, with the two things the store health read (#9224) changes about it. + /// is the expression for an index's size (the collector's own is pg_catalog.pg_relation_size(c.oid), a file stat per index). + /// (empty for the collector) is one more condition on the index set, before the column and + /// statistics joins: the store read keeps the estimate to the indexes it reads and takes their sizes from its own list, so no + /// size function runs a second time. + /// + public static string BuildQueryText(string indexBytesSql, string indexFilterSql) => $@"WITH params AS ( SELECT pg_catalog.current_setting('block_size')::int AS bs, CASE WHEN pg_catalog.version() ~ '64-bit|x86_64|ppc64|ia64|amd64|aarch64|arm64' @@ -192,7 +201,7 @@ idx AS ( c.relname AS index_name, t.oid AS table_oid, c.relpages::bigint AS index_pages, - pg_catalog.pg_relation_size(c.oid) AS index_bytes, + {indexBytesSql} AS index_bytes, t.reltuples AS table_rows, x.indnatts, x.indkey, @@ -208,7 +217,7 @@ FROM pg_catalog.pg_class AS c AND am.amname = 'btree' AND x.indisvalid AND x.indisready - AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast') + AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast'){indexFilterSql} OFFSET 0 ), keycols AS (