From 0668e6a59b8ffa96a497d12b9262275d609e8fbb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 14:50:34 +0000 Subject: [PATCH 1/4] fix(metrics): port pgwatch2 top-N + 'other' bucket to pg_stat/statio_all_* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four per-relation metrics (pg_stat_all_indexes, pg_stat_all_tables, pg_statio_all_tables, pg_statio_all_indexes) had no schema filter and a flat LIMIT 5000 truncation. On extension- or schema-heavy databases this overran prometheus.yml's sample_limit (10000) so the entire scrape was silently rejected, and the LIMIT tail was dropped without any aggregate row left behind — dashboard sums drifted. Port the gen2 (gitlab.com/postgres-ai/pgwatch2) approach faithfully instead of reinventing it: - Read pg_stat_user_*/pg_statio_user_* — pg_catalog, information_schema and pg_toast are excluded by the Postgres view itself, so we don't maintain a hand-curated nspname LIKE pattern that has to grow every time a new extension ships its own schema. - row_number() OVER (ORDER BY ) <= 100 per database. Rank by pg_total_relation_size for tables (big tables are the interesting ones; n_live_tup+n_dead_tup starved big-but-static tables) and by activity for indexes/IO views. - UNION ALL an 'other' row that sums the tail so totals stay correct under the cap. HAVING count(*) > 0 suppresses the row when nothing was truncated. - Skip rows with no I/O activity in the statio views — most of the tail on schema-heavy DBs is dead-cold relations. - Filter pg_temp% from index metrics so leftover temp objects from dead sessions stop leaking samples. Metric names and exposed tag_* labels are unchanged so Dashboards 8–11 keep working. Adds two compliance-vector tests that pin the pattern. --- config/pgwatch-prometheus/metrics.yml | 228 ++++++++++++++---- .../test_mr219_monitoring_guards.py | 46 ++++ 2 files changed, 226 insertions(+), 48 deletions(-) diff --git a/config/pgwatch-prometheus/metrics.yml b/config/pgwatch-prometheus/metrics.yml index 4e2343c..436e02f 100644 --- a/config/pgwatch-prometheus/metrics.yml +++ b/config/pgwatch-prometheus/metrics.yml @@ -1552,9 +1552,26 @@ metrics: - total_relation_size_bytes statement_timeout_seconds: 15 pg_stat_all_indexes: + # Top-N + "other" bucket pattern ported from pgwatch2 (gen2). Reads + # pg_stat_user_indexes so pg_catalog/information_schema/pg_toast are + # excluded by the Postgres view itself, no hand-curated nspname pattern. + # The "other" row aggregates the tail so totals stay correct under a + # hard cardinality cap. sqls: 11: | - select /* pgwatch_generated */ + with ranked as ( /* pgwatch_generated */ + select + row_number() over (order by idx_scan desc nulls last) as rownum, + schemaname, + relname, + indexrelname, + idx_scan, + idx_tup_read, + idx_tup_fetch + from pg_stat_user_indexes + where not schemaname like E'pg\\_temp%' + ) + select current_database() as tag_datname, schemaname as tag_schemaname, relname as tag_relname, @@ -1562,18 +1579,58 @@ metrics: idx_scan, idx_tup_read, idx_tup_fetch - from pg_stat_all_indexes - order by idx_scan desc - limit 5000 + from ranked + where rownum <= 100 + union all + select + current_database() as tag_datname, + 'other'::text as tag_schemaname, + 'other'::text as tag_relname, + 'other'::text as tag_indexrelname, + coalesce(sum(idx_scan), 0)::int8 as idx_scan, + coalesce(sum(idx_tup_read), 0)::int8 as idx_tup_read, + coalesce(sum(idx_tup_fetch), 0)::int8 as idx_tup_fetch + from ranked + where rownum > 100 + having count(*) > 0 gauges: - idx_scan - idx_tup_read - idx_tup_fetch statement_timeout_seconds: 15 pg_stat_all_tables: + # Top-N + "other" bucket pattern ported from pgwatch2 (gen2). Ranks by + # pg_total_relation_size — large tables are usually the interesting ones, + # which avoids starving big-but-static tables out of the top-N (the old + # n_live_tup+n_dead_tup ordering did exactly that). sqls: 11: | - select /* pgwatch_generated */ + with ranked as ( /* pgwatch_generated */ + select + row_number() over (order by pg_total_relation_size(relid) desc nulls last) as rownum, + schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_tup_ins, + n_tup_upd, + n_tup_del, + n_tup_hot_upd, + n_live_tup, + n_dead_tup, + last_vacuum, + last_autovacuum, + last_analyze, + last_autoanalyze, + vacuum_count, + autovacuum_count, + analyze_count, + autoanalyze_count + from pg_stat_user_tables + ) + select current_database() as tag_datname, schemaname as tag_schemaname, relname as tag_relname, @@ -1592,10 +1649,30 @@ metrics: extract(epoch from greatest(last_autoanalyze, last_analyze, '1970-01-01Z'))::int8 as last_analyze, (vacuum_count + autovacuum_count) as vacuum_count, (analyze_count + autoanalyze_count) as analyze_count - from - pg_stat_all_tables - order by n_live_tup + n_dead_tup desc - limit 5000 + from ranked + where rownum <= 100 + union all + select + current_database() as tag_datname, + 'other'::text as tag_schemaname, + 'other'::text as tag_relname, + coalesce(sum(seq_scan), 0)::int8 as seq_scan, + coalesce(sum(seq_tup_read), 0)::int8 as seq_tup_read, + coalesce(sum(idx_scan), 0)::int8 as idx_scan, + coalesce(sum(idx_tup_fetch), 0)::int8 as idx_tup_fetch, + coalesce(sum(n_tup_ins), 0)::int8 as n_tup_ins, + coalesce(sum(n_tup_upd), 0)::int8 as n_tup_upd, + coalesce(sum(n_tup_del), 0)::int8 as n_tup_del, + coalesce(sum(n_tup_hot_upd), 0)::int8 as n_tup_hot_upd, + coalesce(sum(n_live_tup), 0)::int8 as n_live_tup, + coalesce(sum(n_dead_tup), 0)::int8 as n_dead_tup, + 0::int8 as last_vacuum, + 0::int8 as last_analyze, + coalesce(sum(vacuum_count + autovacuum_count), 0)::int8 as vacuum_count, + coalesce(sum(analyze_count + autoanalyze_count), 0)::int8 as analyze_count + from ranked + where rownum > 100 + having count(*) > 0 gauges: - seq_scan - seq_tup_read @@ -2881,60 +2958,115 @@ metrics: statement_timeout_seconds: 15 pg_statio_all_tables: description: > - Retrieves table-level I/O statistics from the PostgreSQL `pg_statio_all_tables` view, providing insights into I/O operations for all tables. - It returns block-level read and hit statistics for heap, index, TOAST, and TOAST index operations broken down by schema and table. - Joined with pg_class for efficient ordering by table size. - This metric helps administrators monitor table-level I/O performance and identify which tables are generating the most I/O activity. + Retrieves table-level I/O statistics from `pg_statio_user_tables`, returning + block-level read and hit counters for heap, index, TOAST and TOAST-index pages. + Ports the pgwatch2 (gen2) top-N + `'other'` bucket pattern: ranks tables by + heap_blks_read, keeps the top 100, and folds the tail into a single `'other'` + row so totals remain accurate while cardinality stays bounded. Drops rows + with no I/O activity at all (every counter zero). Compatible with all PostgreSQL versions. sqls: 11: |- - select /* pgwatch_generated */ + with ranked as ( /* pgwatch_generated */ + select + row_number() over (order by heap_blks_read desc nulls last) as rownum, + schemaname, + relname, + heap_blks_read, + heap_blks_hit, + idx_blks_read, + idx_blks_hit, + toast_blks_read, + toast_blks_hit, + tidx_blks_read, + tidx_blks_hit + from pg_statio_user_tables + where + heap_blks_read > 0 or heap_blks_hit > 0 + or idx_blks_read > 0 or idx_blks_hit > 0 + or toast_blks_read > 0 or toast_blks_hit > 0 + or tidx_blks_read > 0 or tidx_blks_hit > 0 + ) + select (extract(epoch from now()) * 1e9)::int8 as epoch_ns, current_database() as tag_datname, - s.schemaname as tag_schemaname, - s.relname as tag_relname, - s.heap_blks_read, - s.heap_blks_hit, - s.idx_blks_read, - s.idx_blks_hit, - s.toast_blks_read, - s.toast_blks_hit, - s.tidx_blks_read, - s.tidx_blks_hit - from - pg_statio_all_tables as s - join pg_class as c on - s.relname = c.relname - and s.schemaname = c.relnamespace::regnamespace::name - order by c.relpages desc - limit 5000; + schemaname as tag_schemaname, + relname as tag_relname, + heap_blks_read, + heap_blks_hit, + idx_blks_read, + idx_blks_hit, + toast_blks_read, + toast_blks_hit, + tidx_blks_read, + tidx_blks_hit + from ranked + where rownum <= 100 + union all + select + (extract(epoch from now()) * 1e9)::int8 as epoch_ns, + current_database() as tag_datname, + 'other'::text as tag_schemaname, + 'other'::text as tag_relname, + coalesce(sum(heap_blks_read), 0)::int8 as heap_blks_read, + coalesce(sum(heap_blks_hit), 0)::int8 as heap_blks_hit, + coalesce(sum(idx_blks_read), 0)::int8 as idx_blks_read, + coalesce(sum(idx_blks_hit), 0)::int8 as idx_blks_hit, + coalesce(sum(toast_blks_read), 0)::int8 as toast_blks_read, + coalesce(sum(toast_blks_hit), 0)::int8 as toast_blks_hit, + coalesce(sum(tidx_blks_read), 0)::int8 as tidx_blks_read, + coalesce(sum(tidx_blks_hit), 0)::int8 as tidx_blks_hit + from ranked + where rownum > 100 + having count(*) > 0; gauges: - '*' statement_timeout_seconds: 15 pg_statio_all_indexes: description: > - Retrieves index-level I/O statistics from the PostgreSQL `pg_statio_all_indexes` view, providing insights into I/O operations for all indexes. - It returns block-level read and hit statistics for index operations broken down by schema, table, and index name. - Joined with pg_class for efficient ordering by index size. - This metric helps administrators monitor index-level I/O performance and identify which indexes are generating the most I/O activity. + Retrieves index-level I/O statistics from `pg_statio_user_indexes`, returning + block-level read and hit counters per index. Ports the pgwatch2 (gen2) + top-N + `'other'` bucket pattern: ranks indexes by idx_blks_read, keeps the + top 100, folds the tail into a single `'other'` row, and drops indexes with + no I/O activity. Filters temp schemas. Compatible with all PostgreSQL versions. sqls: 11: |- - select /* pgwatch_generated */ + with ranked as ( /* pgwatch_generated */ + select + row_number() over (order by idx_blks_read desc nulls last) as rownum, + schemaname, + relname, + indexrelname, + idx_blks_read, + idx_blks_hit + from pg_statio_user_indexes + where + not schemaname like E'pg\\_temp%' + and (idx_blks_read > 0 or idx_blks_hit > 0) + ) + select (extract(epoch from now()) * 1e9)::int8 as epoch_ns, current_database() as tag_datname, - s.schemaname as tag_schemaname, - s.relname as tag_relname, - s.indexrelname as tag_indexrelname, - s.idx_blks_read, - s.idx_blks_hit - from - pg_statio_all_indexes as s - join pg_class as c on - s.indexrelname = c.relname - and s.schemaname = c.relnamespace::regnamespace::name - order by c.relpages desc - limit 5000; + schemaname as tag_schemaname, + relname as tag_relname, + indexrelname as tag_indexrelname, + idx_blks_read, + idx_blks_hit + from ranked + where rownum <= 100 + union all + select + (extract(epoch from now()) * 1e9)::int8 as epoch_ns, + current_database() as tag_datname, + 'other'::text as tag_schemaname, + 'other'::text as tag_relname, + 'other'::text as tag_indexrelname, + coalesce(sum(idx_blks_read), 0)::int8 as idx_blks_read, + coalesce(sum(idx_blks_hit), 0)::int8 as idx_blks_hit + from ranked + where rownum > 100 + having count(*) > 0; gauges: - '*' statement_timeout_seconds: 15 diff --git a/tests/compliance_vectors/test_mr219_monitoring_guards.py b/tests/compliance_vectors/test_mr219_monitoring_guards.py index 7a00715..48c6fc6 100644 --- a/tests/compliance_vectors/test_mr219_monitoring_guards.py +++ b/tests/compliance_vectors/test_mr219_monitoring_guards.py @@ -80,6 +80,52 @@ def test_pgwatch_metrics_yml_pg_stat_statements_has_top_n_filter(): assert "limit 100" in compact_sql +def test_pgwatch_stat_views_use_topn_and_other_bucket(): + """High-cardinality per-relation metrics must port the pgwatch2 (gen2) + pattern: read pg_stat_user_*/pg_statio_user_* (so pg_catalog, + information_schema and pg_toast are excluded by the Postgres view + itself, no hand-curated nspname pattern), keep the top 100 by relevance, + and aggregate the tail into a single `'other'` tag row so dashboard + totals stay correct under a hard cardinality cap. Hand-rolled nspname + LIKE filters or LIMIT-only truncation silently drop the tail and break + sums on extension-heavy or schema-heavy databases. + """ + metrics = yaml.safe_load( + (PROJECT_ROOT / "config/pgwatch-prometheus/metrics.yml").read_text() + ) + expectations = { + "pg_stat_all_indexes": "pg_stat_user_indexes", + "pg_stat_all_tables": "pg_stat_user_tables", + "pg_statio_all_tables": "pg_statio_user_tables", + "pg_statio_all_indexes": "pg_statio_user_indexes", + } + for metric_name, base_view in expectations.items(): + for sql in metrics["metrics"][metric_name]["sqls"].values(): + compact_sql = _compact_sql(sql) + assert base_view in compact_sql, metric_name + # Top-N window + tail aggregation + assert "row_number() over" in compact_sql, metric_name + assert "rownum <= 100" in compact_sql, metric_name + assert "rownum > 100" in compact_sql, metric_name + assert "'other'" in compact_sql, metric_name + # No unfiltered LIMIT-only truncation left in place + assert "limit 5000" not in compact_sql, metric_name + + +def test_pgwatch_statio_skips_zero_activity_rows(): + """pg_statio_user_* tail is mostly zero-I/O rows on schema-heavy DBs. + Filtering them out (pgwatch2 behavior) cuts cardinality before the + top-N cap is even reached and keeps the `'other'` bucket meaningful. + """ + metrics = yaml.safe_load( + (PROJECT_ROOT / "config/pgwatch-prometheus/metrics.yml").read_text() + ) + for sql in metrics["metrics"]["pg_statio_all_tables"]["sqls"].values(): + assert "heap_blks_read > 0" in _compact_sql(sql) + for sql in metrics["metrics"]["pg_statio_all_indexes"]["sqls"].values(): + assert "idx_blks_read > 0" in _compact_sql(sql) + + def test_pgwatch_dockerfile_sha_pin_and_patch_present(): dockerfile = (PROJECT_ROOT / "pgwatch/Dockerfile").read_text() From 2c966fcddc5b449ac1ac0da9bf0e03ff86ba6a45 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 15:20:19 +0000 Subject: [PATCH 2/4] =?UTF-8?q?ci:=20drop=20custom=20CodeQL=20workflow=20?= =?UTF-8?q?=E2=80=94=20default=20setup=20already=20covers=20python+javascr?= =?UTF-8?q?ipt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo has GitHub's default CodeQL setup enabled (it scans python, javascript-typescript, ruby and actions on every PR). The custom .github/workflows/codeql-analysis.yml runs a second 'Analyze (python|javascript)' matrix on the same commit; GitHub rejects its SARIF upload with 'CodeQL analyses from advanced configurations cannot be processed when the default setup is enabled', so the custom workflow has been failing on every PR since the default setup was turned on. Default setup already covers the same language set (and more), so deleting the custom workflow leaves us with one working CodeQL run instead of one working + one always-red. --- .github/workflows/codeql-analysis.yml | 40 --------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml deleted file mode 100644 index 329a5d3..0000000 --- a/.github/workflows/codeql-analysis.yml +++ /dev/null @@ -1,40 +0,0 @@ -# NOTE: This repo is mirrored to GitHub where this workflow runs automatically. -name: "CodeQL" - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - - cron: '40 17 * * 5' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: ['python', 'javascript'] - # CodeQL supports: 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' - - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - - - name: Initialize CodeQL - uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 - with: - languages: ${{ matrix.language }} - - - name: Autobuild - uses: github/codeql-action/autobuild@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 From 10c153ec75596b903acd8ea1e996679658d9a8b8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 15:21:07 +0000 Subject: [PATCH 3/4] =?UTF-8?q?docs:=20clarify=20pgwatch2=20attribution=20?= =?UTF-8?q?=E2=80=94=20postgres.ai=20edition=20(fork=20of=20Cybertec's)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'pgwatch2 (gen2)' shorthand was ambiguous. The pattern we ported lives in gitlab.com/postgres-ai/pgwatch2 — postgres.ai's fork of Cybertec's pgwatch2 — which was the previous generation of our monitoring stack before postgresai. No code or SQL changes; comment/docstring wording only. --- config/pgwatch-prometheus/metrics.yml | 13 +++++++++---- .../test_mr219_monitoring_guards.py | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/config/pgwatch-prometheus/metrics.yml b/config/pgwatch-prometheus/metrics.yml index 436e02f..2368e7b 100644 --- a/config/pgwatch-prometheus/metrics.yml +++ b/config/pgwatch-prometheus/metrics.yml @@ -1552,7 +1552,9 @@ metrics: - total_relation_size_bytes statement_timeout_seconds: 15 pg_stat_all_indexes: - # Top-N + "other" bucket pattern ported from pgwatch2 (gen2). Reads + # Top-N + "other" bucket pattern ported from pgwatch2 postgres.ai edition + # (gitlab.com/postgres-ai/pgwatch2 — our fork of Cybertec's pgwatch2, + # used as gen2 of our monitoring stack before postgresai). Reads # pg_stat_user_indexes so pg_catalog/information_schema/pg_toast are # excluded by the Postgres view itself, no hand-curated nspname pattern. # The "other" row aggregates the tail so totals stay correct under a @@ -1599,7 +1601,8 @@ metrics: - idx_tup_fetch statement_timeout_seconds: 15 pg_stat_all_tables: - # Top-N + "other" bucket pattern ported from pgwatch2 (gen2). Ranks by + # Top-N + "other" bucket pattern ported from pgwatch2 postgres.ai edition + # (gitlab.com/postgres-ai/pgwatch2). Ranks by # pg_total_relation_size — large tables are usually the interesting ones, # which avoids starving big-but-static tables out of the top-N (the old # n_live_tup+n_dead_tup ordering did exactly that). @@ -2960,7 +2963,8 @@ metrics: description: > Retrieves table-level I/O statistics from `pg_statio_user_tables`, returning block-level read and hit counters for heap, index, TOAST and TOAST-index pages. - Ports the pgwatch2 (gen2) top-N + `'other'` bucket pattern: ranks tables by + Ports the top-N + `'other'` bucket pattern from pgwatch2 postgres.ai + edition (gitlab.com/postgres-ai/pgwatch2): ranks tables by heap_blks_read, keeps the top 100, and folds the tail into a single `'other'` row so totals remain accurate while cardinality stays bounded. Drops rows with no I/O activity at all (every counter zero). @@ -3025,7 +3029,8 @@ metrics: pg_statio_all_indexes: description: > Retrieves index-level I/O statistics from `pg_statio_user_indexes`, returning - block-level read and hit counters per index. Ports the pgwatch2 (gen2) + block-level read and hit counters per index. Ports the pgwatch2 + postgres.ai edition (gitlab.com/postgres-ai/pgwatch2) top-N + `'other'` bucket pattern: ranks indexes by idx_blks_read, keeps the top 100, folds the tail into a single `'other'` row, and drops indexes with no I/O activity. Filters temp schemas. diff --git a/tests/compliance_vectors/test_mr219_monitoring_guards.py b/tests/compliance_vectors/test_mr219_monitoring_guards.py index 48c6fc6..032611d 100644 --- a/tests/compliance_vectors/test_mr219_monitoring_guards.py +++ b/tests/compliance_vectors/test_mr219_monitoring_guards.py @@ -81,8 +81,10 @@ def test_pgwatch_metrics_yml_pg_stat_statements_has_top_n_filter(): def test_pgwatch_stat_views_use_topn_and_other_bucket(): - """High-cardinality per-relation metrics must port the pgwatch2 (gen2) - pattern: read pg_stat_user_*/pg_statio_user_* (so pg_catalog, + """High-cardinality per-relation metrics must port the pattern from + pgwatch2 postgres.ai edition (gitlab.com/postgres-ai/pgwatch2, our + fork of Cybertec's pgwatch2 used as the previous generation of our + monitoring stack): read pg_stat_user_*/pg_statio_user_* (so pg_catalog, information_schema and pg_toast are excluded by the Postgres view itself, no hand-curated nspname pattern), keep the top 100 by relevance, and aggregate the tail into a single `'other'` tag row so dashboard From bd2f16c5b6129173fada1ecd530296d1f1ce0c1d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 15 May 2026 16:25:44 +0000 Subject: [PATCH 4/4] =?UTF-8?q?fix(metrics):=20rank=20don't=20filter=20?= =?UTF-8?q?=E2=80=94=20keep=20pg=5Fcatalog/pg=5Ftoast/timescale=20visible?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first revision read from pg_stat_user_*/pg_statio_user_*, which the Postgres views define as 'pg_stat_all_* WHERE schemaname NOT IN (pg_catalog, information_schema) AND schemaname !~ ^pg_toast'. That's identity-based filtering wearing a different hat: it silently hides bloat in pg_toast, hot scans in pg_catalog, and any issue inside _timescaledb_internal. If a TOAST table is bloated or a catalog index is being hammered, the operator wouldn't see it. Rework the four metrics to read pg_stat_all_*/pg_statio_all_* directly and rely PURELY on cardinality control: - Top 100 by relevance per database (idx_scan / pg_total_relation_size / heap_blks_read / idx_blks_read). - Tail aggregated into a single 'other' row so totals stay correct. - No pg_temp%, no pg_toast%, no _timescaledb% schema filtering anywhere. A relation enters the top-N by activity or by size; if it's not in the top-N, it's in 'other'. The only WHERE filter kept is the zero-counter row skip on the two statio metrics — those rows literally carry no information (every gauge is 0) and cannot mask any issue, so dropping them is information-preserving, not identity-based. Smoke-tested against PG16: - pg_stat_all_tables: 101 rows, 75 from pg_catalog/etc. in top-100. - pg_stat_all_indexes: 101 rows, 98 from system schemas. - pg_statio_all_tables / pg_statio_all_indexes: catalog/toast rows appear in top-N once they have any I/O. Regression tests updated to assert: reads pg_stat_all_*/pg_statio_all_*, no schemaname/nspname LIKE patterns, no 'pg_toast'/'pg_catalog'/ '_timescaledb' literals — top-N + 'other' is the only mechanism. --- config/pgwatch-prometheus/metrics.yml | 73 +++++++++++-------- .../test_mr219_monitoring_guards.py | 50 ++++++++----- 2 files changed, 73 insertions(+), 50 deletions(-) diff --git a/config/pgwatch-prometheus/metrics.yml b/config/pgwatch-prometheus/metrics.yml index 2368e7b..4b7264c 100644 --- a/config/pgwatch-prometheus/metrics.yml +++ b/config/pgwatch-prometheus/metrics.yml @@ -1552,13 +1552,15 @@ metrics: - total_relation_size_bytes statement_timeout_seconds: 15 pg_stat_all_indexes: - # Top-N + "other" bucket pattern ported from pgwatch2 postgres.ai edition - # (gitlab.com/postgres-ai/pgwatch2 — our fork of Cybertec's pgwatch2, - # used as gen2 of our monitoring stack before postgresai). Reads - # pg_stat_user_indexes so pg_catalog/information_schema/pg_toast are - # excluded by the Postgres view itself, no hand-curated nspname pattern. - # The "other" row aggregates the tail so totals stay correct under a - # hard cardinality cap. + # Bound cardinality by ranking — NOT by identity. Reads pg_stat_all_indexes + # directly (NOT pg_stat_user_indexes) so pg_catalog, pg_toast and + # _timescaledb_internal indexes stay visible: a heavily-scanned catalog + # index or a hot Timescale chunk index will naturally rank into the + # top-N. Everything below the cap is aggregated into a single `'other'` + # row so dashboard totals stay correct. Pattern adapted from pgwatch2 + # postgres.ai edition (gitlab.com/postgres-ai/pgwatch2 — our fork of + # Cybertec's pgwatch2), but without that edition's pg_temp%/user-view + # filters which would silently hide system-schema problems. sqls: 11: | with ranked as ( /* pgwatch_generated */ @@ -1570,8 +1572,7 @@ metrics: idx_scan, idx_tup_read, idx_tup_fetch - from pg_stat_user_indexes - where not schemaname like E'pg\\_temp%' + from pg_stat_all_indexes ) select current_database() as tag_datname, @@ -1601,11 +1602,15 @@ metrics: - idx_tup_fetch statement_timeout_seconds: 15 pg_stat_all_tables: - # Top-N + "other" bucket pattern ported from pgwatch2 postgres.ai edition - # (gitlab.com/postgres-ai/pgwatch2). Ranks by - # pg_total_relation_size — large tables are usually the interesting ones, - # which avoids starving big-but-static tables out of the top-N (the old - # n_live_tup+n_dead_tup ordering did exactly that). + # Bound cardinality by ranking — NOT by identity. Reads pg_stat_all_tables + # directly (NOT pg_stat_user_tables) so pg_catalog, pg_toast and + # _timescaledb_internal tables stay visible: a bloated TOAST table or a + # huge Timescale chunk will naturally rank into the top-N by + # pg_total_relation_size. Everything below the cap is summed into a + # single `'other'` row. + # + # Ordering by total relation size (vs the previous n_live_tup+n_dead_tup) + # keeps big-but-static tables — including pg_toast — in scope. sqls: 11: | with ranked as ( /* pgwatch_generated */ @@ -1631,7 +1636,7 @@ metrics: autovacuum_count, analyze_count, autoanalyze_count - from pg_stat_user_tables + from pg_stat_all_tables ) select current_database() as tag_datname, @@ -2961,13 +2966,16 @@ metrics: statement_timeout_seconds: 15 pg_statio_all_tables: description: > - Retrieves table-level I/O statistics from `pg_statio_user_tables`, returning - block-level read and hit counters for heap, index, TOAST and TOAST-index pages. - Ports the top-N + `'other'` bucket pattern from pgwatch2 postgres.ai - edition (gitlab.com/postgres-ai/pgwatch2): ranks tables by - heap_blks_read, keeps the top 100, and folds the tail into a single `'other'` - row so totals remain accurate while cardinality stays bounded. Drops rows - with no I/O activity at all (every counter zero). + Retrieves table-level I/O statistics from `pg_statio_all_tables`, returning + block-level read and hit counters for heap, index, TOAST and TOAST-index + pages. Adapts the top-N + `'other'` bucket pattern from pgwatch2 postgres.ai + edition (gitlab.com/postgres-ai/pgwatch2): ranks tables by heap_blks_read, + keeps the top 100, and folds the tail into a single `'other'` row so totals + remain accurate while cardinality stays bounded. + Reads pg_statio_all_tables (not pg_statio_user_tables) so I/O on pg_catalog, + pg_toast and _timescaledb_internal stays visible — those tables enter the + top-N by activity, not by schema membership. The zero-counter row skip is + kept (those rows literally carry no information and are not identity-based). Compatible with all PostgreSQL versions. sqls: 11: |- @@ -2984,7 +2992,7 @@ metrics: toast_blks_hit, tidx_blks_read, tidx_blks_hit - from pg_statio_user_tables + from pg_statio_all_tables where heap_blks_read > 0 or heap_blks_hit > 0 or idx_blks_read > 0 or idx_blks_hit > 0 @@ -3028,12 +3036,15 @@ metrics: statement_timeout_seconds: 15 pg_statio_all_indexes: description: > - Retrieves index-level I/O statistics from `pg_statio_user_indexes`, returning - block-level read and hit counters per index. Ports the pgwatch2 - postgres.ai edition (gitlab.com/postgres-ai/pgwatch2) - top-N + `'other'` bucket pattern: ranks indexes by idx_blks_read, keeps the + Retrieves index-level I/O statistics from `pg_statio_all_indexes`, returning + block-level read and hit counters per index. Adapts the top-N + `'other'` + bucket pattern from pgwatch2 postgres.ai edition + (gitlab.com/postgres-ai/pgwatch2): ranks indexes by idx_blks_read, keeps the top 100, folds the tail into a single `'other'` row, and drops indexes with - no I/O activity. Filters temp schemas. + no I/O activity (zero-counter rows carry no information). + Reads pg_statio_all_indexes (not pg_statio_user_indexes) so catalog, + pg_toast and _timescaledb_internal indexes stay visible: a hot catalog + index will rank into the top-N by activity, not be hidden by schema name. Compatible with all PostgreSQL versions. sqls: 11: |- @@ -3045,10 +3056,8 @@ metrics: indexrelname, idx_blks_read, idx_blks_hit - from pg_statio_user_indexes - where - not schemaname like E'pg\\_temp%' - and (idx_blks_read > 0 or idx_blks_hit > 0) + from pg_statio_all_indexes + where idx_blks_read > 0 or idx_blks_hit > 0 ) select (extract(epoch from now()) * 1e9)::int8 as epoch_ns, diff --git a/tests/compliance_vectors/test_mr219_monitoring_guards.py b/tests/compliance_vectors/test_mr219_monitoring_guards.py index 032611d..6bc6c33 100644 --- a/tests/compliance_vectors/test_mr219_monitoring_guards.py +++ b/tests/compliance_vectors/test_mr219_monitoring_guards.py @@ -81,30 +81,36 @@ def test_pgwatch_metrics_yml_pg_stat_statements_has_top_n_filter(): def test_pgwatch_stat_views_use_topn_and_other_bucket(): - """High-cardinality per-relation metrics must port the pattern from - pgwatch2 postgres.ai edition (gitlab.com/postgres-ai/pgwatch2, our - fork of Cybertec's pgwatch2 used as the previous generation of our - monitoring stack): read pg_stat_user_*/pg_statio_user_* (so pg_catalog, - information_schema and pg_toast are excluded by the Postgres view - itself, no hand-curated nspname pattern), keep the top 100 by relevance, - and aggregate the tail into a single `'other'` tag row so dashboard - totals stay correct under a hard cardinality cap. Hand-rolled nspname - LIKE filters or LIMIT-only truncation silently drop the tail and break - sums on extension-heavy or schema-heavy databases. + """High-cardinality per-relation metrics must bound cardinality by + RANKING, not by IDENTITY. Read pg_stat_all_*/pg_statio_all_* directly + (NOT the pg_stat_user_*/pg_statio_user_* views, which silently exclude + pg_catalog/pg_toast and would hide bloat or hot scans in those + relations), keep the top 100 by relevance, and aggregate the tail into + a single `'other'` tag row so dashboard totals stay correct. + + The principle: a bloated pg_toast or a heavy _timescaledb_internal + chunk should appear in the top-N when its activity/size warrants it. + Schema-name filtering (`pg_stat_user_*` views, `NOT LIKE 'pg_toast%'`, + `NOT LIKE '_timescaledb%'`) makes those issues invisible. Hand-rolled + nspname LIKE filters or LIMIT-only truncation likewise silently drop + the tail and break sums on extension-heavy or schema-heavy databases. """ metrics = yaml.safe_load( (PROJECT_ROOT / "config/pgwatch-prometheus/metrics.yml").read_text() ) expectations = { - "pg_stat_all_indexes": "pg_stat_user_indexes", - "pg_stat_all_tables": "pg_stat_user_tables", - "pg_statio_all_tables": "pg_statio_user_tables", - "pg_statio_all_indexes": "pg_statio_user_indexes", + "pg_stat_all_indexes": "pg_stat_all_indexes", + "pg_stat_all_tables": "pg_stat_all_tables", + "pg_statio_all_tables": "pg_statio_all_tables", + "pg_statio_all_indexes": "pg_statio_all_indexes", } for metric_name, base_view in expectations.items(): for sql in metrics["metrics"][metric_name]["sqls"].values(): compact_sql = _compact_sql(sql) - assert base_view in compact_sql, metric_name + # Reads the _all_ view, not the _user_ view — keeps catalog/toast/timescale visible. + assert f"from {base_view}" in compact_sql, metric_name + user_view = base_view.replace("_all_", "_user_") + assert user_view not in compact_sql, metric_name # Top-N window + tail aggregation assert "row_number() over" in compact_sql, metric_name assert "rownum <= 100" in compact_sql, metric_name @@ -112,12 +118,20 @@ def test_pgwatch_stat_views_use_topn_and_other_bucket(): assert "'other'" in compact_sql, metric_name # No unfiltered LIMIT-only truncation left in place assert "limit 5000" not in compact_sql, metric_name + # No identity-based schema exclusions sneaking back in. + assert "schemaname like" not in compact_sql, metric_name + assert "nspname like" not in compact_sql, metric_name + assert "'pg_toast'" not in compact_sql, metric_name + assert "'pg_catalog'" not in compact_sql, metric_name + assert "_timescaledb" not in compact_sql, metric_name def test_pgwatch_statio_skips_zero_activity_rows(): - """pg_statio_user_* tail is mostly zero-I/O rows on schema-heavy DBs. - Filtering them out (pgwatch2 behavior) cuts cardinality before the - top-N cap is even reached and keeps the `'other'` bucket meaningful. + """pg_statio tail is mostly zero-I/O rows on schema-heavy DBs. Skipping + them cuts cardinality before the top-N cap is even reached and keeps + the `'other'` bucket meaningful. This is NOT identity-based filtering: + a row with every counter zero literally carries no information and + cannot mask any issue. """ metrics = yaml.safe_load( (PROJECT_ROOT / "config/pgwatch-prometheus/metrics.yml").read_text()