From fb8f6ef6bfc108bef912df75392a544b6c091cf6 Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Thu, 25 Jun 2026 15:42:26 -0500 Subject: [PATCH 1/3] catalog: add per-timeframe console cluster utilization overview views The Console's cluster-detail page polls a replica-utilization rollup that, for every timeframe except "Last 14 days", was recomputed ad-hoc on each request: it builds the whole-fleet rollup (per-replica metrics aggregation, five per-bucket arg-max top-1s, a multi-way join) and only filters to the one selected cluster at the very end. That recompute is CPU-bound on mz_catalog_server, so it serializes under concurrent users and its cost grows linearly with the number of clusters in the deployment. This adds two new indexed views alongside the existing 14-day overview so the Console can read a maintained, per-cluster indexed lookup for every timeframe it offers instead of recomputing: - mz_console_cluster_utilization_overview_3h (1-minute buckets, 3h window) - mz_console_cluster_utilization_overview_24h (5-minute buckets, 24h window) - mz_console_cluster_utilization_overview (now 1-hour buckets, 14d; previously 8-hour buckets) All three share one SQL body (console_cluster_utilization_overview_sql) and output relation so they stay in sync, and each is indexed on cluster_id in mz_catalog_server. The view body is also cleaned up to drop a redundant re-join of replica_history that the binning step never needed. The two new views read only a 3h/24h window of metrics, so they are cheap to maintain (~10MB and ~510MB of arrangements, ~1-10s hydration at 100 replicas). The 14-day view dominates cost (input-bound on 14 days of 1-minute samples); the 8h->1h change roughly doubles its arrangement memory but leaves hydration unchanged. The Console gates use of these views on the environment version, falling back to the ad-hoc query on older environments. Updates the catalog snapshot tests (oid, information_schema_tables, mz_catalog_server_index_accounting, catalog_server_explain, autogenerated mz_internal) and the catalog/indexes/explain-analyze testdrive files to cover the new objects. --- src/catalog/src/builtin.rs | 4 + src/catalog/src/builtin/mz_internal.rs | 432 +++++++------ src/pgrepr-consts/src/oid.rs | 4 + .../autogenerated/mz_internal.slt | 2 + test/sqllogictest/catalog_server_explain.slt | 568 +++++++++++++++--- .../information_schema_tables.slt | 8 + .../mz_catalog_server_index_accounting.slt | 60 +- test/sqllogictest/oid.slt | 4 + test/testdrive/catalog.td | 4 +- test/testdrive/explain-analyze.td | 65 +- test/testdrive/indexes.td | 2 + test/workload-replay/objects.txt | 4 + 12 files changed, 836 insertions(+), 321 deletions(-) diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index aae323a94a838..7395f04b4ac64 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1395,6 +1395,8 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::View(&MZ_MATERIALIZATION_DEPENDENCIES), Builtin::View(&MZ_MATERIALIZATION_LAG), Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW), + Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H), + Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H), Builtin::View(&MZ_COMPUTE_ERROR_COUNTS_PER_WORKER), Builtin::View(&MZ_COMPUTE_ERROR_COUNTS), Builtin::Source(&MZ_COMPUTE_ERROR_COUNTS_RAW_UNIFIED), @@ -1469,6 +1471,8 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Index(&MZ_SECRETS_IND), Builtin::Index(&MZ_VIEWS_IND), Builtin::Index(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_IND), + Builtin::Index(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND), + Builtin::Index(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND), Builtin::Index(&MZ_CLUSTER_DEPLOYMENT_LINEAGE_IND), Builtin::Index(&MZ_CLUSTER_REPLICA_FRONTIERS_IND), Builtin::Index(&MZ_COMPUTE_HYDRATION_TIMES_IND), diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index c291e9a679d14..d9e86066e832a 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -6990,78 +6990,88 @@ JOIN root_times r USING (id)", }, }), }); -/** - * This view is used to display the cluster utilization over 14 days bucketed by 8 hours. - * It's specifically for the Console's environment overview page to speed up load times. - * This query should be kept in sync with MaterializeInc/console/src/api/materialize/cluster/replicaUtilizationHistory.ts - */ -pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW: LazyLock = LazyLock::new(|| { - BuiltinView { - name: "mz_console_cluster_utilization_overview", - schema: MZ_INTERNAL_SCHEMA, - oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_OID, - desc: RelationDesc::builder() - .with_column( - "bucket_start", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("replica_id", SqlScalarType::String.nullable(false)) - .with_column("memory_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_memory_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("disk_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_disk_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column( - "memory_and_disk_percent", - SqlScalarType::Float64.nullable(true), - ) - .with_column( - "max_memory_and_disk_memory_percent", - SqlScalarType::Float64.nullable(true), - ) - .with_column( - "max_memory_and_disk_disk_percent", - SqlScalarType::Float64.nullable(true), - ) - .with_column( - "max_memory_and_disk_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("heap_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_heap_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("max_cpu_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_cpu_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("offline_events", SqlScalarType::Jsonb.nullable(true)) - .with_column( - "bucket_end", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("name", SqlScalarType::String.nullable(true)) - .with_column("cluster_id", SqlScalarType::String.nullable(true)) - .with_column("size", SqlScalarType::String.nullable(true)) - .finish(), - column_comments: BTreeMap::new(), - sql: r#"WITH replica_history AS ( - SELECT replica_id, - size, - cluster_id +/// The output relation shared by all `mz_console_cluster_utilization_overview*` +/// views. Every (bucket size, retention) variant produces the same columns so +/// the Console can swap between them based on the selected time range. +fn console_cluster_utilization_overview_desc() -> RelationDesc { + RelationDesc::builder() + .with_column( + "bucket_start", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("replica_id", SqlScalarType::String.nullable(false)) + .with_column("memory_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_memory_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("disk_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_disk_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column( + "memory_and_disk_percent", + SqlScalarType::Float64.nullable(true), + ) + .with_column( + "max_memory_and_disk_memory_percent", + SqlScalarType::Float64.nullable(true), + ) + .with_column( + "max_memory_and_disk_disk_percent", + SqlScalarType::Float64.nullable(true), + ) + .with_column( + "max_memory_and_disk_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("heap_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_heap_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("max_cpu_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_cpu_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("offline_events", SqlScalarType::Jsonb.nullable(true)) + .with_column( + "bucket_end", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("name", SqlScalarType::String.nullable(true)) + .with_column("cluster_id", SqlScalarType::String.nullable(true)) + .with_column("size", SqlScalarType::String.nullable(true)) + .finish() +} + +/// Builds the SQL body shared by the `mz_console_cluster_utilization_overview*` +/// views, which power the Console's cluster utilization graphs. +/// +/// There is one view per (bucket width, retention window) pair so the Console +/// can read a pre-materialized, indexed rollup for each time range it offers +/// instead of recomputing this (expensive) query on every page load. The bodies +/// must be kept in sync with the equivalent ad-hoc query in the Console +/// (`buildReplicaUtilizationHistoryQuery` in +/// `console/src/api/materialize/cluster/replicaUtilizationHistory.ts`). +/// +/// * `bin`: the `date_bin` bucket width, e.g. `1 MINUTE`. +/// * `retention`: how much history the view retains, e.g. `3 HOURS`, enforced +/// with a temporal `mz_now()` filter so the maintained arrangement stays +/// bounded. +/// * `group_size`: the expected number of metric samples per (replica, bucket), +/// used for the `DISTINCT ON INPUT GROUP SIZE` top-k hint. Replica metrics are +/// scraped roughly once per minute, so this is the bucket width in minutes. +fn console_cluster_utilization_overview_sql(bin: &str, retention: &str, group_size: u32) -> String { + format!( + r#"WITH replica_history AS ( + SELECT replica_id, size, cluster_id FROM mz_internal.mz_cluster_replica_history UNION - -- We need to union the current set of cluster replicas since mz_cluster_replica_history doesn't include system clusters - SELECT id AS replica_id, - size, - cluster_id + -- We union the current set of cluster replicas since mz_cluster_replica_history doesn't include system clusters. + SELECT id AS replica_id, size, cluster_id FROM mz_catalog.mz_cluster_replicas ), replica_metrics_history AS ( @@ -7069,14 +7079,22 @@ replica_metrics_history AS ( m.occurred_at, m.replica_id, r.size, - (SUM(m.cpu_nano_cores::float8) / NULLIF(s.cpu_nano_cores, 0)) / NULLIF(s.processes, 0) AS cpu_percent, - (SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0)) / NULLIF(s.processes, 0) AS memory_percent, - (SUM(m.disk_bytes::float8) / NULLIF(s.disk_bytes, 0)) / NULLIF(s.processes, 0) AS disk_percent, - (SUM(m.heap_bytes::float8) / NULLIF(m.heap_limit, 0)) / NULLIF(s.processes, 0) AS heap_percent, + (SUM(m.cpu_nano_cores::float8) / (NULLIF(s.cpu_nano_cores, 0) * s.processes)) AS cpu_percent, + (SUM(m.memory_bytes::float8) / (NULLIF(s.memory_bytes, 0) * s.processes)) AS memory_percent, + (SUM(m.disk_bytes::float8) / (NULLIF(s.disk_bytes, 0) * s.processes)) AS disk_percent, SUM(m.disk_bytes::float8) AS disk_bytes, SUM(m.memory_bytes::float8) AS memory_bytes, - s.disk_bytes::numeric * s.processes AS total_disk_bytes, - s.memory_bytes::numeric * s.processes AS total_memory_bytes + s.disk_bytes * s.processes AS total_disk_bytes, + s.memory_bytes * s.processes AS total_memory_bytes, + MAX(m.heap_bytes::float8) AS heap_bytes, + MAX(m.heap_limit) AS heap_limit, + -- heap_limit is NULL when clusterd isn't launched with --heap-limit (e.g. + -- the emulator's process orchestrator). Fall back to the size-based memory + -- percent so the chart still renders. + COALESCE( + MAX(m.heap_bytes::float8 / NULLIF(m.heap_limit, 0)), + SUM(m.memory_bytes::float8) / (NULLIF(s.memory_bytes, 0) * s.processes) + ) AS heap_percent FROM replica_history AS r INNER JOIN mz_catalog.mz_cluster_replica_sizes AS s ON r.size = s.size @@ -7088,154 +7106,113 @@ replica_metrics_history AS ( s.cpu_nano_cores, s.memory_bytes, s.disk_bytes, - m.heap_limit, s.processes ), replica_utilization_history_binned AS ( - SELECT m.occurred_at, + -- NOTE: we read directly from replica_metrics_history rather than re-joining + -- replica_history; every replica_id here already came from replica_history, + -- so the join was redundant (and could fan out a replica that changed size). + SELECT + m.occurred_at, m.replica_id, m.cpu_percent, m.memory_percent, m.memory_bytes, m.disk_percent, m.disk_bytes, - m.heap_percent, m.total_disk_bytes, m.total_memory_bytes, + m.heap_bytes, + m.heap_percent, m.size, - date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) AS bucket_start - FROM replica_history AS r - JOIN replica_metrics_history AS m ON m.replica_id = r.replica_id - WHERE mz_now() <= date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) + INTERVAL '14 DAYS' + date_bin('{bin}', m.occurred_at, '1970-01-01'::timestamp) AS bucket_start + FROM replica_metrics_history AS m + WHERE mz_now() <= date_bin('{bin}', m.occurred_at, '1970-01-01'::timestamp) + INTERVAL '{retention}' ), --- For each (replica, bucket), take the (replica, bucket) with the highest memory +-- For each (replica, bucket), take the sample with the highest memory. max_memory AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - memory_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, memory_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(memory_bytes, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(memory_bytes, 0) DESC ), +-- For each (replica, bucket), take the sample with the highest disk. max_disk AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - disk_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, disk_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(disk_bytes, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(disk_bytes, 0) DESC ), +-- For each (replica, bucket), take the sample with the highest cpu. max_cpu AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - cpu_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, cpu_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(cpu_percent, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(cpu_percent, 0) DESC ), /* - This is different - from adding max_memory - and max_disk per bucket because both - values may not occur at the same time if the bucket interval is large. - */ + For each (replica, bucket), take the sample with the highest combined memory + and disk. This is different from adding max_memory and max_disk per bucket + because both values may not occur at the same time if the bucket interval is + large. +*/ max_memory_and_disk AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - memory_percent, - disk_percent, - memory_and_disk_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, memory_percent, disk_percent, memory_and_disk_percent, occurred_at FROM ( - SELECT *, - CASE - WHEN disk_bytes IS NULL - AND memory_bytes IS NULL THEN NULL - ELSE (COALESCE(disk_bytes, 0) + COALESCE(memory_bytes, 0)) - / (total_disk_bytes::numeric + total_memory_bytes::numeric) - END AS memory_and_disk_percent - FROM replica_utilization_history_binned - ) AS max_memory_and_disk_inner - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(memory_and_disk_percent, 0) DESC + SELECT *, + CASE + WHEN disk_bytes IS NULL AND memory_bytes IS NULL THEN NULL + ELSE (COALESCE(memory_bytes, 0) + COALESCE(disk_bytes, 0)) / NULLIF((total_memory_bytes + total_disk_bytes), 0) + END AS memory_and_disk_percent + FROM replica_utilization_history_binned + ) AS max_memory_and_disk_inner + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(memory_and_disk_percent, 0) DESC ), +-- For each (replica, bucket), take the sample with the highest heap. max_heap AS ( - SELECT DISTINCT ON (bucket_start, replica_id) - bucket_start, - replica_id, - heap_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, heap_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, replica_id, COALESCE(heap_percent, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(heap_bytes, 0) DESC ), --- For each (replica, bucket), get its offline events at that time +-- For each (replica, bucket), collect its offline events at that time. replica_offline_event_history AS ( - SELECT date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) AS bucket_start, + SELECT + date_bin('{bin}', occurred_at, '1970-01-01'::timestamp) AS bucket_start, replica_id, jsonb_agg( jsonb_build_object( - 'replicaId', - rsh.replica_id, - 'occurredAt', - rsh.occurred_at, - 'status', - rsh.status, - 'reason', - rsh.reason + 'replicaId', rsh.replica_id, + 'occurredAt', rsh.occurred_at, + 'status', rsh.status, + 'reason', rsh.reason ) ) AS offline_events - FROM mz_internal.mz_cluster_replica_status_history AS rsh -- We assume the statuses for process 0 are the same as all processes + FROM mz_internal.mz_cluster_replica_status_history AS rsh + -- We assume the statuses for process 0 are the same as all processes. WHERE process_id = '0' AND status = 'offline' - AND mz_now() <= date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) + INTERVAL '14 DAYS' - GROUP BY bucket_start, - replica_id + AND mz_now() <= date_bin('{bin}', occurred_at, '1970-01-01'::timestamp) + INTERVAL '{retention}' + GROUP BY bucket_start, replica_id ) SELECT bucket_start, replica_id, max_memory.memory_percent, - max_memory.occurred_at as max_memory_at, + max_memory.occurred_at AS max_memory_at, max_disk.disk_percent, - max_disk.occurred_at as max_disk_at, - max_memory_and_disk.memory_and_disk_percent as memory_and_disk_percent, - max_memory_and_disk.memory_percent as max_memory_and_disk_memory_percent, - max_memory_and_disk.disk_percent as max_memory_and_disk_disk_percent, - max_memory_and_disk.occurred_at as max_memory_and_disk_at, + max_disk.occurred_at AS max_disk_at, + max_memory_and_disk.memory_and_disk_percent AS memory_and_disk_percent, + max_memory_and_disk.memory_percent AS max_memory_and_disk_memory_percent, + max_memory_and_disk.disk_percent AS max_memory_and_disk_disk_percent, + max_memory_and_disk.occurred_at AS max_memory_and_disk_at, max_heap.heap_percent, - max_heap.occurred_at as max_heap_at, - max_cpu.cpu_percent as max_cpu_percent, - max_cpu.occurred_at as max_cpu_at, + max_heap.occurred_at AS max_heap_at, + max_cpu.cpu_percent AS max_cpu_percent, + max_cpu.occurred_at AS max_cpu_at, replica_offline_event_history.offline_events, - bucket_start + INTERVAL '8 HOURS' as bucket_end, + bucket_start + INTERVAL '{bin}' AS bucket_end, replica_name_history.new_name AS name, replica_history.cluster_id, replica_history.size @@ -7245,22 +7222,83 @@ JOIN max_cpu USING (bucket_start, replica_id) JOIN max_memory_and_disk USING (bucket_start, replica_id) JOIN max_heap USING (bucket_start, replica_id) JOIN replica_history USING (replica_id) +/* + TOP k=1 over the name history via a LATERAL subquery + LIMIT: for each bucket, + get the most recent replica name as of the end of the bucket. +*/ CROSS JOIN LATERAL ( SELECT new_name - FROM mz_internal.mz_cluster_replica_name_history as replica_name_history - WHERE replica_id = replica_name_history.id -- We treat NULLs as the beginning of time - AND bucket_start + INTERVAL '8 HOURS' >= COALESCE( - replica_name_history.occurred_at, - '1970-01-01'::timestamp - ) + FROM mz_internal.mz_cluster_replica_name_history AS replica_name_history + WHERE replica_id = replica_name_history.id + -- We treat NULLs as the beginning of time. + AND bucket_start + INTERVAL '{bin}' >= COALESCE(replica_name_history.occurred_at, '1970-01-01'::timestamp) ORDER BY replica_name_history.occurred_at DESC - LIMIT '1' + LIMIT 1 ) AS replica_name_history LEFT JOIN replica_offline_event_history USING (bucket_start, replica_id)"#, + bin = bin, + retention = retention, + group_size = group_size, + ) +} + +/** + * Displays cluster utilization over 14 days bucketed by 1 hour, for the + * Console's environment overview and cluster pages, to speed up load times. + * This view (and its `_3h`/`_24h` siblings) is kept in sync with + * MaterializeInc/console/src/api/materialize/cluster/replicaUtilizationHistory.ts + */ +pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW: LazyLock = + LazyLock::new(|| BuiltinView { + name: "mz_console_cluster_utilization_overview", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_OID, + desc: console_cluster_utilization_overview_desc(), + column_comments: BTreeMap::new(), + sql: Box::leak( + console_cluster_utilization_overview_sql("1 HOUR", "14 DAYS", 60).into_boxed_str(), + ), access: vec![PUBLIC_SELECT], ontology: None, - } -}); + }); + +/** + * Cluster utilization over the last 3 hours bucketed by 1 minute, for the + * Console's "Last hour" / "Last 3 hours" cluster utilization graphs. See + * `console_cluster_utilization_overview_sql` for details. + */ +pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H: LazyLock = + LazyLock::new(|| BuiltinView { + name: "mz_console_cluster_utilization_overview_3h", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_OID, + desc: console_cluster_utilization_overview_desc(), + column_comments: BTreeMap::new(), + sql: Box::leak( + console_cluster_utilization_overview_sql("1 MINUTE", "3 HOURS", 1).into_boxed_str(), + ), + access: vec![PUBLIC_SELECT], + ontology: None, + }); + +/** + * Cluster utilization over the last 24 hours bucketed by 5 minutes, for the + * Console's "Last 6 hours" / "Last 24 hours" cluster utilization graphs. See + * `console_cluster_utilization_overview_sql` for details. + */ +pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H: LazyLock = + LazyLock::new(|| BuiltinView { + name: "mz_console_cluster_utilization_overview_24h", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_OID, + desc: console_cluster_utilization_overview_desc(), + column_comments: BTreeMap::new(), + sql: Box::leak( + console_cluster_utilization_overview_sql("5 MINUTES", "24 HOURS", 5).into_boxed_str(), + ), + access: vec![PUBLIC_SELECT], + ontology: None, + }); /** * Traces the blue/green deployment lineage in the audit log to determine all cluster * IDs that are logically the same cluster. @@ -7566,6 +7604,24 @@ ON mz_internal.mz_console_cluster_utilization_overview (cluster_id)", is_retained_metrics_object: false, }; +pub const MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND: BuiltinIndex = BuiltinIndex { + name: "mz_console_cluster_utilization_overview_3h_ind", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND_OID, + sql: "IN CLUSTER mz_catalog_server +ON mz_internal.mz_console_cluster_utilization_overview_3h (cluster_id)", + is_retained_metrics_object: false, +}; + +pub const MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND: BuiltinIndex = BuiltinIndex { + name: "mz_console_cluster_utilization_overview_24h_ind", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND_OID, + sql: "IN CLUSTER mz_catalog_server +ON mz_internal.mz_console_cluster_utilization_overview_24h (cluster_id)", + is_retained_metrics_object: false, +}; + pub const MZ_CLUSTER_DEPLOYMENT_LINEAGE_IND: BuiltinIndex = BuiltinIndex { name: "mz_cluster_deployment_lineage_ind", schema: MZ_INTERNAL_SCHEMA, diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 31f248d4cf76f..30bbdec2b4375 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -806,3 +806,7 @@ pub const MV_MZ_REPLICA_SYSTEM_PARAMETERS_OID: u32 = 17092; pub const FUNC_MZ_GEN_SERIES_UNOPT_OID: u32 = 17093; pub const FUNC_MZ_GEN_SERIES_UNOPT_STEP_OID: u32 = 17094; pub const MV_MZ_OVERRIDDEN_SYSTEM_PARAMETERS_OID: u32 = 17095; +pub const VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_OID: u32 = 17096; +pub const INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND_OID: u32 = 17097; +pub const VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_OID: u32 = 17098; +pub const INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND_OID: u32 = 17099; diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index fc434766efe61..9e8a620548b87 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -784,6 +784,8 @@ mz_compute_hydration_statuses mz_compute_hydration_times mz_compute_operator_hydration_statuses mz_console_cluster_utilization_overview +mz_console_cluster_utilization_overview_24h +mz_console_cluster_utilization_overview_3h mz_frontiers mz_global_frontiers mz_history_retention_strategies diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 259f2d6c91bd8..1009661e95144 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -813,13 +813,13 @@ Target cluster: mz_catalog_server EOF query T multiline -EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_ind"; +EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_24h_ind"; ---- -mz_internal.mz_console_cluster_utilization_overview_ind: +mz_internal.mz_console_cluster_utilization_overview_24h_ind: →Arrange (#17{cluster_id}) - →Stream mz_internal.mz_console_cluster_utilization_overview + →Stream mz_internal.mz_console_cluster_utilization_overview_24h -mz_internal.mz_console_cluster_utilization_overview: +mz_internal.mz_console_cluster_utilization_overview_24h: →With cte l0 = →Distinct GroupAggregate @@ -834,97 +834,278 @@ mz_internal.mz_console_cluster_utilization_overview: →Arranged mz_catalog.mz_cluster_replicas Key: (#0{id}) cte l1 = - →Differential Join %0:l0[#0{replica_id}] » %1[#1{replica_id}] + →Map/Filter/Project + Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:05:00, #9, 1970-01-01 00:00:00 UTC) + 24:00:00))) + →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] + →Arrange (#0{replica_id}) (#1{size}) + →Fused with Child Map/Filter/Project + Project: #0, #1 + Filter: (#1{size}) IS NOT NULL + →Arranged l0 + Key: (#0..=#2) + →Arranged mz_catalog.mz_cluster_replica_sizes + →Arranged mz_internal.mz_cluster_replica_metrics_history + cte l2 = + →Differential Join %0[#0..=#6] » %1[#0..=#6] after %1: - Project: #0..=#10 - Map: timestamptz_bin(08:00:00, #1{occurred_at}, 1970-01-01 00:00:00 UTC) - →Arrange (#0{replica_id}) + Project: #0, #1, #7, #10..=#18 + Map: (#9 / uint8_to_double((case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end * #6{processes}))), (#10 / uint8_to_double((case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end * #6{processes}))), (#11 / uint8_to_double((case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end * #6{processes}))), (#5{disk_bytes} * #6{processes}), (#4{memory_bytes} * #6{processes}), coalesce(#8, #13), timestamptz_bin(00:05:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) + Aggregations: max, max + Key: + Project: #6, #0, #1, #3..=#5, #2 →Fused with Child Map/Filter/Project - Project: #0 - →Arranged l0 - Key: (#0..=#2) - →Arrange (#1{replica_id}) + Project: #0..=#5, #9..=#11 + →Read l1 + →Temporally-Bucketed Accumulable GroupAggregate + Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})) + Key: + Project: #9, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#9 + →Read l1 + cte l3 = + →Delta Join [%0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#1{replica_id}, #4{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}]] + →Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project - Project: #0, #1, #9, #10, #14, #16, #18..=#21 - Map: uint8_to_numeric(#7{processes}), uint8_to_double(case when (#12 = 0) then null else #7{processes} end), ((#8 / uint8_to_double(case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end)) / #13), uint8_to_numeric(#4{memory_bytes}), ((#9 / uint8_to_double(case when (#15 = 0) then null else #4{memory_bytes} end)) / #13), uint8_to_numeric(#5{disk_bytes}), ((#10 / uint8_to_double(case when (#17 = 0) then null else #5{disk_bytes} end)) / #13), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#6{heap_limit})) then null else #6{heap_limit} end)) / #13), (#17 * #12), (#15 * #12) - →Temporally-Bucketed Accumulable GroupAggregate - Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})), sum(uint8_to_double(#10{heap_bytes})) - Key: - Project: #9, #0, #1, #3..=#5, #11, #2 - →Map/Filter/Project - Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(08:00:00, #9, 1970-01-01 00:00:00 UTC) + 14 days))) - →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] - →Arrange (#0{replica_id}) (#1{size}) - →Fused with Child Map/Filter/Project - Project: #0, #1 - Filter: (#1{size}) IS NOT NULL - →Arranged l0 - Key: (#0..=#2) - →Arranged mz_catalog.mz_cluster_replica_sizes - →Arranged mz_internal.mz_cluster_replica_metrics_history + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #6, #11, #12 + Map: coalesce(#3{memory_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #7, #11, #12 + Map: coalesce(#4{disk_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #5, #11, #12 + Map: coalesce(#5{cpu_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #4{bucket_start}) + →Map/Filter/Project + Project: #0..=#5 + →Non-monotonic TopK + Group By #4, #1 + Order By #6 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #6, #7, #11..=#13 + Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / uint8_to_double(case when (0 = uint8_to_numeric((#9{total_memory_bytes} + #8{total_disk_bytes}))) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end)) end, coalesce(#12{memory_and_disk_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #10..=#12 + Map: coalesce(#2{heap_bytes}, 0) + →Read l2 + →Arrange (#0{replica_id}) + →Arranged l0 + cte l4 = + →Differential Join %1[#0, #1] » %0:l3[#1, #3] + →Arrange (#1, #3) + →Stream l3 + →Arrange (#0, #1) + →Map/Filter/Project + Project: #0, #1, #3 + →Non-monotonic TopK + Group By #0, #1 + Order By #2 desc nulls_first + Limit 1 + →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#0{replica_id}] + after %0: + Project: #0, #4, #1, #3 + Filter: ((#4{bucket_start} + 00:05:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) + →Arrange (#0{replica_id}) + →Distinct GroupAggregate + →Fused with Child Map/Filter/Project + Project: #1, #3 + →Read l3 + →Arranged mz_internal.mz_cluster_replica_name_history + cte l5 = + →Differential Join %1[#1, #0] » %0:l4[#1{replica_id}, #3{bucket_start}] + →Arrange (#1{replica_id}, #3{bucket_start}) + →Stream l4 + →Arrange (#1, #0) + →Temporally-Bucketed Non-incremental GroupAggregate + Aggregation: jsonb_agg[order_by=[]](row(coalesce(jsonbable_to_jsonb(jsonb_build_object("replicaId", jsonbable_to_jsonb(#0{replica_id}), "occurredAt", jsonbable_to_jsonb(timestamp_with_time_zone_to_text(#2{occurred_at})), "status", "offline", "reason", jsonbable_to_jsonb(#1{reason}))), json_null))) + Key: + Project: #3, #0 + Map: timestamptz_bin(00:05:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) + →Fused with Child Map/Filter/Project + Project: #0, #3, #4 + Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:05:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 24:00:00))) + →Arranged mz_internal.mz_cluster_replica_status_history + Key: (#0{replica_id}) + →Return + →Map/Filter/Project + Project: #1, #0, #2..=#5, #10, #8, #9, #11..=#13, #6, #7, #17, #18, #16, #15, #14 + Map: (#1{bucket_start} + 00:05:00) + →Union + →Map/Filter/Project + Project: #0..=#17 + Map: null + →Consolidating Union + →Negate Diffs + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l5 + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l4 + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 + →Read l5 + +Used Indexes: + - mz_catalog.mz_cluster_replicas_ind (*** full scan ***) + - mz_catalog.mz_cluster_replica_sizes_ind (delta join lookup) + - mz_internal.mz_cluster_replica_status_history_ind (*** full scan ***) + - mz_internal.mz_cluster_replica_metrics_history_ind (delta join lookup) + - mz_internal.mz_cluster_replica_history_ind (*** full scan ***) + - mz_internal.mz_cluster_replica_name_history_ind (differential join) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_3h_ind"; +---- +mz_internal.mz_console_cluster_utilization_overview_3h_ind: + →Arrange (#17{cluster_id}) + →Stream mz_internal.mz_console_cluster_utilization_overview_3h + +mz_internal.mz_console_cluster_utilization_overview_3h: + →With + cte l0 = + →Distinct GroupAggregate + →Union + →Fused with Child Map/Filter/Project + Project: #1..=#3 + Filter: (#1) IS NOT NULL + →Arranged mz_internal.mz_cluster_replica_history + Key: (#6{dropped_at}) + →Fused with Child Map/Filter/Project + Project: #0, #3, #2 + →Arranged mz_catalog.mz_cluster_replicas + Key: (#0{id}) + cte l1 = + →Map/Filter/Project + Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:01:00, #9, 1970-01-01 00:00:00 UTC) + 03:00:00))) + →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] + →Arrange (#0{replica_id}) (#1{size}) + →Fused with Child Map/Filter/Project + Project: #0, #1 + Filter: (#1{size}) IS NOT NULL + →Arranged l0 + Key: (#0..=#2) + →Arranged mz_catalog.mz_cluster_replica_sizes + →Arranged mz_internal.mz_cluster_replica_metrics_history cte l2 = - →Delta Join [%0[#0{replica_id}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#0{replica_id}, #3{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#0{replica_id}, #3{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %1[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#0{replica_id}, #4{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#0{replica_id}, #3{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#0{replica_id}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}]] - →Arrange (#0{replica_id}) (#0{replica_id}, #3{bucket_start}) + →Differential Join %0[#0..=#6] » %1[#0..=#6] + after %1: + Project: #0, #1, #7, #10..=#18 + Map: (#9 / uint8_to_double((case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end * #6{processes}))), (#10 / uint8_to_double((case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end * #6{processes}))), (#11 / uint8_to_double((case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end * #6{processes}))), (#5{disk_bytes} * #6{processes}), (#4{memory_bytes} * #6{processes}), coalesce(#8, #13), timestamptz_bin(00:01:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) + Aggregations: max, max + Key: + Project: #6, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#5, #9..=#11 + →Read l1 + →Temporally-Bucketed Accumulable GroupAggregate + Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})) + Key: + Project: #9, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#9 + →Read l1 + cte l3 = + →Delta Join [%0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#1{replica_id}, #4{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}]] + →Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #5, #10, #11 - Map: coalesce(#2{memory_bytes}, 0) - →Read l1 - →Arrange (#0{replica_id}, #3{bucket_start}) + Project: #0, #1, #6, #11, #12 + Map: coalesce(#3{memory_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #6, #10, #11 - Map: coalesce(#3{disk_bytes}, 0) - →Read l1 - →Arrange (#0{replica_id}, #3{bucket_start}) + Project: #0, #1, #7, #11, #12 + Map: coalesce(#4{disk_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #4, #10, #11 - Map: coalesce(#4{cpu_percent}, 0) - →Read l1 - →Arrange (#0{replica_id}, #4{bucket_start}) + Project: #0, #1, #5, #11, #12 + Map: coalesce(#5{cpu_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #4{bucket_start}) →Map/Filter/Project Project: #0..=#5 →Non-monotonic TopK - Group By #4, #0 + Group By #4, #1 Order By #6 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #5, #6, #10..=#12 - Map: case when ((#2{memory_bytes}) IS NULL AND (#3{disk_bytes}) IS NULL) then null else ((coalesce(#3{disk_bytes}, 0) + coalesce(#2{memory_bytes}, 0)) / numeric_to_double((#8{total_disk_bytes} + #9{total_memory_bytes}))) end, coalesce(#11{memory_and_disk_percent}, 0) - →Read l1 - →Arrange (#0{replica_id}, #3{bucket_start}) + Project: #0, #1, #6, #7, #11..=#13 + Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / uint8_to_double(case when (0 = uint8_to_numeric((#9{total_memory_bytes} + #8{total_disk_bytes}))) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end)) end, coalesce(#12{memory_and_disk_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #7, #10, #11 - Map: coalesce(#7{heap_percent}, 0) - →Read l1 + Project: #0, #1, #10..=#12 + Map: coalesce(#2{heap_bytes}, 0) + →Read l2 →Arrange (#0{replica_id}) →Arranged l0 - cte l3 = - →Differential Join %1[#0, #1] » %0:l2[#0, #3] - →Arrange (#0, #3) - →Stream l2 + cte l4 = + →Differential Join %1[#0, #1] » %0:l3[#1, #3] + →Arrange (#1, #3) + →Stream l3 →Arrange (#0, #1) →Map/Filter/Project Project: #0, #1, #3 @@ -935,32 +1116,211 @@ mz_internal.mz_console_cluster_utilization_overview: →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#0{replica_id}] after %0: Project: #0, #4, #1, #3 - Filter: ((#4{bucket_start} + 08:00:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) + Filter: ((#4{bucket_start} + 00:01:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) →Arrange (#0{replica_id}) →Distinct GroupAggregate →Fused with Child Map/Filter/Project - Project: #0, #3 - →Read l2 + Project: #1, #3 + →Read l3 →Arranged mz_internal.mz_cluster_replica_name_history + cte l5 = + →Differential Join %1[#1, #0] » %0:l4[#1{replica_id}, #3{bucket_start}] + →Arrange (#1{replica_id}, #3{bucket_start}) + →Stream l4 + →Arrange (#1, #0) + →Temporally-Bucketed Non-incremental GroupAggregate + Aggregation: jsonb_agg[order_by=[]](row(coalesce(jsonbable_to_jsonb(jsonb_build_object("replicaId", jsonbable_to_jsonb(#0{replica_id}), "occurredAt", jsonbable_to_jsonb(timestamp_with_time_zone_to_text(#2{occurred_at})), "status", "offline", "reason", jsonbable_to_jsonb(#1{reason}))), json_null))) + Key: + Project: #3, #0 + Map: timestamptz_bin(00:01:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) + →Fused with Child Map/Filter/Project + Project: #0, #3, #4 + Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:01:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 03:00:00))) + →Arranged mz_internal.mz_cluster_replica_status_history + Key: (#0{replica_id}) + →Return + →Map/Filter/Project + Project: #1, #0, #2..=#5, #10, #8, #9, #11..=#13, #6, #7, #17, #18, #16, #15, #14 + Map: (#1{bucket_start} + 00:01:00) + →Union + →Map/Filter/Project + Project: #0..=#17 + Map: null + →Consolidating Union + →Negate Diffs + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l5 + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l4 + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 + →Read l5 + +Used Indexes: + - mz_catalog.mz_cluster_replicas_ind (*** full scan ***) + - mz_catalog.mz_cluster_replica_sizes_ind (delta join lookup) + - mz_internal.mz_cluster_replica_status_history_ind (*** full scan ***) + - mz_internal.mz_cluster_replica_metrics_history_ind (delta join lookup) + - mz_internal.mz_cluster_replica_history_ind (*** full scan ***) + - mz_internal.mz_cluster_replica_name_history_ind (differential join) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_ind"; +---- +mz_internal.mz_console_cluster_utilization_overview_ind: + →Arrange (#17{cluster_id}) + →Stream mz_internal.mz_console_cluster_utilization_overview + +mz_internal.mz_console_cluster_utilization_overview: + →With + cte l0 = + →Distinct GroupAggregate + →Union + →Fused with Child Map/Filter/Project + Project: #1..=#3 + Filter: (#1) IS NOT NULL + →Arranged mz_internal.mz_cluster_replica_history + Key: (#6{dropped_at}) + →Fused with Child Map/Filter/Project + Project: #0, #3, #2 + →Arranged mz_catalog.mz_cluster_replicas + Key: (#0{id}) + cte l1 = + →Map/Filter/Project + Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(01:00:00, #9, 1970-01-01 00:00:00 UTC) + 14 days))) + →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] + →Arrange (#0{replica_id}) (#1{size}) + →Fused with Child Map/Filter/Project + Project: #0, #1 + Filter: (#1{size}) IS NOT NULL + →Arranged l0 + Key: (#0..=#2) + →Arranged mz_catalog.mz_cluster_replica_sizes + →Arranged mz_internal.mz_cluster_replica_metrics_history + cte l2 = + →Differential Join %0[#0..=#6] » %1[#0..=#6] + after %1: + Project: #0, #1, #7, #10..=#18 + Map: (#9 / uint8_to_double((case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end * #6{processes}))), (#10 / uint8_to_double((case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end * #6{processes}))), (#11 / uint8_to_double((case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end * #6{processes}))), (#5{disk_bytes} * #6{processes}), (#4{memory_bytes} * #6{processes}), coalesce(#8, #13), timestamptz_bin(01:00:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) + Aggregations: max, max + Key: + Project: #6, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#5, #9..=#11 + →Read l1 + →Temporally-Bucketed Accumulable GroupAggregate + Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})) + Key: + Project: #9, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#9 + →Read l1 + cte l3 = + →Delta Join [%0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#1{replica_id}, #4{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}]] + →Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #6, #11, #12 + Map: coalesce(#3{memory_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #7, #11, #12 + Map: coalesce(#4{disk_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #5, #11, #12 + Map: coalesce(#5{cpu_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #4{bucket_start}) + →Map/Filter/Project + Project: #0..=#5 + →Non-monotonic TopK + Group By #4, #1 + Order By #6 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #6, #7, #11..=#13 + Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / uint8_to_double(case when (0 = uint8_to_numeric((#9{total_memory_bytes} + #8{total_disk_bytes}))) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end)) end, coalesce(#12{memory_and_disk_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #10..=#12 + Map: coalesce(#2{heap_bytes}, 0) + →Read l2 + →Arrange (#0{replica_id}) + →Arranged l0 cte l4 = - →Differential Join %1[#1, #0] » %0:l3[#0{replica_id}, #3{bucket_start}] - →Arrange (#0{replica_id}, #3{bucket_start}) + →Differential Join %1[#0, #1] » %0:l3[#1, #3] + →Arrange (#1, #3) →Stream l3 + →Arrange (#0, #1) + →Map/Filter/Project + Project: #0, #1, #3 + →Non-monotonic TopK + Group By #0, #1 + Order By #2 desc nulls_first + Limit 1 + →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#0{replica_id}] + after %0: + Project: #0, #4, #1, #3 + Filter: ((#4{bucket_start} + 01:00:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) + →Arrange (#0{replica_id}) + →Distinct GroupAggregate + →Fused with Child Map/Filter/Project + Project: #1, #3 + →Read l3 + →Arranged mz_internal.mz_cluster_replica_name_history + cte l5 = + →Differential Join %1[#1, #0] » %0:l4[#1{replica_id}, #3{bucket_start}] + →Arrange (#1{replica_id}, #3{bucket_start}) + →Stream l4 →Arrange (#1, #0) →Temporally-Bucketed Non-incremental GroupAggregate Aggregation: jsonb_agg[order_by=[]](row(coalesce(jsonbable_to_jsonb(jsonb_build_object("replicaId", jsonbable_to_jsonb(#0{replica_id}), "occurredAt", jsonbable_to_jsonb(timestamp_with_time_zone_to_text(#2{occurred_at})), "status", "offline", "reason", jsonbable_to_jsonb(#1{reason}))), json_null))) Key: Project: #3, #0 - Map: timestamptz_bin(08:00:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) + Map: timestamptz_bin(01:00:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) →Fused with Child Map/Filter/Project Project: #0, #3, #4 - Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(08:00:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 14 days))) + Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(01:00:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 14 days))) →Arranged mz_internal.mz_cluster_replica_status_history Key: (#0{replica_id}) →Return →Map/Filter/Project Project: #1, #0, #2..=#5, #10, #8, #9, #11..=#13, #6, #7, #17, #18, #16, #15, #14 - Map: (#1{bucket_start} + 08:00:00) + Map: (#1{bucket_start} + 01:00:00) →Union →Map/Filter/Project Project: #0..=#17 @@ -968,14 +1328,14 @@ mz_internal.mz_console_cluster_utilization_overview: →Consolidating Union →Negate Diffs →Fused with Child Map/Filter/Project - Project: #0, #3, #2, #1, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 - →Read l4 + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l5 →Fused with Child Map/Filter/Project - Project: #0, #3, #2, #1, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 - →Read l3 + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l4 →Fused with Child Map/Filter/Project - Project: #0, #3, #2, #1, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 - →Read l4 + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 + →Read l5 Used Indexes: - mz_catalog.mz_cluster_replicas_ind (*** full scan ***) @@ -4294,7 +4654,7 @@ mz_catalog.mz_indexes: Project: #2, #0, #1, #3, #4, #7, #5, #6 Map: "s1" →Arrange (#1{name}) (#2{on_schema}, #3{on_name}) - →Constant (76 rows) + →Constant (78 rows) →Arrange (empty key) (#1{name}) →Fused with Child Map/Filter/Project Project: #4, #3 @@ -4795,45 +5155,45 @@ Target cluster: mz_catalog_server EOF query T multiline -EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_pending_cluster_replicas"; +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_overridden_system_parameters"; ---- -mz_internal.mz_pending_cluster_replicas: +mz_internal.mz_overridden_system_parameters: →Read mz_internal.mz_catalog_raw Source mz_internal.mz_catalog_raw - project=(#1) - filter=((true = text_to_boolean((((((#0{data} -> "value") -> "config") -> "location") -> "Managed") ->> "pending"))) AND ("ClusterReplica" = (#0{data} ->> "kind"))) - map=(parse_catalog_id(((#0{data} -> "key") -> "id"))) + project=(#1, #2) + filter=(("ServerConfiguration" = (#0{data} ->> "kind"))) + map=(((#0{data} -> "key") ->> "name"), ((#0{data} -> "value") ->> "value")) Target cluster: mz_catalog_server EOF query T multiline -EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_replica_system_parameters"; +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_pending_cluster_replicas"; ---- -mz_internal.mz_replica_system_parameters: +mz_internal.mz_pending_cluster_replicas: →Read mz_internal.mz_catalog_raw Source mz_internal.mz_catalog_raw - project=(#2..=#4) - filter=(("ReplicaSystemConfiguration" = (#0{data} ->> "kind"))) - map=((#0{data} -> "key"), parse_catalog_id((#1 -> "replica_id")), (#1 ->> "name"), ((#0{data} -> "value") ->> "value")) + project=(#1) + filter=((true = text_to_boolean((((((#0{data} -> "value") -> "config") -> "location") -> "Managed") ->> "pending"))) AND ("ClusterReplica" = (#0{data} ->> "kind"))) + map=(parse_catalog_id(((#0{data} -> "key") -> "id"))) Target cluster: mz_catalog_server EOF query T multiline -EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_overridden_system_parameters"; +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_replica_system_parameters"; ---- -mz_internal.mz_overridden_system_parameters: +mz_internal.mz_replica_system_parameters: →Read mz_internal.mz_catalog_raw Source mz_internal.mz_catalog_raw - project=(#1, #2) - filter=(("ServerConfiguration" = (#0{data} ->> "kind"))) - map=(((#0{data} -> "key") ->> "name"), ((#0{data} -> "value") ->> "value")) + project=(#2..=#4) + filter=(("ReplicaSystemConfiguration" = (#0{data} ->> "kind"))) + map=((#0{data} -> "key"), parse_catalog_id((#1 -> "replica_id")), (#1 ->> "name"), ((#0{data} -> "value") ->> "value")) Target cluster: mz_catalog_server @@ -7257,6 +7617,36 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_console_cluster_utilization_overview_24h"; +---- +Explained Query (fast path): + →Map/Filter/Project + Project: #1..=#17, #0, #18 + →Indexed mz_internal.mz_console_cluster_utilization_overview_24h (using mz_internal.mz_console_cluster_utilization_overview_24h_ind) + +Used Indexes: + - mz_internal.mz_console_cluster_utilization_overview_24h_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_console_cluster_utilization_overview_3h"; +---- +Explained Query (fast path): + →Map/Filter/Project + Project: #1..=#17, #0, #18 + →Indexed mz_internal.mz_console_cluster_utilization_overview_3h (using mz_internal.mz_console_cluster_utilization_overview_3h_ind) + +Used Indexes: + - mz_internal.mz_console_cluster_utilization_overview_3h_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_global_frontiers"; ---- diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index a2c8bb57f7d9d..69d7ef2d9b457 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -381,6 +381,14 @@ mz_console_cluster_utilization_overview VIEW materialize mz_internal +mz_console_cluster_utilization_overview_24h +VIEW +materialize +mz_internal +mz_console_cluster_utilization_overview_3h +VIEW +materialize +mz_internal mz_frontiers SOURCE materialize diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 980f20663c1b9..dc46f37a5875a 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -37,9 +37,9 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangemen mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id") mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") @@ -57,13 +57,15 @@ mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s738␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s740␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s510␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s736␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s735␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s734␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s475␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") @@ -71,7 +73,7 @@ mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_ch mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s722␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s478␠AS␠"mz_catalog"."mz_indexes"]␠("id") mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s472␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s534␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") @@ -79,10 +81,10 @@ mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_messag mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s819␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s739␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s823␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s473␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s545␠AS␠"mz_internal"."mz_object_history"]␠("id") mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s546␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") @@ -91,14 +93,14 @@ mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[ mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s706␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s702␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s812␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s816␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s496␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s476␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s509␠AS␠"mz_catalog"."mz_secrets"]␠("name") mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s590␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s592␠AS␠"mz_internal"."mz_show_clusters"]␠("name") mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s591␠AS␠"mz_internal"."mz_show_columns"]␠("id") mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") @@ -381,6 +383,44 @@ mz_console_cluster_utilization_overview name mz_console_cluster_utilization_overview offline_events mz_console_cluster_utilization_overview replica_id mz_console_cluster_utilization_overview size +mz_console_cluster_utilization_overview_24h bucket_end +mz_console_cluster_utilization_overview_24h bucket_start +mz_console_cluster_utilization_overview_24h cluster_id +mz_console_cluster_utilization_overview_24h disk_percent +mz_console_cluster_utilization_overview_24h heap_percent +mz_console_cluster_utilization_overview_24h max_cpu_at +mz_console_cluster_utilization_overview_24h max_cpu_percent +mz_console_cluster_utilization_overview_24h max_disk_at +mz_console_cluster_utilization_overview_24h max_heap_at +mz_console_cluster_utilization_overview_24h max_memory_and_disk_at +mz_console_cluster_utilization_overview_24h max_memory_and_disk_disk_percent +mz_console_cluster_utilization_overview_24h max_memory_and_disk_memory_percent +mz_console_cluster_utilization_overview_24h max_memory_at +mz_console_cluster_utilization_overview_24h memory_and_disk_percent +mz_console_cluster_utilization_overview_24h memory_percent +mz_console_cluster_utilization_overview_24h name +mz_console_cluster_utilization_overview_24h offline_events +mz_console_cluster_utilization_overview_24h replica_id +mz_console_cluster_utilization_overview_24h size +mz_console_cluster_utilization_overview_3h bucket_end +mz_console_cluster_utilization_overview_3h bucket_start +mz_console_cluster_utilization_overview_3h cluster_id +mz_console_cluster_utilization_overview_3h disk_percent +mz_console_cluster_utilization_overview_3h heap_percent +mz_console_cluster_utilization_overview_3h max_cpu_at +mz_console_cluster_utilization_overview_3h max_cpu_percent +mz_console_cluster_utilization_overview_3h max_disk_at +mz_console_cluster_utilization_overview_3h max_heap_at +mz_console_cluster_utilization_overview_3h max_memory_and_disk_at +mz_console_cluster_utilization_overview_3h max_memory_and_disk_disk_percent +mz_console_cluster_utilization_overview_3h max_memory_and_disk_memory_percent +mz_console_cluster_utilization_overview_3h max_memory_at +mz_console_cluster_utilization_overview_3h memory_and_disk_percent +mz_console_cluster_utilization_overview_3h memory_percent +mz_console_cluster_utilization_overview_3h name +mz_console_cluster_utilization_overview_3h offline_events +mz_console_cluster_utilization_overview_3h replica_id +mz_console_cluster_utilization_overview_3h size mz_databases id mz_databases name mz_databases oid diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index 5e49b9b5cef09..6e34708a2b546 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1200,3 +1200,7 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17093 generate_series_unoptimized 17094 generate_series_unoptimized 17095 mz_overridden_system_parameters +17096 mz_console_cluster_utilization_overview_3h +17097 mz_console_cluster_utilization_overview_3h_ind +17098 mz_console_cluster_utilization_overview_24h +17099 mz_console_cluster_utilization_overview_24h_ind diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 16e86981ecad9..c65347e1635d8 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -640,6 +640,8 @@ mz_cluster_replica_utilization "" mz_cluster_replica_utilization_history "" mz_compute_hydration_statuses "" mz_console_cluster_utilization_overview "" +mz_console_cluster_utilization_overview_24h "" +mz_console_cluster_utilization_overview_3h "" mz_global_frontiers "" mz_hydration_statuses "" mz_index_advice "" @@ -830,7 +832,7 @@ test_table "" # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' -268 +270 # Create a second schema with the same table name as above > CREATE SCHEMA tester2 diff --git a/test/testdrive/explain-analyze.td b/test/testdrive/explain-analyze.td index 53d7ffefd6e67..c7b719bd4c2d4 100644 --- a/test/testdrive/explain-analyze.td +++ b/test/testdrive/explain-analyze.td @@ -72,19 +72,19 @@ operator total_elapsed " Stream " "Returning Map/Filter/Project" " Union" -" Read l4" +" Read l5" " Map/Filter/Project" " Consolidating Union" -" Read l3" +" Read l4" " Negate Diffs" -" Read l4" -"With l4 = Differential Join %1 » %0" +" Read l5" +"With l5 = Differential Join %1 » %0" " Arrange (#1, #0)" " Non-incremental GroupAggregate" " Arranged " -" Arrange (#0{replica_id}, #3{bucket_start})" -" Stream l3" -"l3 = Differential Join %1 » %0" +" Arrange (#1{replica_id}, #3{bucket_start})" +" Stream l4" +"l4 = Differential Join %1 » %0" " Arrange (#0, #1)" " Map/Filter/Project" " Non-monotonic TopK" @@ -92,44 +92,43 @@ operator total_elapsed " Arranged " " Arrange (#0{replica_id})" " Distinct GroupAggregate" -" Read l2" -" Arrange (#0, #3)" -" Stream l2" -"l2 = Delta Join [%0 » %1 » %2 » %3 » %4 » %5] [%1 » %0 » %2 » %3 » %4 » %5] [%2 » %0 » %1 » %3 » %4 » %5] [%3 » %0 » %1 » %2 » %4 » %5] [%4 » %0 » %1 » %2 » %3 » %5] [%5 » %0 » %1 » %2 » %3 » %4]" +" Read l3" +" Arrange (#1, #3)" +" Stream l3" +"l3 = Delta Join [%0 » %1 » %2 » %3 » %4 » %5] [%1 » %0 » %2 » %3 » %4 » %5] [%2 » %0 » %1 » %3 » %4 » %5] [%3 » %0 » %1 » %2 » %4 » %5] [%4 » %0 » %1 » %2 » %3 » %5] [%5 » %0 » %1 » %2 » %3 » %4]" " Arrange (#0{replica_id})" " Arranged l0" -" Arrange (#0{replica_id}, #3{bucket_start})" +" Arrange (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}, #4{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}, #4{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}, #3{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}, #3{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}) (#0{replica_id}, #3{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -"l1 = Differential Join %0 » %1" -" Arrange (#1{replica_id})" -" Map/Filter/Project" -" Accumulable GroupAggregate" -" Map/Filter/Project" -" Delta Join [%0 » %2 » %1] [%1 » %0 » %2] [%2 » %0 » %1]" -" Arranged " -" Arranged " -" Arrange (#0{replica_id}) (#1{size})" -" Arranged l0" -" Arrange (#0{replica_id})" -" Arranged l0" +" Read l2" +"l2 = Differential Join %0 » %1" +" Accumulable GroupAggregate" +" Read l1" +" Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16)" +" Read l1" +"l1 = Map/Filter/Project" +" Delta Join [%0 » %2 » %1] [%1 » %0 » %2] [%2 » %0 » %1]" +" Arranged " +" Arranged " +" Arrange (#0{replica_id}) (#1{size})" +" Arranged l0" "l0 = Distinct GroupAggregate" " Union" " Arranged " diff --git a/test/testdrive/indexes.td b/test/testdrive/indexes.td index 07aa9746dbe31..a8089707e43f6 100644 --- a/test/testdrive/indexes.td +++ b/test/testdrive/indexes.td @@ -322,6 +322,8 @@ mz_compute_operator_durations_histogram_raw_s2_primary_idx mz_compute_operator_ mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx mz_compute_operator_hydration_statuses_per_worker mz_catalog_server {export_id,lir_id,worker_id} "" mz_cluster_prometheus_metrics_s2_primary_idx mz_cluster_prometheus_metrics mz_catalog_server {process_id,metric_name,labels} "" mz_connections_ind mz_connections mz_catalog_server {schema_id} "" +mz_console_cluster_utilization_overview_24h_ind mz_console_cluster_utilization_overview_24h mz_catalog_server {cluster_id} "" +mz_console_cluster_utilization_overview_3h_ind mz_console_cluster_utilization_overview_3h mz_catalog_server {cluster_id} "" mz_console_cluster_utilization_overview_ind mz_console_cluster_utilization_overview mz_catalog_server {cluster_id} "" mz_databases_ind mz_databases mz_catalog_server {name} "" mz_dataflow_addresses_per_worker_s2_primary_idx mz_dataflow_addresses_per_worker mz_catalog_server {id,worker_id} "" diff --git a/test/workload-replay/objects.txt b/test/workload-replay/objects.txt index 24fb3e4a016e0..8bdd9443e0bd7 100644 --- a/test/workload-replay/objects.txt +++ b/test/workload-replay/objects.txt @@ -479,6 +479,10 @@ mz_connection_oid mz_connections mz_connections_ind mz_console_cluster_utilization_overview +mz_console_cluster_utilization_overview_24h +mz_console_cluster_utilization_overview_24h_ind +mz_console_cluster_utilization_overview_3h +mz_console_cluster_utilization_overview_3h_ind mz_console_cluster_utilization_overview_ind mz_database_oid mz_databases From b9ea499a1e86f50f3b18fc3d78c0491f184714dd Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Fri, 26 Jun 2026 18:15:30 -0400 Subject: [PATCH 2/3] catalog: flag-gate builtin index creation at bootstrap Adds ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX dyncfg + a bootstrap filter that skips gated builtin indexes when off (read from txn and system_parameter_defaults). Indexes are ephemeral, so flag-off drops the arrangement; default-on preserves current behavior. Co-Authored-By: Claude Opus 4.8 --- src/adapter-types/src/dyncfgs.rs | 12 ++++++++ src/adapter/src/catalog/open.rs | 39 ++++++++++++++++++++++++-- src/catalog/src/durable/transaction.rs | 11 ++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/adapter-types/src/dyncfgs.rs b/src/adapter-types/src/dyncfgs.rs index 9c0873c530ca4..e16d87986e278 100644 --- a/src/adapter-types/src/dyncfgs.rs +++ b/src/adapter-types/src/dyncfgs.rs @@ -127,6 +127,17 @@ pub const ENABLE_PASSWORD_AUTH: Config = Config::new( "Enable password authentication.", ); +/// Whether to create the `mz_console_cluster_utilization_recent` index on +/// `mz_catalog_server`. Gating only the index (the view is free) lets an +/// operator drop its maintained arrangement without a binary change, e.g. on a +/// resource-constrained self-managed environment where it would otherwise grow +/// `mz_catalog_server`'s memory or stall an upgrade at hydration. +pub const ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX: Config = Config::new( + "enable_console_cluster_utilization_recent_index", + true, + "Create the mz_console_cluster_utilization_recent index on mz_catalog_server.", +); + /// OIDC issuer URL. pub const OIDC_ISSUER: Config> = Config::new("oidc_issuer", None, "OIDC issuer URL."); @@ -333,6 +344,7 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&PLAN_INSIGHTS_NOTICE_FAST_PATH_CLUSTERS_OPTIMIZE_DURATION) .add(&ENABLE_EXPRESSION_CACHE) .add(&ENABLE_PASSWORD_AUTH) + .add(&ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX) .add(&OIDC_ISSUER) .add(&OIDC_AUDIENCE) .add(&OIDC_AUTHENTICATION_CLAIM) diff --git a/src/adapter/src/catalog/open.rs b/src/adapter/src/catalog/open.rs index 706f7b5ac0bf3..45bfa1644ed66 100644 --- a/src/adapter/src/catalog/open.rs +++ b/src/adapter/src/catalog/open.rs @@ -19,7 +19,9 @@ use std::time::{Duration, Instant}; use futures::future::{BoxFuture, FutureExt}; use itertools::{Either, Itertools}; use mz_adapter_types::bootstrap_builtin_cluster_config::BootstrapBuiltinClusterConfig; -use mz_adapter_types::dyncfgs::ENABLE_EXPRESSION_CACHE; +use mz_adapter_types::dyncfgs::{ + ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX, ENABLE_EXPRESSION_CACHE, +}; use mz_audit_log::{ CreateOrDropClusterReplicaReasonV1, EventDetails, EventType, ObjectType, VersionedEvent, }; @@ -195,7 +197,10 @@ impl Catalog { txn.set_system_config_synced_once()?; } // Add any new builtin objects and remove old ones. - let new_builtin_collections = add_new_remove_old_builtin_items_migration(&mut txn)?; + let new_builtin_collections = add_new_remove_old_builtin_items_migration( + &mut txn, + &config.system_parameter_defaults, + )?; let builtin_bootstrap_cluster_config_map = BuiltinBootstrapClusterConfigMap { system_cluster: config.builtin_system_cluster_config, catalog_server_cluster: config.builtin_catalog_server_cluster_config, @@ -747,6 +752,7 @@ impl CatalogState { /// Returns the list of new builtin [`GlobalId`]s. fn add_new_remove_old_builtin_items_migration( txn: &mut mz_catalog::durable::Transaction<'_>, + system_parameter_defaults: &BTreeMap, ) -> Result, mz_catalog::durable::CatalogError> { let mut new_builtin_mappings = Vec::new(); // Used to validate unique descriptions. @@ -755,7 +761,36 @@ fn add_new_remove_old_builtin_items_migration( // We compare the builtin items that are compiled into the binary with the builtin items that // are persisted in the catalog to discover new and deleted builtin items. let mut builtins = Vec::new(); + + // Builtin indexes whose creation is gated behind a system flag. When the + // flag is off at boot the index is skipped here, so an existing one is + // dropped (indexes are exempt from the deletion guard below) and a new one + // is not created, removing its maintained arrangement and hydration cost. + let gated_off_indexes: BTreeSet<&str> = [( + "mz_console_cluster_utilization_recent_ind", + ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX.name(), + )] + .into_iter() + .filter(|&(_, flag)| { + // Effective value at boot: persisted or LaunchDarkly-synced system + // config, else a launch-time default, else the dyncfg default + // (enabled). Launch defaults aren't in the catalog txn yet at this + // point, so consult both sources. + let value = txn + .get_system_config(flag) + .or_else(|| system_parameter_defaults.get(flag).cloned()); + match value { + Some(v) => !matches!(v.trim().to_lowercase().as_str(), "true" | "on" | "t" | "1"), + None => false, + } + }) + .map(|(index_name, _)| index_name) + .collect(); + for builtin in BUILTINS::iter() { + if matches!(builtin, Builtin::Index(_)) && gated_off_indexes.contains(builtin.name()) { + continue; + } let desc = SystemObjectDescription { schema_name: builtin.schema().to_string(), object_type: builtin.catalog_item_type(), diff --git a/src/catalog/src/durable/transaction.rs b/src/catalog/src/durable/transaction.rs index fd454927ad2f6..4b4856310aff3 100644 --- a/src/catalog/src/durable/transaction.rs +++ b/src/catalog/src/durable/transaction.rs @@ -2140,6 +2140,17 @@ impl<'a> Transaction<'a> { self.system_configurations.delete(|_k, _v| true, self.op_id); } + /// Returns the persisted value of system configuration `name`, if any. Used + /// during bootstrap to read a system parameter before the in-memory catalog + /// (and its `SystemVars`) has been seeded. + pub fn get_system_config(&self, name: &str) -> Option { + self.system_configurations + .items() + .into_iter() + .find(|(k, _)| k.name == name) + .map(|(_, v)| v.value.clone()) + } + /// Returns the persisted cluster-coherent scoped system configurations. pub fn get_cluster_system_configurations( &self, From f8a5cf7cec8e352043aa82a58a70a9b7ef8869b1 Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Mon, 29 Jun 2026 12:51:39 -0400 Subject: [PATCH 3/3] catalog: un-binned 3h console utilization view + gated 24h index Short cluster-detail windows (<=3h) now read an un-binned base view that the Console bins client-side, replacing the server-side five-pass top-k. This drops the maintained binned arrangement for that tier and lets the Console SUBSCRIBE to the live window. - Repoint mz_console_cluster_utilization_overview_3h to a new un-binned base (one row per replica metric sample, no date_bin or top-k). replica_history is deduped to one row per replica so the Console SUBSCRIBE upsert key (replica_id, occurred_at) stays unique even if a replica changed size. - Gate the 24h overview index behind enable_console_cluster_utilization_24h_index (default off) so a resource-constrained or self-managed environment does not build its maintained arrangement or stall an upgrade at hydration. The bootstrap gate uses the flag's effective value and falls back to the compiled default. Cloud enables it via system parameter. - sqllogictest enables the gated index in its test defaults so the catalog-server snapshots still cover it. Regenerates catalog_server_explain.slt and mz_catalog_server_index_accounting.slt. --- .../reference/system-catalog/mz_internal.md | 2 + src/adapter-types/src/dyncfgs.rs | 12 - src/adapter/src/catalog/open.rs | 38 +--- .../catalog/open/builtin_schema_migration.rs | 10 + src/catalog/src/builtin/mz_internal.rs | 135 +++++++++-- src/catalog/src/durable/transaction.rs | 11 - test/sqllogictest/catalog_server_explain.slt | 212 +++++------------- test/sqllogictest/cluster.slt | 4 +- .../mz_catalog_server_index_accounting.slt | 13 +- 9 files changed, 200 insertions(+), 237 deletions(-) diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index 6dfc5af594d17..dfab4b9222dc8 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -1478,6 +1478,8 @@ The `mz_webhook_sources` table contains a row for each webhook source in the sys + + diff --git a/src/adapter-types/src/dyncfgs.rs b/src/adapter-types/src/dyncfgs.rs index e16d87986e278..9c0873c530ca4 100644 --- a/src/adapter-types/src/dyncfgs.rs +++ b/src/adapter-types/src/dyncfgs.rs @@ -127,17 +127,6 @@ pub const ENABLE_PASSWORD_AUTH: Config = Config::new( "Enable password authentication.", ); -/// Whether to create the `mz_console_cluster_utilization_recent` index on -/// `mz_catalog_server`. Gating only the index (the view is free) lets an -/// operator drop its maintained arrangement without a binary change, e.g. on a -/// resource-constrained self-managed environment where it would otherwise grow -/// `mz_catalog_server`'s memory or stall an upgrade at hydration. -pub const ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX: Config = Config::new( - "enable_console_cluster_utilization_recent_index", - true, - "Create the mz_console_cluster_utilization_recent index on mz_catalog_server.", -); - /// OIDC issuer URL. pub const OIDC_ISSUER: Config> = Config::new("oidc_issuer", None, "OIDC issuer URL."); @@ -344,7 +333,6 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&PLAN_INSIGHTS_NOTICE_FAST_PATH_CLUSTERS_OPTIMIZE_DURATION) .add(&ENABLE_EXPRESSION_CACHE) .add(&ENABLE_PASSWORD_AUTH) - .add(&ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX) .add(&OIDC_ISSUER) .add(&OIDC_AUDIENCE) .add(&OIDC_AUTHENTICATION_CLAIM) diff --git a/src/adapter/src/catalog/open.rs b/src/adapter/src/catalog/open.rs index 45bfa1644ed66..d8f9718adb124 100644 --- a/src/adapter/src/catalog/open.rs +++ b/src/adapter/src/catalog/open.rs @@ -19,9 +19,7 @@ use std::time::{Duration, Instant}; use futures::future::{BoxFuture, FutureExt}; use itertools::{Either, Itertools}; use mz_adapter_types::bootstrap_builtin_cluster_config::BootstrapBuiltinClusterConfig; -use mz_adapter_types::dyncfgs::{ - ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX, ENABLE_EXPRESSION_CACHE, -}; +use mz_adapter_types::dyncfgs::ENABLE_EXPRESSION_CACHE; use mz_audit_log::{ CreateOrDropClusterReplicaReasonV1, EventDetails, EventType, ObjectType, VersionedEvent, }; @@ -197,10 +195,7 @@ impl Catalog { txn.set_system_config_synced_once()?; } // Add any new builtin objects and remove old ones. - let new_builtin_collections = add_new_remove_old_builtin_items_migration( - &mut txn, - &config.system_parameter_defaults, - )?; + let new_builtin_collections = add_new_remove_old_builtin_items_migration(&mut txn)?; let builtin_bootstrap_cluster_config_map = BuiltinBootstrapClusterConfigMap { system_cluster: config.builtin_system_cluster_config, catalog_server_cluster: config.builtin_catalog_server_cluster_config, @@ -752,7 +747,6 @@ impl CatalogState { /// Returns the list of new builtin [`GlobalId`]s. fn add_new_remove_old_builtin_items_migration( txn: &mut mz_catalog::durable::Transaction<'_>, - system_parameter_defaults: &BTreeMap, ) -> Result, mz_catalog::durable::CatalogError> { let mut new_builtin_mappings = Vec::new(); // Used to validate unique descriptions. @@ -762,35 +756,7 @@ fn add_new_remove_old_builtin_items_migration( // are persisted in the catalog to discover new and deleted builtin items. let mut builtins = Vec::new(); - // Builtin indexes whose creation is gated behind a system flag. When the - // flag is off at boot the index is skipped here, so an existing one is - // dropped (indexes are exempt from the deletion guard below) and a new one - // is not created, removing its maintained arrangement and hydration cost. - let gated_off_indexes: BTreeSet<&str> = [( - "mz_console_cluster_utilization_recent_ind", - ENABLE_CONSOLE_CLUSTER_UTILIZATION_RECENT_INDEX.name(), - )] - .into_iter() - .filter(|&(_, flag)| { - // Effective value at boot: persisted or LaunchDarkly-synced system - // config, else a launch-time default, else the dyncfg default - // (enabled). Launch defaults aren't in the catalog txn yet at this - // point, so consult both sources. - let value = txn - .get_system_config(flag) - .or_else(|| system_parameter_defaults.get(flag).cloned()); - match value { - Some(v) => !matches!(v.trim().to_lowercase().as_str(), "true" | "on" | "t" | "1"), - None => false, - } - }) - .map(|(index_name, _)| index_name) - .collect(); - for builtin in BUILTINS::iter() { - if matches!(builtin, Builtin::Index(_)) && gated_off_indexes.contains(builtin.name()) { - continue; - } let desc = SystemObjectDescription { schema_name: builtin.schema().to_string(), object_type: builtin.catalog_item_type(), diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index edbc0208b7062..2e744887f4dd7 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -252,6 +252,16 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_INTERNAL_SCHEMA, "mz_comments", ), + // Required because we added the console cluster-utilization overview builtin + // indexes (overview/_3h/_24h). make_mz_indexes inlines the builtin-index set + // as VALUES, so any add/remove changes its SQL fingerprint and requires an + // explicit replacement step. + MigrationStep::replacement( + "26.32.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_indexes", + ), ] }); diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index d9e86066e832a..19c498e47414c 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -7079,13 +7079,13 @@ replica_metrics_history AS ( m.occurred_at, m.replica_id, r.size, - (SUM(m.cpu_nano_cores::float8) / (NULLIF(s.cpu_nano_cores, 0) * s.processes)) AS cpu_percent, - (SUM(m.memory_bytes::float8) / (NULLIF(s.memory_bytes, 0) * s.processes)) AS memory_percent, - (SUM(m.disk_bytes::float8) / (NULLIF(s.disk_bytes, 0) * s.processes)) AS disk_percent, + (SUM(m.cpu_nano_cores::float8) / NULLIF(s.cpu_nano_cores, 0) / NULLIF(s.processes, 0)) AS cpu_percent, + (SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0)) AS memory_percent, + (SUM(m.disk_bytes::float8) / NULLIF(s.disk_bytes, 0) / NULLIF(s.processes, 0)) AS disk_percent, SUM(m.disk_bytes::float8) AS disk_bytes, SUM(m.memory_bytes::float8) AS memory_bytes, - s.disk_bytes * s.processes AS total_disk_bytes, - s.memory_bytes * s.processes AS total_memory_bytes, + s.disk_bytes::float8 * s.processes AS total_disk_bytes, + s.memory_bytes::float8 * s.processes AS total_memory_bytes, MAX(m.heap_bytes::float8) AS heap_bytes, MAX(m.heap_limit) AS heap_limit, -- heap_limit is NULL when clusterd isn't launched with --heap-limit (e.g. @@ -7093,7 +7093,7 @@ replica_metrics_history AS ( -- percent so the chart still renders. COALESCE( MAX(m.heap_bytes::float8 / NULLIF(m.heap_limit, 0)), - SUM(m.memory_bytes::float8) / (NULLIF(s.memory_bytes, 0) * s.processes) + SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0) ) AS heap_percent FROM replica_history AS r @@ -7242,6 +7242,116 @@ LEFT JOIN replica_offline_event_history USING (bucket_start, replica_id)"#, ) } +/// Schema for the un-binned 3-hour console cluster utilization base. Unlike the +/// binned `_overview*` views, this exposes raw per-(replica, sample) metrics so +/// the Console can bin client-side. +fn console_cluster_utilization_unbinned_3h_desc() -> RelationDesc { + RelationDesc::builder() + .with_column("replica_id", SqlScalarType::String.nullable(false)) + .with_column("cluster_id", SqlScalarType::String.nullable(true)) + .with_column("size", SqlScalarType::String.nullable(false)) + .with_column("name", SqlScalarType::String.nullable(true)) + .with_column( + "occurred_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("cpu_percent", SqlScalarType::Float64.nullable(true)) + .with_column("memory_percent", SqlScalarType::Float64.nullable(true)) + .with_column("disk_percent", SqlScalarType::Float64.nullable(true)) + .with_column("heap_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "memory_and_disk_percent", + SqlScalarType::Float64.nullable(true), + ) + .finish() +} + +/// Builds the SQL for the un-binned 3-hour console cluster utilization base: one +/// row per (replica, metric sample) over `retention`, with no `date_bin`/top-k, +/// so the Console bins it client-side. The binned `_overview*` views handle the +/// longer windows. A temporal `mz_now()` filter bounds the maintained +/// arrangement. Kept in sync with the Console +/// (`buildConsoleClusterUtilizationUnbinned3hQuery` in +/// `replicaUtilizationHistory.ts`). +fn console_cluster_utilization_unbinned_3h_sql(retention: &str) -> String { + format!( + r#"WITH replica_history AS ( + -- Dedup to one row per replica (prefer the current size). Size is fixed per + -- replica so this is normally a no-op, but a stray duplicate size in history + -- would fan out the metrics join; with no Top-1 dedup here that would emit two + -- rows per (replica_id, occurred_at) and break the Console SUBSCRIBE upsert key. + SELECT DISTINCT ON (replica_id) replica_id, size, cluster_id + FROM ( + -- We union the current set of cluster replicas since mz_cluster_replica_history doesn't include system clusters. + SELECT id AS replica_id, size, cluster_id, 0 AS source_rank + FROM mz_catalog.mz_cluster_replicas + UNION ALL + SELECT replica_id, size, cluster_id, 1 AS source_rank + FROM mz_internal.mz_cluster_replica_history + ) all_replicas + ORDER BY replica_id, source_rank +), +replica_metrics AS ( + SELECT + m.occurred_at, + m.replica_id, + r.cluster_id, + r.size, + (SUM(m.cpu_nano_cores::float8) / NULLIF(s.cpu_nano_cores, 0) / NULLIF(s.processes, 0)) AS cpu_percent, + (SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0)) AS memory_percent, + (SUM(m.disk_bytes::float8) / NULLIF(s.disk_bytes, 0) / NULLIF(s.processes, 0)) AS disk_percent, + COALESCE( + MAX(m.heap_bytes::float8 / NULLIF(m.heap_limit, 0)), + SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0) + ) AS heap_percent, + CASE + WHEN SUM(m.disk_bytes::float8) IS NULL AND SUM(m.memory_bytes::float8) IS NULL THEN NULL + ELSE (COALESCE(SUM(m.memory_bytes::float8), 0) + COALESCE(SUM(m.disk_bytes::float8), 0)) + / NULLIF((s.memory_bytes::float8 + s.disk_bytes::float8) * s.processes, 0) + END AS memory_and_disk_percent + FROM replica_history AS r + INNER JOIN mz_catalog.mz_cluster_replica_sizes AS s ON r.size = s.size + INNER JOIN mz_internal.mz_cluster_replica_metrics_history AS m ON m.replica_id = r.replica_id + -- No aggregation over time: one row per (replica, sample) so the Console bins + -- client-side. The temporal mz_now() filter keeps the maintained arrangement + -- bounded to the retention window. + WHERE mz_now() <= m.occurred_at + INTERVAL '{retention}' + GROUP BY + m.occurred_at, + m.replica_id, + r.cluster_id, + r.size, + s.cpu_nano_cores, + s.memory_bytes, + s.disk_bytes, + s.processes +) +SELECT + m.replica_id, + m.cluster_id, + m.size, + replica_name_history.new_name AS name, + m.occurred_at, + m.cpu_percent, + m.memory_percent, + m.disk_percent, + m.heap_percent, + m.memory_and_disk_percent +FROM replica_metrics AS m +/* Most recent replica name as of the sample time. */ +CROSS JOIN LATERAL ( + SELECT new_name + FROM mz_internal.mz_cluster_replica_name_history AS replica_name_history + WHERE m.replica_id = replica_name_history.id + -- We treat NULLs as the beginning of time. + AND m.occurred_at >= COALESCE(replica_name_history.occurred_at, '1970-01-01'::timestamp) + ORDER BY replica_name_history.occurred_at DESC + LIMIT 1 +) AS replica_name_history"#, + retention = retention, + ) +} + /** * Displays cluster utilization over 14 days bucketed by 1 hour, for the * Console's environment overview and cluster pages, to speed up load times. @@ -7263,20 +7373,19 @@ pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW: LazyLock = }); /** - * Cluster utilization over the last 3 hours bucketed by 1 minute, for the - * Console's "Last hour" / "Last 3 hours" cluster utilization graphs. See - * `console_cluster_utilization_overview_sql` for details. + * Un-binned cluster utilization over the last 3 hours, for the Console's "Last + * hour" / "Last 3 hours" graphs. Unlike the binned `_overview*` views, this + * exposes raw per-(replica, sample) metrics and the Console bins client-side. + * See `console_cluster_utilization_unbinned_3h_sql` for details. */ pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H: LazyLock = LazyLock::new(|| BuiltinView { name: "mz_console_cluster_utilization_overview_3h", schema: MZ_INTERNAL_SCHEMA, oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_OID, - desc: console_cluster_utilization_overview_desc(), + desc: console_cluster_utilization_unbinned_3h_desc(), column_comments: BTreeMap::new(), - sql: Box::leak( - console_cluster_utilization_overview_sql("1 MINUTE", "3 HOURS", 1).into_boxed_str(), - ), + sql: Box::leak(console_cluster_utilization_unbinned_3h_sql("3 HOURS").into_boxed_str()), access: vec![PUBLIC_SELECT], ontology: None, }); diff --git a/src/catalog/src/durable/transaction.rs b/src/catalog/src/durable/transaction.rs index 4b4856310aff3..fd454927ad2f6 100644 --- a/src/catalog/src/durable/transaction.rs +++ b/src/catalog/src/durable/transaction.rs @@ -2140,17 +2140,6 @@ impl<'a> Transaction<'a> { self.system_configurations.delete(|_k, _v| true, self.op_id); } - /// Returns the persisted value of system configuration `name`, if any. Used - /// during bootstrap to read a system parameter before the in-memory catalog - /// (and its `SystemVars`) has been seeded. - pub fn get_system_config(&self, name: &str) -> Option { - self.system_configurations - .items() - .into_iter() - .find(|(k, _)| k.name == name) - .map(|(_, v)| v.value.clone()) - } - /// Returns the persisted cluster-coherent scoped system configurations. pub fn get_cluster_system_configurations( &self, diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 1009661e95144..52de5cc30c89d 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -848,8 +848,8 @@ mz_internal.mz_console_cluster_utilization_overview_24h: cte l2 = →Differential Join %0[#0..=#6] » %1[#0..=#6] after %1: - Project: #0, #1, #7, #10..=#18 - Map: (#9 / uint8_to_double((case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end * #6{processes}))), (#10 / uint8_to_double((case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end * #6{processes}))), (#11 / uint8_to_double((case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end * #6{processes}))), (#5{disk_bytes} * #6{processes}), (#4{memory_bytes} * #6{processes}), coalesce(#8, #13), timestamptz_bin(00:05:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + Project: #0, #1, #7, #10, #11, #13..=#15, #17..=#20 + Map: uint8_to_double(case when (0 = uint8_to_numeric(#6{processes})) then null else #6{processes} end), ((#9 / uint8_to_double(case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end)) / #12), ((#10 / uint8_to_double(case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end)) / #12), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end)) / #12), uint8_to_double(#6{processes}), (uint8_to_double(#5{disk_bytes}) * #16), (uint8_to_double(#4{memory_bytes}) * #16), coalesce(#8, #14), timestamptz_bin(00:05:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) Aggregations: max, max Key: @@ -908,7 +908,7 @@ mz_internal.mz_console_cluster_utilization_overview_24h: Limit 1 →Fused with Child Map/Filter/Project Project: #0, #1, #6, #7, #11..=#13 - Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / uint8_to_double(case when (0 = uint8_to_numeric((#9{total_memory_bytes} + #8{total_disk_bytes}))) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end)) end, coalesce(#12{memory_and_disk_percent}, 0) + Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / case when (0 = (#9{total_memory_bytes} + #8{total_disk_bytes})) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end) end, coalesce(#12{memory_and_disk_percent}, 0) →Read l2 →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project @@ -995,173 +995,81 @@ query T multiline EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_3h_ind"; ---- mz_internal.mz_console_cluster_utilization_overview_3h_ind: - →Arrange (#17{cluster_id}) + →Arrange (#1{cluster_id}) →Stream mz_internal.mz_console_cluster_utilization_overview_3h mz_internal.mz_console_cluster_utilization_overview_3h: →With cte l0 = - →Distinct GroupAggregate - →Union - →Fused with Child Map/Filter/Project - Project: #1..=#3 - Filter: (#1) IS NOT NULL - →Arranged mz_internal.mz_cluster_replica_history - Key: (#6{dropped_at}) - →Fused with Child Map/Filter/Project - Project: #0, #3, #2 - →Arranged mz_catalog.mz_cluster_replicas - Key: (#0{id}) - cte l1 = →Map/Filter/Project - Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:01:00, #9, 1970-01-01 00:00:00 UTC) + 03:00:00))) - →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] + Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((#10{occurred_at} + 03:00:00))) + →Delta Join [%0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] →Arrange (#0{replica_id}) (#1{size}) - →Fused with Child Map/Filter/Project - Project: #0, #1 + →Map/Filter/Project + Project: #0..=#2 Filter: (#1{size}) IS NOT NULL - →Arranged l0 - Key: (#0..=#2) + →Non-monotonic TopK + Group By #0 + Order By #3 asc nulls_last + Limit 1 + →Union + →Fused with Child Map/Filter/Project + Project: #0, #3, #2, #7 + Map: 0 + →Arranged mz_catalog.mz_cluster_replicas + Key: (#0{id}) + →Fused with Child Map/Filter/Project + Project: #1..=#3, #8 + Filter: (#1{replica_id}) IS NOT NULL + Map: 1 + →Arranged mz_internal.mz_cluster_replica_history + Key: (#6{dropped_at}) →Arranged mz_catalog.mz_cluster_replica_sizes →Arranged mz_internal.mz_cluster_replica_metrics_history - cte l2 = - →Differential Join %0[#0..=#6] » %1[#0..=#6] + cte l1 = + →Differential Join %0[#0..=#7] » %1[#0..=#7] after %1: - Project: #0, #1, #7, #10..=#18 - Map: (#9 / uint8_to_double((case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end * #6{processes}))), (#10 / uint8_to_double((case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end * #6{processes}))), (#11 / uint8_to_double((case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end * #6{processes}))), (#5{disk_bytes} * #6{processes}), (#4{memory_bytes} * #6{processes}), coalesce(#8, #13), timestamptz_bin(00:01:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + Project: #0..=#3, #13..=#17 + Map: uint8_to_double(case when (0 = uint8_to_numeric(#7{processes})) then null else #7{processes} end), ((#9 / uint8_to_double(case when (0 = uint8_to_numeric(#4{cpu_nano_cores})) then null else #4{cpu_nano_cores} end)) / #12), ((#10 / uint8_to_double(case when (0 = uint8_to_numeric(#5{memory_bytes})) then null else #5{memory_bytes} end)) / #12), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#6{disk_bytes})) then null else #6{disk_bytes} end)) / #12), coalesce(#8, #14), case when ((#10) IS NULL AND (#11) IS NULL) then null else ((coalesce(#10, 0) + coalesce(#11, 0)) / case when (0 = ((uint8_to_double(#5{memory_bytes}) + uint8_to_double(#6{disk_bytes})) * uint8_to_double(#7{processes}))) then null else ((uint8_to_double(#5{memory_bytes}) + uint8_to_double(#6{disk_bytes})) * uint8_to_double(#7{processes})) end) end →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) - Aggregations: max, max + Aggregations: max Key: - Project: #6, #0, #1, #3..=#5, #2 + Project: #7, #0, #2, #1, #4..=#6, #3 →Fused with Child Map/Filter/Project - Project: #0..=#5, #9..=#11 - →Read l1 + Project: #0..=#6, #10..=#12 + →Read l0 →Temporally-Bucketed Accumulable GroupAggregate - Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})) + Simple aggregates: sum(uint8_to_double(#7{cpu_nano_cores})), sum(uint8_to_double(#8{memory_bytes})), sum(uint8_to_double(#9{disk_bytes})) Key: - Project: #9, #0, #1, #3..=#5, #2 + Project: #10, #0, #2, #1, #4..=#6, #3 →Fused with Child Map/Filter/Project - Project: #0..=#9 - →Read l1 - cte l3 = - →Delta Join [%0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#1{replica_id}, #4{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}]] - →Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start}) - →Map/Filter/Project - Project: #0..=#3 - →Non-monotonic TopK - Group By #3, #1 - Order By #4 desc nulls_first - Limit 1 - →Fused with Child Map/Filter/Project - Project: #0, #1, #6, #11, #12 - Map: coalesce(#3{memory_bytes}, 0) - →Read l2 - →Arrange (#1{replica_id}, #3{bucket_start}) - →Map/Filter/Project - Project: #0..=#3 - →Non-monotonic TopK - Group By #3, #1 - Order By #4 desc nulls_first - Limit 1 - →Fused with Child Map/Filter/Project - Project: #0, #1, #7, #11, #12 - Map: coalesce(#4{disk_bytes}, 0) - →Read l2 - →Arrange (#1{replica_id}, #3{bucket_start}) - →Map/Filter/Project - Project: #0..=#3 - →Non-monotonic TopK - Group By #3, #1 - Order By #4 desc nulls_first - Limit 1 - →Fused with Child Map/Filter/Project - Project: #0, #1, #5, #11, #12 - Map: coalesce(#5{cpu_percent}, 0) - →Read l2 - →Arrange (#1{replica_id}, #4{bucket_start}) - →Map/Filter/Project - Project: #0..=#5 - →Non-monotonic TopK - Group By #4, #1 - Order By #6 desc nulls_first - Limit 1 - →Fused with Child Map/Filter/Project - Project: #0, #1, #6, #7, #11..=#13 - Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / uint8_to_double(case when (0 = uint8_to_numeric((#9{total_memory_bytes} + #8{total_disk_bytes}))) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end)) end, coalesce(#12{memory_and_disk_percent}, 0) - →Read l2 - →Arrange (#1{replica_id}, #3{bucket_start}) - →Map/Filter/Project - Project: #0..=#3 - →Non-monotonic TopK - Group By #3, #1 - Order By #4 desc nulls_first - Limit 1 - →Fused with Child Map/Filter/Project - Project: #0, #1, #10..=#12 - Map: coalesce(#2{heap_bytes}, 0) - →Read l2 - →Arrange (#0{replica_id}) - →Arranged l0 - cte l4 = - →Differential Join %1[#0, #1] » %0:l3[#1, #3] - →Arrange (#1, #3) - →Stream l3 - →Arrange (#0, #1) - →Map/Filter/Project - Project: #0, #1, #3 - →Non-monotonic TopK - Group By #0, #1 - Order By #2 desc nulls_first - Limit 1 - →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#0{replica_id}] - after %0: - Project: #0, #4, #1, #3 - Filter: ((#4{bucket_start} + 00:01:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) - →Arrange (#0{replica_id}) - →Distinct GroupAggregate - →Fused with Child Map/Filter/Project - Project: #1, #3 - →Read l3 - →Arranged mz_internal.mz_cluster_replica_name_history - cte l5 = - →Differential Join %1[#1, #0] » %0:l4[#1{replica_id}, #3{bucket_start}] - →Arrange (#1{replica_id}, #3{bucket_start}) - →Stream l4 - →Arrange (#1, #0) - →Temporally-Bucketed Non-incremental GroupAggregate - Aggregation: jsonb_agg[order_by=[]](row(coalesce(jsonbable_to_jsonb(jsonb_build_object("replicaId", jsonbable_to_jsonb(#0{replica_id}), "occurredAt", jsonbable_to_jsonb(timestamp_with_time_zone_to_text(#2{occurred_at})), "status", "offline", "reason", jsonbable_to_jsonb(#1{reason}))), json_null))) - Key: - Project: #3, #0 - Map: timestamptz_bin(00:01:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) - →Fused with Child Map/Filter/Project - Project: #0, #3, #4 - Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:01:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 03:00:00))) - →Arranged mz_internal.mz_cluster_replica_status_history - Key: (#0{replica_id}) + Project: #0..=#10 + →Read l0 →Return - →Map/Filter/Project - Project: #1, #0, #2..=#5, #10, #8, #9, #11..=#13, #6, #7, #17, #18, #16, #15, #14 - Map: (#1{bucket_start} + 00:01:00) - →Union - →Map/Filter/Project - Project: #0..=#17 - Map: null - →Consolidating Union - →Negate Diffs - →Fused with Child Map/Filter/Project - Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 - →Read l5 - →Fused with Child Map/Filter/Project - Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 - →Read l4 - →Fused with Child Map/Filter/Project - Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 - →Read l5 + →Differential Join %1[#0, #1] » %0:l1[#0, #1] + →Arrange (#0, #1) + →Stream l1 + →Arrange (#0, #1) + →Map/Filter/Project + Project: #0, #1, #3 + →Non-monotonic TopK + Group By #0, #1 + Order By #2 desc nulls_first + Limit 1 + →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#1{replica_id}] + after %0: + Project: #4, #0, #1, #3 + Filter: (#4{occurred_at} >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) + →Arrange (#1{replica_id}) + →Distinct GroupAggregate + →Fused with Child Map/Filter/Project + Project: #0, #1 + →Read l1 + →Arranged mz_internal.mz_cluster_replica_name_history Used Indexes: - mz_catalog.mz_cluster_replicas_ind (*** full scan ***) - mz_catalog.mz_cluster_replica_sizes_ind (delta join lookup) - - mz_internal.mz_cluster_replica_status_history_ind (*** full scan ***) - mz_internal.mz_cluster_replica_metrics_history_ind (delta join lookup) - mz_internal.mz_cluster_replica_history_ind (*** full scan ***) - mz_internal.mz_cluster_replica_name_history_ind (differential join) @@ -1206,8 +1114,8 @@ mz_internal.mz_console_cluster_utilization_overview: cte l2 = →Differential Join %0[#0..=#6] » %1[#0..=#6] after %1: - Project: #0, #1, #7, #10..=#18 - Map: (#9 / uint8_to_double((case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end * #6{processes}))), (#10 / uint8_to_double((case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end * #6{processes}))), (#11 / uint8_to_double((case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end * #6{processes}))), (#5{disk_bytes} * #6{processes}), (#4{memory_bytes} * #6{processes}), coalesce(#8, #13), timestamptz_bin(01:00:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + Project: #0, #1, #7, #10, #11, #13..=#15, #17..=#20 + Map: uint8_to_double(case when (0 = uint8_to_numeric(#6{processes})) then null else #6{processes} end), ((#9 / uint8_to_double(case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end)) / #12), ((#10 / uint8_to_double(case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end)) / #12), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end)) / #12), uint8_to_double(#6{processes}), (uint8_to_double(#5{disk_bytes}) * #16), (uint8_to_double(#4{memory_bytes}) * #16), coalesce(#8, #14), timestamptz_bin(01:00:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) Aggregations: max, max Key: @@ -1266,7 +1174,7 @@ mz_internal.mz_console_cluster_utilization_overview: Limit 1 →Fused with Child Map/Filter/Project Project: #0, #1, #6, #7, #11..=#13 - Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / uint8_to_double(case when (0 = uint8_to_numeric((#9{total_memory_bytes} + #8{total_disk_bytes}))) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end)) end, coalesce(#12{memory_and_disk_percent}, 0) + Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / case when (0 = (#9{total_memory_bytes} + #8{total_disk_bytes})) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end) end, coalesce(#12{memory_and_disk_percent}, 0) →Read l2 →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project @@ -7637,7 +7545,7 @@ EXPLAIN SELECT * FROM "mz_internal"."mz_console_cluster_utilization_overview_3h" ---- Explained Query (fast path): →Map/Filter/Project - Project: #1..=#17, #0, #18 + Project: #1, #0, #2..=#9 →Indexed mz_internal.mz_console_cluster_utilization_overview_3h (using mz_internal.mz_console_cluster_utilization_overview_3h_ind) Used Indexes: diff --git a/test/sqllogictest/cluster.slt b/test/sqllogictest/cluster.slt index b3015f21d86b8..0b932d6279276 100644 --- a/test/sqllogictest/cluster.slt +++ b/test/sqllogictest/cluster.slt @@ -420,7 +420,7 @@ CREATE CLUSTER test REPLICAS (foo (SIZE 'scale=1,workers=1')); query I SELECT COUNT(name) FROM mz_indexes; ---- -300 +302 statement ok DROP CLUSTER test CASCADE @@ -428,7 +428,7 @@ DROP CLUSTER test CASCADE query T SELECT COUNT(name) FROM mz_indexes; ---- -268 +270 simple conn=mz_system,user=mz_system ALTER CLUSTER quickstart OWNER TO materialize diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index dc46f37a5875a..3d180a65d6359 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -402,23 +402,14 @@ mz_console_cluster_utilization_overview_24h name mz_console_cluster_utilization_overview_24h offline_events mz_console_cluster_utilization_overview_24h replica_id mz_console_cluster_utilization_overview_24h size -mz_console_cluster_utilization_overview_3h bucket_end -mz_console_cluster_utilization_overview_3h bucket_start mz_console_cluster_utilization_overview_3h cluster_id +mz_console_cluster_utilization_overview_3h cpu_percent mz_console_cluster_utilization_overview_3h disk_percent mz_console_cluster_utilization_overview_3h heap_percent -mz_console_cluster_utilization_overview_3h max_cpu_at -mz_console_cluster_utilization_overview_3h max_cpu_percent -mz_console_cluster_utilization_overview_3h max_disk_at -mz_console_cluster_utilization_overview_3h max_heap_at -mz_console_cluster_utilization_overview_3h max_memory_and_disk_at -mz_console_cluster_utilization_overview_3h max_memory_and_disk_disk_percent -mz_console_cluster_utilization_overview_3h max_memory_and_disk_memory_percent -mz_console_cluster_utilization_overview_3h max_memory_at mz_console_cluster_utilization_overview_3h memory_and_disk_percent mz_console_cluster_utilization_overview_3h memory_percent mz_console_cluster_utilization_overview_3h name -mz_console_cluster_utilization_overview_3h offline_events +mz_console_cluster_utilization_overview_3h occurred_at mz_console_cluster_utilization_overview_3h replica_id mz_console_cluster_utilization_overview_3h size mz_databases id