diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index 6dfc5af594d17..dfab4b9222dc8 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -1478,6 +1478,8 @@ The `mz_webhook_sources` table contains a row for each webhook source in the sys + + diff --git a/src/adapter/src/catalog/open.rs b/src/adapter/src/catalog/open.rs index 706f7b5ac0bf3..d8f9718adb124 100644 --- a/src/adapter/src/catalog/open.rs +++ b/src/adapter/src/catalog/open.rs @@ -755,6 +755,7 @@ fn add_new_remove_old_builtin_items_migration( // We compare the builtin items that are compiled into the binary with the builtin items that // are persisted in the catalog to discover new and deleted builtin items. let mut builtins = Vec::new(); + for builtin in BUILTINS::iter() { let desc = SystemObjectDescription { schema_name: builtin.schema().to_string(), diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index edbc0208b7062..2e744887f4dd7 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -252,6 +252,16 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_INTERNAL_SCHEMA, "mz_comments", ), + // Required because we added the console cluster-utilization overview builtin + // indexes (overview/_3h/_24h). make_mz_indexes inlines the builtin-index set + // as VALUES, so any add/remove changes its SQL fingerprint and requires an + // explicit replacement step. + MigrationStep::replacement( + "26.32.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_indexes", + ), ] }); diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index aae323a94a838..7395f04b4ac64 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1395,6 +1395,8 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::View(&MZ_MATERIALIZATION_DEPENDENCIES), Builtin::View(&MZ_MATERIALIZATION_LAG), Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW), + Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H), + Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H), Builtin::View(&MZ_COMPUTE_ERROR_COUNTS_PER_WORKER), Builtin::View(&MZ_COMPUTE_ERROR_COUNTS), Builtin::Source(&MZ_COMPUTE_ERROR_COUNTS_RAW_UNIFIED), @@ -1469,6 +1471,8 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Index(&MZ_SECRETS_IND), Builtin::Index(&MZ_VIEWS_IND), Builtin::Index(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_IND), + Builtin::Index(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND), + Builtin::Index(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND), Builtin::Index(&MZ_CLUSTER_DEPLOYMENT_LINEAGE_IND), Builtin::Index(&MZ_CLUSTER_REPLICA_FRONTIERS_IND), Builtin::Index(&MZ_COMPUTE_HYDRATION_TIMES_IND), diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index c291e9a679d14..19c498e47414c 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -6990,78 +6990,88 @@ JOIN root_times r USING (id)", }, }), }); -/** - * This view is used to display the cluster utilization over 14 days bucketed by 8 hours. - * It's specifically for the Console's environment overview page to speed up load times. - * This query should be kept in sync with MaterializeInc/console/src/api/materialize/cluster/replicaUtilizationHistory.ts - */ -pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW: LazyLock = LazyLock::new(|| { - BuiltinView { - name: "mz_console_cluster_utilization_overview", - schema: MZ_INTERNAL_SCHEMA, - oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_OID, - desc: RelationDesc::builder() - .with_column( - "bucket_start", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("replica_id", SqlScalarType::String.nullable(false)) - .with_column("memory_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_memory_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("disk_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_disk_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column( - "memory_and_disk_percent", - SqlScalarType::Float64.nullable(true), - ) - .with_column( - "max_memory_and_disk_memory_percent", - SqlScalarType::Float64.nullable(true), - ) - .with_column( - "max_memory_and_disk_disk_percent", - SqlScalarType::Float64.nullable(true), - ) - .with_column( - "max_memory_and_disk_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("heap_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_heap_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("max_cpu_percent", SqlScalarType::Float64.nullable(true)) - .with_column( - "max_cpu_at", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("offline_events", SqlScalarType::Jsonb.nullable(true)) - .with_column( - "bucket_end", - SqlScalarType::TimestampTz { precision: None }.nullable(false), - ) - .with_column("name", SqlScalarType::String.nullable(true)) - .with_column("cluster_id", SqlScalarType::String.nullable(true)) - .with_column("size", SqlScalarType::String.nullable(true)) - .finish(), - column_comments: BTreeMap::new(), - sql: r#"WITH replica_history AS ( - SELECT replica_id, - size, - cluster_id +/// The output relation shared by all `mz_console_cluster_utilization_overview*` +/// views. Every (bucket size, retention) variant produces the same columns so +/// the Console can swap between them based on the selected time range. +fn console_cluster_utilization_overview_desc() -> RelationDesc { + RelationDesc::builder() + .with_column( + "bucket_start", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("replica_id", SqlScalarType::String.nullable(false)) + .with_column("memory_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_memory_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("disk_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_disk_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column( + "memory_and_disk_percent", + SqlScalarType::Float64.nullable(true), + ) + .with_column( + "max_memory_and_disk_memory_percent", + SqlScalarType::Float64.nullable(true), + ) + .with_column( + "max_memory_and_disk_disk_percent", + SqlScalarType::Float64.nullable(true), + ) + .with_column( + "max_memory_and_disk_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("heap_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_heap_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("max_cpu_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "max_cpu_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("offline_events", SqlScalarType::Jsonb.nullable(true)) + .with_column( + "bucket_end", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("name", SqlScalarType::String.nullable(true)) + .with_column("cluster_id", SqlScalarType::String.nullable(true)) + .with_column("size", SqlScalarType::String.nullable(true)) + .finish() +} + +/// Builds the SQL body shared by the `mz_console_cluster_utilization_overview*` +/// views, which power the Console's cluster utilization graphs. +/// +/// There is one view per (bucket width, retention window) pair so the Console +/// can read a pre-materialized, indexed rollup for each time range it offers +/// instead of recomputing this (expensive) query on every page load. The bodies +/// must be kept in sync with the equivalent ad-hoc query in the Console +/// (`buildReplicaUtilizationHistoryQuery` in +/// `console/src/api/materialize/cluster/replicaUtilizationHistory.ts`). +/// +/// * `bin`: the `date_bin` bucket width, e.g. `1 MINUTE`. +/// * `retention`: how much history the view retains, e.g. `3 HOURS`, enforced +/// with a temporal `mz_now()` filter so the maintained arrangement stays +/// bounded. +/// * `group_size`: the expected number of metric samples per (replica, bucket), +/// used for the `DISTINCT ON INPUT GROUP SIZE` top-k hint. Replica metrics are +/// scraped roughly once per minute, so this is the bucket width in minutes. +fn console_cluster_utilization_overview_sql(bin: &str, retention: &str, group_size: u32) -> String { + format!( + r#"WITH replica_history AS ( + SELECT replica_id, size, cluster_id FROM mz_internal.mz_cluster_replica_history UNION - -- We need to union the current set of cluster replicas since mz_cluster_replica_history doesn't include system clusters - SELECT id AS replica_id, - size, - cluster_id + -- We union the current set of cluster replicas since mz_cluster_replica_history doesn't include system clusters. + SELECT id AS replica_id, size, cluster_id FROM mz_catalog.mz_cluster_replicas ), replica_metrics_history AS ( @@ -7069,14 +7079,22 @@ replica_metrics_history AS ( m.occurred_at, m.replica_id, r.size, - (SUM(m.cpu_nano_cores::float8) / NULLIF(s.cpu_nano_cores, 0)) / NULLIF(s.processes, 0) AS cpu_percent, - (SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0)) / NULLIF(s.processes, 0) AS memory_percent, - (SUM(m.disk_bytes::float8) / NULLIF(s.disk_bytes, 0)) / NULLIF(s.processes, 0) AS disk_percent, - (SUM(m.heap_bytes::float8) / NULLIF(m.heap_limit, 0)) / NULLIF(s.processes, 0) AS heap_percent, + (SUM(m.cpu_nano_cores::float8) / NULLIF(s.cpu_nano_cores, 0) / NULLIF(s.processes, 0)) AS cpu_percent, + (SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0)) AS memory_percent, + (SUM(m.disk_bytes::float8) / NULLIF(s.disk_bytes, 0) / NULLIF(s.processes, 0)) AS disk_percent, SUM(m.disk_bytes::float8) AS disk_bytes, SUM(m.memory_bytes::float8) AS memory_bytes, - s.disk_bytes::numeric * s.processes AS total_disk_bytes, - s.memory_bytes::numeric * s.processes AS total_memory_bytes + s.disk_bytes::float8 * s.processes AS total_disk_bytes, + s.memory_bytes::float8 * s.processes AS total_memory_bytes, + MAX(m.heap_bytes::float8) AS heap_bytes, + MAX(m.heap_limit) AS heap_limit, + -- heap_limit is NULL when clusterd isn't launched with --heap-limit (e.g. + -- the emulator's process orchestrator). Fall back to the size-based memory + -- percent so the chart still renders. + COALESCE( + MAX(m.heap_bytes::float8 / NULLIF(m.heap_limit, 0)), + SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0) + ) AS heap_percent FROM replica_history AS r INNER JOIN mz_catalog.mz_cluster_replica_sizes AS s ON r.size = s.size @@ -7088,154 +7106,113 @@ replica_metrics_history AS ( s.cpu_nano_cores, s.memory_bytes, s.disk_bytes, - m.heap_limit, s.processes ), replica_utilization_history_binned AS ( - SELECT m.occurred_at, + -- NOTE: we read directly from replica_metrics_history rather than re-joining + -- replica_history; every replica_id here already came from replica_history, + -- so the join was redundant (and could fan out a replica that changed size). + SELECT + m.occurred_at, m.replica_id, m.cpu_percent, m.memory_percent, m.memory_bytes, m.disk_percent, m.disk_bytes, - m.heap_percent, m.total_disk_bytes, m.total_memory_bytes, + m.heap_bytes, + m.heap_percent, m.size, - date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) AS bucket_start - FROM replica_history AS r - JOIN replica_metrics_history AS m ON m.replica_id = r.replica_id - WHERE mz_now() <= date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) + INTERVAL '14 DAYS' + date_bin('{bin}', m.occurred_at, '1970-01-01'::timestamp) AS bucket_start + FROM replica_metrics_history AS m + WHERE mz_now() <= date_bin('{bin}', m.occurred_at, '1970-01-01'::timestamp) + INTERVAL '{retention}' ), --- For each (replica, bucket), take the (replica, bucket) with the highest memory +-- For each (replica, bucket), take the sample with the highest memory. max_memory AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - memory_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, memory_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(memory_bytes, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(memory_bytes, 0) DESC ), +-- For each (replica, bucket), take the sample with the highest disk. max_disk AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - disk_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, disk_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(disk_bytes, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(disk_bytes, 0) DESC ), +-- For each (replica, bucket), take the sample with the highest cpu. max_cpu AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - cpu_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, cpu_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(cpu_percent, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(cpu_percent, 0) DESC ), /* - This is different - from adding max_memory - and max_disk per bucket because both - values may not occur at the same time if the bucket interval is large. - */ + For each (replica, bucket), take the sample with the highest combined memory + and disk. This is different from adding max_memory and max_disk per bucket + because both values may not occur at the same time if the bucket interval is + large. +*/ max_memory_and_disk AS ( - SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, - replica_id, - memory_percent, - disk_percent, - memory_and_disk_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, memory_percent, disk_percent, memory_and_disk_percent, occurred_at FROM ( - SELECT *, - CASE - WHEN disk_bytes IS NULL - AND memory_bytes IS NULL THEN NULL - ELSE (COALESCE(disk_bytes, 0) + COALESCE(memory_bytes, 0)) - / (total_disk_bytes::numeric + total_memory_bytes::numeric) - END AS memory_and_disk_percent - FROM replica_utilization_history_binned - ) AS max_memory_and_disk_inner - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, - replica_id, - COALESCE(memory_and_disk_percent, 0) DESC + SELECT *, + CASE + WHEN disk_bytes IS NULL AND memory_bytes IS NULL THEN NULL + ELSE (COALESCE(memory_bytes, 0) + COALESCE(disk_bytes, 0)) / NULLIF((total_memory_bytes + total_disk_bytes), 0) + END AS memory_and_disk_percent + FROM replica_utilization_history_binned + ) AS max_memory_and_disk_inner + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(memory_and_disk_percent, 0) DESC ), +-- For each (replica, bucket), take the sample with the highest heap. max_heap AS ( - SELECT DISTINCT ON (bucket_start, replica_id) - bucket_start, - replica_id, - heap_percent, - occurred_at + SELECT DISTINCT ON (bucket_start, replica_id) bucket_start, replica_id, heap_percent, occurred_at FROM replica_utilization_history_binned - OPTIONS (DISTINCT ON INPUT GROUP SIZE = 480) - ORDER BY bucket_start, replica_id, COALESCE(heap_percent, 0) DESC + OPTIONS (DISTINCT ON INPUT GROUP SIZE = {group_size}) + ORDER BY bucket_start, replica_id, COALESCE(heap_bytes, 0) DESC ), --- For each (replica, bucket), get its offline events at that time +-- For each (replica, bucket), collect its offline events at that time. replica_offline_event_history AS ( - SELECT date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) AS bucket_start, + SELECT + date_bin('{bin}', occurred_at, '1970-01-01'::timestamp) AS bucket_start, replica_id, jsonb_agg( jsonb_build_object( - 'replicaId', - rsh.replica_id, - 'occurredAt', - rsh.occurred_at, - 'status', - rsh.status, - 'reason', - rsh.reason + 'replicaId', rsh.replica_id, + 'occurredAt', rsh.occurred_at, + 'status', rsh.status, + 'reason', rsh.reason ) ) AS offline_events - FROM mz_internal.mz_cluster_replica_status_history AS rsh -- We assume the statuses for process 0 are the same as all processes + FROM mz_internal.mz_cluster_replica_status_history AS rsh + -- We assume the statuses for process 0 are the same as all processes. WHERE process_id = '0' AND status = 'offline' - AND mz_now() <= date_bin( - '8 HOURS', - occurred_at, - '1970-01-01'::timestamp - ) + INTERVAL '14 DAYS' - GROUP BY bucket_start, - replica_id + AND mz_now() <= date_bin('{bin}', occurred_at, '1970-01-01'::timestamp) + INTERVAL '{retention}' + GROUP BY bucket_start, replica_id ) SELECT bucket_start, replica_id, max_memory.memory_percent, - max_memory.occurred_at as max_memory_at, + max_memory.occurred_at AS max_memory_at, max_disk.disk_percent, - max_disk.occurred_at as max_disk_at, - max_memory_and_disk.memory_and_disk_percent as memory_and_disk_percent, - max_memory_and_disk.memory_percent as max_memory_and_disk_memory_percent, - max_memory_and_disk.disk_percent as max_memory_and_disk_disk_percent, - max_memory_and_disk.occurred_at as max_memory_and_disk_at, + max_disk.occurred_at AS max_disk_at, + max_memory_and_disk.memory_and_disk_percent AS memory_and_disk_percent, + max_memory_and_disk.memory_percent AS max_memory_and_disk_memory_percent, + max_memory_and_disk.disk_percent AS max_memory_and_disk_disk_percent, + max_memory_and_disk.occurred_at AS max_memory_and_disk_at, max_heap.heap_percent, - max_heap.occurred_at as max_heap_at, - max_cpu.cpu_percent as max_cpu_percent, - max_cpu.occurred_at as max_cpu_at, + max_heap.occurred_at AS max_heap_at, + max_cpu.cpu_percent AS max_cpu_percent, + max_cpu.occurred_at AS max_cpu_at, replica_offline_event_history.offline_events, - bucket_start + INTERVAL '8 HOURS' as bucket_end, + bucket_start + INTERVAL '{bin}' AS bucket_end, replica_name_history.new_name AS name, replica_history.cluster_id, replica_history.size @@ -7245,22 +7222,192 @@ JOIN max_cpu USING (bucket_start, replica_id) JOIN max_memory_and_disk USING (bucket_start, replica_id) JOIN max_heap USING (bucket_start, replica_id) JOIN replica_history USING (replica_id) +/* + TOP k=1 over the name history via a LATERAL subquery + LIMIT: for each bucket, + get the most recent replica name as of the end of the bucket. +*/ CROSS JOIN LATERAL ( SELECT new_name - FROM mz_internal.mz_cluster_replica_name_history as replica_name_history - WHERE replica_id = replica_name_history.id -- We treat NULLs as the beginning of time - AND bucket_start + INTERVAL '8 HOURS' >= COALESCE( - replica_name_history.occurred_at, - '1970-01-01'::timestamp - ) + FROM mz_internal.mz_cluster_replica_name_history AS replica_name_history + WHERE replica_id = replica_name_history.id + -- We treat NULLs as the beginning of time. + AND bucket_start + INTERVAL '{bin}' >= COALESCE(replica_name_history.occurred_at, '1970-01-01'::timestamp) ORDER BY replica_name_history.occurred_at DESC - LIMIT '1' + LIMIT 1 ) AS replica_name_history LEFT JOIN replica_offline_event_history USING (bucket_start, replica_id)"#, + bin = bin, + retention = retention, + group_size = group_size, + ) +} + +/// Schema for the un-binned 3-hour console cluster utilization base. Unlike the +/// binned `_overview*` views, this exposes raw per-(replica, sample) metrics so +/// the Console can bin client-side. +fn console_cluster_utilization_unbinned_3h_desc() -> RelationDesc { + RelationDesc::builder() + .with_column("replica_id", SqlScalarType::String.nullable(false)) + .with_column("cluster_id", SqlScalarType::String.nullable(true)) + .with_column("size", SqlScalarType::String.nullable(false)) + .with_column("name", SqlScalarType::String.nullable(true)) + .with_column( + "occurred_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column("cpu_percent", SqlScalarType::Float64.nullable(true)) + .with_column("memory_percent", SqlScalarType::Float64.nullable(true)) + .with_column("disk_percent", SqlScalarType::Float64.nullable(true)) + .with_column("heap_percent", SqlScalarType::Float64.nullable(true)) + .with_column( + "memory_and_disk_percent", + SqlScalarType::Float64.nullable(true), + ) + .finish() +} + +/// Builds the SQL for the un-binned 3-hour console cluster utilization base: one +/// row per (replica, metric sample) over `retention`, with no `date_bin`/top-k, +/// so the Console bins it client-side. The binned `_overview*` views handle the +/// longer windows. A temporal `mz_now()` filter bounds the maintained +/// arrangement. Kept in sync with the Console +/// (`buildConsoleClusterUtilizationUnbinned3hQuery` in +/// `replicaUtilizationHistory.ts`). +fn console_cluster_utilization_unbinned_3h_sql(retention: &str) -> String { + format!( + r#"WITH replica_history AS ( + -- Dedup to one row per replica (prefer the current size). Size is fixed per + -- replica so this is normally a no-op, but a stray duplicate size in history + -- would fan out the metrics join; with no Top-1 dedup here that would emit two + -- rows per (replica_id, occurred_at) and break the Console SUBSCRIBE upsert key. + SELECT DISTINCT ON (replica_id) replica_id, size, cluster_id + FROM ( + -- We union the current set of cluster replicas since mz_cluster_replica_history doesn't include system clusters. + SELECT id AS replica_id, size, cluster_id, 0 AS source_rank + FROM mz_catalog.mz_cluster_replicas + UNION ALL + SELECT replica_id, size, cluster_id, 1 AS source_rank + FROM mz_internal.mz_cluster_replica_history + ) all_replicas + ORDER BY replica_id, source_rank +), +replica_metrics AS ( + SELECT + m.occurred_at, + m.replica_id, + r.cluster_id, + r.size, + (SUM(m.cpu_nano_cores::float8) / NULLIF(s.cpu_nano_cores, 0) / NULLIF(s.processes, 0)) AS cpu_percent, + (SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0)) AS memory_percent, + (SUM(m.disk_bytes::float8) / NULLIF(s.disk_bytes, 0) / NULLIF(s.processes, 0)) AS disk_percent, + COALESCE( + MAX(m.heap_bytes::float8 / NULLIF(m.heap_limit, 0)), + SUM(m.memory_bytes::float8) / NULLIF(s.memory_bytes, 0) / NULLIF(s.processes, 0) + ) AS heap_percent, + CASE + WHEN SUM(m.disk_bytes::float8) IS NULL AND SUM(m.memory_bytes::float8) IS NULL THEN NULL + ELSE (COALESCE(SUM(m.memory_bytes::float8), 0) + COALESCE(SUM(m.disk_bytes::float8), 0)) + / NULLIF((s.memory_bytes::float8 + s.disk_bytes::float8) * s.processes, 0) + END AS memory_and_disk_percent + FROM replica_history AS r + INNER JOIN mz_catalog.mz_cluster_replica_sizes AS s ON r.size = s.size + INNER JOIN mz_internal.mz_cluster_replica_metrics_history AS m ON m.replica_id = r.replica_id + -- No aggregation over time: one row per (replica, sample) so the Console bins + -- client-side. The temporal mz_now() filter keeps the maintained arrangement + -- bounded to the retention window. + WHERE mz_now() <= m.occurred_at + INTERVAL '{retention}' + GROUP BY + m.occurred_at, + m.replica_id, + r.cluster_id, + r.size, + s.cpu_nano_cores, + s.memory_bytes, + s.disk_bytes, + s.processes +) +SELECT + m.replica_id, + m.cluster_id, + m.size, + replica_name_history.new_name AS name, + m.occurred_at, + m.cpu_percent, + m.memory_percent, + m.disk_percent, + m.heap_percent, + m.memory_and_disk_percent +FROM replica_metrics AS m +/* Most recent replica name as of the sample time. */ +CROSS JOIN LATERAL ( + SELECT new_name + FROM mz_internal.mz_cluster_replica_name_history AS replica_name_history + WHERE m.replica_id = replica_name_history.id + -- We treat NULLs as the beginning of time. + AND m.occurred_at >= COALESCE(replica_name_history.occurred_at, '1970-01-01'::timestamp) + ORDER BY replica_name_history.occurred_at DESC + LIMIT 1 +) AS replica_name_history"#, + retention = retention, + ) +} + +/** + * Displays cluster utilization over 14 days bucketed by 1 hour, for the + * Console's environment overview and cluster pages, to speed up load times. + * This view (and its `_3h`/`_24h` siblings) is kept in sync with + * MaterializeInc/console/src/api/materialize/cluster/replicaUtilizationHistory.ts + */ +pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW: LazyLock = + LazyLock::new(|| BuiltinView { + name: "mz_console_cluster_utilization_overview", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_OID, + desc: console_cluster_utilization_overview_desc(), + column_comments: BTreeMap::new(), + sql: Box::leak( + console_cluster_utilization_overview_sql("1 HOUR", "14 DAYS", 60).into_boxed_str(), + ), access: vec![PUBLIC_SELECT], ontology: None, - } -}); + }); + +/** + * Un-binned cluster utilization over the last 3 hours, for the Console's "Last + * hour" / "Last 3 hours" graphs. Unlike the binned `_overview*` views, this + * exposes raw per-(replica, sample) metrics and the Console bins client-side. + * See `console_cluster_utilization_unbinned_3h_sql` for details. + */ +pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H: LazyLock = + LazyLock::new(|| BuiltinView { + name: "mz_console_cluster_utilization_overview_3h", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_OID, + desc: console_cluster_utilization_unbinned_3h_desc(), + column_comments: BTreeMap::new(), + sql: Box::leak(console_cluster_utilization_unbinned_3h_sql("3 HOURS").into_boxed_str()), + access: vec![PUBLIC_SELECT], + ontology: None, + }); + +/** + * Cluster utilization over the last 24 hours bucketed by 5 minutes, for the + * Console's "Last 6 hours" / "Last 24 hours" cluster utilization graphs. See + * `console_cluster_utilization_overview_sql` for details. + */ +pub static MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H: LazyLock = + LazyLock::new(|| BuiltinView { + name: "mz_console_cluster_utilization_overview_24h", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_OID, + desc: console_cluster_utilization_overview_desc(), + column_comments: BTreeMap::new(), + sql: Box::leak( + console_cluster_utilization_overview_sql("5 MINUTES", "24 HOURS", 5).into_boxed_str(), + ), + access: vec![PUBLIC_SELECT], + ontology: None, + }); /** * Traces the blue/green deployment lineage in the audit log to determine all cluster * IDs that are logically the same cluster. @@ -7566,6 +7713,24 @@ ON mz_internal.mz_console_cluster_utilization_overview (cluster_id)", is_retained_metrics_object: false, }; +pub const MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND: BuiltinIndex = BuiltinIndex { + name: "mz_console_cluster_utilization_overview_3h_ind", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND_OID, + sql: "IN CLUSTER mz_catalog_server +ON mz_internal.mz_console_cluster_utilization_overview_3h (cluster_id)", + is_retained_metrics_object: false, +}; + +pub const MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND: BuiltinIndex = BuiltinIndex { + name: "mz_console_cluster_utilization_overview_24h_ind", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND_OID, + sql: "IN CLUSTER mz_catalog_server +ON mz_internal.mz_console_cluster_utilization_overview_24h (cluster_id)", + is_retained_metrics_object: false, +}; + pub const MZ_CLUSTER_DEPLOYMENT_LINEAGE_IND: BuiltinIndex = BuiltinIndex { name: "mz_cluster_deployment_lineage_ind", schema: MZ_INTERNAL_SCHEMA, diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 31f248d4cf76f..30bbdec2b4375 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -806,3 +806,7 @@ pub const MV_MZ_REPLICA_SYSTEM_PARAMETERS_OID: u32 = 17092; pub const FUNC_MZ_GEN_SERIES_UNOPT_OID: u32 = 17093; pub const FUNC_MZ_GEN_SERIES_UNOPT_STEP_OID: u32 = 17094; pub const MV_MZ_OVERRIDDEN_SYSTEM_PARAMETERS_OID: u32 = 17095; +pub const VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_OID: u32 = 17096; +pub const INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H_IND_OID: u32 = 17097; +pub const VIEW_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_OID: u32 = 17098; +pub const INDEX_MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H_IND_OID: u32 = 17099; diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index fc434766efe61..9e8a620548b87 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -784,6 +784,8 @@ mz_compute_hydration_statuses mz_compute_hydration_times mz_compute_operator_hydration_statuses mz_console_cluster_utilization_overview +mz_console_cluster_utilization_overview_24h +mz_console_cluster_utilization_overview_3h mz_frontiers mz_global_frontiers mz_history_retention_strategies diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 259f2d6c91bd8..52de5cc30c89d 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -812,6 +812,272 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_24h_ind"; +---- +mz_internal.mz_console_cluster_utilization_overview_24h_ind: + →Arrange (#17{cluster_id}) + →Stream mz_internal.mz_console_cluster_utilization_overview_24h + +mz_internal.mz_console_cluster_utilization_overview_24h: + →With + cte l0 = + →Distinct GroupAggregate + →Union + →Fused with Child Map/Filter/Project + Project: #1..=#3 + Filter: (#1) IS NOT NULL + →Arranged mz_internal.mz_cluster_replica_history + Key: (#6{dropped_at}) + →Fused with Child Map/Filter/Project + Project: #0, #3, #2 + →Arranged mz_catalog.mz_cluster_replicas + Key: (#0{id}) + cte l1 = + →Map/Filter/Project + Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:05:00, #9, 1970-01-01 00:00:00 UTC) + 24:00:00))) + →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] + →Arrange (#0{replica_id}) (#1{size}) + →Fused with Child Map/Filter/Project + Project: #0, #1 + Filter: (#1{size}) IS NOT NULL + →Arranged l0 + Key: (#0..=#2) + →Arranged mz_catalog.mz_cluster_replica_sizes + →Arranged mz_internal.mz_cluster_replica_metrics_history + cte l2 = + →Differential Join %0[#0..=#6] » %1[#0..=#6] + after %1: + Project: #0, #1, #7, #10, #11, #13..=#15, #17..=#20 + Map: uint8_to_double(case when (0 = uint8_to_numeric(#6{processes})) then null else #6{processes} end), ((#9 / uint8_to_double(case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end)) / #12), ((#10 / uint8_to_double(case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end)) / #12), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end)) / #12), uint8_to_double(#6{processes}), (uint8_to_double(#5{disk_bytes}) * #16), (uint8_to_double(#4{memory_bytes}) * #16), coalesce(#8, #14), timestamptz_bin(00:05:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) + Aggregations: max, max + Key: + Project: #6, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#5, #9..=#11 + →Read l1 + →Temporally-Bucketed Accumulable GroupAggregate + Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})) + Key: + Project: #9, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#9 + →Read l1 + cte l3 = + →Delta Join [%0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#1{replica_id}, #4{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}]] + →Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #6, #11, #12 + Map: coalesce(#3{memory_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #7, #11, #12 + Map: coalesce(#4{disk_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #5, #11, #12 + Map: coalesce(#5{cpu_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #4{bucket_start}) + →Map/Filter/Project + Project: #0..=#5 + →Non-monotonic TopK + Group By #4, #1 + Order By #6 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #6, #7, #11..=#13 + Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / case when (0 = (#9{total_memory_bytes} + #8{total_disk_bytes})) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end) end, coalesce(#12{memory_and_disk_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) + →Map/Filter/Project + Project: #0..=#3 + →Non-monotonic TopK + Group By #3, #1 + Order By #4 desc nulls_first + Limit 1 + →Fused with Child Map/Filter/Project + Project: #0, #1, #10..=#12 + Map: coalesce(#2{heap_bytes}, 0) + →Read l2 + →Arrange (#0{replica_id}) + →Arranged l0 + cte l4 = + →Differential Join %1[#0, #1] » %0:l3[#1, #3] + →Arrange (#1, #3) + →Stream l3 + →Arrange (#0, #1) + →Map/Filter/Project + Project: #0, #1, #3 + →Non-monotonic TopK + Group By #0, #1 + Order By #2 desc nulls_first + Limit 1 + →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#0{replica_id}] + after %0: + Project: #0, #4, #1, #3 + Filter: ((#4{bucket_start} + 00:05:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) + →Arrange (#0{replica_id}) + →Distinct GroupAggregate + →Fused with Child Map/Filter/Project + Project: #1, #3 + →Read l3 + →Arranged mz_internal.mz_cluster_replica_name_history + cte l5 = + →Differential Join %1[#1, #0] » %0:l4[#1{replica_id}, #3{bucket_start}] + →Arrange (#1{replica_id}, #3{bucket_start}) + →Stream l4 + →Arrange (#1, #0) + →Temporally-Bucketed Non-incremental GroupAggregate + Aggregation: jsonb_agg[order_by=[]](row(coalesce(jsonbable_to_jsonb(jsonb_build_object("replicaId", jsonbable_to_jsonb(#0{replica_id}), "occurredAt", jsonbable_to_jsonb(timestamp_with_time_zone_to_text(#2{occurred_at})), "status", "offline", "reason", jsonbable_to_jsonb(#1{reason}))), json_null))) + Key: + Project: #3, #0 + Map: timestamptz_bin(00:05:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) + →Fused with Child Map/Filter/Project + Project: #0, #3, #4 + Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(00:05:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 24:00:00))) + →Arranged mz_internal.mz_cluster_replica_status_history + Key: (#0{replica_id}) + →Return + →Map/Filter/Project + Project: #1, #0, #2..=#5, #10, #8, #9, #11..=#13, #6, #7, #17, #18, #16, #15, #14 + Map: (#1{bucket_start} + 00:05:00) + →Union + →Map/Filter/Project + Project: #0..=#17 + Map: null + →Consolidating Union + →Negate Diffs + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l5 + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l4 + →Fused with Child Map/Filter/Project + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 + →Read l5 + +Used Indexes: + - mz_catalog.mz_cluster_replicas_ind (*** full scan ***) + - mz_catalog.mz_cluster_replica_sizes_ind (delta join lookup) + - mz_internal.mz_cluster_replica_status_history_ind (*** full scan ***) + - mz_internal.mz_cluster_replica_metrics_history_ind (delta join lookup) + - mz_internal.mz_cluster_replica_history_ind (*** full scan ***) + - mz_internal.mz_cluster_replica_name_history_ind (differential join) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_3h_ind"; +---- +mz_internal.mz_console_cluster_utilization_overview_3h_ind: + →Arrange (#1{cluster_id}) + →Stream mz_internal.mz_console_cluster_utilization_overview_3h + +mz_internal.mz_console_cluster_utilization_overview_3h: + →With + cte l0 = + →Map/Filter/Project + Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((#10{occurred_at} + 03:00:00))) + →Delta Join [%0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] + →Arrange (#0{replica_id}) (#1{size}) + →Map/Filter/Project + Project: #0..=#2 + Filter: (#1{size}) IS NOT NULL + →Non-monotonic TopK + Group By #0 + Order By #3 asc nulls_last + Limit 1 + →Union + →Fused with Child Map/Filter/Project + Project: #0, #3, #2, #7 + Map: 0 + →Arranged mz_catalog.mz_cluster_replicas + Key: (#0{id}) + →Fused with Child Map/Filter/Project + Project: #1..=#3, #8 + Filter: (#1{replica_id}) IS NOT NULL + Map: 1 + →Arranged mz_internal.mz_cluster_replica_history + Key: (#6{dropped_at}) + →Arranged mz_catalog.mz_cluster_replica_sizes + →Arranged mz_internal.mz_cluster_replica_metrics_history + cte l1 = + →Differential Join %0[#0..=#7] » %1[#0..=#7] + after %1: + Project: #0..=#3, #13..=#17 + Map: uint8_to_double(case when (0 = uint8_to_numeric(#7{processes})) then null else #7{processes} end), ((#9 / uint8_to_double(case when (0 = uint8_to_numeric(#4{cpu_nano_cores})) then null else #4{cpu_nano_cores} end)) / #12), ((#10 / uint8_to_double(case when (0 = uint8_to_numeric(#5{memory_bytes})) then null else #5{memory_bytes} end)) / #12), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#6{disk_bytes})) then null else #6{disk_bytes} end)) / #12), coalesce(#8, #14), case when ((#10) IS NULL AND (#11) IS NULL) then null else ((coalesce(#10, 0) + coalesce(#11, 0)) / case when (0 = ((uint8_to_double(#5{memory_bytes}) + uint8_to_double(#6{disk_bytes})) * uint8_to_double(#7{processes}))) then null else ((uint8_to_double(#5{memory_bytes}) + uint8_to_double(#6{disk_bytes})) * uint8_to_double(#7{processes})) end) end + →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) + Aggregations: max + Key: + Project: #7, #0, #2, #1, #4..=#6, #3 + →Fused with Child Map/Filter/Project + Project: #0..=#6, #10..=#12 + →Read l0 + →Temporally-Bucketed Accumulable GroupAggregate + Simple aggregates: sum(uint8_to_double(#7{cpu_nano_cores})), sum(uint8_to_double(#8{memory_bytes})), sum(uint8_to_double(#9{disk_bytes})) + Key: + Project: #10, #0, #2, #1, #4..=#6, #3 + →Fused with Child Map/Filter/Project + Project: #0..=#10 + →Read l0 + →Return + →Differential Join %1[#0, #1] » %0:l1[#0, #1] + →Arrange (#0, #1) + →Stream l1 + →Arrange (#0, #1) + →Map/Filter/Project + Project: #0, #1, #3 + →Non-monotonic TopK + Group By #0, #1 + Order By #2 desc nulls_first + Limit 1 + →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#1{replica_id}] + after %0: + Project: #4, #0, #1, #3 + Filter: (#4{occurred_at} >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) + →Arrange (#1{replica_id}) + →Distinct GroupAggregate + →Fused with Child Map/Filter/Project + Project: #0, #1 + →Read l1 + →Arranged mz_internal.mz_cluster_replica_name_history + +Used Indexes: + - mz_catalog.mz_cluster_replicas_ind (*** full scan ***) + - mz_catalog.mz_cluster_replica_sizes_ind (delta join lookup) + - mz_internal.mz_cluster_replica_metrics_history_ind (delta join lookup) + - mz_internal.mz_cluster_replica_history_ind (*** full scan ***) + - mz_internal.mz_cluster_replica_name_history_ind (differential join) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN INDEX "mz_internal"."mz_console_cluster_utilization_overview_ind"; ---- @@ -834,97 +1100,99 @@ mz_internal.mz_console_cluster_utilization_overview: →Arranged mz_catalog.mz_cluster_replicas Key: (#0{id}) cte l1 = - →Differential Join %0:l0[#0{replica_id}] » %1[#1{replica_id}] + →Map/Filter/Project + Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(01:00:00, #9, 1970-01-01 00:00:00 UTC) + 14 days))) + →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] + →Arrange (#0{replica_id}) (#1{size}) + →Fused with Child Map/Filter/Project + Project: #0, #1 + Filter: (#1{size}) IS NOT NULL + →Arranged l0 + Key: (#0..=#2) + →Arranged mz_catalog.mz_cluster_replica_sizes + →Arranged mz_internal.mz_cluster_replica_metrics_history + cte l2 = + →Differential Join %0[#0..=#6] » %1[#0..=#6] after %1: - Project: #0..=#10 - Map: timestamptz_bin(08:00:00, #1{occurred_at}, 1970-01-01 00:00:00 UTC) - →Arrange (#0{replica_id}) + Project: #0, #1, #7, #10, #11, #13..=#15, #17..=#20 + Map: uint8_to_double(case when (0 = uint8_to_numeric(#6{processes})) then null else #6{processes} end), ((#9 / uint8_to_double(case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end)) / #12), ((#10 / uint8_to_double(case when (0 = uint8_to_numeric(#4{memory_bytes})) then null else #4{memory_bytes} end)) / #12), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#5{disk_bytes})) then null else #5{disk_bytes} end)) / #12), uint8_to_double(#6{processes}), (uint8_to_double(#5{disk_bytes}) * #16), (uint8_to_double(#4{memory_bytes}) * #16), coalesce(#8, #14), timestamptz_bin(01:00:00, #0{occurred_at}, 1970-01-01 00:00:00 UTC) + →Temporally-Bucketed Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16) + Aggregations: max, max + Key: + Project: #6, #0, #1, #3..=#5, #2 →Fused with Child Map/Filter/Project - Project: #0 - →Arranged l0 - Key: (#0..=#2) - →Arrange (#1{replica_id}) - →Map/Filter/Project - Project: #0, #1, #9, #10, #14, #16, #18..=#21 - Map: uint8_to_numeric(#7{processes}), uint8_to_double(case when (#12 = 0) then null else #7{processes} end), ((#8 / uint8_to_double(case when (0 = uint8_to_numeric(#3{cpu_nano_cores})) then null else #3{cpu_nano_cores} end)) / #13), uint8_to_numeric(#4{memory_bytes}), ((#9 / uint8_to_double(case when (#15 = 0) then null else #4{memory_bytes} end)) / #13), uint8_to_numeric(#5{disk_bytes}), ((#10 / uint8_to_double(case when (#17 = 0) then null else #5{disk_bytes} end)) / #13), ((#11 / uint8_to_double(case when (0 = uint8_to_numeric(#6{heap_limit})) then null else #6{heap_limit} end)) / #13), (#17 * #12), (#15 * #12) - →Temporally-Bucketed Accumulable GroupAggregate - Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})), sum(uint8_to_double(#10{heap_bytes})) - Key: - Project: #9, #0, #1, #3..=#5, #11, #2 - →Map/Filter/Project - Filter: (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(08:00:00, #9, 1970-01-01 00:00:00 UTC) + 14 days))) - →Delta Join [%0:l0[#0{replica_id}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] [%1:mz_cluster_replica_sizes[#0{size}] » %0:l0[#1{size}] » %2:mz_cluster_replica_metrics_history[#0{replica_id}]] [%2:mz_cluster_replica_metrics_history[#0{replica_id}] » %0:l0[#0{replica_id}] » %1:mz_cluster_replica_sizes[#0{size}]] - →Arrange (#0{replica_id}) (#1{size}) - →Fused with Child Map/Filter/Project - Project: #0, #1 - Filter: (#1{size}) IS NOT NULL - →Arranged l0 - Key: (#0..=#2) - →Arranged mz_catalog.mz_cluster_replica_sizes - →Arranged mz_internal.mz_cluster_replica_metrics_history - cte l2 = - →Delta Join [%0[#0{replica_id}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#0{replica_id}, #3{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#0{replica_id}, #3{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %1[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#0{replica_id}, #4{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#0{replica_id}, #3{bucket_start}] » %0[#0{replica_id}, #3{bucket_start}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#0{replica_id}] » %1[#0{replica_id}, #3{bucket_start}] » %2[#0{replica_id}, #3{bucket_start}] » %3[#0{replica_id}, #4{bucket_start}] » %4[#0{replica_id}, #3{bucket_start}]] - →Arrange (#0{replica_id}) (#0{replica_id}, #3{bucket_start}) + Project: #0..=#5, #9..=#11 + →Read l1 + →Temporally-Bucketed Accumulable GroupAggregate + Simple aggregates: sum(uint8_to_double(#6{cpu_nano_cores})), sum(uint8_to_double(#7{memory_bytes})), sum(uint8_to_double(#8{disk_bytes})) + Key: + Project: #9, #0, #1, #3..=#5, #2 + →Fused with Child Map/Filter/Project + Project: #0..=#9 + →Read l1 + cte l3 = + →Delta Join [%0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%1[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%2[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%3[#1{replica_id}, #4{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}] » %5:l0[#0{replica_id}]] [%4[#1{replica_id}, #3{bucket_start}] » %0[#1{replica_id}, #3{bucket_start}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %5:l0[#0{replica_id}]] [%5:l0[#0{replica_id}] » %0[#1{replica_id}] » %1[#1{replica_id}, #3{bucket_start}] » %2[#1{replica_id}, #3{bucket_start}] » %3[#1{replica_id}, #4{bucket_start}] » %4[#1{replica_id}, #3{bucket_start}]] + →Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #5, #10, #11 - Map: coalesce(#2{memory_bytes}, 0) - →Read l1 - →Arrange (#0{replica_id}, #3{bucket_start}) + Project: #0, #1, #6, #11, #12 + Map: coalesce(#3{memory_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #6, #10, #11 - Map: coalesce(#3{disk_bytes}, 0) - →Read l1 - →Arrange (#0{replica_id}, #3{bucket_start}) + Project: #0, #1, #7, #11, #12 + Map: coalesce(#4{disk_bytes}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #4, #10, #11 - Map: coalesce(#4{cpu_percent}, 0) - →Read l1 - →Arrange (#0{replica_id}, #4{bucket_start}) + Project: #0, #1, #5, #11, #12 + Map: coalesce(#5{cpu_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #4{bucket_start}) →Map/Filter/Project Project: #0..=#5 →Non-monotonic TopK - Group By #4, #0 + Group By #4, #1 Order By #6 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #5, #6, #10..=#12 - Map: case when ((#2{memory_bytes}) IS NULL AND (#3{disk_bytes}) IS NULL) then null else ((coalesce(#3{disk_bytes}, 0) + coalesce(#2{memory_bytes}, 0)) / numeric_to_double((#8{total_disk_bytes} + #9{total_memory_bytes}))) end, coalesce(#11{memory_and_disk_percent}, 0) - →Read l1 - →Arrange (#0{replica_id}, #3{bucket_start}) + Project: #0, #1, #6, #7, #11..=#13 + Map: case when ((#3{memory_bytes}) IS NULL AND (#4{disk_bytes}) IS NULL) then null else ((coalesce(#3{memory_bytes}, 0) + coalesce(#4{disk_bytes}, 0)) / case when (0 = (#9{total_memory_bytes} + #8{total_disk_bytes})) then null else (#9{total_memory_bytes} + #8{total_disk_bytes}) end) end, coalesce(#12{memory_and_disk_percent}, 0) + →Read l2 + →Arrange (#1{replica_id}, #3{bucket_start}) →Map/Filter/Project Project: #0..=#3 →Non-monotonic TopK - Group By #3, #0 + Group By #3, #1 Order By #4 desc nulls_first Limit 1 →Fused with Child Map/Filter/Project - Project: #0, #1, #7, #10, #11 - Map: coalesce(#7{heap_percent}, 0) - →Read l1 + Project: #0, #1, #10..=#12 + Map: coalesce(#2{heap_bytes}, 0) + →Read l2 →Arrange (#0{replica_id}) →Arranged l0 - cte l3 = - →Differential Join %1[#0, #1] » %0:l2[#0, #3] - →Arrange (#0, #3) - →Stream l2 + cte l4 = + →Differential Join %1[#0, #1] » %0:l3[#1, #3] + →Arrange (#1, #3) + →Stream l3 →Arrange (#0, #1) →Map/Filter/Project Project: #0, #1, #3 @@ -935,32 +1203,32 @@ mz_internal.mz_console_cluster_utilization_overview: →Differential Join %1:mz_cluster_replica_name_history[#1{id}] » %0[#0{replica_id}] after %0: Project: #0, #4, #1, #3 - Filter: ((#4{bucket_start} + 08:00:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) + Filter: ((#4{bucket_start} + 01:00:00) >= coalesce(#1{occurred_at}, 1970-01-01 00:00:00 UTC)) →Arrange (#0{replica_id}) →Distinct GroupAggregate →Fused with Child Map/Filter/Project - Project: #0, #3 - →Read l2 + Project: #1, #3 + →Read l3 →Arranged mz_internal.mz_cluster_replica_name_history - cte l4 = - →Differential Join %1[#1, #0] » %0:l3[#0{replica_id}, #3{bucket_start}] - →Arrange (#0{replica_id}, #3{bucket_start}) - →Stream l3 + cte l5 = + →Differential Join %1[#1, #0] » %0:l4[#1{replica_id}, #3{bucket_start}] + →Arrange (#1{replica_id}, #3{bucket_start}) + →Stream l4 →Arrange (#1, #0) →Temporally-Bucketed Non-incremental GroupAggregate Aggregation: jsonb_agg[order_by=[]](row(coalesce(jsonbable_to_jsonb(jsonb_build_object("replicaId", jsonbable_to_jsonb(#0{replica_id}), "occurredAt", jsonbable_to_jsonb(timestamp_with_time_zone_to_text(#2{occurred_at})), "status", "offline", "reason", jsonbable_to_jsonb(#1{reason}))), json_null))) Key: Project: #3, #0 - Map: timestamptz_bin(08:00:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) + Map: timestamptz_bin(01:00:00, #2{occurred_at}, 1970-01-01 00:00:00 UTC) →Fused with Child Map/Filter/Project Project: #0, #3, #4 - Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(08:00:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 14 days))) + Filter: (#1{process_id} = 0) AND (#2{status} = "offline") AND (mz_now() <= timestamp_tz_to_mz_timestamp((timestamptz_bin(01:00:00, #4{occurred_at}, 1970-01-01 00:00:00 UTC) + 14 days))) →Arranged mz_internal.mz_cluster_replica_status_history Key: (#0{replica_id}) →Return →Map/Filter/Project Project: #1, #0, #2..=#5, #10, #8, #9, #11..=#13, #6, #7, #17, #18, #16, #15, #14 - Map: (#1{bucket_start} + 08:00:00) + Map: (#1{bucket_start} + 01:00:00) →Union →Map/Filter/Project Project: #0..=#17 @@ -968,14 +1236,14 @@ mz_internal.mz_console_cluster_utilization_overview: →Consolidating Union →Negate Diffs →Fused with Child Map/Filter/Project - Project: #0, #3, #2, #1, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 - →Read l4 + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l5 →Fused with Child Map/Filter/Project - Project: #0, #3, #2, #1, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 - →Read l3 + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#16 + →Read l4 →Fused with Child Map/Filter/Project - Project: #0, #3, #2, #1, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 - →Read l4 + Project: #1, #3, #2, #0, #5, #4, #7, #6, #9..=#11, #8, #13, #12, #14..=#17 + →Read l5 Used Indexes: - mz_catalog.mz_cluster_replicas_ind (*** full scan ***) @@ -4294,7 +4562,7 @@ mz_catalog.mz_indexes: Project: #2, #0, #1, #3, #4, #7, #5, #6 Map: "s1" →Arrange (#1{name}) (#2{on_schema}, #3{on_name}) - →Constant (76 rows) + →Constant (78 rows) →Arrange (empty key) (#1{name}) →Fused with Child Map/Filter/Project Project: #4, #3 @@ -4795,45 +5063,45 @@ Target cluster: mz_catalog_server EOF query T multiline -EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_pending_cluster_replicas"; +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_overridden_system_parameters"; ---- -mz_internal.mz_pending_cluster_replicas: +mz_internal.mz_overridden_system_parameters: →Read mz_internal.mz_catalog_raw Source mz_internal.mz_catalog_raw - project=(#1) - filter=((true = text_to_boolean((((((#0{data} -> "value") -> "config") -> "location") -> "Managed") ->> "pending"))) AND ("ClusterReplica" = (#0{data} ->> "kind"))) - map=(parse_catalog_id(((#0{data} -> "key") -> "id"))) + project=(#1, #2) + filter=(("ServerConfiguration" = (#0{data} ->> "kind"))) + map=(((#0{data} -> "key") ->> "name"), ((#0{data} -> "value") ->> "value")) Target cluster: mz_catalog_server EOF query T multiline -EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_replica_system_parameters"; +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_pending_cluster_replicas"; ---- -mz_internal.mz_replica_system_parameters: +mz_internal.mz_pending_cluster_replicas: →Read mz_internal.mz_catalog_raw Source mz_internal.mz_catalog_raw - project=(#2..=#4) - filter=(("ReplicaSystemConfiguration" = (#0{data} ->> "kind"))) - map=((#0{data} -> "key"), parse_catalog_id((#1 -> "replica_id")), (#1 ->> "name"), ((#0{data} -> "value") ->> "value")) + project=(#1) + filter=((true = text_to_boolean((((((#0{data} -> "value") -> "config") -> "location") -> "Managed") ->> "pending"))) AND ("ClusterReplica" = (#0{data} ->> "kind"))) + map=(parse_catalog_id(((#0{data} -> "key") -> "id"))) Target cluster: mz_catalog_server EOF query T multiline -EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_overridden_system_parameters"; +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_replica_system_parameters"; ---- -mz_internal.mz_overridden_system_parameters: +mz_internal.mz_replica_system_parameters: →Read mz_internal.mz_catalog_raw Source mz_internal.mz_catalog_raw - project=(#1, #2) - filter=(("ServerConfiguration" = (#0{data} ->> "kind"))) - map=(((#0{data} -> "key") ->> "name"), ((#0{data} -> "value") ->> "value")) + project=(#2..=#4) + filter=(("ReplicaSystemConfiguration" = (#0{data} ->> "kind"))) + map=((#0{data} -> "key"), parse_catalog_id((#1 -> "replica_id")), (#1 ->> "name"), ((#0{data} -> "value") ->> "value")) Target cluster: mz_catalog_server @@ -7257,6 +7525,36 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_console_cluster_utilization_overview_24h"; +---- +Explained Query (fast path): + →Map/Filter/Project + Project: #1..=#17, #0, #18 + →Indexed mz_internal.mz_console_cluster_utilization_overview_24h (using mz_internal.mz_console_cluster_utilization_overview_24h_ind) + +Used Indexes: + - mz_internal.mz_console_cluster_utilization_overview_24h_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_console_cluster_utilization_overview_3h"; +---- +Explained Query (fast path): + →Map/Filter/Project + Project: #1, #0, #2..=#9 + →Indexed mz_internal.mz_console_cluster_utilization_overview_3h (using mz_internal.mz_console_cluster_utilization_overview_3h_ind) + +Used Indexes: + - mz_internal.mz_console_cluster_utilization_overview_3h_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_global_frontiers"; ---- diff --git a/test/sqllogictest/cluster.slt b/test/sqllogictest/cluster.slt index b3015f21d86b8..0b932d6279276 100644 --- a/test/sqllogictest/cluster.slt +++ b/test/sqllogictest/cluster.slt @@ -420,7 +420,7 @@ CREATE CLUSTER test REPLICAS (foo (SIZE 'scale=1,workers=1')); query I SELECT COUNT(name) FROM mz_indexes; ---- -300 +302 statement ok DROP CLUSTER test CASCADE @@ -428,7 +428,7 @@ DROP CLUSTER test CASCADE query T SELECT COUNT(name) FROM mz_indexes; ---- -268 +270 simple conn=mz_system,user=mz_system ALTER CLUSTER quickstart OWNER TO materialize diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index a2c8bb57f7d9d..69d7ef2d9b457 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -381,6 +381,14 @@ mz_console_cluster_utilization_overview VIEW materialize mz_internal +mz_console_cluster_utilization_overview_24h +VIEW +materialize +mz_internal +mz_console_cluster_utilization_overview_3h +VIEW +materialize +mz_internal mz_frontiers SOURCE materialize diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 980f20663c1b9..3d180a65d6359 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -37,9 +37,9 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangemen mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id") mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") @@ -57,13 +57,15 @@ mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s738␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s740␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s510␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s736␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s735␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s734␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s475␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") @@ -71,7 +73,7 @@ mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_ch mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s722␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s478␠AS␠"mz_catalog"."mz_indexes"]␠("id") mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s472␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s534␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") @@ -79,10 +81,10 @@ mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_messag mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s819␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s739␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s823␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s473␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s545␠AS␠"mz_internal"."mz_object_history"]␠("id") mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s546␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") @@ -91,14 +93,14 @@ mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[ mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s706␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s702␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s812␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s816␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s496␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s476␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s509␠AS␠"mz_catalog"."mz_secrets"]␠("name") mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s590␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s592␠AS␠"mz_internal"."mz_show_clusters"]␠("name") mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s591␠AS␠"mz_internal"."mz_show_columns"]␠("id") mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") @@ -381,6 +383,35 @@ mz_console_cluster_utilization_overview name mz_console_cluster_utilization_overview offline_events mz_console_cluster_utilization_overview replica_id mz_console_cluster_utilization_overview size +mz_console_cluster_utilization_overview_24h bucket_end +mz_console_cluster_utilization_overview_24h bucket_start +mz_console_cluster_utilization_overview_24h cluster_id +mz_console_cluster_utilization_overview_24h disk_percent +mz_console_cluster_utilization_overview_24h heap_percent +mz_console_cluster_utilization_overview_24h max_cpu_at +mz_console_cluster_utilization_overview_24h max_cpu_percent +mz_console_cluster_utilization_overview_24h max_disk_at +mz_console_cluster_utilization_overview_24h max_heap_at +mz_console_cluster_utilization_overview_24h max_memory_and_disk_at +mz_console_cluster_utilization_overview_24h max_memory_and_disk_disk_percent +mz_console_cluster_utilization_overview_24h max_memory_and_disk_memory_percent +mz_console_cluster_utilization_overview_24h max_memory_at +mz_console_cluster_utilization_overview_24h memory_and_disk_percent +mz_console_cluster_utilization_overview_24h memory_percent +mz_console_cluster_utilization_overview_24h name +mz_console_cluster_utilization_overview_24h offline_events +mz_console_cluster_utilization_overview_24h replica_id +mz_console_cluster_utilization_overview_24h size +mz_console_cluster_utilization_overview_3h cluster_id +mz_console_cluster_utilization_overview_3h cpu_percent +mz_console_cluster_utilization_overview_3h disk_percent +mz_console_cluster_utilization_overview_3h heap_percent +mz_console_cluster_utilization_overview_3h memory_and_disk_percent +mz_console_cluster_utilization_overview_3h memory_percent +mz_console_cluster_utilization_overview_3h name +mz_console_cluster_utilization_overview_3h occurred_at +mz_console_cluster_utilization_overview_3h replica_id +mz_console_cluster_utilization_overview_3h size mz_databases id mz_databases name mz_databases oid diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index 5e49b9b5cef09..6e34708a2b546 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1200,3 +1200,7 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17093 generate_series_unoptimized 17094 generate_series_unoptimized 17095 mz_overridden_system_parameters +17096 mz_console_cluster_utilization_overview_3h +17097 mz_console_cluster_utilization_overview_3h_ind +17098 mz_console_cluster_utilization_overview_24h +17099 mz_console_cluster_utilization_overview_24h_ind diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 16e86981ecad9..c65347e1635d8 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -640,6 +640,8 @@ mz_cluster_replica_utilization "" mz_cluster_replica_utilization_history "" mz_compute_hydration_statuses "" mz_console_cluster_utilization_overview "" +mz_console_cluster_utilization_overview_24h "" +mz_console_cluster_utilization_overview_3h "" mz_global_frontiers "" mz_hydration_statuses "" mz_index_advice "" @@ -830,7 +832,7 @@ test_table "" # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' -268 +270 # Create a second schema with the same table name as above > CREATE SCHEMA tester2 diff --git a/test/testdrive/explain-analyze.td b/test/testdrive/explain-analyze.td index 53d7ffefd6e67..c7b719bd4c2d4 100644 --- a/test/testdrive/explain-analyze.td +++ b/test/testdrive/explain-analyze.td @@ -72,19 +72,19 @@ operator total_elapsed " Stream " "Returning Map/Filter/Project" " Union" -" Read l4" +" Read l5" " Map/Filter/Project" " Consolidating Union" -" Read l3" +" Read l4" " Negate Diffs" -" Read l4" -"With l4 = Differential Join %1 » %0" +" Read l5" +"With l5 = Differential Join %1 » %0" " Arrange (#1, #0)" " Non-incremental GroupAggregate" " Arranged " -" Arrange (#0{replica_id}, #3{bucket_start})" -" Stream l3" -"l3 = Differential Join %1 » %0" +" Arrange (#1{replica_id}, #3{bucket_start})" +" Stream l4" +"l4 = Differential Join %1 » %0" " Arrange (#0, #1)" " Map/Filter/Project" " Non-monotonic TopK" @@ -92,44 +92,43 @@ operator total_elapsed " Arranged " " Arrange (#0{replica_id})" " Distinct GroupAggregate" -" Read l2" -" Arrange (#0, #3)" -" Stream l2" -"l2 = Delta Join [%0 » %1 » %2 » %3 » %4 » %5] [%1 » %0 » %2 » %3 » %4 » %5] [%2 » %0 » %1 » %3 » %4 » %5] [%3 » %0 » %1 » %2 » %4 » %5] [%4 » %0 » %1 » %2 » %3 » %5] [%5 » %0 » %1 » %2 » %3 » %4]" +" Read l3" +" Arrange (#1, #3)" +" Stream l3" +"l3 = Delta Join [%0 » %1 » %2 » %3 » %4 » %5] [%1 » %0 » %2 » %3 » %4 » %5] [%2 » %0 » %1 » %3 » %4 » %5] [%3 » %0 » %1 » %2 » %4 » %5] [%4 » %0 » %1 » %2 » %3 » %5] [%5 » %0 » %1 » %2 » %3 » %4]" " Arrange (#0{replica_id})" " Arranged l0" -" Arrange (#0{replica_id}, #3{bucket_start})" +" Arrange (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}, #4{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}, #4{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}, #3{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}, #3{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -" Arrange (#0{replica_id}) (#0{replica_id}, #3{bucket_start})" +" Read l2" +" Arrange (#1{replica_id}) (#1{replica_id}, #3{bucket_start})" " Map/Filter/Project" " Non-monotonic TopK" -" Read l1" -"l1 = Differential Join %0 » %1" -" Arrange (#1{replica_id})" -" Map/Filter/Project" -" Accumulable GroupAggregate" -" Map/Filter/Project" -" Delta Join [%0 » %2 » %1] [%1 » %0 » %2] [%2 » %0 » %1]" -" Arranged " -" Arranged " -" Arrange (#0{replica_id}) (#1{size})" -" Arranged l0" -" Arrange (#0{replica_id})" -" Arranged l0" +" Read l2" +"l2 = Differential Join %0 » %1" +" Accumulable GroupAggregate" +" Read l1" +" Bucketed Hierarchical GroupAggregate (buckets: 268435456 16777216 1048576 65536 4096 256 16)" +" Read l1" +"l1 = Map/Filter/Project" +" Delta Join [%0 » %2 » %1] [%1 » %0 » %2] [%2 » %0 » %1]" +" Arranged " +" Arranged " +" Arrange (#0{replica_id}) (#1{size})" +" Arranged l0" "l0 = Distinct GroupAggregate" " Union" " Arranged " diff --git a/test/testdrive/indexes.td b/test/testdrive/indexes.td index 07aa9746dbe31..a8089707e43f6 100644 --- a/test/testdrive/indexes.td +++ b/test/testdrive/indexes.td @@ -322,6 +322,8 @@ mz_compute_operator_durations_histogram_raw_s2_primary_idx mz_compute_operator_ mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx mz_compute_operator_hydration_statuses_per_worker mz_catalog_server {export_id,lir_id,worker_id} "" mz_cluster_prometheus_metrics_s2_primary_idx mz_cluster_prometheus_metrics mz_catalog_server {process_id,metric_name,labels} "" mz_connections_ind mz_connections mz_catalog_server {schema_id} "" +mz_console_cluster_utilization_overview_24h_ind mz_console_cluster_utilization_overview_24h mz_catalog_server {cluster_id} "" +mz_console_cluster_utilization_overview_3h_ind mz_console_cluster_utilization_overview_3h mz_catalog_server {cluster_id} "" mz_console_cluster_utilization_overview_ind mz_console_cluster_utilization_overview mz_catalog_server {cluster_id} "" mz_databases_ind mz_databases mz_catalog_server {name} "" mz_dataflow_addresses_per_worker_s2_primary_idx mz_dataflow_addresses_per_worker mz_catalog_server {id,worker_id} "" diff --git a/test/workload-replay/objects.txt b/test/workload-replay/objects.txt index 24fb3e4a016e0..8bdd9443e0bd7 100644 --- a/test/workload-replay/objects.txt +++ b/test/workload-replay/objects.txt @@ -479,6 +479,10 @@ mz_connection_oid mz_connections mz_connections_ind mz_console_cluster_utilization_overview +mz_console_cluster_utilization_overview_24h +mz_console_cluster_utilization_overview_24h_ind +mz_console_cluster_utilization_overview_3h +mz_console_cluster_utilization_overview_3h_ind mz_console_cluster_utilization_overview_ind mz_database_oid mz_databases