From da2f35adcad65b7e0fa9f768dbcebdf23659cfe7 Mon Sep 17 00:00:00 2001 From: Socrates Date: Sat, 13 Jun 2026 18:18:20 +0800 Subject: [PATCH 01/10] feat: add global system tables framework under sys database - Add GetOptions() to Catalog interface for catalog-level config access - FileSystemCatalog stores and exposes catalog_options - New GlobalSystemTableLoader with independent registry for sys tables - Implement sys.catalog_options, sys.all_table_options, sys.tables - Stub for sys.partitions (manifest aggregation to follow) - Extend SystemTablePath with is_global flag - TryParsePath detects sys/ paths for TableScan/TableRead routing - FileSystemCatalog handles sys database in ListTables, DatabaseExists, TableExists, LoadTableSchema Co-Authored-By: Claude Fable 5 --- include/paimon/catalog/catalog.h | 5 + src/paimon/CMakeLists.txt | 1 + src/paimon/core/catalog/catalog.cpp | 2 +- .../core/catalog/file_system_catalog.cpp | 39 +- src/paimon/core/catalog/file_system_catalog.h | 5 +- .../table/system/global_system_tables.cpp | 362 ++++++++++++++++++ .../core/table/system/global_system_tables.h | 117 ++++++ src/paimon/core/table/system/system_table.cpp | 28 +- src/paimon/core/table/system/system_table.h | 2 + 9 files changed, 553 insertions(+), 8 deletions(-) create mode 100644 src/paimon/core/table/system/global_system_tables.cpp create mode 100644 src/paimon/core/table/system/global_system_tables.h diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index 0ff9349bd..745756b71 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -183,6 +183,11 @@ class PAIMON_EXPORT Catalog { /// @return A shared pointer to the file system instance. virtual std::shared_ptr GetFileSystem() const = 0; + /// Returns the catalog-level options that were passed during catalog creation. + /// + /// @return A const reference to the map of catalog options (key-value pairs). + virtual const std::map& GetOptions() const = 0; + /// Loads the latest schema of a specified table. /// /// @note System tables will not be supported. diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index c883ec012..ecf2e20e4 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -341,6 +341,7 @@ set(PAIMON_CORE_SRCS core/table/source/data_evolution_batch_scan.cpp core/table/system/audit_log_system_table.cpp core/table/system/binlog_system_table.cpp + core/table/system/global_system_tables.cpp core/table/system/in_memory_system_table.cpp core/table/system/metadata_system_tables.cpp core/table/system/system_table.cpp diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 08b879c7a..80f93cb29 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -32,7 +32,7 @@ Result> Catalog::Create(const std::string& root_path, const std::map& options, const std::shared_ptr& file_system) { PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); - return std::make_unique(core_options.GetFileSystem(), root_path); + return std::make_unique(core_options.GetFileSystem(), root_path, options); } } // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index db9c2494e..eab4ae666 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/snapshot.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/system_table.h" #include "paimon/core/table/system/system_table_schema.h" #include "paimon/core/utils/branch_manager.h" @@ -47,8 +48,12 @@ struct ArrowSchema; namespace paimon { FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, - const std::string& warehouse) - : fs_(fs), warehouse_(warehouse), logger_(Logger::GetLogger("FileSystemCatalog")) {} + const std::string& warehouse, + const std::map& catalog_options) + : fs_(fs), + warehouse_(warehouse), + catalog_options_(catalog_options), + logger_(Logger::GetLogger("FileSystemCatalog")) {} Status FileSystemCatalog::CreateDatabase(const std::string& db_name, const std::map& options, @@ -88,13 +93,16 @@ Status FileSystemCatalog::CreateDatabaseImpl(const std::string& db_name, Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const { if (IsSystemDatabase(db_name)) { - return Status::NotImplemented( - "do not support checking DatabaseExists for system database."); + return true; } return fs_->Exists(NewDatabasePath(warehouse_, db_name)); } Result FileSystemCatalog::TableExists(const Identifier& identifier) const { + // Handle sys database global tables + if (IsSystemDatabase(identifier.GetDatabaseName())) { + return GlobalSystemTableLoader::IsSupported(identifier.GetTableName()); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, @@ -184,6 +192,10 @@ std::shared_ptr FileSystemCatalog::GetFileSystem() const { return fs_; } +const std::map& FileSystemCatalog::GetOptions() const { + return catalog_options_; +} + bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) { return db_name == SYSTEM_DATABASE_NAME; } @@ -228,7 +240,7 @@ Result> FileSystemCatalog::ListDatabases() const { Result> FileSystemCatalog::ListTables(const std::string& db_name) const { if (IsSystemDatabase(db_name)) { - return Status::NotImplemented("do not support listing tables for system database."); + return GlobalSystemTableLoader::GetSupportedTableNames(); } std::string database_path = NewDatabasePath(warehouse_, db_name); std::vector> file_status_list; @@ -261,6 +273,23 @@ Result FileSystemCatalog::TableExistsInFileSystem(const std::string& table Result> FileSystemCatalog::LoadTableSchema( const Identifier& identifier) const { + // Handle sys database global tables + if (IsSystemDatabase(identifier.GetDatabaseName())) { + if (!GlobalSystemTableLoader::IsSupported(identifier.GetTableName())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = const_cast(this); + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = catalog_options_; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (is_system_table) { PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index b9fbc2b47..3b93dc650 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -38,7 +38,8 @@ class Logger; class FileSystemCatalog : public Catalog { public: - FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse); + FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, + const std::map& catalog_options); Status CreateDatabase(const std::string& db_name, const std::map& options, @@ -61,6 +62,7 @@ class FileSystemCatalog : public Catalog { Result> LoadTableSchema(const Identifier& identifier) const override; std::string GetRootPath() const override; std::shared_ptr GetFileSystem() const override; + const std::map& GetOptions() const override; Result> GetTable(const Identifier& identifier) const override; Result> ListSnapshots(const Identifier& identifier, const std::string& branch) const override; @@ -92,6 +94,7 @@ class FileSystemCatalog : public Catalog { std::shared_ptr fs_; std::string warehouse_; + std::map catalog_options_; std::shared_ptr logger_; }; diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp new file mode 100644 index 000000000..4e91928bb --- /dev/null +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -0,0 +1,362 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/table/system/global_system_tables.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/data/binary_string.h" +#include "paimon/common/data/generic_row.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/status.h" + +namespace paimon { +namespace { + +// ============================================================================= +// Registry +// ============================================================================= + +using GlobalSystemTableFactory = + std::function>(const GlobalSystemTableContext&)>; + +struct GlobalSystemTableRegistryEntry { + std::string name; + GlobalSystemTableFactory factory; +}; + +const std::vector& GlobalSystemTableRegistry() { + static const std::vector registry = { + {CatalogOptionsSystemTable::kName, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + {AllTableOptionsSystemTable::kName, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + {TablesSystemTable::kName, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + {PartitionsSystemTable::kName, + [](const GlobalSystemTableContext& ctx) -> Result> { + return std::make_shared(ctx); + }}, + }; + return registry; +} + +// ============================================================================= +// Helpers for sys.tables and sys.partitions +// ============================================================================= + +VariantType StringValue(const std::string& value) { + return BinaryString::FromString(value, GetDefaultPool().get()); +} + +VariantType OptionalStringValue(const std::optional& value) { + if (!value) { + return NullType(); + } + return StringValue(value.value()); +} + +VariantType OptionalInt64Value(const std::optional& value) { + if (!value) { + return NullType(); + } + return value.value(); +} + +} // namespace + +// ============================================================================= +// GlobalSystemTableLoader +// ============================================================================= + +bool GlobalSystemTableLoader::IsSupported(const std::string& table_name) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return true; + } + } + return false; +} + +Result> GlobalSystemTableLoader::Load( + const std::string& table_name, const GlobalSystemTableContext& context) { + std::string normalized = StringUtils::ToLowerCase(table_name); + for (const auto& entry : GlobalSystemTableRegistry()) { + if (entry.name == normalized) { + return entry.factory(context); + } + } + return Status::NotImplemented("unsupported global system table: ", table_name); +} + +std::vector GlobalSystemTableLoader::GetSupportedTableNames() { + std::vector names; + names.reserve(GlobalSystemTableRegistry().size()); + for (const auto& entry : GlobalSystemTableRegistry()) { + names.push_back(entry.name); + } + return names; +} + +// ============================================================================= +// sys.catalog_options +// ============================================================================= + +CatalogOptionsSystemTable::CatalogOptionsSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/catalog_options"), context_(std::move(context)) {} + +std::string CatalogOptionsSystemTable::Name() const { + return kName; +} + +Result> CatalogOptionsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("key", arrow::utf8(), /*nullable=*/false), + arrow::field("value", arrow::utf8(), /*nullable=*/false), + }); +} + +Result> CatalogOptionsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + rows.reserve(context_.catalog_options.size()); + for (const auto& [key, value] : context_.catalog_options) { + GenericRow row(schema->num_fields()); + row.SetField(0, std::string_view(key)); + row.SetField(1, std::string_view(value)); + rows.push_back(std::move(row)); + } + return rows; +} + +// ============================================================================= +// sys.all_table_options +// ============================================================================= + +AllTableOptionsSystemTable::AllTableOptionsSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/all_table_options"), context_(std::move(context)) {} + +std::string AllTableOptionsSystemTable::Name() const { + return kName; +} + +Result> AllTableOptionsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("database_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_name", arrow::utf8(), /*nullable=*/false), + arrow::field("key", arrow::utf8(), /*nullable=*/false), + arrow::field("value", arrow::utf8(), /*nullable=*/false), + }); +} + +Result> AllTableOptionsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + + PAIMON_ASSIGN_OR_RAISE(std::vector databases, + context_.catalog->ListDatabases()); + for (const auto& db : databases) { + PAIMON_ASSIGN_OR_RAISE(std::vector tables, + context_.catalog->ListTables(db)); + for (const auto& table : tables) { + Identifier id(db, table); + auto schema_result = context_.catalog->LoadTableSchema(id); + if (!schema_result.ok()) { + continue; // skip tables with errors (e.g. dropped concurrently) + } + auto schema_ptr = schema_result.ValueUnsafe(); + auto data_schema = std::dynamic_pointer_cast(schema_ptr); + if (!data_schema) { + continue; + } + for (const auto& [key, value] : data_schema->Options()) { + GenericRow row(schema->num_fields()); + row.SetField(0, std::string_view(db)); + row.SetField(1, std::string_view(table)); + row.SetField(2, std::string_view(key)); + row.SetField(3, std::string_view(value)); + rows.push_back(std::move(row)); + } + } + } + return rows; +} + +// ============================================================================= +// sys.tables +// ============================================================================= + +TablesSystemTable::TablesSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/tables"), context_(std::move(context)) {} + +std::string TablesSystemTable::Name() const { + return kName; +} + +Result> TablesSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("database_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_type", arrow::utf8(), /*nullable=*/false), + arrow::field("partitioned", arrow::boolean(), /*nullable=*/false), + arrow::field("primary_key", arrow::utf8(), /*nullable=*/false), + arrow::field("record_count", arrow::int64(), /*nullable=*/true), + arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true), + arrow::field("file_count", arrow::int64(), /*nullable=*/true), + arrow::field("last_file_creation_time", + arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/true), + }); +} + +Result> TablesSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + + PAIMON_ASSIGN_OR_RAISE(std::vector databases, + context_.catalog->ListDatabases()); + for (const auto& db : databases) { + PAIMON_ASSIGN_OR_RAISE(std::vector tables, + context_.catalog->ListTables(db)); + for (const auto& table : tables) { + Identifier id(db, table); + auto schema_result = context_.catalog->LoadTableSchema(id); + if (!schema_result.ok()) { + continue; + } + auto schema_ptr = schema_result.ValueUnsafe(); + auto data_schema = std::dynamic_pointer_cast(schema_ptr); + if (!data_schema) { + continue; + } + + // Determine table type + std::string table_type_str = "MANAGED"; + // Check if table has external path in options + const auto& options = data_schema->Options(); + // (simplified: could check for external path options) + + bool partitioned = !data_schema->PartitionKeys().empty(); + std::string primary_keys_str; + const auto& pks = data_schema->PrimaryKeys(); + for (size_t i = 0; i < pks.size(); ++i) { + if (i > 0) primary_keys_str += ","; + primary_keys_str += pks[i]; + } + + GenericRow row(schema->num_fields()); + row.SetField(0, std::string_view(db)); + row.SetField(1, std::string_view(table)); + row.SetField(2, StringValue(table_type_str)); + row.SetField(3, partitioned); + row.SetField(4, primary_keys_str.empty() + ? VariantType(NullType()) + : VariantType(StringValue(primary_keys_str))); + + // Try to get stats from latest snapshot + PAIMON_ASSIGN_OR_RAISE(std::string table_path, + context_.catalog->GetTableLocation(id)); + SnapshotManager snapshot_manager(context_.fs, table_path, + BranchManager::DEFAULT_MAIN_BRANCH); + auto snapshot_result = snapshot_manager.LatestSnapshot(); + if (snapshot_result.ok() && snapshot_result.ValueUnsafe()) { + const auto& snapshot = *snapshot_result.ValueUnsafe(); + row.SetField(5, OptionalInt64Value(snapshot.TotalRecordCount())); + // file_size and file_count not available from Snapshot alone; + // leave as null for now + row.SetField(6, NullType()); + row.SetField(7, NullType()); + row.SetField(8, NullType()); + } else { + row.SetField(5, NullType()); + row.SetField(6, NullType()); + row.SetField(7, NullType()); + row.SetField(8, NullType()); + } + + rows.push_back(std::move(row)); + } + } + return rows; +} + +// ============================================================================= +// sys.partitions +// ============================================================================= + +PartitionsSystemTable::PartitionsSystemTable(GlobalSystemTableContext context) + : InMemorySystemTable("sys/partitions"), context_(std::move(context)) {} + +std::string PartitionsSystemTable::Name() const { + return kName; +} + +Result> PartitionsSystemTable::ArrowSchema() const { + return arrow::schema({ + arrow::field("database_name", arrow::utf8(), /*nullable=*/false), + arrow::field("table_name", arrow::utf8(), /*nullable=*/false), + arrow::field("partition_name", arrow::utf8(), /*nullable=*/true), + arrow::field("record_count", arrow::int64(), /*nullable=*/true), + arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true), + arrow::field("file_count", arrow::int64(), /*nullable=*/true), + arrow::field("last_update_time", + arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/true), + }); +} + +Result> PartitionsSystemTable::BuildRows() const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + std::vector rows; + + // TODO(suxiaogang223): Implement partition-level aggregation using + // manifest entry reading (similar to FilesSystemTable::BuildRows() + // but grouped by partition). For now, return empty result set. + // + // The implementation should: + // 1. Enumerate all databases and tables + // 2. For each partitioned table, read latest snapshot's manifest entries + // 3. Group DataFileMeta entries by entry.Partition() + // 4. Aggregate: sum(file_size), sum(record_count), count files, + // max(creation_time) + + return rows; +} + +} // namespace paimon diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h new file mode 100644 index 000000000..b3dc37570 --- /dev/null +++ b/src/paimon/core/table/system/global_system_tables.h @@ -0,0 +1,117 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/core/table/system/in_memory_system_table.h" + +namespace paimon { +class Catalog; +class FileSystem; + +/// Context passed to global system table constructors, providing catalog-level +/// access for enumerating databases, tables, and reading metadata. +struct GlobalSystemTableContext { + Catalog* catalog; // non-owning pointer + std::shared_ptr fs; + std::string warehouse; + std::map catalog_options; +}; + +/// System table for `sys.catalog_options`, exposing catalog-level configuration +/// as key/value rows. +class CatalogOptionsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "catalog_options"; + + explicit CatalogOptionsSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// System table for `sys.all_table_options`, exposing all table options across +/// all databases as (database_name, table_name, key, value) rows. +class AllTableOptionsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "all_table_options"; + + explicit AllTableOptionsSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// System table for `sys.tables`, exposing metadata for all tables across all +/// databases including record counts and file statistics. +class TablesSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "tables"; + + explicit TablesSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// System table for `sys.partitions`, exposing partition-level file statistics +/// for all tables across all databases. +class PartitionsSystemTable : public InMemorySystemTable { + public: + static constexpr const char* kName = "partitions"; + + explicit PartitionsSystemTable(GlobalSystemTableContext context); + + std::string Name() const override; + Result> ArrowSchema() const override; + Result> BuildRows() const override; + + private: + GlobalSystemTableContext context_; +}; + +/// Loader for global system tables under the `sys` database. +/// +/// Maintains its own registry with a factory signature that receives a +/// GlobalSystemTableContext instead of a per-table TableSchema. +class GlobalSystemTableLoader { + public: + static bool IsSupported(const std::string& table_name); + + static Result> Load( + const std::string& table_name, const GlobalSystemTableContext& context); + + static std::vector GetSupportedTableNames(); +}; + +} // namespace paimon diff --git a/src/paimon/core/table/system/system_table.cpp b/src/paimon/core/table/system/system_table.cpp index 87e1c3b81..4d805dfb1 100644 --- a/src/paimon/core/table/system/system_table.cpp +++ b/src/paimon/core/table/system/system_table.cpp @@ -30,6 +30,7 @@ #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/system/audit_log_system_table.h" #include "paimon/core/table/system/binlog_system_table.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/table/system/metadata_system_tables.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/status.h" @@ -179,6 +180,17 @@ Result> SystemTableLoader::Load( Result> SystemTableLoader::TryParsePath(const std::string& path) { std::string table_name = PathUtil::GetName(path); + std::string parent = PathUtil::GetParentDirPath(path); + std::string parent_name = PathUtil::GetName(parent); + + // Detect global system table paths: /sys/ + if (parent_name == "sys") { + SystemTablePath system_table_path; + system_table_path.is_global = true; + system_table_path.system_table_name = table_name; + return std::optional(std::move(system_table_path)); + } + Identifier identifier(table_name); PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); if (!is_system_table) { @@ -188,7 +200,6 @@ Result> SystemTableLoader::TryParsePath(const std PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, identifier.GetSystemTableName()); - std::string parent = PathUtil::GetParentDirPath(path); SystemTablePath system_table_path; system_table_path.table_path = PathUtil::JoinPath(parent, data_table_name); system_table_path.branch = std::move(branch); @@ -204,6 +215,21 @@ Result> SystemTableLoader::LoadFromPath( return Status::Invalid("path is not a system table path: ", path); } const auto& parsed = system_table_path.value(); + + // Handle global system tables (under sys/ directory) + if (parsed.is_global) { + GlobalSystemTableContext context; + context.fs = fs; + // The warehouse is the grandparent of the sys/ path + context.warehouse = PathUtil::GetParentDirPath(PathUtil::GetParentDirPath(path)); + context.catalog_options = dynamic_options; + // Note: context.catalog is intentionally left as nullptr here. + // Global tables loaded from path do not have a Catalog reference and + // cannot enumerate databases/tables. Only tables that don't require + // catalog enumeration (e.g. catalog_options) will work in this path. + return GlobalSystemTableLoader::Load(parsed.system_table_name, context); + } + SchemaManager schema_manager(fs, parsed.table_path, parsed.branch.value_or(BranchManager::DEFAULT_MAIN_BRANCH)); PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, diff --git a/src/paimon/core/table/system/system_table.h b/src/paimon/core/table/system/system_table.h index 5db20789e..3f897460c 100644 --- a/src/paimon/core/table/system/system_table.h +++ b/src/paimon/core/table/system/system_table.h @@ -42,6 +42,8 @@ struct SystemTablePath { std::optional branch; /// System table name, for example `options` or `snapshots`. std::string system_table_name; + /// Whether this is a global system table under the `sys` database. + bool is_global = false; }; /// Base interface for table-scoped system tables such as `T$options` and `T$snapshots`. From 40c237fcdd0e5698dfe8256c3aeb0e675861f0c3 Mon Sep 17 00:00:00 2001 From: Socrates Date: Sun, 14 Jun 2026 00:03:56 +0800 Subject: [PATCH 02/10] fix: remove unused helper functions in global system tables --- .../core/table/system/global_system_tables.cpp | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 4e91928bb..d49e49936 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -86,20 +86,6 @@ VariantType StringValue(const std::string& value) { return BinaryString::FromString(value, GetDefaultPool().get()); } -VariantType OptionalStringValue(const std::optional& value) { - if (!value) { - return NullType(); - } - return StringValue(value.value()); -} - -VariantType OptionalInt64Value(const std::optional& value) { - if (!value) { - return NullType(); - } - return value.value(); -} - } // namespace // ============================================================================= @@ -298,7 +284,9 @@ Result> TablesSystemTable::BuildRows() const { auto snapshot_result = snapshot_manager.LatestSnapshot(); if (snapshot_result.ok() && snapshot_result.ValueUnsafe()) { const auto& snapshot = *snapshot_result.ValueUnsafe(); - row.SetField(5, OptionalInt64Value(snapshot.TotalRecordCount())); + auto total_count = snapshot.TotalRecordCount(); + row.SetField(5, total_count ? VariantType(total_count.value()) + : VariantType(NullType())); // file_size and file_count not available from Snapshot alone; // leave as null for now row.SetField(6, NullType()); From af38088b30743b8b6190c1e2b654c1a728aa3a81 Mon Sep 17 00:00:00 2001 From: Socrates Date: Sun, 14 Jun 2026 00:04:49 +0800 Subject: [PATCH 03/10] fix: replace ValueUnsafe() with value() in global system tables --- src/paimon/core/table/system/global_system_tables.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index d49e49936..2b158c1b1 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -188,7 +188,7 @@ Result> AllTableOptionsSystemTable::BuildRows() const { if (!schema_result.ok()) { continue; // skip tables with errors (e.g. dropped concurrently) } - auto schema_ptr = schema_result.ValueUnsafe(); + auto schema_ptr = schema_result.value(); auto data_schema = std::dynamic_pointer_cast(schema_ptr); if (!data_schema) { continue; @@ -247,7 +247,7 @@ Result> TablesSystemTable::BuildRows() const { if (!schema_result.ok()) { continue; } - auto schema_ptr = schema_result.ValueUnsafe(); + auto schema_ptr = schema_result.value(); auto data_schema = std::dynamic_pointer_cast(schema_ptr); if (!data_schema) { continue; @@ -282,8 +282,8 @@ Result> TablesSystemTable::BuildRows() const { SnapshotManager snapshot_manager(context_.fs, table_path, BranchManager::DEFAULT_MAIN_BRANCH); auto snapshot_result = snapshot_manager.LatestSnapshot(); - if (snapshot_result.ok() && snapshot_result.ValueUnsafe()) { - const auto& snapshot = *snapshot_result.ValueUnsafe(); + if (snapshot_result.ok() && snapshot_result.value()) { + const auto& snapshot = *snapshot_result.value(); auto total_count = snapshot.TotalRecordCount(); row.SetField(5, total_count ? VariantType(total_count.value()) : VariantType(NullType())); From 633ce7b3f6fb2afa9d00d24e3d0804f854237fa3 Mon Sep 17 00:00:00 2001 From: Socrates Date: Sun, 14 Jun 2026 00:06:05 +0800 Subject: [PATCH 04/10] fix: add default value for catalog_options in FileSystemCatalog constructor --- src/paimon/core/catalog/file_system_catalog.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 3b93dc650..2a4656cc1 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -39,7 +39,7 @@ class Logger; class FileSystemCatalog : public Catalog { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options); + const std::map& catalog_options = {}); Status CreateDatabase(const std::string& db_name, const std::map& options, From 6514d0928478efb6eb9457e6d8c1dbc594db23e8 Mon Sep 17 00:00:00 2001 From: Socrates Date: Sun, 14 Jun 2026 00:08:25 +0800 Subject: [PATCH 05/10] fix: update TestInvalidList to expect global system table names --- .../core/catalog/file_system_catalog_test.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 8377493e5..a84296221 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -16,6 +16,8 @@ #include "paimon/core/catalog/file_system_catalog.h" +#include + #include "arrow/api.h" #include "arrow/c/abi.h" #include "arrow/c/bridge.h" @@ -606,8 +608,16 @@ TEST(FileSystemCatalogTest, TestInvalidList) { auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str()); - ASSERT_NOK_WITH_MSG(catalog.ListTables("sys"), - "do not support listing tables for system database."); + ASSERT_OK_AND_ASSIGN(auto sys_tables, catalog.ListTables("sys")); + ASSERT_FALSE(sys_tables.empty()); + // Verify expected global system table names are present + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "catalog_options") != + sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "all_table_options") != + sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "partitions") != + sys_tables.end()); } TEST(FileSystemCatalogTest, TestValidateTableSchema) { From 92de892f5adaae58af3bba558cfb72fb9b604d51 Mon Sep 17 00:00:00 2001 From: Socrates Date: Sun, 14 Jun 2026 00:42:16 +0800 Subject: [PATCH 06/10] improve: add table_type detection and clearer TODO for manifest-dependent fields --- .../core/table/system/global_system_tables.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 2b158c1b1..4da161a30 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -33,6 +33,7 @@ #include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/core_options.h" +#include "paimon/defs.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" @@ -253,11 +254,12 @@ Result> TablesSystemTable::BuildRows() const { continue; } - // Determine table type + // Determine table type: EXTERNAL if data-file.external-paths is set std::string table_type_str = "MANAGED"; - // Check if table has external path in options - const auto& options = data_schema->Options(); - // (simplified: could check for external path options) + const auto& opts = data_schema->Options(); + if (opts.find(Options::DATA_FILE_EXTERNAL_PATHS) != opts.end()) { + table_type_str = "EXTERNAL"; + } bool partitioned = !data_schema->PartitionKeys().empty(); std::string primary_keys_str; @@ -287,8 +289,10 @@ Result> TablesSystemTable::BuildRows() const { auto total_count = snapshot.TotalRecordCount(); row.SetField(5, total_count ? VariantType(total_count.value()) : VariantType(NullType())); - // file_size and file_count not available from Snapshot alone; - // leave as null for now + // TODO(suxiaogang223): Populate file_size_in_bytes, file_count, and + // last_file_creation_time by reading manifest entries. This requires + // the manifest reading infrastructure from the files/manifests system + // tables PR (codex/system-table-files-manifests-pr4). row.SetField(6, NullType()); row.SetField(7, NullType()); row.SetField(8, NullType()); From 94a2781e868d72decdf650bb424cdb2d81b6ed05 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 2 Jul 2026 16:58:15 +0800 Subject: [PATCH 07/10] feat: populate sys.tables file stats via manifest reading Replace the TODO stubs in TablesSystemTable::BuildRows() with actual manifest entry aggregation. The new AggregateFileStats() helper reads the latest snapshot data files and computes record_count, file_size, file_count, and last_file_creation_time. Co-Authored-By: Claude Fable 5 --- .../table/system/global_system_tables.cpp | 153 ++++++++++++++++-- 1 file changed, 137 insertions(+), 16 deletions(-) diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 4da161a30..1121a6820 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -34,11 +34,21 @@ #include "paimon/common/utils/path_util.h" #include "paimon/core/core_options.h" #include "paimon/defs.h" +#include "paimon/core/io/data_file_meta.h" +#include "paimon/core/manifest/file_entry.h" +#include "paimon/core/manifest/file_kind.h" +#include "paimon/core/manifest/manifest_entry.h" +#include "paimon/core/manifest/manifest_file.h" +#include "paimon/core/manifest/manifest_file_meta.h" +#include "paimon/core/manifest/manifest_list.h" #include "paimon/core/schema/schema_manager.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/snapshot.h" #include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/field_mapping.h" +#include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" +#include "paimon/data/timestamp.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -87,6 +97,111 @@ VariantType StringValue(const std::string& value) { return BinaryString::FromString(value, GetDefaultPool().get()); } +// Aggregated file-level statistics for a table or partition. +struct FileStats { + int64_t record_count = 0; + int64_t file_size_in_bytes = 0; + int64_t file_count = 0; + int64_t last_file_creation_time_millis = 0; +}; + +// Read the latest snapshot's data files and aggregate statistics. +// Returns an empty map if no snapshot or no data files exist. +Result> AggregateFileStats( + const std::shared_ptr& fs, const std::string& table_path, + const std::map& options) { + std::map result; + + SnapshotManager snapshot_manager(fs, table_path, + BranchManager::DEFAULT_MAIN_BRANCH); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, + snapshot_manager.LatestSnapshot()); + if (!snapshot) { + return result; + } + + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(options)); + + // Use SchemaManager to load the latest schema for field/partition info + SchemaManager schema_mgr(fs, table_path, BranchManager::DEFAULT_MAIN_BRANCH); + auto latest_schema_result = schema_mgr.Latest(); + if (!latest_schema_result.ok() || !latest_schema_result.value()) { + return result; + } + auto table_schema = *latest_schema_result.value(); + + auto pool = GetDefaultPool(); + + std::shared_ptr arrow_schema = + DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, + core_options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + table_path, arrow_schema, table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), + core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), + core_options.LegacyPartitionNameEnabled(), external_paths, + global_index_external_path, core_options.IndexFileInDataFileDir(), pool)); + + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr manifest_list, + ManifestList::Create(fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, pool)); + + std::vector manifests; + PAIMON_RETURN_NOT_OK( + manifest_list->ReadDataManifests(*snapshot, &manifests)); + + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_schema, + FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr manifest_file, + ManifestFile::Create(fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetManifestTargetFileSize(), pool, + core_options, partition_schema)); + + std::vector entries; + for (const auto& manifest : manifests) { + PAIMON_RETURN_NOT_OK( + manifest_file->Read(manifest.FileName(), /*filter=*/nullptr, &entries)); + } + + std::vector merged_entries; + PAIMON_RETURN_NOT_OK(FileEntry::MergeEntries(entries, &merged_entries)); + + for (const auto& entry : merged_entries) { + if (!(entry.Kind() == FileKind::Add())) { + continue; + } + const auto& file = entry.File(); + + // Use empty string key for unpartitioned tables + std::string partition_key; + if (entry.Partition().GetFieldCount() > 0) { + partition_key = "partitioned"; + } + + auto& stats = result[partition_key]; + stats.record_count += file->row_count; + stats.file_size_in_bytes += file->file_size; + stats.file_count++; + int64_t creation_millis = file->creation_time.GetMillisecond(); + if (creation_millis > stats.last_file_creation_time_millis) { + stats.last_file_creation_time_millis = creation_millis; + } + } + + return result; +} + } // namespace // ============================================================================= @@ -278,24 +393,30 @@ Result> TablesSystemTable::BuildRows() const { ? VariantType(NullType()) : VariantType(StringValue(primary_keys_str))); - // Try to get stats from latest snapshot + // Get table path and aggregate file stats from manifest entries PAIMON_ASSIGN_OR_RAISE(std::string table_path, context_.catalog->GetTableLocation(id)); - SnapshotManager snapshot_manager(context_.fs, table_path, - BranchManager::DEFAULT_MAIN_BRANCH); - auto snapshot_result = snapshot_manager.LatestSnapshot(); - if (snapshot_result.ok() && snapshot_result.value()) { - const auto& snapshot = *snapshot_result.value(); - auto total_count = snapshot.TotalRecordCount(); - row.SetField(5, total_count ? VariantType(total_count.value()) - : VariantType(NullType())); - // TODO(suxiaogang223): Populate file_size_in_bytes, file_count, and - // last_file_creation_time by reading manifest entries. This requires - // the manifest reading infrastructure from the files/manifests system - // tables PR (codex/system-table-files-manifests-pr4). - row.SetField(6, NullType()); - row.SetField(7, NullType()); - row.SetField(8, NullType()); + + auto file_stats_result = + AggregateFileStats(context_.fs, table_path, data_schema->Options()); + if (file_stats_result.ok()) { + auto& all_stats = file_stats_result.value(); + int64_t total_record = 0, total_size = 0, total_files = 0, + max_creation = 0; + for (const auto& [key, stats] : all_stats) { + total_record += stats.record_count; + total_size += stats.file_size_in_bytes; + total_files += stats.file_count; + if (stats.last_file_creation_time_millis > max_creation) { + max_creation = stats.last_file_creation_time_millis; + } + } + row.SetField(5, VariantType(total_record)); + row.SetField(6, VariantType(total_size)); + row.SetField(7, VariantType(total_files)); + row.SetField(8, max_creation > 0 + ? VariantType(Timestamp::FromEpochMillis(max_creation)) + : VariantType(NullType())); } else { row.SetField(5, NullType()); row.SetField(6, NullType()); From d9591183480b2e89d1dade20a86b934a27e62035 Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 2 Jul 2026 17:36:58 +0800 Subject: [PATCH 08/10] feat: implement sys.partitions with partition-level aggregation Replace the stub with actual partition-level file statistics using the AggregateFileStats helper. For each partitioned table, read manifest entries and emit one row per partition with record_count, file_size, file_count, and last_update_time. Co-Authored-By: Claude Fable 5 --- .../table/system/global_system_tables.cpp | 80 ++++++++++++++++--- 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 1121a6820..55bc196df 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -30,6 +30,7 @@ #include "paimon/catalog/identifier.h" #include "paimon/common/data/binary_string.h" #include "paimon/common/data/generic_row.h" +#include "paimon/common/utils/binary_row_partition_computer.h" #include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/core/core_options.h" @@ -152,7 +153,8 @@ Result> AggregateFileStats( PAIMON_ASSIGN_OR_RAISE( std::unique_ptr manifest_list, ManifestList::Create(fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, pool)); + core_options.GetManifestCompression(), path_factory, + core_options.GetCache(), pool)); std::vector manifests; PAIMON_RETURN_NOT_OK( @@ -183,10 +185,16 @@ Result> AggregateFileStats( } const auto& file = entry.File(); - // Use empty string key for unpartitioned tables + // Convert partition BinaryRow to string representation std::string partition_key; if (entry.Partition().GetFieldCount() > 0) { - partition_key = "partitioned"; + PAIMON_ASSIGN_OR_RAISE( + partition_key, + BinaryRowPartitionComputer::PartToSimpleString( + partition_schema, entry.Partition(), ",", + /*max_length=*/255, + /*legacy_partition_name_enabled=*/false)); + partition_key = "{" + partition_key + "}"; } auto& stats = result[partition_key]; @@ -458,17 +466,63 @@ Result> PartitionsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); std::vector rows; - // TODO(suxiaogang223): Implement partition-level aggregation using - // manifest entry reading (similar to FilesSystemTable::BuildRows() - // but grouped by partition). For now, return empty result set. - // - // The implementation should: - // 1. Enumerate all databases and tables - // 2. For each partitioned table, read latest snapshot's manifest entries - // 3. Group DataFileMeta entries by entry.Partition() - // 4. Aggregate: sum(file_size), sum(record_count), count files, - // max(creation_time) + PAIMON_ASSIGN_OR_RAISE(std::vector databases, + context_.catalog->ListDatabases()); + for (const auto& db : databases) { + PAIMON_ASSIGN_OR_RAISE(std::vector tables, + context_.catalog->ListTables(db)); + for (const auto& table : tables) { + Identifier id(db, table); + auto schema_result = context_.catalog->LoadTableSchema(id); + if (!schema_result.ok()) { + continue; + } + auto schema_ptr = schema_result.value(); + auto data_schema = std::dynamic_pointer_cast(schema_ptr); + if (!data_schema) { + continue; + } + // Only emit rows for partitioned tables + if (data_schema->PartitionKeys().empty()) { + continue; + } + + // Get table path and aggregate file stats by partition + auto table_path_result = context_.catalog->GetTableLocation(id); + if (!table_path_result.ok()) { + continue; + } + std::string table_path = table_path_result.value(); + + auto file_stats_result = + AggregateFileStats(context_.fs, table_path, data_schema->Options()); + if (!file_stats_result.ok()) { + continue; + } + + auto& stats_map = file_stats_result.value(); + for (const auto& [partition_key, stats] : stats_map) { + if (stats.file_count == 0) { + continue; + } + GenericRow row(schema->num_fields()); + row.SetField(0, std::string_view(db)); + row.SetField(1, std::string_view(table)); + row.SetField(2, partition_key.empty() + ? VariantType(NullType()) + : VariantType(StringValue(partition_key))); + row.SetField(3, VariantType(stats.record_count)); + row.SetField(4, VariantType(stats.file_size_in_bytes)); + row.SetField(5, VariantType(stats.file_count)); + row.SetField(6, stats.last_file_creation_time_millis > 0 + ? VariantType(Timestamp::FromEpochMillis( + stats.last_file_creation_time_millis)) + : VariantType(NullType())); + rows.push_back(std::move(row)); + } + } + } return rows; } From c5d3439806cc820fac364b461745e795bd970a3b Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 2 Jul 2026 20:59:58 +0800 Subject: [PATCH 09/10] test: add integration tests for global system tables Add 4 tests to SystemTableReadInteTest: - TestReadGlobalCatalogOptions: verifies sys.catalog_options schema and content - TestReadGlobalAllTableOptions: verifies sys.all_table_options with table options - TestReadGlobalTables: verifies sys.tables schema, table_type, partitioned, pk - TestReadGlobalPartitions: verifies sys.partitions returns empty for unpartitioned Tests use a ReadGlobalSystemTable helper that creates the GlobalSystemTableContext with a proper Catalog pointer. Co-Authored-By: Claude Fable 5 --- .../table/system/global_system_tables.cpp | 16 +- test/inte/read_inte_test.cpp | 237 ++++++++++++++++++ 2 files changed, 245 insertions(+), 8 deletions(-) diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index 55bc196df..e0c0a592a 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -319,10 +319,10 @@ Result> AllTableOptionsSystemTable::BuildRows() const { } for (const auto& [key, value] : data_schema->Options()) { GenericRow row(schema->num_fields()); - row.SetField(0, std::string_view(db)); - row.SetField(1, std::string_view(table)); - row.SetField(2, std::string_view(key)); - row.SetField(3, std::string_view(value)); + row.SetField(0, StringValue(db)); + row.SetField(1, StringValue(table)); + row.SetField(2, StringValue(key)); + row.SetField(3, StringValue(value)); rows.push_back(std::move(row)); } } @@ -393,8 +393,8 @@ Result> TablesSystemTable::BuildRows() const { } GenericRow row(schema->num_fields()); - row.SetField(0, std::string_view(db)); - row.SetField(1, std::string_view(table)); + row.SetField(0, StringValue(db)); + row.SetField(1, StringValue(table)); row.SetField(2, StringValue(table_type_str)); row.SetField(3, partitioned); row.SetField(4, primary_keys_str.empty() @@ -507,8 +507,8 @@ Result> PartitionsSystemTable::BuildRows() const { continue; } GenericRow row(schema->num_fields()); - row.SetField(0, std::string_view(db)); - row.SetField(1, std::string_view(table)); + row.SetField(0, StringValue(db)); + row.SetField(1, StringValue(table)); row.SetField(2, partition_key.empty() ? VariantType(NullType()) : VariantType(StringValue(partition_key))); diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index a267bfca2..70a829908 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include #include #include #include @@ -53,6 +54,7 @@ #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/defs.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" @@ -3353,4 +3355,239 @@ TEST_P(ReadInteTest, TestSpecificFs) { ASSERT_GT(io_count, 0); } +// ============================================================================= +// Global System Table Tests +// ============================================================================= + +namespace { + +Result ReadGlobalSystemTable( + const std::string& table_name, Catalog* catalog, + const std::shared_ptr& fs, const std::string& warehouse, + const std::map& options) { + GlobalSystemTableContext ctx; + ctx.catalog = catalog; + ctx.fs = fs; + ctx.warehouse = warehouse; + ctx.catalog_options = options; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(table_name, ctx)); + + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + + std::string sys_path = PathUtil::JoinPath(PathUtil::JoinPath(warehouse, "sys"), table_name); + + ScanContextBuilder scan_context_builder(sys_path); + scan_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scan_context, + scan_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + system_table->NewScan(scan_context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + + ReadContextBuilder read_context_builder(sys_path); + read_context_builder.SetOptions(options); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_context, + read_context_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + system_table->NewRead(read_context)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr batch_reader, + table_read->CreateReader(plan->Splits())); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + ReadResultCollector::CollectResult(batch_reader.get())); + return SystemTableReadResult(std::move(batch_reader), result); +} + +} // namespace + +TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {"custom.catalog.option", "test-value"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("catalog_options", catalog.get(), + catalog->GetFileSystem(), warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + auto key_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto value_array = std::dynamic_pointer_cast(struct_array->field(1)); + ASSERT_TRUE(key_array); + ASSERT_TRUE(value_array); + + // Build a map from the result + std::map result_map; + for (int64_t i = 0; i < struct_array->length(); ++i) { + result_map[key_array->GetString(i)] = value_array->GetString(i); + } + ASSERT_EQ(result_map["file-system"], "local"); + ASSERT_EQ(result_map["file.format"], "orc"); +} + +TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {"table.option.custom", "my-value"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + // Create a database and table + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // Verify basic enumeration works + ASSERT_OK_AND_ASSIGN(auto dbs, catalog->ListDatabases()); + ASSERT_TRUE(std::find(dbs.begin(), dbs.end(), "test_db") != dbs.end()); + ASSERT_OK_AND_ASSIGN(auto tbls, catalog->ListTables("test_db")); + ASSERT_TRUE(std::find(tbls.begin(), tbls.end(), "test_tbl") != tbls.end()); + + // Verify schema loads and has Options + ASSERT_OK_AND_ASSIGN(auto loaded_schema, + catalog->LoadTableSchema(Identifier("test_db", "test_tbl"))); + auto ds = std::dynamic_pointer_cast(loaded_schema); + ASSERT_TRUE(ds != nullptr) << "LoadTableSchema did not return DataSchema"; + ASSERT_FALSE(ds->Options().empty()) << "Table schema has no options"; + + // Directly test BuildRows + { + GlobalSystemTableContext ctx; + ctx.catalog = catalog.get(); + ctx.fs = catalog->GetFileSystem(); + ctx.warehouse = warehouse; + ctx.catalog_options = options; + AllTableOptionsSystemTable table(ctx); + ASSERT_OK_AND_ASSIGN(auto rows, table.BuildRows()); + ASSERT_GT(rows.size(), 0) << "BuildRows returned empty, expected at least 1 row"; + } + + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("all_table_options", catalog.get(), + catalog->GetFileSystem(), warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + ASSERT_GE(struct_array->length(), 1) << "result has " << struct_array->length() << " rows"; + + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto tbl_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto key_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto val_array = std::dynamic_pointer_cast(struct_array->field(3)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(tbl_array); + ASSERT_TRUE(key_array); + ASSERT_TRUE(val_array); + + // Verify that our table's options appear in the result + bool found_db = false; + bool found_format = false; + for (int64_t i = 0; i < struct_array->length(); ++i) { + auto db_name = std::string(db_array->GetString(i)); + auto tbl_name = std::string(tbl_array->GetString(i)); + if (db_name == "test_db" && tbl_name == "test_tbl") { + found_db = true; + auto key_str = std::string(key_array->GetString(i)); + if (key_str == "file.format") { + EXPECT_EQ(std::string(val_array->GetString(i)), "orc"); + found_format = true; + } + } + } + ASSERT_TRUE(found_db) << "test_db.test_tbl not found in sys.all_table_options"; + ASSERT_TRUE(found_format) << "file.format option not found in sys.all_table_options"; +} + +TEST(SystemTableReadInteTest, TestReadGlobalTables) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}, + {Options::BUCKET, "1"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + auto fs = catalog->GetFileSystem(); + + // Create a database and a PK table + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + auto typed_schema = arrow::schema({ + arrow::field("pk", arrow::utf8()), + arrow::field("v", arrow::int32()), + }); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{"pk"}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + ASSERT_OK_AND_ASSIGN(auto result, + ReadGlobalSystemTable("tables", catalog.get(), fs, warehouse, options)); + auto struct_array = SingleStructChunk(result); + ASSERT_TRUE(struct_array); + ASSERT_GE(struct_array->length(), 1); + + auto db_array = std::dynamic_pointer_cast(struct_array->field(0)); + auto tbl_array = std::dynamic_pointer_cast(struct_array->field(1)); + auto type_array = std::dynamic_pointer_cast(struct_array->field(2)); + auto part_array = std::dynamic_pointer_cast(struct_array->field(3)); + auto pk_array = std::dynamic_pointer_cast(struct_array->field(4)); + ASSERT_TRUE(db_array); + ASSERT_TRUE(tbl_array); + ASSERT_TRUE(type_array); + ASSERT_TRUE(part_array); + ASSERT_TRUE(pk_array); + + // Find our table by table name + bool found = false; + for (int64_t i = 0; i < struct_array->length(); ++i) { + if (std::string(tbl_array->GetString(i)) == "test_tbl") { + EXPECT_EQ(std::string(db_array->GetString(i)), "test_db"); + EXPECT_EQ(std::string(type_array->GetString(i)), "MANAGED"); + EXPECT_FALSE(part_array->Value(i)); + // pk is stored as BinaryString; check not null + EXPECT_FALSE(pk_array->IsNull(i)); + found = true; + } + } + ASSERT_TRUE(found) << "table not found in sys.tables"; +} + +TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { + std::map options = {{Options::FILE_SYSTEM, "local"}, + {Options::FILE_FORMAT, "orc"}, + {Options::MANIFEST_FORMAT, "orc"}}; + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); + ASSERT_OK_AND_ASSIGN(auto catalog, Catalog::Create(warehouse, options)); + auto fs = catalog->GetFileSystem(); + + // Create a database and an unpartitioned table (no partitions → empty result) + ASSERT_OK(catalog->CreateDatabase("test_db", options, /*ignore_if_exists=*/false)); + auto typed_schema = arrow::schema({arrow::field("f0", arrow::int32())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("test_db", "test_tbl"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // Unpartitioned tables are skipped by sys.partitions → empty result. + // CollectResult returns null shared_ptr when result is empty. + ASSERT_OK_AND_ASSIGN(auto part_result, + ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_FALSE(part_result.array) << "expected null array for empty partitions result"; +} + } // namespace paimon::test From ed5857aef6a47547ca4a8cc4bdf15c63a8c00d0c Mon Sep 17 00:00:00 2001 From: Socrates Date: Thu, 2 Jul 2026 21:06:19 +0800 Subject: [PATCH 10/10] style: apply clang-format fixes from pre-commit Co-Authored-By: Claude Fable 5 --- .../core/catalog/file_system_catalog.cpp | 5 +- src/paimon/core/catalog/file_system_catalog.h | 2 +- .../core/catalog/file_system_catalog_test.cpp | 3 +- .../table/system/global_system_tables.cpp | 97 ++++++++----------- .../core/table/system/global_system_tables.h | 4 +- test/inte/read_inte_test.cpp | 11 +-- 6 files changed, 50 insertions(+), 72 deletions(-) diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index eab4ae666..a1e9a9f65 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -283,9 +283,8 @@ Result> FileSystemCatalog::LoadTableSchema( context.fs = fs_; context.warehouse = warehouse_; context.catalog_options = catalog_options_; - PAIMON_ASSIGN_OR_RAISE( - std::shared_ptr system_table, - GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, system_table->ArrowSchema()); return std::make_shared(std::move(arrow_schema)); diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 2a4656cc1..7a2ad4a17 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -39,7 +39,7 @@ class Logger; class FileSystemCatalog : public Catalog { public: FileSystemCatalog(const std::shared_ptr& fs, const std::string& warehouse, - const std::map& catalog_options = {}); + const std::map& catalog_options = {}); Status CreateDatabase(const std::string& db_name, const std::map& options, diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index a84296221..f970fddb6 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -616,8 +616,7 @@ TEST(FileSystemCatalogTest, TestInvalidList) { ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "all_table_options") != sys_tables.end()); ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); - ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "partitions") != - sys_tables.end()); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "partitions") != sys_tables.end()); } TEST(FileSystemCatalogTest, TestValidateTableSchema) { diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index e0c0a592a..3e0a3e5df 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -31,10 +31,9 @@ #include "paimon/common/data/binary_string.h" #include "paimon/common/data/generic_row.h" #include "paimon/common/utils/binary_row_partition_computer.h" -#include "paimon/common/utils/string_utils.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" -#include "paimon/defs.h" #include "paimon/core/io/data_file_meta.h" #include "paimon/core/manifest/file_entry.h" #include "paimon/core/manifest/file_kind.h" @@ -50,6 +49,7 @@ #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/data/timestamp.h" +#include "paimon/defs.h" #include "paimon/memory/memory_pool.h" #include "paimon/status.h" @@ -113,16 +113,13 @@ Result> AggregateFileStats( const std::map& options) { std::map result; - SnapshotManager snapshot_manager(fs, table_path, - BranchManager::DEFAULT_MAIN_BRANCH); - PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, - snapshot_manager.LatestSnapshot()); + SnapshotManager snapshot_manager(fs, table_path, BranchManager::DEFAULT_MAIN_BRANCH); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, snapshot_manager.LatestSnapshot()); if (!snapshot) { return result; } - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, - CoreOptions::FromMap(options)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); // Use SchemaManager to load the latest schema for field/partition info SchemaManager schema_mgr(fs, table_path, BranchManager::DEFAULT_MAIN_BRANCH); @@ -144,31 +141,27 @@ Result> AggregateFileStats( std::shared_ptr path_factory, FileStorePathFactory::Create( table_path, arrow_schema, table_schema->PartitionKeys(), - core_options.GetPartitionDefaultName(), - core_options.GetFileFormat()->Identifier(), - core_options.DataFilePrefix(), - core_options.LegacyPartitionNameEnabled(), external_paths, - global_index_external_path, core_options.IndexFileInDataFileDir(), pool)); + core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), + pool)); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr manifest_list, - ManifestList::Create(fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetCache(), pool)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_list, + ManifestList::Create(fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetCache(), pool)); std::vector manifests; - PAIMON_RETURN_NOT_OK( - manifest_list->ReadDataManifests(*snapshot, &manifests)); + PAIMON_RETURN_NOT_OK(manifest_list->ReadDataManifests(*snapshot, &manifests)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr partition_schema, FieldMapping::GetPartitionSchema(arrow_schema, table_schema->PartitionKeys())); - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr manifest_file, - ManifestFile::Create(fs, core_options.GetManifestFormat(), - core_options.GetManifestCompression(), path_factory, - core_options.GetManifestTargetFileSize(), pool, - core_options, partition_schema)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr manifest_file, + ManifestFile::Create(fs, core_options.GetManifestFormat(), + core_options.GetManifestCompression(), path_factory, + core_options.GetManifestTargetFileSize(), pool, + core_options, partition_schema)); std::vector entries; for (const auto& manifest : manifests) { @@ -188,12 +181,10 @@ Result> AggregateFileStats( // Convert partition BinaryRow to string representation std::string partition_key; if (entry.Partition().GetFieldCount() > 0) { - PAIMON_ASSIGN_OR_RAISE( - partition_key, - BinaryRowPartitionComputer::PartToSimpleString( - partition_schema, entry.Partition(), ",", - /*max_length=*/255, - /*legacy_partition_name_enabled=*/false)); + PAIMON_ASSIGN_OR_RAISE(partition_key, BinaryRowPartitionComputer::PartToSimpleString( + partition_schema, entry.Partition(), ",", + /*max_length=*/255, + /*legacy_partition_name_enabled=*/false)); partition_key = "{" + partition_key + "}"; } @@ -301,11 +292,9 @@ Result> AllTableOptionsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); std::vector rows; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, - context_.catalog->ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, context_.catalog->ListDatabases()); for (const auto& db : databases) { - PAIMON_ASSIGN_OR_RAISE(std::vector tables, - context_.catalog->ListTables(db)); + PAIMON_ASSIGN_OR_RAISE(std::vector tables, context_.catalog->ListTables(db)); for (const auto& table : tables) { Identifier id(db, table); auto schema_result = context_.catalog->LoadTableSchema(id); @@ -351,8 +340,8 @@ Result> TablesSystemTable::ArrowSchema() const { arrow::field("record_count", arrow::int64(), /*nullable=*/true), arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true), arrow::field("file_count", arrow::int64(), /*nullable=*/true), - arrow::field("last_file_creation_time", - arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/true), + arrow::field("last_file_creation_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/true), }); } @@ -360,11 +349,9 @@ Result> TablesSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); std::vector rows; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, - context_.catalog->ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, context_.catalog->ListDatabases()); for (const auto& db : databases) { - PAIMON_ASSIGN_OR_RAISE(std::vector tables, - context_.catalog->ListTables(db)); + PAIMON_ASSIGN_OR_RAISE(std::vector tables, context_.catalog->ListTables(db)); for (const auto& table : tables) { Identifier id(db, table); auto schema_result = context_.catalog->LoadTableSchema(id); @@ -397,20 +384,17 @@ Result> TablesSystemTable::BuildRows() const { row.SetField(1, StringValue(table)); row.SetField(2, StringValue(table_type_str)); row.SetField(3, partitioned); - row.SetField(4, primary_keys_str.empty() - ? VariantType(NullType()) - : VariantType(StringValue(primary_keys_str))); + row.SetField(4, primary_keys_str.empty() ? VariantType(NullType()) + : VariantType(StringValue(primary_keys_str))); // Get table path and aggregate file stats from manifest entries - PAIMON_ASSIGN_OR_RAISE(std::string table_path, - context_.catalog->GetTableLocation(id)); + PAIMON_ASSIGN_OR_RAISE(std::string table_path, context_.catalog->GetTableLocation(id)); auto file_stats_result = AggregateFileStats(context_.fs, table_path, data_schema->Options()); if (file_stats_result.ok()) { auto& all_stats = file_stats_result.value(); - int64_t total_record = 0, total_size = 0, total_files = 0, - max_creation = 0; + int64_t total_record = 0, total_size = 0, total_files = 0, max_creation = 0; for (const auto& [key, stats] : all_stats) { total_record += stats.record_count; total_size += stats.file_size_in_bytes; @@ -457,8 +441,8 @@ Result> PartitionsSystemTable::ArrowSchema() cons arrow::field("record_count", arrow::int64(), /*nullable=*/true), arrow::field("file_size_in_bytes", arrow::int64(), /*nullable=*/true), arrow::field("file_count", arrow::int64(), /*nullable=*/true), - arrow::field("last_update_time", - arrow::timestamp(arrow::TimeUnit::MILLI), /*nullable=*/true), + arrow::field("last_update_time", arrow::timestamp(arrow::TimeUnit::MILLI), + /*nullable=*/true), }); } @@ -466,11 +450,9 @@ Result> PartitionsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); std::vector rows; - PAIMON_ASSIGN_OR_RAISE(std::vector databases, - context_.catalog->ListDatabases()); + PAIMON_ASSIGN_OR_RAISE(std::vector databases, context_.catalog->ListDatabases()); for (const auto& db : databases) { - PAIMON_ASSIGN_OR_RAISE(std::vector tables, - context_.catalog->ListTables(db)); + PAIMON_ASSIGN_OR_RAISE(std::vector tables, context_.catalog->ListTables(db)); for (const auto& table : tables) { Identifier id(db, table); auto schema_result = context_.catalog->LoadTableSchema(id); @@ -509,9 +491,8 @@ Result> PartitionsSystemTable::BuildRows() const { GenericRow row(schema->num_fields()); row.SetField(0, StringValue(db)); row.SetField(1, StringValue(table)); - row.SetField(2, partition_key.empty() - ? VariantType(NullType()) - : VariantType(StringValue(partition_key))); + row.SetField(2, partition_key.empty() ? VariantType(NullType()) + : VariantType(StringValue(partition_key))); row.SetField(3, VariantType(stats.record_count)); row.SetField(4, VariantType(stats.file_size_in_bytes)); row.SetField(5, VariantType(stats.file_count)); diff --git a/src/paimon/core/table/system/global_system_tables.h b/src/paimon/core/table/system/global_system_tables.h index b3dc37570..5615abf84 100644 --- a/src/paimon/core/table/system/global_system_tables.h +++ b/src/paimon/core/table/system/global_system_tables.h @@ -108,8 +108,8 @@ class GlobalSystemTableLoader { public: static bool IsSupported(const std::string& table_name); - static Result> Load( - const std::string& table_name, const GlobalSystemTableContext& context); + static Result> Load(const std::string& table_name, + const GlobalSystemTableContext& context); static std::vector GetSupportedTableNames(); }; diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 70a829908..bb331ea64 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -50,11 +50,11 @@ #include "paimon/core/table/source/data_split_impl.h" #include "paimon/core/table/source/deletion_file.h" #include "paimon/core/table/source/fallback_data_split.h" +#include "paimon/core/table/system/global_system_tables.h" #include "paimon/core/tag/tag.h" #include "paimon/data/decimal.h" #include "paimon/data/timestamp.h" #include "paimon/defs.h" -#include "paimon/core/table/system/global_system_tables.h" #include "paimon/fs/file_system.h" #include "paimon/fs/local/local_file_system.h" #include "paimon/memory/memory_pool.h" @@ -3362,9 +3362,8 @@ TEST_P(ReadInteTest, TestSpecificFs) { namespace { Result ReadGlobalSystemTable( - const std::string& table_name, Catalog* catalog, - const std::shared_ptr& fs, const std::string& warehouse, - const std::map& options) { + const std::string& table_name, Catalog* catalog, const std::shared_ptr& fs, + const std::string& warehouse, const std::map& options) { GlobalSystemTableContext ctx; ctx.catalog = catalog; ctx.fs = fs; @@ -3585,8 +3584,8 @@ TEST(SystemTableReadInteTest, TestReadGlobalPartitions) { // Unpartitioned tables are skipped by sys.partitions → empty result. // CollectResult returns null shared_ptr when result is empty. - ASSERT_OK_AND_ASSIGN(auto part_result, - ReadGlobalSystemTable("partitions", catalog.get(), fs, warehouse, options)); + ASSERT_OK_AND_ASSIGN(auto part_result, ReadGlobalSystemTable("partitions", catalog.get(), fs, + warehouse, options)); ASSERT_FALSE(part_result.array) << "expected null array for empty partitions result"; }