From 9010fa178e52fe7a51364c379d40023d2048b511 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Wed, 9 Sep 2026 12:33:48 +0800 Subject: [PATCH] [feature](fe) Govern external metadata cache memory ### What problem does this PR solve? Issue Number: None Related PR: #66717 Problem Summary: External catalog metadata caches were bounded primarily by entry count, so large Hive, Iceberg, and Paimon metadata objects could consume unpredictable FE heap and cause OOM. Add bounded retained-size estimation, hierarchical global/catalog/entry budgets, coordinated local reclamation, rejection observability, and catalog cache statistics. Keep estimation off the cache-hit path, preserve unbounded behavior when no weight limit is configured, and prevent managed cache owners from escaping connector lifecycle or registry concurrency. Include the Hive gateway's Hudi sibling HMS caches in shared aggregate accounting and statistics while preserving their existing entry-local configuration namespace. ### Release note Add configurable memory limits and usage statistics for external metadata caches. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.connector.cache.MetaCacheWeightGovernanceTest,org.apache.doris.connector.hive.HiveConnectorSiblingTest,org.apache.doris.connector.paimon.PaimonCatalogFactoryTest - EXTRA_FE_MODULES=hudi=fe-connector/fe-connector-hudi ./run-fe-ut.sh --run HudiConnectorHmsCacheTest,HudiResourceLifecycleTest,MetaCacheWeightGovernanceTest,HiveConnectorSiblingTest - Paimon partition weighting: 571 connector unit tests passed; one opt-in live-connectivity test skipped. FE/plugin packaging and Checkstyle passed. Three fresh JVM microbenchmark runs and independent local JOL calibration completed. - Behavior changed: Yes (external metadata caches can be governed by retained-size limits and expose usage/rejection statistics) - Does this need documentation: Yes (maintained in the Doris website documentation change) --- ...chema_catalog_meta_cache_stats_scanner.cpp | 66 +- .../schema_catalog_meta_cache_stats_scanner.h | 3 + .../java/org/apache/doris/common/Config.java | 5 + .../doris/connector/cache/CacheSpec.java | 202 ++++- .../connector/cache/CatalogMetaCache.java | 97 ++- .../cache/ConnectorMetadataCache.java | 18 +- .../doris/connector/cache/JvmSizeUtils.java | 233 ++++++ .../doris/connector/cache/MetaCache.java | 5 + .../cache/MetaCacheBudgetManager.java | 605 +++++++++++++++ .../connector/cache/MetaCacheDefinition.java | 25 + .../connector/cache/MetaCacheGovernance.java | 90 +++ .../cache/MetaCacheSizeEstimate.java | 60 ++ .../cache/MetaCacheSizeEstimator.java | 39 + .../cache/MetaCacheSizeEstimators.java | 35 + .../cache/ReflectiveObjectSizeEstimator.java | 428 +++++++++++ .../connector/cache/ScopedMetaCache.java | 588 ++++++++++++--- .../cache/ScopedMetaCacheRegistry.java | 60 +- .../doris/connector/cache/CacheSpecTest.java | 46 ++ .../cache/MetaCacheEstimatorBenchmark.java | 139 ++++ .../cache/MetaCacheWeightGovernanceTest.java | 463 ++++++++++++ .../ReflectiveObjectSizeEstimatorTest.java | 139 ++++ .../cache/ScopedMetaCacheConcurrencyTest.java | 43 ++ .../connector/hive/HiveCatalogProperties.java | 4 + .../doris/connector/hive/HiveConnector.java | 162 ++-- .../connector/hive/HiveFileListingCache.java | 33 +- .../hive/HiveFileListingSizeEstimator.java | 77 ++ .../hive/HivePartitionViewSizeEstimator.java | 116 +++ .../hive/HiveCatalogPropertiesTest.java | 13 + ...nnectorMetadataPartitionViewCacheTest.java | 16 + .../hive/HiveConnectorSiblingTest.java | 13 + .../hive/HiveFileListingCacheTest.java | 17 + .../doris/connector/hms/CachingHmsClient.java | 18 +- .../connector/hms/HmsCacheSizeEstimator.java | 76 ++ .../connector/hms/CachingHmsClientTest.java | 47 +- .../hms/HmsCacheSizeEstimatorBenchmark.java | 113 +++ .../doris/connector/hudi/HudiConnector.java | 3 +- .../hudi/HudiConnectorHmsCacheTest.java | 89 ++- .../iceberg/IcebergCacheSizeEstimator.java | 702 ++++++++++++++++++ .../iceberg/IcebergCatalogProperties.java | 2 + .../iceberg/IcebergCommentCache.java | 2 + .../connector/iceberg/IcebergConnector.java | 26 +- .../connector/iceberg/IcebergFormatCache.java | 2 + .../iceberg/IcebergLatestSnapshotCache.java | 2 + .../iceberg/IcebergManifestCache.java | 35 +- .../iceberg/IcebergPartitionCache.java | 33 +- .../iceberg/IcebergPartitionUtils.java | 16 + .../iceberg/IcebergStatementScope.java | 4 +- .../connector/iceberg/IcebergTableCache.java | 70 +- .../connector/iceberg/ManifestCacheValue.java | 27 +- ...nnectorMetadataPartitionViewCacheTest.java | 26 + .../iceberg/IcebergManifestCacheTest.java | 18 + .../iceberg/IcebergPartitionCacheTest.java | 25 + .../iceberg/IcebergTableCacheBenchmark.java | 167 +++++ .../iceberg/IcebergTableCacheTest.java | 204 +++++ .../paimon/PaimonCatalogFactory.java | 8 + .../paimon/PaimonCatalogProperties.java | 1 + .../connector/paimon/PaimonConnector.java | 17 +- .../paimon/PaimonConnectorMetadata.java | 7 +- .../paimon/PaimonLatestSnapshotCache.java | 2 + .../connector/paimon/PaimonPartitionView.java | 56 ++ .../PaimonPartitionViewSizeEstimator.java | 126 ++++ .../connector/paimon/PaimonSchemaAtMemo.java | 2 + .../paimon/PaimonCacheSizeBenchmark.java | 171 +++++ .../paimon/PaimonCacheSizeBenchmarkTest.java | 145 ++++ .../paimon/PaimonCatalogFactoryTest.java | 39 + ...nnectorMetadataPartitionViewCacheTest.java | 73 ++ .../org/apache/doris/catalog/SchemaTable.java | 4 + .../doris/datasource/ExternalCatalog.java | 33 +- .../datasource/ExternalMetaCacheMgr.java | 30 + .../doris/RemoteDorisExternalCatalog.java | 7 + .../metacache/CatalogMetaCacheRuntime.java | 33 +- .../metacache/ExternalCatalogMetaCache.java | 13 +- .../metacache/FeMetaCacheEntry.java | 19 +- .../metacache/MetaCacheEntryStats.java | 50 +- .../tablefunction/MetadataGenerator.java | 4 + .../ExternalCatalogMetaCacheWeightTest.java | 77 ++ .../metacache/FeMetaCacheEntryTest.java | 23 + ...nDrivenExternalCatalogConcurrencyTest.java | 41 + .../external_table_p0/test_catalog_ddl.groovy | 102 +++ 79 files changed, 6330 insertions(+), 300 deletions(-) create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheBudgetManager.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheGovernance.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimate.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimators.java create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEstimatorBenchmark.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java create mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HivePartitionViewSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsCacheSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsCacheSizeEstimatorBenchmark.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheBenchmark.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmark.java create mode 100644 fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmarkTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogMetaCacheWeightTest.java diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index e33590fc69e857..7dde0c92e0d8bb 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -53,8 +53,14 @@ std::vector SchemaCatalogMetaCacheStatsScanner::_s_tb {"LAST_LOAD_SUCCESS_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_LOAD_FAILURE_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_ERROR", TYPE_STRING, sizeof(StringRef), true}, + {"MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"WEIGHT_REJECT_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"LAST_WEIGHT_REJECT_REASON", TYPE_STRING, sizeof(StringRef), true}, }; +static constexpr size_t kLegacyMetaCacheStatsColumnCount = 24; + SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS) {} @@ -67,9 +73,12 @@ Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { +Status SchemaCatalogMetaCacheStatsScanner::_fetch_from_fe(size_t column_count, + TFetchSchemaTableDataResult* result, + bool* fe_rejected) { + *fe_rejected = false; TSchemaTableRequestParams schema_table_request_params; - for (int i = 0; i < _s_tbls_columns.size(); i++) { + for (size_t i = 0; i < column_count; i++) { schema_table_request_params.__isset.columns_name = true; schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); } @@ -79,20 +88,36 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); request.__set_schema_table_params(schema_table_request_params); - TFetchSchemaTableDataResult result; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( _fe_addr.hostname, _fe_addr.port, - [&request, &result](FrontendServiceConnection& client) { - client->fetchSchemaTableData(result, request); + [&request, result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(*result, request); }, _rpc_timeout)); - Status status(Status::create(result.status)); + Status fe_status = Status::create(result->status); + *fe_rejected = !fe_status.ok(); + return fe_status; +} + +Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { + TFetchSchemaTableDataResult result; + bool fe_rejected = false; + Status status = _fetch_from_fe(_s_tbls_columns.size(), &result, &fe_rejected); if (!status.ok()) { - LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname - << ") failed, errmsg=" << status; - return status; + if (!fe_rejected) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << status; + return status; + } + Status first_status = status; + result = TFetchSchemaTableDataResult(); + status = _fetch_from_fe(kLegacyMetaCacheStatsColumnCount, &result, &fe_rejected); + if (!status.ok()) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << first_status; + return first_status; + } } std::vector result_data = result.data_batch; @@ -106,19 +131,28 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { _block->reserve(_block_rows_limit); - if (result_data.size() > 0) { - auto col_size = result_data[0].column_value.size(); - if (col_size != _s_tbls_columns.size()) { + size_t col_size = _s_tbls_columns.size(); + if (!result_data.empty()) { + col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size() && col_size != kLegacyMetaCacheStatsColumnCount) { return Status::InternalError( "catalog meta cache stats schema is not match for FE and BE"); } } + int available_columns = static_cast(col_size); + int total_columns = static_cast(_s_tbls_columns.size()); for (int i = 0; i < result_data.size(); i++) { TRow row = result_data[i]; - for (int j = 0; j < _s_tbls_columns.size(); j++) { - RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), - _s_tbls_columns[j].type)); + for (int j = 0; j < total_columns; j++) { + if (j < available_columns) { + RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), + _s_tbls_columns[j].type)); + } else { + auto column_guard = _block->mutate_column_scoped(j); + column_guard.mutable_column()->insert_default(); + column_guard.restore(); + } } } return Status::OK(); diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h index 836500fd97de85..006e1459edfa91 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h @@ -25,6 +25,7 @@ namespace doris { class RuntimeState; class Block; +class TFetchSchemaTableDataResult; class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { ENABLE_FACTORY_CREATOR(SchemaCatalogMetaCacheStatsScanner); @@ -40,6 +41,8 @@ class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { private: Status _get_meta_cache_from_fe(); + Status _fetch_from_fe(size_t column_count, TFetchSchemaTableDataResult* result, + bool* fe_rejected); TNetworkAddress _fe_addr; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 54a16d46685cae..82ab488829e1af 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2204,6 +2204,11 @@ public class Config extends ConfigBase { @ConfField(description = "The auto-refresh interval of the external meta cache.") public static long external_cache_refresh_time_minutes = 10; // 10 mins + @ConfField(mutable = false, masterOnly = false, + description = "FE-wide maximum weight for managed external metadata caches. Supports byte units " + + "or a percentage of the JVM max heap; 0 disables the global quota.") + public static String external_meta_cache_max_weight = "0"; + // Enable manual miss load for external meta cache to avoid blocking replayer on slow loaders. @ConfField(mutable = true, masterOnly = false, description = "Whether external meta cache uses manual miss load instead of Caffeine sync load.") diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java index 01ee88a1fc905b..50ca5f914c1f64 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java @@ -17,10 +17,21 @@ package org.apache.doris.connector.cache; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Common cache specification for external metadata caches. @@ -37,29 +48,44 @@ *
    *
  • enable=false disables cache
    • *
  • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
    • - *
  • capacity=0 disables cache; capacity is count-based
    • + *
  • capacity=0 disables cache; a positive capacity remains the count safety limit
    • + *
  • max-weight is an optional estimated retained-byte admission limit
    • *
*/ public final class CacheSpec { + private static final Logger LOG = LogManager.getLogger(CacheSpec.class); public static final long CACHE_NO_TTL = -1L; public static final long CACHE_TTL_DISABLE_CACHE = 0L; private static final String META_CACHE_PREFIX = "meta.cache."; private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; + private static final Pattern DATA_VOLUME_PATTERN = Pattern.compile( + "^([0-9]+)\\s*(B|KB|MB|GB|TB|PB)?$", Pattern.CASE_INSENSITIVE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + if (maxWeight <= 0L) { + throw new IllegalArgumentException("maxWeight must be positive: " + maxWeight); + } + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); } /** @@ -98,7 +124,8 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getWeightProperty(properties, propertySpec.getMaxWeightKey()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** @@ -116,6 +143,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT) .build(); } @@ -168,10 +196,115 @@ public static void checkLongProperty(String value, long minValue, String key) { } } + /** Strict CREATE/ALTER-time validation for the catalog weight property. */ + public static OptionalLong checkCatalogWeightProperty(Map properties) { + if (properties == null) { + return OptionalLong.empty(); + } + String catalogValue = properties.get(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + if (catalogValue == null) { + return OptionalLong.empty(); + } + long parsed = parseWeight(catalogValue, + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0L) { + throw new IllegalArgumentException( + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } + + /** Strict CREATE/ALTER-time validation for catalog and known engine entry weight properties. */ + public static void checkWeightProperties( + Map properties, String engine, String... knownEntries) { + OptionalLong catalogMax = checkCatalogWeightProperty(properties); + if (properties == null) { + return; + } + Set entries = new HashSet<>(); + Collections.addAll(entries, knownEntries); + String prefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (!key.startsWith(prefix) || !key.endsWith(KEY_MAX_WEIGHT)) { + continue; + } + String entry = key.substring(prefix.length(), key.length() - KEY_MAX_WEIGHT.length()); + if (!entries.contains(entry)) { + throw new IllegalArgumentException("Unknown metadata cache weight property: " + key); + } + long parsed = parseWeight(property.getValue(), key, false, 0L); + if (parsed <= 0L) { + throw new IllegalArgumentException(key + " must be positive"); + } + if (catalogMax.isPresent() && parsed > catalogMax.getAsLong()) { + throw new IllegalArgumentException(key + " can not exceed " + + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + } + } + } + public static boolean isCacheEnabled(boolean enable, long ttlSecond, long capacity) { return enable && ttlSecond != 0 && capacity != 0; } + /** Parse bytes with an optional binary unit, or a heap percentage when explicitly allowed. */ + public static long parseWeight(String value, String key, boolean allowPercent, long maxHeapBytes) { + String normalized = Objects.requireNonNull(value, "value").trim(); + if (normalized.isEmpty()) { + throw invalidWeight(key, value); + } + if (normalized.endsWith("%")) { + if (!allowPercent || maxHeapBytes <= 0L) { + throw invalidWeight(key, value); + } + try { + BigDecimal percentage = new BigDecimal( + normalized.substring(0, normalized.length() - 1).trim()); + if (percentage.signum() < 0 || percentage.compareTo(BigDecimal.valueOf(100L)) > 0) { + throw invalidWeight(key, value); + } + return checkedLong(BigDecimal.valueOf(maxHeapBytes) + .multiply(percentage) + .divide(BigDecimal.valueOf(100L)) + .toBigInteger(), key, value); + } catch (NumberFormatException e) { + throw invalidWeight(key, value); + } + } + Matcher matcher = DATA_VOLUME_PATTERN.matcher(normalized); + if (!matcher.matches()) { + throw invalidWeight(key, value); + } + BigInteger amount = new BigInteger(matcher.group(1)); + String rawUnit = matcher.group(2); + String unit = rawUnit == null ? "B" : rawUnit.toUpperCase(Locale.ROOT); + int power; + switch (unit) { + case "B": + power = 0; + break; + case "KB": + power = 1; + break; + case "MB": + power = 2; + break; + case "GB": + power = 3; + break; + case "TB": + power = 4; + break; + case "PB": + power = 5; + break; + default: + throw invalidWeight(key, value); + } + return checkedLong(amount.multiply(BigInteger.valueOf(1024L).pow(power)), key, value); + } + /** * Build standard external meta cache key prefix for one engine. * Example: {@code meta.cache.iceberg.} @@ -229,6 +362,37 @@ private static long getLongProperty(Map properties, String key, } } + private static OptionalLong getWeightProperty(Map properties, String key) { + if (key == null) { + return OptionalLong.empty(); + } + String value = properties.get(key); + if (value == null) { + return OptionalLong.empty(); + } + try { + long parsed = parseWeight(value, key, false, 0L); + if (parsed <= 0L) { + throw invalidWeight(key, value); + } + return OptionalLong.of(parsed); + } catch (IllegalArgumentException e) { + LOG.warn("Ignoring invalid persisted metadata cache weight property {}={}", key, value); + return OptionalLong.empty(); + } + } + + private static long checkedLong(BigInteger value, String key, String rawValue) { + if (value.signum() < 0 || value.compareTo(LONG_MAX) > 0) { + throw invalidWeight(key, rawValue); + } + return value.longValue(); + } + + private static IllegalArgumentException invalidWeight(String key, String value) { + return new IllegalArgumentException("Invalid cache weight for '" + key + "': " + value); + } + public boolean isEnable() { return enable; } @@ -241,6 +405,19 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return isCacheEnabled(enable, ttlSecond, capacity) + && (!maxWeight.isPresent() || maxWeight.getAsLong() != 0L); + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -248,15 +425,17 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, String maxWeightKey) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; } public String getEnableKey() { @@ -283,6 +462,10 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -290,6 +473,7 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -309,6 +493,11 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key) { + this.maxWeightKey = Objects.requireNonNull(key, "key"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -316,7 +505,8 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey); } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java index 199f516582ebf5..f3f76361cbbf97 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -20,7 +20,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -33,15 +35,48 @@ */ public final class CatalogMetaCache implements AutoCloseable { private final ScopedMetaCacheRegistry registry; + private final MetaCacheBudgetManager budgetManager; + private final long catalogId; + private final String engine; + private final OptionalLong catalogMaxWeight; + private final boolean managed; private final Set names = ConcurrentHashMap.newKeySet(); + private final Map> entries = new ConcurrentHashMap<>(); private final AtomicBoolean closed = new AtomicBoolean(false); public CatalogMetaCache() { - this(new ScopedMetaCacheRegistry()); + this(new ScopedMetaCacheRegistry(), new MetaCacheBudgetManager(OptionalLong.empty()), + 0L, "standalone", OptionalLong.empty(), false); } CatalogMetaCache(ScopedMetaCacheRegistry registry) { + this(registry, new MetaCacheBudgetManager(OptionalLong.empty()), + 0L, "standalone", OptionalLong.empty(), false); + } + + public CatalogMetaCache(MetaCacheBudgetManager budgetManager, long catalogId, + String engine, Map catalogProperties) { + this(new ScopedMetaCacheRegistry(), budgetManager, catalogId, engine, + budgetManager.parseCatalogMaxWeight(catalogProperties), false); + } + + public static CatalogMetaCache managed(long catalogId, String engine, + Map catalogProperties) { + MetaCacheBudgetManager manager = MetaCacheGovernance.budgetManager(); + CatalogMetaCache cache = new CatalogMetaCache(new ScopedMetaCacheRegistry(), manager, + catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true); + MetaCacheGovernance.register(cache); + return cache; + } + + CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, + long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed) { this.registry = Objects.requireNonNull(registry, "registry can not be null"); + this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); + this.catalogId = catalogId; + this.engine = Objects.requireNonNull(engine, "engine can not be null"); + this.catalogMaxWeight = Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight can not be null"); + this.managed = managed; } public MetaCache create(MetaCacheDefinition definition) { @@ -51,14 +86,39 @@ public MetaCache create(MetaCacheDefinition definition) { if (!names.add(nonNullDefinition.name())) { throw new IllegalArgumentException("Duplicate meta cache name: " + nonNullDefinition.name()); } + MetaCacheBudgetManager.EntryBudget entryBudget = null; try { - return new MetaCache<>(nonNullDefinition, + boolean weightLimited = budgetManager.hasLimit( + catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); + if (weightLimited && nonNullDefinition.sizeEstimator() == null) { + throw new IllegalArgumentException("Weighted metadata cache requires a size estimator: " + + nonNullDefinition.name()); + } + if (weightLimited) { + entryBudget = budgetManager.createEntryBudget(catalogId, engine, + nonNullDefinition.name(), nonNullDefinition.budgetGroup(), catalogMaxWeight, + nonNullDefinition.cacheSpec().getMaxWeight()); + } + MetaCache created = new MetaCache<>(nonNullDefinition, registry.createCacheWithMetaRemovalListener(nonNullDefinition.name(), nonNullDefinition.cacheSpec(), nonNullDefinition.removalListener(), nonNullDefinition.discardListener(), - nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor())); + nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor(), + entryBudget == null ? null : nonNullDefinition.sizeEstimator(), entryBudget)); + entries.put(nonNullDefinition.name(), created); + return created; } catch (RuntimeException | Error throwable) { + if (entryBudget != null) { + entryBudget.close(); + } names.remove(nonNullDefinition.name()); + if (managed) { + try { + close(); + } catch (RuntimeException | Error closeFailure) { + throwable.addSuppressed(closeFailure); + } + } throw throwable; } } @@ -100,11 +160,38 @@ public ScopedMetaCacheRegistry.ScopeMetrics metrics() { return registry.metrics(); } + public Map> entries() { + return java.util.Collections.unmodifiableMap(entries); + } + + public long catalogId() { + return catalogId; + } + + public String engine() { + return engine; + } + + public OptionalLong catalogMaxWeight() { + return catalogMaxWeight; + } + + public boolean hasEnclosingWeightLimit() { + return budgetManager.hasLimit(catalogMaxWeight, OptionalLong.empty()); + } + @Override public void close() { if (closed.compareAndSet(false, true)) { - registry.close(); - names.clear(); + try { + registry.close(); + } finally { + names.clear(); + entries.clear(); + if (managed) { + MetaCacheGovernance.unregister(this); + } + } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java index da47360f862225..adaa6a1a711f83 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java @@ -73,15 +73,25 @@ public ConnectorMetadataCache( public ConnectorMetadataCache(CatalogMetaCache owner, String cacheName, String engine, String entryName, Map props, Function scopeResolver) { + this(owner, cacheName, engine, entryName, props, scopeResolver, MetaCacheSizeEstimators.reflective()); + } + + public ConnectorMetadataCache(CatalogMetaCache owner, String cacheName, String engine, String entryName, + Map props, Function scopeResolver, + MetaCacheSizeEstimator sizeEstimator) { this.owner = Objects.requireNonNull(owner, "owner can not be null"); Objects.requireNonNull(engine, "engine can not be null"); Objects.requireNonNull(entryName, "entryName can not be null"); Map properties = props == null ? Collections.emptyMap() : props; CacheSpec spec = CacheSpec.fromProperties(properties, engine, entryName, CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY)); - this.entry = owner.create(MetaCacheDefinition + MetaCacheDefinition.Builder builder = MetaCacheDefinition .builder(cacheName, spec, scopeResolver) - .build()); + .budgetGroup(entryName); + if (sizeEstimator != null) { + builder.sizeEstimator(sizeEstimator); + } + this.entry = owner.create(builder.build()); } /** Caching is on only when the resolved {@link CacheSpec} is effectively enabled (see {@link #entry}'s spec). */ @@ -89,6 +99,10 @@ public boolean isEnabled() { return entry.isEnabled(); } + public boolean isWeightBounded() { + return entry.isWeightBounded(); + } + /** * Returns the cached value for {@code key} if present, else runs {@code loader}, caches and returns it. * Disabled cache -> {@code loader} runs on every call. The loader runs OUTSIDE Caffeine's compute lock diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java new file mode 100644 index 00000000000000..6dd064a563b390 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import com.sun.management.HotSpotDiagnosticMXBean; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.List; +import java.util.Objects; +import java.util.function.ToLongFunction; + +/** + * Low-cost JVM heap layout formulas for type-specific connector cache estimators. + * + *

This class reflects class declarations once to calculate shallow sizes. It never reads fields from a runtime + * object and never walks an object graph. + */ +public final class JvmSizeUtils { + private static final VmLayout VM_LAYOUT = VmLayout.detect(); + private static final boolean COMPACT_STRINGS = vmBoolean("CompactStrings", true); + + private static final ClassValue SHALLOW_SIZES = new ClassValue<>() { + @Override + protected Long computeValue(Class type) { + if (type.isArray()) { + throw new IllegalArgumentException("Array size depends on its length: " + type); + } + long size = VM_LAYOUT.objectHeaderBytes(); + for (Class current = type; current != null; current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers())) { + size = saturatedAdd(size, fieldSize(field.getType())); + } + } + } + return align(size); + } + }; + private static final long STRING_SHALLOW_BYTES = instanceSize(String.class); + private static final long ARRAY_LIST_SHALLOW_BYTES = instanceSize(java.util.ArrayList.class); + + private JvmSizeUtils() { + } + + public static long instanceSize(Class type) { + return SHALLOW_SIZES.get(type); + } + + public static long objectArraySize(int length) { + return arraySize(length, VM_LAYOUT.referenceBytes()); + } + + public static long byteArraySize(int length) { + return arraySize(length, Byte.BYTES); + } + + public static long intArraySize(int length) { + return arraySize(length, Integer.BYTES); + } + + public static long longArraySize(int length) { + return arraySize(length, Long.BYTES); + } + + /** Estimate an array whose component is a primitive type. */ + public static long primitiveArraySize(Class componentType, int length) { + if (!componentType.isPrimitive() || componentType == void.class) { + throw new IllegalArgumentException("Not an array component primitive: " + componentType); + } + return arraySize(length, Math.toIntExact(fieldSize(componentType))); + } + + public static long arrayListSize(int backingArrayCapacity) { + return saturatedAdd(ARRAY_LIST_SHALLOW_BYTES, objectArraySize(backingArrayCapacity)); + } + + /** Estimates list element payload from evenly spaced samples, including both ends of the list. */ + public static long sampledListPayload( + List values, int sampleCount, ToLongFunction estimator) { + Objects.requireNonNull(values, "values"); + Objects.requireNonNull(estimator, "estimator"); + if (sampleCount <= 0) { + throw new IllegalArgumentException("sampleCount must be positive: " + sampleCount); + } + int size = values.size(); + if (size == 0) { + return 0L; + } + int samples = Math.min(size, sampleCount); + long sampledBytes = 0L; + for (int sample = 0; sample < samples; sample++) { + int index = samples == 1 + ? 0 : (int) ((long) sample * (size - 1) / (samples - 1)); + sampledBytes = saturatedAdd(sampledBytes, estimator.applyAsLong(values.get(index))); + } + if (samples == size) { + return sampledBytes; + } + double scaled = (double) sampledBytes * size / samples; + return scaled >= Long.MAX_VALUE ? Long.MAX_VALUE : (long) scaled; + } + + /** Estimate the heap retained by a Java 17 String and its compact-string byte array. */ + public static long stringSize(String value) { + if (value == null) { + return 0L; + } + int bytesPerCharacter = COMPACT_STRINGS && isLatin1(value) ? Byte.BYTES : Character.BYTES; + long valueBytes = saturatedMultiply(value.length(), bytesPerCharacter); + int arrayLength = valueBytes >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) valueBytes; + return saturatedAdd(STRING_SHALLOW_BYTES, byteArraySize(arrayLength)); + } + + public static long saturatedAdd(long left, long right) { + if (right > 0L && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + return left + right; + } + + public static long saturatedMultiply(long left, long right) { + if (left == 0L || right == 0L) { + return 0L; + } + if (left > Long.MAX_VALUE / right) { + return Long.MAX_VALUE; + } + return left * right; + } + + private static long arraySize(int length, int elementBytes) { + long elements = saturatedMultiply(length, elementBytes); + return align(saturatedAdd(VM_LAYOUT.arrayHeaderBytes(), elements)); + } + + private static long fieldSize(Class type) { + if (!type.isPrimitive()) { + return VM_LAYOUT.referenceBytes(); + } + if (type == long.class || type == double.class) { + return Long.BYTES; + } + if (type == int.class || type == float.class) { + return Integer.BYTES; + } + if (type == short.class || type == char.class) { + return Short.BYTES; + } + return Byte.BYTES; + } + + private static boolean isLatin1(String value) { + for (int i = 0; i < value.length(); i++) { + if (value.charAt(i) > 0xff) { + return false; + } + } + return true; + } + + private static long align(long value) { + long remainder = value % VM_LAYOUT.objectAlignmentBytes(); + return remainder == 0L + ? value + : saturatedAdd(value, VM_LAYOUT.objectAlignmentBytes() - remainder); + } + + private static boolean vmBoolean(String option, boolean defaultValue) { + try { + HotSpotDiagnosticMXBean diagnostic = hotSpotDiagnostic(); + return diagnostic == null + ? defaultValue + : Boolean.parseBoolean(diagnostic.getVMOption(option).getValue()); + } catch (RuntimeException | LinkageError ignored) { + return defaultValue; + } + } + + private static int vmInt(String option, int defaultValue) { + try { + HotSpotDiagnosticMXBean diagnostic = hotSpotDiagnostic(); + return diagnostic == null + ? defaultValue + : Integer.parseInt(diagnostic.getVMOption(option).getValue()); + } catch (RuntimeException | LinkageError ignored) { + return defaultValue; + } + } + + private static HotSpotDiagnosticMXBean hotSpotDiagnostic() { + return ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class); + } + + private record VmLayout( + int referenceBytes, + int objectHeaderBytes, + int arrayHeaderBytes, + int objectAlignmentBytes) { + + private static VmLayout detect() { + int referenceBytes = vmBoolean("UseCompressedOops", true) ? Integer.BYTES : Long.BYTES; + int classPointerBytes = vmBoolean("UseCompressedClassPointers", true) + ? Integer.BYTES : Long.BYTES; + int alignment = vmInt("ObjectAlignmentInBytes", Long.BYTES); + int objectHeaderBytes = Long.BYTES + classPointerBytes; + int arrayHeaderBytes = Math.toIntExact( + alignWithoutLayout(objectHeaderBytes + Integer.BYTES, alignment)); + return new VmLayout(referenceBytes, objectHeaderBytes, arrayHeaderBytes, alignment); + } + + private static long alignWithoutLayout(long value, int alignment) { + long remainder = value % alignment; + return remainder == 0L ? value : value + alignment - remainder; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java index 3b57fef2364dc0..d55b93d4bcdea4 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java @@ -109,6 +109,11 @@ public boolean isEnabled() { return metrics().isEffectiveEnabled(); } + /** Whether this entry currently participates in byte-based memory governance. */ + public boolean isWeightBounded() { + return metrics().isWeightBounded(); + } + public long size() { return metrics().getPhysicalEntryCount(); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheBudgetManager.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheBudgetManager.java new file mode 100644 index 00000000000000..40b22ec76c7e88 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheBudgetManager.java @@ -0,0 +1,605 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongUnaryOperator; + +/** + * Process-wide admission accounting shared by FE and connector metadata caches. + * Estimation happens before this class is entered; its lock covers only arithmetic and small maps. + */ +public final class MetaCacheBudgetManager { + private static final Logger LOG = LogManager.getLogger(MetaCacheBudgetManager.class); + private static final ExecutorService PEER_RECLAIM_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-peer-reclaim"); + thread.setDaemon(true); + return thread; + }); + + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; + + private final Object lock = new Object(); + private final OptionalLong globalMaxWeight; + private final Map catalogBuckets = new HashMap<>(); + private final Map entryGroupBuckets = new HashMap<>(); + private final Map entryBudgets = new HashMap<>(); + private final AtomicLong nextEntryBudgetId = new AtomicLong(); + private long globalUsedWeight; + + public MetaCacheBudgetManager(OptionalLong globalMaxWeight) { + this.globalMaxWeight = Objects.requireNonNull(globalMaxWeight, "globalMaxWeight"); + if (globalMaxWeight.isPresent() && globalMaxWeight.getAsLong() <= 0L) { + throw new IllegalArgumentException("global max weight must be positive when enabled"); + } + } + + public OptionalLong parseCatalogMaxWeight(Map catalogProperties) { + String configured = catalogProperties == null ? null + : catalogProperties.get(CATALOG_MAX_WEIGHT_PROPERTY); + if (configured == null) { + return OptionalLong.empty(); + } + try { + long parsed = CacheSpec.parseWeight(configured, CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0L) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } catch (IllegalArgumentException e) { + LOG.warn("Ignoring invalid persisted metadata cache property {}={}", + CATALOG_MAX_WEIGHT_PROPERTY, configured); + return OptionalLong.empty(); + } + } + + public boolean hasLimit(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + return globalMaxWeight.isPresent() || catalogMaxWeight.isPresent() || entryMaxWeight.isPresent(); + } + + public EntryBudget createEntryBudget(long catalogId, String engine, String entryName, String budgetGroup, + OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + Objects.requireNonNull(engine, "engine"); + Objects.requireNonNull(entryName, "entryName"); + Objects.requireNonNull(budgetGroup, "budgetGroup"); + OptionalLong effectiveMax = minimumPresent(globalMaxWeight, catalogMaxWeight, entryMaxWeight); + if (!effectiveMax.isPresent()) { + throw new IllegalArgumentException("entry budget requires at least one configured weight bound"); + } + + EntryGroupScope groupScope = new EntryGroupScope(catalogId, engine, budgetGroup); + EntryScope scope = new EntryScope(nextEntryBudgetId.incrementAndGet(), groupScope, entryName); + synchronized (lock) { + Bucket catalogBucket = catalogBuckets.get(catalogId); + long catalogLimit = minimumLimit(globalMaxWeight, catalogMaxWeight); + if (catalogBucket == null) { + catalogBucket = new Bucket(catalogLimit); + catalogBuckets.put(catalogId, catalogBucket); + } else if (catalogBucket.maxWeight != catalogLimit) { + throw new IllegalStateException("Conflicting catalog cache max weight for catalog " + catalogId); + } + Bucket groupBucket = entryGroupBuckets.get(groupScope); + if (groupBucket == null) { + groupBucket = new Bucket(effectiveMax.getAsLong()); + entryGroupBuckets.put(groupScope, groupBucket); + } else if (groupBucket.maxWeight != effectiveMax.getAsLong()) { + throw new IllegalStateException("Conflicting metadata cache entry max weight for " + groupScope); + } + Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + EntryBudget budget = new EntryBudget(this, scope, catalogBucket, groupBucket, entryBucket, + effectiveMax.getAsLong()); + entryBudgets.put(scope, budget); + catalogBucket.liveEntries++; + groupBucket.liveEntries++; + return budget; + } + } + + public long getGlobalUsedWeight() { + synchronized (lock) { + return globalUsedWeight; + } + } + + private Optional tryReserve(EntryBudget budget, long bytes) { + checkWeight(bytes); + synchronized (lock) { + if (budget.closed) { + return Optional.empty(); + } + if (!fits(limitOf(globalMaxWeight), globalUsedWeight, bytes) + || !fits(budget.catalogBucket.maxWeight, budget.catalogBucket.usedWeight, bytes) + || !fits(budget.groupBucket.maxWeight, budget.groupBucket.usedWeight, bytes)) { + return Optional.empty(); + } + addUsed(budget, bytes); + return Optional.of(new AdmissionReservation(this, budget, bytes)); + } + } + + private Optional tryReplace( + AdmissionReservation previous, long newBytes) { + checkWeight(newBytes); + synchronized (lock) { + if (!previous.active || previous.entryBudget.closed) { + return Optional.empty(); + } + EntryBudget budget = previous.entryBudget; + long heldBytes = Math.max(previous.accountedBytes, newBytes); + long delta = heldBytes - previous.accountedBytes; + if (delta > 0L && (!fits(limitOf(globalMaxWeight), globalUsedWeight, delta) + || !fits(budget.catalogBucket.maxWeight, budget.catalogBucket.usedWeight, delta) + || !fits(budget.groupBucket.maxWeight, budget.groupBucket.usedWeight, delta))) { + return Optional.empty(); + } + addUsed(budget, delta); + previous.active = false; + AdmissionReservation replacement = new AdmissionReservation( + this, budget, newBytes, heldBytes); + return Optional.of(new ReservationReplacement(this, previous, replacement)); + } + } + + private void commitReplacement(ReservationReplacement replacement) { + synchronized (lock) { + replacement.checkPending(); + AdmissionReservation current = replacement.current; + subtractUsed(current.entryBudget, current.accountedBytes - current.bytes); + current.accountedBytes = current.bytes; + replacement.finished = true; + } + } + + private void rollbackReplacement(ReservationReplacement replacement) { + synchronized (lock) { + replacement.checkPending(); + AdmissionReservation previous = replacement.previous; + AdmissionReservation current = replacement.current; + subtractUsed(current.entryBudget, current.accountedBytes - previous.accountedBytes); + current.bytes = 0L; + current.accountedBytes = 0L; + current.active = false; + previous.active = true; + replacement.finished = true; + } + } + + private void release(AdmissionReservation reservation) { + synchronized (lock) { + if (!reservation.active) { + return; + } + if (!reservation.entryBudget.closed) { + subtractUsed(reservation.entryBudget, reservation.accountedBytes); + } + reservation.bytes = 0L; + reservation.accountedBytes = 0L; + reservation.active = false; + } + } + + private void close(EntryBudget budget) { + synchronized (lock) { + if (budget.closed) { + return; + } + long leaked = budget.entryBucket.usedWeight; + if (leaked != 0L) { + LOG.error("Force-closing metadata cache budget {} with {} bytes still reserved", + budget.scope, leaked); + globalUsedWeight = Math.max(0L, globalUsedWeight - leaked); + budget.catalogBucket.usedWeight = Math.max(0L, budget.catalogBucket.usedWeight - leaked); + budget.groupBucket.usedWeight = Math.max(0L, budget.groupBucket.usedWeight - leaked); + budget.entryBucket.usedWeight = 0L; + } + budget.closed = true; + budget.reclaimer = null; + entryBudgets.remove(budget.scope, budget); + budget.catalogBucket.liveEntries--; + if (budget.catalogBucket.liveEntries == 0) { + catalogBuckets.remove(budget.scope.groupScope.catalogId, budget.catalogBucket); + } + budget.groupBucket.liveEntries--; + if (budget.groupBucket.liveEntries == 0) { + entryGroupBuckets.remove(budget.scope.groupScope, budget.groupBucket); + } + } + } + + private void addUsed(EntryBudget budget, long bytes) { + globalUsedWeight += bytes; + budget.catalogBucket.usedWeight += bytes; + budget.groupBucket.usedWeight += bytes; + budget.entryBucket.usedWeight += bytes; + } + + private void subtractUsed(EntryBudget budget, long bytes) { + if (bytes > globalUsedWeight || bytes > budget.catalogBucket.usedWeight + || bytes > budget.groupBucket.usedWeight + || bytes > budget.entryBucket.usedWeight) { + throw new IllegalStateException("metadata cache budget accounting underflow"); + } + globalUsedWeight -= bytes; + budget.catalogBucket.usedWeight -= bytes; + budget.groupBucket.usedWeight -= bytes; + budget.entryBucket.usedWeight -= bytes; + } + + private void requestPeerReclaim(EntryBudget requester, long additionalBytes) { + if (additionalBytes <= 0L || requester.closed) { + return; + } + synchronized (lock) { + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + long groupDeficit = deficit( + requester.groupBucket.maxWeight, requester.groupBucket.usedWeight, additionalBytes); + if (Math.max(groupDeficit, Math.max(globalDeficit, catalogDeficit)) == 0L) { + return; + } + } + requester.requestedAdmissionBytes.accumulateAndGet(additionalBytes, Math::max); + schedulePeerReclaim(requester); + } + + private void schedulePeerReclaim(EntryBudget requester) { + if (!requester.reclaimScheduled.compareAndSet(false, true)) { + return; + } + try { + PEER_RECLAIM_EXECUTOR.execute(() -> drainPeerReclaim(requester)); + } catch (RejectedExecutionException e) { + requester.reclaimScheduled.set(false); + LOG.warn("Failed to schedule metadata cache peer reclamation for {}", requester.scope, e); + } + } + + private void drainPeerReclaim(EntryBudget requester) { + try { + long requested = requester.requestedAdmissionBytes.getAndSet(0L); + if (requested <= 0L || requester.closed) { + return; + } + List candidates; + synchronized (lock) { + candidates = new ArrayList<>(); + for (EntryBudget candidate : entryBudgets.values()) { + if (!candidate.closed && candidate.reclaimer != null + && candidate.entryBucket.usedWeight > 0L) { + candidates.add(candidate); + } + } + candidates.sort((left, right) -> { + boolean leftSibling = left.scope.groupScope.catalogId + == requester.scope.groupScope.catalogId; + boolean rightSibling = right.scope.groupScope.catalogId + == requester.scope.groupScope.catalogId; + if (leftSibling != rightSibling) { + return leftSibling ? -1 : 1; + } + return Long.compare(right.entryBucket.usedWeight, left.entryBucket.usedWeight); + }); + } + long remaining = currentDeficit(requester, requested); + for (EntryBudget candidate : candidates) { + if (currentGroupDeficit(requester, requested) > 0L + && candidate.groupBucket != requester.groupBucket) { + continue; + } + boolean sibling = candidate.scope.groupScope.catalogId + == requester.scope.groupScope.catalogId; + if (!sibling && currentCatalogDeficit(requester, requested) > 0L) { + continue; + } + try { + candidate.reclaimer.applyAsLong(remaining); + } catch (RuntimeException e) { + LOG.warn("Failed to reclaim metadata cache budget from peer {}", candidate.scope, e); + } + remaining = currentDeficit(requester, requested); + if (remaining == 0L) { + break; + } + } + } finally { + requester.reclaimScheduled.set(false); + if (!requester.closed && requester.requestedAdmissionBytes.get() > 0L) { + schedulePeerReclaim(requester); + } + } + } + + private long currentDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + if (requester.closed) { + return 0L; + } + return Math.max( + deficit(requester.groupBucket.maxWeight, + requester.groupBucket.usedWeight, additionalBytes), + Math.max(deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes), + deficit(requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes))); + } + } + + private long currentGroupDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit(requester.groupBucket.maxWeight, + requester.groupBucket.usedWeight, additionalBytes); + } + } + + private long currentCatalogDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit(requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes); + } + } + + private static long deficit(long maxWeight, long usedWeight, long additionalBytes) { + if (maxWeight == Long.MAX_VALUE || additionalBytes <= maxWeight - Math.min(usedWeight, maxWeight)) { + return 0L; + } + return JvmSizeUtils.saturatedAdd(usedWeight, additionalBytes) - maxWeight; + } + + private static boolean fits(long maxWeight, long usedWeight, long delta) { + return delta >= 0L && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; + } + + private static long limitOf(OptionalLong configured) { + return configured.isPresent() ? configured.getAsLong() : Long.MAX_VALUE; + } + + private static long minimumLimit(OptionalLong first, OptionalLong second) { + return Math.min(limitOf(first), limitOf(second)); + } + + private static OptionalLong minimumPresent(OptionalLong first, OptionalLong second, OptionalLong third) { + if (!first.isPresent() && !second.isPresent() && !third.isPresent()) { + return OptionalLong.empty(); + } + return OptionalLong.of(Math.min(limitOf(first), Math.min(limitOf(second), limitOf(third)))); + } + + private static void checkWeight(long bytes) { + if (bytes < 0L) { + throw new IllegalArgumentException("cache reservation can not be negative: " + bytes); + } + } + + private static final class Bucket { + private final long maxWeight; + private long usedWeight; + private int liveEntries; + + private Bucket(long maxWeight) { + this.maxWeight = maxWeight; + } + } + + private static final class EntryGroupScope { + private final long catalogId; + private final String engine; + private final String groupName; + + private EntryGroupScope(long catalogId, String engine, String groupName) { + this.catalogId = catalogId; + this.engine = engine; + this.groupName = groupName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryGroupScope)) { + return false; + } + EntryGroupScope that = (EntryGroupScope) other; + return catalogId == that.catalogId && engine.equals(that.engine) && groupName.equals(that.groupName); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, engine, groupName); + } + + @Override + public String toString() { + return catalogId + "/" + engine + "/" + groupName; + } + } + + private static final class EntryScope { + private final long budgetId; + private final EntryGroupScope groupScope; + private final String entryName; + + private EntryScope(long budgetId, EntryGroupScope groupScope, String entryName) { + this.budgetId = budgetId; + this.groupScope = groupScope; + this.entryName = entryName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryScope)) { + return false; + } + EntryScope that = (EntryScope) other; + return budgetId == that.budgetId; + } + + @Override + public int hashCode() { + return Long.hashCode(budgetId); + } + + @Override + public String toString() { + return groupScope + "/" + entryName + "#" + budgetId; + } + } + + public static final class EntryBudget implements AutoCloseable { + private final MetaCacheBudgetManager manager; + private final EntryScope scope; + private final Bucket catalogBucket; + private final Bucket groupBucket; + private final Bucket entryBucket; + private final long effectiveMaxWeight; + private final AtomicLong requestedAdmissionBytes = new AtomicLong(); + private final AtomicBoolean reclaimScheduled = new AtomicBoolean(); + private volatile LongUnaryOperator reclaimer; + private volatile boolean closed; + + private EntryBudget(MetaCacheBudgetManager manager, EntryScope scope, Bucket catalogBucket, + Bucket groupBucket, Bucket entryBucket, long effectiveMaxWeight) { + this.manager = manager; + this.scope = scope; + this.catalogBucket = catalogBucket; + this.groupBucket = groupBucket; + this.entryBucket = entryBucket; + this.effectiveMaxWeight = effectiveMaxWeight; + } + + public Optional tryReserve(long bytes) { + return manager.tryReserve(this, bytes); + } + + public Optional tryReplace( + AdmissionReservation previous, long newBytes) { + Objects.requireNonNull(previous, "previous reservation"); + if (previous.entryBudget != this) { + throw new IllegalArgumentException("replacement reservation belongs to another entry"); + } + return manager.tryReplace(previous, newBytes); + } + + public void setReclaimer(LongUnaryOperator reclaimer) { + this.reclaimer = Objects.requireNonNull(reclaimer, "reclaimer"); + } + + public void requestPeerReclaim(long additionalBytes) { + manager.requestPeerReclaim(this, additionalBytes); + } + + public long getEffectiveMaxWeight() { + return effectiveMaxWeight; + } + + public long getUsedWeight() { + synchronized (manager.lock) { + return entryBucket.usedWeight; + } + } + + @Override + public void close() { + manager.close(this); + } + } + + public static final class AdmissionReservation { + private final MetaCacheBudgetManager manager; + private final EntryBudget entryBudget; + private long bytes; + private long accountedBytes; + private boolean active = true; + + private AdmissionReservation(MetaCacheBudgetManager manager, EntryBudget entryBudget, long bytes) { + this(manager, entryBudget, bytes, bytes); + } + + private AdmissionReservation(MetaCacheBudgetManager manager, EntryBudget entryBudget, + long bytes, long accountedBytes) { + this.manager = manager; + this.entryBudget = entryBudget; + this.bytes = bytes; + this.accountedBytes = accountedBytes; + } + + public void release() { + manager.release(this); + } + + public long getBytes() { + synchronized (manager.lock) { + return bytes; + } + } + } + + /** + * Atomic accounting hand-off between two generations of the same cache key. + * The manager temporarily holds max(old, new), never old + new, and keeps enough + * accounting to restore the old generation until publication commits. + */ + public static final class ReservationReplacement { + private final MetaCacheBudgetManager manager; + private final AdmissionReservation previous; + private final AdmissionReservation current; + private boolean finished; + + private ReservationReplacement(MetaCacheBudgetManager manager, + AdmissionReservation previous, AdmissionReservation current) { + this.manager = manager; + this.previous = previous; + this.current = current; + } + + public AdmissionReservation current() { + return current; + } + + public void commit() { + manager.commitReplacement(this); + } + + public void rollback() { + manager.rollbackReplacement(this); + } + + private void checkPending() { + if (finished) { + throw new IllegalStateException("reservation replacement is already finished"); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java index ab8c0e3615e749..a461b8879e44a6 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java @@ -32,6 +32,7 @@ */ public final class MetaCacheDefinition { private final String name; + private final String budgetGroup; private final CacheSpec cacheSpec; private final Function scopeResolver; private final Function loader; @@ -39,9 +40,11 @@ public final class MetaCacheDefinition { private final BiConsumer discardListener; private final Duration refreshAfterWrite; private final Executor refreshExecutor; + private final MetaCacheSizeEstimator sizeEstimator; private MetaCacheDefinition(Builder builder) { name = requireName(builder.name); + budgetGroup = requireName(builder.budgetGroup == null ? builder.name : builder.budgetGroup); cacheSpec = Objects.requireNonNull(builder.cacheSpec, "cacheSpec can not be null"); scopeResolver = Objects.requireNonNull(builder.scopeResolver, "scopeResolver can not be null"); loader = builder.loader; @@ -49,6 +52,7 @@ private MetaCacheDefinition(Builder builder) { discardListener = builder.discardListener; refreshAfterWrite = builder.refreshAfterWrite; refreshExecutor = builder.refreshExecutor; + sizeEstimator = builder.sizeEstimator; if (refreshAfterWrite != null && loader == null) { throw new IllegalArgumentException("refresh-after-write requires a default loader"); } @@ -63,6 +67,10 @@ public String name() { return name; } + String budgetGroup() { + return budgetGroup; + } + CacheSpec cacheSpec() { return cacheSpec; } @@ -93,6 +101,10 @@ Executor refreshExecutor() { return refreshExecutor; } + MetaCacheSizeEstimator sizeEstimator() { + return sizeEstimator; + } + private static String requireName(String name) { String nonNullName = Objects.requireNonNull(name, "name can not be null"); if (nonNullName.isEmpty()) { @@ -110,6 +122,8 @@ public static final class Builder { private BiConsumer discardListener; private Duration refreshAfterWrite; private Executor refreshExecutor; + private MetaCacheSizeEstimator sizeEstimator; + private String budgetGroup; private Builder(String name, CacheSpec cacheSpec, Function scopeResolver) { this.name = name; @@ -146,6 +160,17 @@ public Builder refreshAfterWrite(Duration duration, Executor executor) { return this; } + public Builder sizeEstimator(MetaCacheSizeEstimator estimator) { + this.sizeEstimator = Objects.requireNonNull(estimator, "estimator can not be null"); + return this; + } + + /** Shares one configured weight limit across physical caches in the same logical entry group. */ + public Builder budgetGroup(String budgetGroup) { + this.budgetGroup = requireName(budgetGroup); + return this; + } + public MetaCacheDefinition build() { return new MetaCacheDefinition<>(this); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheGovernance.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheGovernance.java new file mode 100644 index 00000000000000..a19d2f2eafda15 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheGovernance.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** Process-wide bridge used because connector-cache is parent-first for every connector plugin. */ +public final class MetaCacheGovernance { + private static final Object CONFIG_LOCK = new Object(); + private static final ConcurrentHashMap> CATALOG_CACHES = + new ConcurrentHashMap<>(); + private static volatile MetaCacheBudgetManager budgetManager = + new MetaCacheBudgetManager(OptionalLong.empty()); + private static volatile OptionalLong configuredGlobalMaxWeight = OptionalLong.empty(); + + private MetaCacheGovernance() { + } + + public static void configureGlobalMaxWeight(OptionalLong globalMaxWeight) { + synchronized (CONFIG_LOCK) { + OptionalLong requested = globalMaxWeight == null ? OptionalLong.empty() : globalMaxWeight; + if (same(configuredGlobalMaxWeight, requested)) { + return; + } + if (!CATALOG_CACHES.isEmpty()) { + throw new IllegalStateException( + "Can not change external metadata cache global weight after catalogs are initialized"); + } + budgetManager = new MetaCacheBudgetManager(requested); + configuredGlobalMaxWeight = requested; + } + } + + static MetaCacheBudgetManager budgetManager() { + return budgetManager; + } + + static void register(CatalogMetaCache cache) { + CATALOG_CACHES.compute(cache.catalogId(), (ignored, caches) -> { + Set updated = caches == null ? ConcurrentHashMap.newKeySet() : caches; + updated.add(cache); + return updated; + }); + } + + static void unregister(CatalogMetaCache cache) { + CATALOG_CACHES.computeIfPresent(cache.catalogId(), (ignored, caches) -> { + caches.remove(cache); + return caches.isEmpty() ? null : caches; + }); + } + + public static List catalogCaches(long catalogId) { + Set caches = CATALOG_CACHES.get(catalogId); + return caches == null ? Collections.emptyList() : new ArrayList<>(caches); + } + + public static OptionalLong globalMaxWeight() { + return configuredGlobalMaxWeight; + } + + public static long globalEstimatedWeight() { + return budgetManager.getGlobalUsedWeight(); + } + + private static boolean same(OptionalLong left, OptionalLong right) { + return left.isPresent() == right.isPresent() + && (!left.isPresent() || left.getAsLong() == right.getAsLong()); + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimate.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimate.java new file mode 100644 index 00000000000000..c9f25bce80d3a6 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimate.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.Objects; + +/** Immutable result of estimating one metadata-cache entry. */ +public final class MetaCacheSizeEstimate { + private final long bytes; + private final boolean complete; + private final String incompleteReason; + + private MetaCacheSizeEstimate(long bytes, boolean complete, String incompleteReason) { + this.bytes = bytes; + this.complete = complete; + this.incompleteReason = Objects.requireNonNull(incompleteReason, "incompleteReason"); + } + + public static MetaCacheSizeEstimate complete(long bytes) { + if (bytes < 0L) { + throw new IllegalArgumentException("cache size estimate can not be negative: " + bytes); + } + return new MetaCacheSizeEstimate(bytes, true, ""); + } + + public static MetaCacheSizeEstimate incomplete(String reason) { + String safeReason = Objects.requireNonNull(reason, "reason").trim(); + if (safeReason.isEmpty()) { + throw new IllegalArgumentException("incomplete cache size estimate requires a reason"); + } + return new MetaCacheSizeEstimate(0L, false, safeReason); + } + + public long getBytes() { + return bytes; + } + + public boolean isComplete() { + return complete; + } + + public String getIncompleteReason() { + return incompleteReason; + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..e6af814d1cd049 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.Objects; +import java.util.function.Supplier; + +/** Estimates the retained bytes owned by one connector metadata-cache entry. */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + MetaCacheSizeEstimate estimate(K key, V value); + + /** Estimator incompatibility rejects weighted admission without failing the metadata load. */ + static MetaCacheSizeEstimate estimateSafely( + String failureReason, Supplier estimation) { + Objects.requireNonNull(failureReason, "failureReason"); + Objects.requireNonNull(estimation, "estimation"); + try { + return Objects.requireNonNull(estimation.get(), "size estimate"); + } catch (RuntimeException | LinkageError e) { + return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimators.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimators.java new file mode 100644 index 00000000000000..8ec78638fa0f38 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimators.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +/** Shared estimators for small or structurally unknown metadata-cache entries. */ +public final class MetaCacheSizeEstimators { + private MetaCacheSizeEstimators() { + } + + /** + * Returns a visit-bounded complete graph estimator. Values that exceed the visit budget are rejected from a + * weighted cache instead of admitting an arbitrarily low sample. Large connector values should use a + * type-specific construction-time estimator that counts their payload in a cheap linear pass. + */ + public static MetaCacheSizeEstimator reflective() { + return (key, value) -> MetaCacheSizeEstimate.complete(JvmSizeUtils.saturatedAdd( + ReflectiveObjectSizeEstimator.estimateComplete(key), + ReflectiveObjectSizeEstimator.estimateComplete(value))); + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java new file mode 100644 index 00000000000000..7577f83e5aa219 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java @@ -0,0 +1,428 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.RandomAccess; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Bounded reflective object-graph estimator for validating type-specific cache estimators. + * + *

The expensive class discovery is cached in a {@link ClassValue}: each class hierarchy is reflected once to + * retain its shallow size and accessible reference fields. Runtime estimation still reads each new root's actual + * fields because two instances of the same class can retain very different lists, maps, arrays, and strings. + * The diagnostic API samples collections and object arrays. The admission API instead traverses every element + * up to a fixed visit budget: this catches skewed tail values while bounding CPU and temporary identity-set + * memory. Strongly encapsulated reference fields, depth truncation, and visit-budget exhaustion fail the + * estimate so admission can reject the value without turning an incomplete graph into a false low weight. + * + *

This is intentionally a construction-time safety net, not a Caffeine hit-path weigher. Cache values should + * combine it with a type-specific estimate once, store the larger result, and expose that stored number to + * admission accounting in O(1). + */ +public final class ReflectiveObjectSizeEstimator { + private static final int DEFAULT_SAMPLE_SIZE = 5; + private static final int DEFAULT_MAX_DEPTH = 20; + private static final int DEFAULT_COMPLETE_VISIT_BUDGET = 10_000; + + private static final long HASH_MAP_NODE_BYTES = classSize("java.util.HashMap$Node"); + private static final long LINKED_HASH_MAP_ENTRY_BYTES = classSize("java.util.LinkedHashMap$Entry"); + private static final long TREE_MAP_ENTRY_BYTES = classSize("java.util.TreeMap$Entry"); + private static final long LINKED_LIST_NODE_BYTES = classSize("java.util.LinkedList$Node"); + private static final long CONCURRENT_HASH_MAP_NODE_BYTES = classSize("java.util.concurrent.ConcurrentHashMap$Node"); + + private static final Set> SHALLOW_LEAF_TYPES = Set.of( + Boolean.class, + Byte.class, + Character.class, + Short.class, + Integer.class, + Float.class, + Long.class, + Double.class); + + private static final ClassValue CLASS_PLANS = new ClassValue<>() { + @Override + protected ClassPlan computeValue(Class type) { + List referenceFields = new ArrayList<>(); + boolean inaccessibleReferenceField = false; + for (Class current = type; current != null; current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) + || field.getType().isPrimitive() + || field.getType().isEnum()) { + continue; + } + try { + if (field.trySetAccessible()) { + referenceFields.add(field); + } else { + inaccessibleReferenceField = true; + } + } catch (RuntimeException ignored) { + inaccessibleReferenceField = true; + } + } + } + return new ClassPlan( + JvmSizeUtils.instanceSize(type), + referenceFields.toArray(new Field[0]), + inaccessibleReferenceField); + } + }; + + private ReflectiveObjectSizeEstimator() { + } + + public static long estimate(Object root) { + return estimate(root, DEFAULT_SAMPLE_SIZE, DEFAULT_MAX_DEPTH); + } + + public static long estimate(Object root, int sampleSize, int maxDepth) { + if (sampleSize <= 0) { + throw new IllegalArgumentException("sampleSize must be positive: " + sampleSize); + } + if (maxDepth < 0) { + throw new IllegalArgumentException("maxDepth can not be negative: " + maxDepth); + } + return new Walker(sampleSize, Integer.MAX_VALUE).estimate(root, maxDepth); + } + + /** + * Completely traverses a small or medium object graph for cache admission. The visit budget bounds the + * temporary identity set and CPU cost; exceeding it makes the estimate incomplete so weighted admission + * rejects the value instead of treating a sample as an exact retained size. + */ + public static long estimateComplete(Object root) { + return estimateComplete(root, DEFAULT_COMPLETE_VISIT_BUDGET, DEFAULT_MAX_DEPTH); + } + + public static long estimateComplete(Object root, int visitBudget, int maxDepth) { + if (visitBudget <= 0) { + throw new IllegalArgumentException("visitBudget must be positive: " + visitBudget); + } + if (maxDepth < 0) { + throw new IllegalArgumentException("maxDepth can not be negative: " + maxDepth); + } + return new Walker(Integer.MAX_VALUE, visitBudget).estimate(root, maxDepth); + } + + private static final class Walker { + private final int sampleSize; + private final int visitBudget; + private final Map visited = new IdentityHashMap<>(); + + private Walker(int sampleSize, int visitBudget) { + this.sampleSize = sampleSize; + this.visitBudget = visitBudget; + } + + private long estimate(Object value, int depth) { + if (value == null || visited.put(value, Boolean.TRUE) != null) { + return 0L; + } + if (visited.size() > visitBudget) { + throw new IllegalStateException( + "Complete estimate exceeded visit budget " + visitBudget); + } + + Class type = value.getClass(); + if (type.isEnum() || value instanceof Class) { + return 0L; + } + if (value instanceof String) { + return JvmSizeUtils.stringSize((String) value); + } + if (SHALLOW_LEAF_TYPES.contains(type)) { + return JvmSizeUtils.instanceSize(type); + } + if (type.isHidden()) { + throw new IllegalStateException("Can not completely estimate hidden class: " + type.getName()); + } + if (type.isArray()) { + return estimateArray(value, depth); + } + if (value instanceof ByteBuffer) { + return estimateByteBuffer((ByteBuffer) value, depth); + } + if (value instanceof Optional) { + return estimateOptional((Optional) value, depth); + } + if (value instanceof Map) { + return estimateMap((Map) value, depth); + } + if (value instanceof Collection) { + return estimateCollection((Collection) value, depth); + } + + ClassPlan plan = CLASS_PLANS.get(type); + if (plan.inaccessibleReferenceField) { + throw new IllegalStateException( + "Can not completely estimate strongly encapsulated class: " + type.getName()); + } + long bytes = plan.shallowBytes; + if (depth == 0) { + if (plan.referenceFields.length > 0) { + throw incomplete(type); + } + return bytes; + } + for (Field field : plan.referenceFields) { + bytes = add(bytes, estimateField(value, field, depth - 1)); + } + return bytes; + } + + private long estimateArray(Object value, int depth) { + int length = Array.getLength(value); + Class componentType = value.getClass().getComponentType(); + if (componentType.isPrimitive()) { + return JvmSizeUtils.primitiveArraySize(componentType, length); + } + + long bytes = JvmSizeUtils.objectArraySize(length); + if (length == 0) { + return bytes; + } + if (depth == 0) { + throw incomplete(value.getClass()); + } + int samples = Math.min(length, sampleSize); + long sampledBytes = 0L; + for (int i = 0; i < samples; i++) { + int index = sampleIndex(i, samples, length); + sampledBytes = add(sampledBytes, estimate(Array.get(value, index), depth - 1)); + } + return add(bytes, scale(sampledBytes, samples, length)); + } + + private long estimateByteBuffer(ByteBuffer value, int depth) { + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + if (value.hasArray()) { + if (depth == 0) { + throw incomplete(value.getClass()); + } + bytes = add(bytes, estimate(value.array(), depth - 1)); + } + return bytes; + } + + private long estimateOptional(Optional value, int depth) { + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + if (value.isEmpty()) { + return bytes; + } + if (depth == 0) { + throw incomplete(value.getClass()); + } + return add(bytes, estimate(value.get(), depth - 1)); + } + + private long estimateCollection(Collection values, int depth) { + int size = values.size(); + long bytes = add( + JvmSizeUtils.instanceSize(values.getClass()), + collectionStorageBytes(values, size)); + if (size == 0) { + return bytes; + } + if (depth == 0) { + throw incomplete(values.getClass()); + } + + int samples = Math.min(size, sampleSize); + long sampledBytes = 0L; + if (values instanceof List && values instanceof RandomAccess) { + List list = (List) values; + for (int i = 0; i < samples; i++) { + sampledBytes = add(sampledBytes, + estimate(list.get(sampleIndex(i, samples, size)), depth - 1)); + } + } else { + sampledBytes = estimateIterableSamples(values, samples, depth - 1); + } + return add(bytes, scale(sampledBytes, samples, size)); + } + + private long estimateMap(Map values, int depth) { + int size = values.size(); + long bytes = add( + JvmSizeUtils.instanceSize(values.getClass()), + mapStorageBytes(values, size)); + if (size == 0) { + return bytes; + } + if (depth == 0) { + throw incomplete(values.getClass()); + } + + int samples = Math.min(size, sampleSize); + long sampledBytes = 0L; + Iterator> iterator = values.entrySet().iterator(); + for (int i = 0; i < samples; i++) { + Map.Entry entry = iterator.next(); + sampledBytes = add(sampledBytes, estimate(entry.getKey(), depth - 1)); + sampledBytes = add(sampledBytes, estimate(entry.getValue(), depth - 1)); + } + return add(bytes, scale(sampledBytes, samples, size)); + } + + private long estimateIterableSamples(Collection values, int samples, int depth) { + Iterator iterator = values.iterator(); + long sampledBytes = 0L; + for (int i = 0; i < samples; i++) { + sampledBytes = add(sampledBytes, estimate(iterator.next(), depth)); + } + return sampledBytes; + } + + private long estimateField(Object owner, Field field, int depth) { + try { + return estimate(field.get(owner), depth); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Cached reference field is no longer accessible: " + field, e); + } + } + } + + private static long collectionStorageBytes(Collection values, int size) { + if (size == 0) { + return 0L; + } + if (values instanceof LinkedHashSet) { + return hashStorageBytes(size, LINKED_HASH_MAP_ENTRY_BYTES); + } + if (values instanceof HashSet) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + if (values instanceof TreeSet) { + return multiply(size, TREE_MAP_ENTRY_BYTES); + } + if (values instanceof LinkedList) { + return multiply(size, LINKED_LIST_NODE_BYTES); + } + if (values instanceof Set) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + return JvmSizeUtils.objectArraySize(size); + } + + private static long mapStorageBytes(Map values, int size) { + if (size == 0) { + return 0L; + } + if (values instanceof LinkedHashMap) { + return hashStorageBytes(size, LINKED_HASH_MAP_ENTRY_BYTES); + } + if (values instanceof HashMap) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + if (values instanceof ConcurrentHashMap) { + return hashStorageBytes(size, CONCURRENT_HASH_MAP_NODE_BYTES); + } + if (values instanceof TreeMap) { + return multiply(size, TREE_MAP_ENTRY_BYTES); + } + return add(JvmSizeUtils.objectArraySize(saturatedDouble(size)), + multiply(size, HASH_MAP_NODE_BYTES)); + } + + private static long hashStorageBytes(int size, long nodeBytes) { + return add( + JvmSizeUtils.objectArraySize(hashCapacity(size)), + multiply(size, nodeBytes)); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static int sampleIndex(int sample, int sampleCount, int totalCount) { + return sampleCount == 1 + ? 0 + : (int) ((long) sample * (totalCount - 1) / (sampleCount - 1)); + } + + private static long scale(long sampledBytes, int sampleCount, int totalCount) { + if (sampleCount == totalCount) { + return sampledBytes; + } + double scaled = (double) sampledBytes * totalCount / sampleCount; + return scaled >= Long.MAX_VALUE ? Long.MAX_VALUE : (long) scaled; + } + + private static int saturatedDouble(int value) { + return value > Integer.MAX_VALUE / 2 ? Integer.MAX_VALUE : value * 2; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static IllegalStateException incomplete(Class type) { + return new IllegalStateException("Object graph depth limit reached at: " + type.getName()); + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } + + private static final class ClassPlan { + private final long shallowBytes; + private final Field[] referenceFields; + private final boolean inaccessibleReferenceField; + + private ClassPlan(long shallowBytes, Field[] referenceFields, boolean inaccessibleReferenceField) { + this.shallowBytes = shallowBytes; + this.referenceFields = referenceFields; + this.inaccessibleReferenceField = inaccessibleReferenceField; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index 4f8ac0924e79c9..a4353067106ab9 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; @@ -48,6 +49,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; @@ -65,6 +67,8 @@ */ public final class ScopedMetaCache implements AutoCloseable { private static final Logger LOG = LogManager.getLogger(ScopedMetaCache.class); + private static final long FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES = 512L; + private static final long WEIGHT_REJECT_WARN_INTERVAL_NANOS = Duration.ofMinutes(1).toNanos(); private static final Runnable NO_OP = () -> { }; @@ -90,6 +94,9 @@ public final class ScopedMetaCache implements AutoCloseable { private final AtomicReference lastLoadSuccessTimeMs = new AtomicReference<>(-1L); private final AtomicReference lastLoadFailureTimeMs = new AtomicReference<>(-1L); private final AtomicReference lastError = new AtomicReference<>(""); + private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final LongAdder weightRejectCount = new LongAdder(); + private final AtomicLong lastWeightRejectWarnNanos = new AtomicLong(Long.MIN_VALUE); private final RemovalListener beforeRemoval; private final BiConsumer discardListener; private final Ticker ticker; @@ -98,8 +105,12 @@ public final class ScopedMetaCache implements AutoCloseable { private final Runnable afterLoadElection; private final Runnable afterBulkStage; private final Runnable afterRefreshRegistration; + private final MetaCacheSizeEstimator sizeEstimator; + private final MetaCacheBudgetManager.EntryBudget entryBudget; + private final boolean weightBounded; private final ThreadLocal> removalDeferrals = ThreadLocal.withInitial(RemovalDeferral::new); + private final ThreadLocal budgetEvictionRegistration = new ThreadLocal<>(); private BigInteger exactInvalidationSequence = BigInteger.ZERO; ScopedMetaCache( @@ -113,7 +124,9 @@ public final class ScopedMetaCache implements AutoCloseable { Executor refreshExecutor, Runnable afterLoadElection, Runnable afterBulkStage, - Runnable afterRefreshRegistration) { + Runnable afterRefreshRegistration, + MetaCacheSizeEstimator sizeEstimator, + MetaCacheBudgetManager.EntryBudget entryBudget) { this.registry = Objects.requireNonNull(registry, "registry can not be null"); this.name = Objects.requireNonNull(name, "name can not be null"); Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); @@ -127,8 +140,17 @@ public final class ScopedMetaCache implements AutoCloseable { this.afterBulkStage = Objects.requireNonNull(afterBulkStage, "afterBulkStage can not be null"); this.afterRefreshRegistration = Objects.requireNonNull( afterRefreshRegistration, "afterRefreshRegistration can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); + this.sizeEstimator = sizeEstimator; + this.entryBudget = entryBudget; + if ((sizeEstimator == null) != (entryBudget == null)) { + throw new IllegalArgumentException( + "Weighted metadata cache requires both estimator and budget: " + name); + } + this.weightBounded = entryBudget != null; + if (weightBounded) { + entryBudget.setReclaimer(this::reclaimForPeer); + } + this.effectiveEnabled = cacheSpec.isCacheEnabled(); Caffeine builder = Caffeine.newBuilder() .maximumSize(effectiveEnabled ? cacheSpec.getCapacity() : 0L) @@ -284,10 +306,46 @@ public void put(K key, ScopePath path, V value) { return; } try (PublicationLease lease = acquirePublicationLease(key, path, false)) { - guardedCommit(lease, () -> { - lease.keyNode.loadPublicationState.set(new Object()); - return publishCommitted(lease, key, value) != null; - }); + AtomicReference> preparedRef = new AtomicReference<>(); + AtomicReference> replacementRef = new AtomicReference<>(); + AtomicBoolean admissionRejected = new AtomicBoolean(false); + boolean retained = false; + try { + retained = guardedCommit(lease, () -> { + VersionedValue current = currentVersionedValue(key, path); + if (current != null) { + ReplacementValue replacement = + prepareReplacementVersionedValue(lease, key, value, current); + replacementRef.set(replacement); + if (replacement == null) { + admissionRejected.set(true); + lease.keyNode.loadPublicationState.set(new Object()); + removeCurrentVersion(key, current); + return false; + } + lease.keyNode.loadPublicationState.set(new Object()); + return installReplacement(replacement); + } + VersionedValue prepared = prepareVersionedValue(lease, key, value); + preparedRef.set(prepared); + if (prepared == null) { + admissionRejected.set(true); + return false; + } + lease.keyNode.loadPublicationState.set(new Object()); + install(prepared, lease); + return true; + }); + } finally { + if (!retained && replacementRef.get() != null) { + replacementRef.get().rollback(); + } else if (!retained && preparedRef.get() != null) { + releaseReservation(preparedRef.get()); + } + if (admissionRejected.get()) { + notifyDiscarded(key, value); + } + } } } @@ -306,29 +364,74 @@ public boolean compareAndSet( return true; } try (PublicationLease lease = acquirePublicationLease(key, path, false)) { - return guardedCommit(lease, () -> { - VersionedValue current = currentVersionedValue(key, path); - V currentValue = current == null ? null : current.value; - if (currentValue != expectedValue) { - return false; - } - lease.keyNode.loadPublicationState.set(new Object()); - action.run(); - if (updatedValue == currentValue) { + AtomicReference> preparedRef = new AtomicReference<>(); + AtomicReference> replacementRef = new AtomicReference<>(); + AtomicBoolean admissionRejected = new AtomicBoolean(false); + boolean committed = false; + try { + committed = guardedCommit(lease, () -> { + VersionedValue current = currentVersionedValue(key, path); + V currentValue = current == null ? null : current.value; + if (currentValue != expectedValue) { + return false; + } + if (updatedValue != null && updatedValue != currentValue) { + if (current != null) { + ReplacementValue replacement = + prepareReplacementVersionedValue(lease, key, updatedValue, current); + replacementRef.set(replacement); + if (replacement == null) { + admissionRejected.set(true); + lease.keyNode.loadPublicationState.set(new Object()); + action.run(); + removeCurrentVersion(key, current); + return true; + } + } else { + VersionedValue prepared = prepareVersionedValue(lease, key, updatedValue); + preparedRef.set(prepared); + if (prepared == null) { + admissionRejected.set(true); + lease.keyNode.loadPublicationState.set(new Object()); + action.run(); + return true; + } + } + } + lease.keyNode.loadPublicationState.set(new Object()); + action.run(); + if (updatedValue == currentValue) { + lease.keyNode.loadPublicationState.set(new Object()); + return true; + } + if (updatedValue != null) { + if (replacementRef.get() != null) { + lease.keyNode.loadPublicationState.set(new Object()); + return installReplacement(replacementRef.get()); + } + lease.keyNode.loadPublicationState.set(new Object()); + install(preparedRef.get(), lease); + return true; + } + if (updatedValue == null) { + if (current != null) { + removeCurrentVersion(key, current); + } + } lease.keyNode.loadPublicationState.set(new Object()); return true; + }); + } finally { + if (!committed && replacementRef.get() != null) { + replacementRef.get().rollback(); + } else if (!committed && preparedRef.get() != null) { + releaseReservation(preparedRef.get()); } - if (updatedValue == null) { - if (current != null) { - data.asMap().remove(key, current); - lease.keyNode.registration.compareAndSet(current, null); - } - } else { - publishCommitted(lease, key, updatedValue); + if (admissionRejected.get()) { + notifyDiscarded(key, updatedValue); } - lease.keyNode.loadPublicationState.set(new Object()); - return true; - }); + } + return committed; } } @@ -366,10 +469,14 @@ void invalidateKey( return; } afterStateReplacement.run(); - VersionedValue registered = invalidated.node.registration.get(); - if (registered != null && registered.keyState == invalidated.keyState) { - data.asMap().remove(key, registered); - invalidated.node.registration.compareAndSet(registered, null); + Registration registered = invalidated.node.registration.get(); + VersionedValue current = data.getIfPresent(key); + if (registered != null && current != null && current.registration == registered + && current.keyState == invalidated.keyState) { + data.asMap().remove(key, current); + } + if (registered != null && invalidated.node.registration.compareAndSet(registered, null)) { + releaseRegistration(registered); } tryPruneKey(key, invalidated.node); } @@ -414,9 +521,20 @@ public boolean publish( return false; } try (PublicationLease lease = acquirePublicationLease(key, actualScope, false)) { - VersionedValue staged = newVersionedValue(lease, key, value); - afterBulkStage.run(); - return handle.tryCommit(key, lease, staged); + VersionedValue staged = prepareVersionedValue(lease, key, value); + if (staged == null) { + return false; + } + boolean retained = false; + try { + afterBulkStage.run(); + retained = handle.tryCommit(key, lease, staged); + return retained; + } finally { + if (!retained) { + releaseReservation(staged); + } + } } } @@ -438,7 +556,12 @@ public CacheMetrics metrics() { invalidateCount.sum(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), - lastError.get())); + lastError.get(), + weightBounded, + entryBudget == null ? -1L : entryBudget.getEffectiveMaxWeight(), + entryBudget == null ? 0L : entryBudget.getUsedWeight(), + weightRejectCount.sum(), + lastWeightRejectReason.get())); } int refreshingCountForTest() { @@ -504,9 +627,11 @@ void closeFromRegistry() { void removeExpectedRaw(Object rawKey, Object expectedValue) { @SuppressWarnings("unchecked") K key = (K) rawKey; - @SuppressWarnings("unchecked") - VersionedValue versionedValue = (VersionedValue) expectedValue; - data.asMap().remove(key, versionedValue); + Registration expectedRegistration = (Registration) expectedValue; + VersionedValue current = data.getIfPresent(key); + if (current != null && current.registration == expectedRegistration) { + data.asMap().remove(key, current); + } } private PublicationLease acquirePublicationLease( @@ -527,29 +652,25 @@ private PublicationLease acquirePublicationLease( } } - private VersionedValue publish(PublicationLease lease, K key, V value) { - if (!lease.isCurrent()) { - return null; - } - VersionedValue versioned = newVersionedValue(lease, key, value); - install(versioned, lease); - if (!lease.isCurrent()) { - data.asMap().remove(key, versioned); - return null; - } - return versioned; - } - - private VersionedValue publishCommitted(PublicationLease lease, K key, V value) { - return publish(lease, key, value); - } - private boolean commitLoaded(PublicationLease lease, K key, V value, Runnable beforePublication) { Runnable action = Objects.requireNonNull(beforePublication, "beforePublication can not be null"); - return guardedCommit(lease, () -> { - action.run(); - return publishCommitted(lease, key, value) != null; - }); + VersionedValue prepared = prepareVersionedValue(lease, key, value); + boolean retained = false; + try { + retained = guardedCommit(lease, () -> { + action.run(); + if (prepared == null) { + return false; + } + install(prepared, lease); + return true; + }); + return retained; + } finally { + if (!retained && prepared != null) { + releaseReservation(prepared); + } + } } private boolean guardedCommit(PublicationLease lease, BooleanSupplier commitAction) { @@ -562,11 +683,167 @@ private boolean guardedCommit(PublicationLease lease, BooleanSupplier comm }))); } - private VersionedValue newVersionedValue( + private VersionedValue prepareVersionedValue( PublicationLease lease, K key, V value) { + MetaCacheBudgetManager.AdmissionReservation reservation = reserve(key, value); + if (weightBounded && reservation == null) { + return null; + } CacheAddress address = new CacheAddress(this, key); + ScopeSnapshot scopeSnapshot = lease.scopeLease.snapshot(); + Registration registration = new Registration(reservation, address, scopeSnapshot); return new VersionedValue<>( - key, value, address, lease.scopeLease.snapshot(), lease.keyNode, lease.keyState, ticker.read()); + key, value, address, scopeSnapshot, lease.keyNode, lease.keyState, + ticker.read(), registration); + } + + private MetaCacheBudgetManager.AdmissionReservation reserve(K key, V value) { + if (!weightBounded) { + return null; + } + long bytes = estimateWeight(key, value); + return bytes < 0L ? null : reserveEstimated(bytes); + } + + private long estimateWeight(K key, V value) { + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "estimator_failure", () -> sizeEstimator.estimate(key, value)); + if (!estimate.isComplete()) { + rejectWeight("incomplete_estimate:" + estimate.getIncompleteReason()); + return -1L; + } + if (estimate.getBytes() == 0L) { + rejectWeight("invalid_zero_estimate"); + return -1L; + } + long bytes = JvmSizeUtils.saturatedAdd( + estimate.getBytes(), FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES); + if (bytes > entryBudget.getEffectiveMaxWeight()) { + rejectWeight("entry_too_large"); + return -1L; + } + return bytes; + } + + private MetaCacheBudgetManager.AdmissionReservation reserveEstimated(long bytes) { + Optional reservation = entryBudget.tryReserve(bytes); + if (!reservation.isPresent()) { + entryBudget.requestPeerReclaim(bytes); + rejectWeight("budget_exceeded"); + return null; + } + return reservation.get(); + } + + private ReplacementValue prepareReplacementVersionedValue( + PublicationLease lease, K key, V value, VersionedValue previous) { + if (!weightBounded) { + VersionedValue versioned = prepareVersionedValue(lease, key, value); + return versioned == null ? null : new ReplacementValue<>(versioned, previous, null); + } + long bytes = estimateWeight(key, value); + if (bytes < 0L) { + return null; + } + Optional replacement = + entryBudget.tryReplace(previous.registration.reservation, bytes); + if (!replacement.isPresent()) { + long additionalBytes = Math.max(0L, + bytes - previous.registration.reservation.getBytes()); + entryBudget.requestPeerReclaim(additionalBytes); + rejectWeight("budget_exceeded"); + return null; + } + MetaCacheBudgetManager.ReservationReplacement accounting = replacement.get(); + if (!previous.registration.released.compareAndSet(false, true)) { + accounting.rollback(); + return null; + } + CacheAddress address = new CacheAddress(this, key); + ScopeSnapshot scopeSnapshot = lease.scopeLease.snapshot(); + Registration registration = new Registration(accounting.current(), address, scopeSnapshot); + VersionedValue versioned = new VersionedValue<>( + key, value, address, scopeSnapshot, lease.keyNode, lease.keyState, + ticker.read(), registration); + return new ReplacementValue<>(versioned, previous, accounting); + } + + private boolean installReplacement(ReplacementValue replacement) { + VersionedValue previous = replacement.previous; + VersionedValue current = replacement.current; + registry.register(current.address, current.registration, current.scopeSnapshot); + if (!current.keyNode.registration.compareAndSet( + previous.registration, current.registration)) { + registry.register(previous.address, previous.registration, previous.scopeSnapshot); + replacement.rollback(); + return false; + } + if (!data.asMap().replace(current.key, previous, current)) { + current.keyNode.registration.compareAndSet(current.registration, previous.registration); + registry.register(previous.address, previous.registration, previous.scopeSnapshot); + replacement.rollback(); + return false; + } + replacement.commit(); + return true; + } + + private long evictLocalColdest() { + if (!data.policy().eviction().isPresent()) { + return 0L; + } + Map> coldest = data.policy().eviction().get().coldest(1); + for (Map.Entry> candidate : coldest.entrySet()) { + VersionedValue current = data.getIfPresent(candidate.getKey()); + if (current == candidate.getValue()) { + long weight = current.registration.reservation.getBytes(); + budgetEvictionRegistration.set(current.registration); + try { + if (data.asMap().remove(candidate.getKey(), current)) { + return weight; + } + } finally { + budgetEvictionRegistration.remove(); + } + } + } + return 0L; + } + + private long reclaimForPeer(long targetBytes) { + if (targetBytes <= 0L || closed.get()) { + return 0L; + } + long reclaimed = 0L; + long removed; + while (reclaimed < targetBytes && (removed = evictLocalColdest()) > 0L) { + reclaimed = JvmSizeUtils.saturatedAdd(reclaimed, removed); + } + return reclaimed; + } + + private void rejectWeight(String reason) { + weightRejectCount.increment(); + lastWeightRejectReason.set(reason); + long now = System.nanoTime(); + long last = lastWeightRejectWarnNanos.get(); + if ((last == Long.MIN_VALUE || now - last >= WEIGHT_REJECT_WARN_INTERVAL_NANOS) + && lastWeightRejectWarnNanos.compareAndSet(last, now)) { + LOG.warn("Metadata cache entry '{}' rejected a value by weight: reason={}, used={}, max={}", + name, reason, entryBudget == null ? 0L : entryBudget.getUsedWeight(), + entryBudget == null ? -1L : entryBudget.getEffectiveMaxWeight()); + } + } + + private static void releaseReservation(VersionedValue versioned) { + releaseRegistration(versioned.registration); + } + + private static void releaseRegistration(Registration registration) { + if (registration != null && registration.released.compareAndSet(false, true) + && registration.reservation != null) { + registration.reservation.release(); + } } private void scheduleRefresh(K key, ScopePath path, Function loader, VersionedValue current) { @@ -620,34 +897,45 @@ private void scheduleRefresh(K key, ScopePath path, Function loader, Versi private void install( VersionedValue versioned, PublicationLease lease) { - registry.register(versioned.address, versioned, versioned.scopeSnapshot); - lease.keyNode.registration.set(versioned); + registry.register(versioned.address, versioned.registration, versioned.scopeSnapshot); + lease.keyNode.registration.set(versioned.registration); data.asMap().put(versioned.key, versioned); } private boolean replaceRefreshExpected(PublicationLease lease, K key, VersionedValue expected, V refreshed) { - return guardedCommit(lease, () -> { - if (data.getIfPresent(key) != expected || !expected.isCurrent(registry, keyNodes)) { - return false; - } - if (refreshed == expected.value) { + if (refreshed == expected.value) { + return guardedCommit(lease, () -> { + if (data.getIfPresent(key) != expected || !expected.isCurrent(registry, keyNodes)) { + return false; + } expected.writeTimeNanos = ticker.read(); return true; + }); + } + AtomicReference> replacementRef = new AtomicReference<>(); + boolean retained = false; + try { + retained = guardedCommit(lease, () -> { + if (data.getIfPresent(key) != expected || !expected.isCurrent(registry, keyNodes)) { + return false; + } + ReplacementValue replacement = + prepareReplacementVersionedValue(lease, key, refreshed, expected); + replacementRef.set(replacement); + if (replacement == null) { + lease.keyNode.loadPublicationState.set(new Object()); + removeCurrentVersion(key, expected); + return false; + } + return installReplacement(replacement); + }); + return retained; + } finally { + if (!retained && replacementRef.get() != null) { + replacementRef.get().rollback(); } - VersionedValue replacement = newVersionedValue(lease, key, refreshed); - registry.register(replacement.address, replacement, replacement.scopeSnapshot); - if (!lease.keyNode.registration.compareAndSet(expected, replacement)) { - registry.unregister(replacement.address, replacement, replacement.scopeSnapshot); - return false; - } - if (!data.asMap().replace(key, expected, replacement)) { - lease.keyNode.registration.compareAndSet(replacement, null); - registry.unregister(replacement.address, replacement, replacement.scopeSnapshot); - return false; - } - return true; - }); + } } private boolean tryCommitBulk( @@ -679,6 +967,13 @@ private boolean tryCommitBulk( })); } + private void removeCurrentVersion(K key, VersionedValue current) { + data.asMap().remove(key, current); + if (current.keyNode.registration.compareAndSet(current.registration, null)) { + releaseRegistration(current.registration); + } + } + private boolean deferRemovals(BooleanSupplier action) { RemovalDeferral removalDeferral = removalDeferrals.get(); removalDeferral.depth++; @@ -752,25 +1047,30 @@ private void closePhysicalState() { bulkInvalidationGate.write(exactInvalidations::clear); keyNodes.forEach((key, node) -> { replaceKeyState(node); - VersionedValue versioned = node.registration.get(); - if (versioned != null) { + Registration registered = node.registration.get(); + VersionedValue versioned = data.getIfPresent(key); + if (versioned != null && versioned.registration == registered) { data.asMap().remove(key, versioned); - node.registration.compareAndSet(versioned, null); + } + if (registered != null && node.registration.compareAndSet(registered, null)) { + releaseRegistration(registered); } tryPruneKey(key, node); }); data.invalidateAll(); data.cleanUp(); + if (entryBudget != null) { + entryBudget.close(); + } } private void onRemoval( Object rawKey, Object rawValue, RemovalCause cause) { - Objects.requireNonNull(rawKey, "removed cache key can not be null"); - Objects.requireNonNull(rawValue, "removed cache value can not be null"); @SuppressWarnings("unchecked") VersionedValue versioned = (VersionedValue) rawValue; RemovalDeferral removalDeferral = removalDeferrals.get(); - removalDeferral.removals.addLast(new DeferredRemoval<>(versioned, cause)); + removalDeferral.removals.addLast(new DeferredRemoval<>( + versioned, cause, budgetEvictionRegistration.get() == versioned.registration)); if (removalDeferral.depth > 0 || removalDeferral.draining) { return; } @@ -784,7 +1084,7 @@ private void drainDeferredRemovals(RemovalDeferral removalDeferral) { DeferredRemoval removal; while ((removal = removalDeferral.removals.pollFirst()) != null) { try { - completeRemoval(removal.versioned, removal.cause); + completeRemoval(removal.versioned, removal.cause, removal.budgetEviction); } catch (RuntimeException | Error e) { if (failure == null) { failure = e; @@ -806,8 +1106,9 @@ private void drainDeferredRemovals(RemovalDeferral removalDeferral) { } } - private void completeRemoval(VersionedValue versioned, RemovalCause cause) { - if (cause.wasEvicted()) { + private void completeRemoval( + VersionedValue versioned, RemovalCause cause, boolean budgetEviction) { + if (cause.wasEvicted() || budgetEviction) { evictionCount.increment(); } else if (cause == RemovalCause.EXPLICIT) { invalidateCount.increment(); @@ -820,9 +1121,12 @@ private void completeRemoval(VersionedValue versioned, RemovalCause cause) LOG.warn("Scoped metadata cache removal callback failed", t); } } - registry.unregister(versioned.address, versioned, versioned.scopeSnapshot); - versioned.keyNode.registration.compareAndSet(versioned, null); - tryPruneKey(key, versioned.keyNode); + synchronized (versioned.keyNode) { + registry.unregister(versioned.address, versioned.registration, versioned.scopeSnapshot); + versioned.keyNode.registration.compareAndSet(versioned.registration, null); + releaseRegistration(versioned.registration); + tryPruneKey(key, versioned.keyNode); + } } private void notifyDiscarded(K key, V value) { @@ -845,10 +1149,13 @@ private static final class RemovalDeferral { private static final class DeferredRemoval { private final VersionedValue versioned; private final RemovalCause cause; + private final boolean budgetEviction; - private DeferredRemoval(VersionedValue versioned, RemovalCause cause) { + private DeferredRemoval( + VersionedValue versioned, RemovalCause cause, boolean budgetEviction) { this.versioned = versioned; this.cause = cause; + this.budgetEviction = budgetEviction; } } @@ -956,6 +1263,11 @@ public static final class CacheMetrics { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final boolean weightBounded; + private final long maxWeight; + private final long estimatedWeight; + private final long weightRejectCount; + private final String lastWeightRejectReason; private CacheMetrics( long physicalEntryCount, @@ -974,7 +1286,12 @@ private CacheMetrics( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + boolean weightBounded, + long maxWeight, + long estimatedWeight, + long weightRejectCount, + String lastWeightRejectReason) { this.physicalEntryCount = physicalEntryCount; this.keyNodeCount = keyNodeCount; this.inFlightLoadCount = inFlightLoadCount; @@ -992,6 +1309,11 @@ private CacheMetrics( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = lastError; + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.weightRejectCount = weightRejectCount; + this.lastWeightRejectReason = lastWeightRejectReason; } public long getPhysicalEntryCount() { @@ -1061,6 +1383,26 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public boolean isWeightBounded() { + return weightBounded; + } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getWeightRejectCount() { + return weightRejectCount; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } private static final class PublicationLease implements AutoCloseable { @@ -1115,6 +1457,7 @@ private static final class VersionedValue { private final KeyNode keyNode; private final KeyState keyState; private volatile long writeTimeNanos; + private final Registration registration; private VersionedValue( K key, @@ -1123,7 +1466,8 @@ private VersionedValue( ScopeSnapshot scopeSnapshot, KeyNode keyNode, KeyState keyState, - long writeTimeNanos) { + long writeTimeNanos, + Registration registration) { this.key = key; this.value = value; this.address = address; @@ -1131,6 +1475,7 @@ private VersionedValue( this.keyNode = keyNode; this.keyState = keyState; this.writeTimeNanos = writeTimeNanos; + this.registration = registration; } private boolean isCurrent( @@ -1139,14 +1484,63 @@ private boolean isCurrent( return scopeSnapshot.isCurrent(registry) && currentKeyNodes.get(key) == keyNode && keyNode.current.get() == keyState - && keyNode.registration.get() == this; + && keyNode.registration.get() == registration; + } + } + + private static final class ReplacementValue { + private final VersionedValue current; + private final VersionedValue previous; + private final MetaCacheBudgetManager.ReservationReplacement accounting; + private boolean finished; + + private ReplacementValue(VersionedValue current, + VersionedValue previous, + MetaCacheBudgetManager.ReservationReplacement accounting) { + this.current = current; + this.previous = previous; + this.accounting = accounting; + } + + private void commit() { + if (accounting != null) { + accounting.commit(); + } + finished = true; + } + + private void rollback() { + if (finished) { + return; + } + if (accounting != null) { + current.registration.released.set(true); + previous.registration.released.set(false); + accounting.rollback(); + } + finished = true; + } + } + + /** Lightweight ownership token: deliberately does not retain the cached value. */ + private static final class Registration { + private final MetaCacheBudgetManager.AdmissionReservation reservation; + private final CacheAddress address; + private final ScopeSnapshot scopeSnapshot; + private final AtomicBoolean released = new AtomicBoolean(false); + + private Registration(MetaCacheBudgetManager.AdmissionReservation reservation, + CacheAddress address, ScopeSnapshot scopeSnapshot) { + this.reservation = reservation; + this.address = address; + this.scopeSnapshot = scopeSnapshot; } } private static final class KeyNode { private final AtomicReference current = new AtomicReference<>(new KeyState()); private final AtomicReference loadPublicationState = new AtomicReference<>(new Object()); - private final AtomicReference> registration = new AtomicReference<>(); + private final AtomicReference registration = new AtomicReference<>(); private final AtomicInteger activeLoads = new AtomicInteger(); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index d08fcdb97202ea..918b2829a7927b 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -19,6 +19,8 @@ import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.time.Duration; import java.util.ArrayList; @@ -46,6 +48,7 @@ * live values and in-flight loads rather than by every name ever observed. */ public final class ScopedMetaCacheRegistry implements AutoCloseable { + private static final Logger LOG = LogManager.getLogger(ScopedMetaCacheRegistry.class); private static final Runnable NO_OP = () -> { }; private static final BiConsumer NO_OP_SCOPE = (level, key) -> { @@ -81,19 +84,23 @@ ScopedMetaCache createCacheWithRemovalListener( String name, CacheSpec cacheSpec, RemovalListener removalListener, Duration refreshAfterWrite, Executor refreshExecutor) { return createCacheWithRemovalListener(name, cacheSpec, null, removalListener, - null, refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP); + null, refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP, + null, null); } ScopedMetaCache createCacheWithMetaRemovalListener( String name, CacheSpec cacheSpec, MetaCacheRemovalListener removalListener, BiConsumer discardListener, - Duration refreshAfterWrite, Executor refreshExecutor) { + Duration refreshAfterWrite, Executor refreshExecutor, + MetaCacheSizeEstimator sizeEstimator, + MetaCacheBudgetManager.EntryBudget entryBudget) { RemovalListener caffeineListener = removalListener == null ? null : (key, value, cause) -> removalListener.onRemoval( key, value, MetaCacheRemovalReason.valueOf(cause.name())); return createCacheWithRemovalListener( name, cacheSpec, null, caffeineListener, discardListener, - refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP); + refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP, + sizeEstimator, entryBudget); } ScopedMetaCache createCache( @@ -125,7 +132,7 @@ ScopedMetaCache createCache( : (key, value, cause) -> beforeRemoval.accept(key, value); return createCacheWithRemovalListener( name, cacheSpec, ticker, listener, null, null, null, - afterLoadElection, afterBulkStage, NO_OP); + afterLoadElection, afterBulkStage, NO_OP, null, null); } ScopedMetaCache createCacheWithRefresh( @@ -136,7 +143,7 @@ ScopedMetaCache createCacheWithRefresh( Runnable afterRefreshRegistration) { return createCacheWithRemovalListener( name, cacheSpec, null, null, null, refreshAfterWrite, refreshExecutor, - NO_OP, NO_OP, afterRefreshRegistration); + NO_OP, NO_OP, afterRefreshRegistration, null, null); } private ScopedMetaCache createCacheWithRemovalListener( @@ -149,7 +156,9 @@ private ScopedMetaCache createCacheWithRemovalListener( Executor refreshExecutor, Runnable afterLoadElection, Runnable afterBulkStage, - Runnable afterRefreshRegistration) { + Runnable afterRefreshRegistration, + MetaCacheSizeEstimator sizeEstimator, + MetaCacheBudgetManager.EntryBudget entryBudget) { checkOpen(); ScopedMetaCache cache = new ScopedMetaCache<>( this, @@ -162,7 +171,9 @@ private ScopedMetaCache createCacheWithRemovalListener( refreshExecutor, afterLoadElection, afterBulkStage, - afterRefreshRegistration); + afterRefreshRegistration, + sizeEstimator, + entryBudget); caches.add(cache); if (closed.get()) { caches.remove(cache); @@ -252,7 +263,25 @@ public void close() { cleanDetachedState(oldState); List> snapshot = new ArrayList<>(caches); caches.clear(); - snapshot.forEach(ScopedMetaCache::closeFromRegistry); + Throwable failure = null; + for (ScopedMetaCache cache : snapshot) { + try { + cache.closeFromRegistry(); + } catch (RuntimeException | Error throwable) { + LOG.error("Failed to close scoped metadata cache", throwable); + if (failure == null) { + failure = throwable; + } else { + failure.addSuppressed(throwable); + } + } + } + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure != null) { + throw (Error) failure; + } } ScopeLease acquire(ScopePath path) { @@ -280,12 +309,12 @@ void removeCache(ScopedMetaCache cache) { caches.remove(cache); } - void register(CacheAddress address, Object versionedValue, ScopeSnapshot snapshot) { - snapshot.leafState().entries.put(address, versionedValue); + void register(CacheAddress address, Object ownershipToken, ScopeSnapshot snapshot) { + snapshot.leafState().entries.put(address, ownershipToken); } - void unregister(CacheAddress address, Object versionedValue, ScopeSnapshot snapshot) { - snapshot.leafState().entries.remove(address, versionedValue); + void unregister(CacheAddress address, Object ownershipToken, ScopeSnapshot snapshot) { + snapshot.leafState().entries.remove(address, ownershipToken); tryPrune(snapshot); } @@ -404,9 +433,9 @@ private void bumpPublicationStates(List nodes) { } private void cleanDetachedState(ScopeState state) { - state.entries.forEach((address, value) -> { - if (state.entries.remove(address, value)) { - address.removeExpected(value); + state.entries.forEach((address, ownershipToken) -> { + if (state.entries.remove(address, ownershipToken)) { + address.removeExpected(ownershipToken); } }); state.children.values().forEach(child -> cleanDetachedState(child.current.get())); @@ -859,6 +888,7 @@ private void cancelPrune() { private static final class ScopeState { private final long generation; private final ConcurrentMap children = new ConcurrentHashMap<>(); + // Ownership tokens contain no cached value, so the registry does not become a second value owner. private final ConcurrentMap entries = new ConcurrentHashMap<>(); private volatile ScopeSnapshot scopeSnapshot; diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java index c0cde92bd92229..08d5808096eb9b 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java @@ -246,4 +246,50 @@ public void isMetaCacheKeyForEngine() { CacheSpec.isMetaCacheKeyForEngine("meta.cache.paimon.table.ttl-second", "iceberg")); Assertions.assertFalse(CacheSpec.isMetaCacheKeyForEngine(null, "iceberg")); } + + @Test + public void parsesAndValidatesWeightProperties() { + Assertions.assertEquals(1024L, CacheSpec.parseWeight("1KB", "weight", false, 0L)); + Assertions.assertEquals(512L, CacheSpec.parseWeight("25%", "weight", true, 2048L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1.5GB", "weight", false, 0L)); + + Map properties = new HashMap<>(); + properties.put("meta.cache.max-weight", "2GB"); + properties.put("meta.cache.iceberg.table.max-weight", "1GB"); + CacheSpec.checkWeightProperties(properties, "iceberg", "table"); + + properties.put("meta.cache.iceberg.table.max-weight", "3GB"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkWeightProperties(properties, "iceberg", "table")); + + properties.put("meta.cache.iceberg.table.max-weight", "0"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkWeightProperties(properties, "iceberg", "table")); + + properties.put("meta.cache.iceberg.table.max-weight", "1GB"); + properties.put("meta.cache.iceberg.unknown.max-weight", "1MB"); + IllegalArgumentException unknownEntry = Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkWeightProperties(properties, "iceberg", "table")); + Assertions.assertTrue(unknownEntry.getMessage().contains("meta.cache.iceberg.unknown.max-weight")); + + properties.clear(); + properties.put("meta.cache.max-weight", "invalid"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkCatalogWeightProperty(properties)); + } + + @Test + public void invalidPersistedEntryWeightFallsBackToUnbounded() { + Map properties = new HashMap<>(); + properties.put("meta.cache.hive.file.max-weight", "invalid"); + CacheSpec spec = CacheSpec.fromProperties( + properties, "hive", "file", CacheSpec.of(true, 60L, 100L)); + Assertions.assertFalse(spec.getMaxWeight().isPresent()); + + properties.put("meta.cache.hive.file.max-weight", "0"); + spec = CacheSpec.fromProperties(properties, "hive", "file", + CacheSpec.of(true, 10L, 100L)); + Assertions.assertFalse(spec.getMaxWeight().isPresent()); + } } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEstimatorBenchmark.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEstimatorBenchmark.java new file mode 100644 index 00000000000000..44182b954b4140 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEstimatorBenchmark.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Dependency-free microbenchmark for bounded estimators versus an exact list traversal. + * Run after test compilation with the module's target/classes and target/test-classes on the classpath. + */ +public final class MetaCacheEstimatorBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static volatile long blackhole; + + private MetaCacheEstimatorBenchmark() { + } + + public static void main(String[] args) { + for (int size : new int[] {1_000, 100_000}) { + List values = fixture(size); + Result sampled = measure(() -> ReflectiveObjectSizeEstimator.estimate(values), 10_000); + Result typedSampled = measure(() -> JvmSizeUtils.sampledListPayload( + values, 16, JvmSizeUtils::stringSize), 10_000); + int exactOperationsPerWindow = size <= 1_000 ? 1_000 : 20; + Result exact = measure(() -> exactStringListSize(values), exactOperationsPerWindow); + Result boundedAdmission = measure( + () -> completeOrRejected(values), exactOperationsPerWindow); + System.out.printf( + "uniform_size=%d reflective_sampled_ns_op=%d typed_sampled_ns_op=%d " + + "exact_ns_op=%d bounded_admission_ns_op=%d reflective_sampled_ops=%d " + + "typed_sampled_ops=%d exact_ops=%d bounded_admission_ops=%d%n", + size, sampled.medianNanos, typedSampled.medianNanos, exact.medianNanos, + boundedAdmission.medianNanos, sampled.operations, typedSampled.operations, + exact.operations, boundedAdmission.operations); + } + + Map skewed = skewedFixture(); + long sampledBytes = ReflectiveObjectSizeEstimator.estimate(skewed); + long completeBytes = ReflectiveObjectSizeEstimator.estimateComplete(skewed); + Result sampled = measure(() -> ReflectiveObjectSizeEstimator.estimate(skewed), 10_000); + Result complete = measure(() -> ReflectiveObjectSizeEstimator.estimateComplete(skewed), 1_000); + System.out.printf( + "skewed_map_size=%d sampled_bytes=%d complete_bytes=%d sampled_ns_op=%d " + + "complete_ns_op=%d sampled_ops=%d complete_ops=%d%n", + skewed.size(), sampledBytes, completeBytes, sampled.medianNanos, complete.medianNanos, + sampled.operations, complete.operations); + } + + private static List fixture(int size) { + List values = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + values.add("partition-" + i + "-value-0123456789"); + } + return values; + } + + private static long exactStringListSize(List values) { + long bytes = JvmSizeUtils.arrayListSize(values.size()); + for (String value : values) { + bytes = JvmSizeUtils.saturatedAdd(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long completeOrRejected(Object value) { + try { + return ReflectiveObjectSizeEstimator.estimateComplete(value); + } catch (IllegalStateException expected) { + return -1L; + } + } + + private static Map skewedFixture() { + Map values = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + values.put("small-" + i, "x"); + } + values.put("large-tail", "x".repeat(2 * 1024 * 1024)); + return values; + } + + private static Result measure(LongOperation operation, int operationsPerWindow) { + for (int i = 0; i < WARMUP_WINDOWS; i++) { + runWindow(operation, operationsPerWindow); + } + long[] nanosPerOperation = new long[MEASURE_WINDOWS]; + for (int i = 0; i < MEASURE_WINDOWS; i++) { + long start = System.nanoTime(); + runWindow(operation, operationsPerWindow); + nanosPerOperation[i] = (System.nanoTime() - start) / operationsPerWindow; + } + Arrays.sort(nanosPerOperation); + return new Result(nanosPerOperation[MEASURE_WINDOWS / 2], + (long) operationsPerWindow * MEASURE_WINDOWS); + } + + private static void runWindow(LongOperation operation, int operations) { + long value = 0L; + for (int i = 0; i < operations; i++) { + value ^= operation.run(); + } + blackhole = value; + } + + @FunctionalInterface + private interface LongOperation { + long run(); + } + + private static final class Result { + private final long medianNanos; + private final long operations; + + private Result(long medianNanos, long operations) { + this.medianNanos = medianNanos; + this.operations = operations; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java new file mode 100644 index 00000000000000..b7983a4a420e5d --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java @@ -0,0 +1,463 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.time.Duration; +import java.util.AbstractSet; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +class MetaCacheWeightGovernanceTest { + @Test + void admissionRemovalAndCloseKeepHierarchicalAccountingBalanced() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(4096L)); + CatalogMetaCache owner = new CatalogMetaCache(manager, 7L, "iceberg", Collections.emptyMap()); + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.ofWeight(true, -1L, 100L, 2048L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build()); + + entry.put("one", "value"); + Assertions.assertEquals(1024L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(1024L, manager.getGlobalUsedWeight()); + + entry.invalidateKey("one"); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + owner.close(); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + void oversizedValueIsReturnedButNotCached() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 8L, "paimon", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("partition", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(1024L)) + .build()); + + Assertions.assertEquals("value", entry.get("key", ignored -> "value")); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(1L, entry.metrics().getWeightRejectCount()); + Assertions.assertEquals("entry_too_large", entry.metrics().getLastWeightRejectReason()); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void sameKeyReplacementUsesOnlyTheLargerGenerationWeight() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 9L, "iceberg", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build()); + + entry.put("key", "generation-one"); + entry.put("key", "generation-two"); + + Assertions.assertEquals("generation-two", entry.getIfPresent("key")); + Assertions.assertEquals(1024L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(0L, entry.metrics().getWeightRejectCount()); + entry.invalidateKey("key"); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void budgetPressureReclaimsOneColdEntryBeforeTheNextAdmission() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(2048L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 12L, "iceberg", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("partition", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build()); + + entry.put("one", "value-one"); + entry.put("two", "value-two"); + entry.put("three", "value-three"); + + Assertions.assertNull(entry.getIfPresent("three")); + awaitSize(entry, 1L); + entry.put("three", "value-three"); + Assertions.assertEquals("value-three", entry.getIfPresent("three")); + Assertions.assertEquals(2L, entry.size()); + Assertions.assertEquals(2048L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(1L, entry.metrics().getWeightRejectCount()); + Assertions.assertEquals(1L, entry.metrics().getEvictionCount()); + } + } + + @Test + void managedOwnerClosesWhenPhysicalCacheConstructionFails() { + long catalogId = Long.MIN_VALUE + 31L; + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); + CatalogMetaCache owner = CatalogMetaCache.managed(catalogId, "iceberg", Collections.emptyMap()); + + Assertions.assertThrows(IllegalArgumentException.class, () -> owner.create(MetaCacheDefinition + .builder("invalid", CacheSpec.of(true, -1L, -1L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build())); + + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); + } + + @Test + void concurrentRegistrationCannotBeDetachedByLastOwnerRemoval() throws Exception { + long catalogId = Long.MIN_VALUE + 32L; + CatalogMetaCache retiring = CatalogMetaCache.managed( + catalogId, "iceberg", Collections.emptyMap()); + CatalogMetaCache replacement = new CatalogMetaCache( + new ScopedMetaCacheRegistry(), MetaCacheGovernance.budgetManager(), catalogId, + "iceberg", OptionalLong.empty(), true); + BlockingEmptySet owners = new BlockingEmptySet<>(); + owners.add(retiring); + owners.arm(); + catalogCacheRegistry().put(catalogId, owners); + ExecutorService executor = Executors.newFixedThreadPool(2); + Future unregister = null; + Future register = null; + try { + unregister = executor.submit(retiring::close); + Assertions.assertTrue(owners.awaitEmptyObservation(), + "the retiring owner must reach its empty-set decision"); + CountDownLatch registerStarted = new CountDownLatch(1); + register = executor.submit(() -> { + registerStarted.countDown(); + MetaCacheGovernance.register(replacement); + }); + Assertions.assertTrue(registerStarted.await(10L, TimeUnit.SECONDS)); + boolean registeredBeforeRemovalCompleted = owners.awaitAdd(); + + owners.releaseEmptyObservation(); + unregister.get(10L, TimeUnit.SECONDS); + register.get(10L, TimeUnit.SECONDS); + + Assertions.assertFalse(registeredBeforeRemovalCompleted, + "registration must serialize with removal for the same catalog id"); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).contains(replacement), + "the replacement owner must remain discoverable after the retiring owner closes"); + } finally { + owners.releaseEmptyObservation(); + if (unregister != null) { + unregister.cancel(true); + } + if (register != null) { + register.cancel(true); + } + retiring.close(); + replacement.close(); + executor.shutdownNow(); + } + } + + @Test + void estimatorIsNotInvokedWhenNoWeightLimitIsConfigured() { + AtomicInteger estimates = new AtomicInteger(); + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.empty()); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 10L, "hive", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("file", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> { + estimates.incrementAndGet(); + return MetaCacheSizeEstimate.complete(512L); + }) + .build()); + + entry.put("key", "value"); + Assertions.assertEquals("value", entry.getIfPresent("key")); + Assertions.assertFalse(entry.isWeightBounded()); + Assertions.assertEquals(0, estimates.get()); + } + } + + @Test + void estimatorFailureReturnsTheLoadedValueWithoutCachingIt() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(4096L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 11L, "hive", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> { + throw new IllegalStateException("unsupported value layout"); + }) + .build()); + + Assertions.assertEquals("value", entry.get("key", ignored -> "value")); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(1L, entry.metrics().getWeightRejectCount()); + Assertions.assertTrue(entry.metrics().getLastWeightRejectReason() + .startsWith("incomplete_estimate:estimator_failure:")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void overlappingCatalogOwnersShareCatalogBudgetWithoutNameCollision() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(4096L)); + CatalogMetaCache firstOwner = new CatalogMetaCache( + manager, 12L, "iceberg", Collections.emptyMap()); + CatalogMetaCache secondOwner = new CatalogMetaCache( + manager, 12L, "iceberg", Collections.emptyMap()); + try { + MetaCache first = weightedStringEntry(firstOwner, "table", 512L); + MetaCache second = weightedStringEntry(secondOwner, "table", 512L); + + first.put("first", "value"); + second.put("second", "value"); + Assertions.assertEquals(2048L, manager.getGlobalUsedWeight()); + + firstOwner.close(); + Assertions.assertEquals(1024L, manager.getGlobalUsedWeight()); + } finally { + firstOwner.close(); + secondOwner.close(); + } + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + void physicalCachesInOneBudgetGroupShareEntryLimit() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.empty()); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 18L, "iceberg", Collections.emptyMap())) { + CacheSpec spec = CacheSpec.ofWeight(true, -1L, 100L, 1536L); + MetaCache first = owner.create(MetaCacheDefinition + .builder("mvcc-partition-view", spec, ignored -> ScopePath.catalog()) + .budgetGroup("partition_view") + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(1024L)) + .build()); + MetaCache second = owner.create(MetaCacheDefinition + .builder("list-partitions-view", spec, ignored -> ScopePath.catalog()) + .budgetGroup("partition_view") + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(1024L)) + .build()); + + first.put("first", "value"); + second.put("second", "value"); + + Assertions.assertNull(second.getIfPresent("second")); + Assertions.assertTrue(manager.getGlobalUsedWeight() <= 1536L); + } + } + + @Test + void weightedCompareAndSetRejectionCommitsAndInvalidatesOldValue() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + AtomicInteger commits = new AtomicInteger(); + AtomicReference discarded = new AtomicReference<>(); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 13L, "default", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .discardListener((key, value) -> discarded.set(value)) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete( + "old".equals(value) ? 512L : 1024L)) + .build()); + entry.put("key", "old"); + + Assertions.assertTrue(entry.compareAndSet("key", "old", "new", commits::incrementAndGet)); + Assertions.assertEquals(1, commits.get()); + Assertions.assertEquals("new", discarded.get()); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void weightedPutRejectionInvalidatesOldValue() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + AtomicReference discarded = new AtomicReference<>(); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 14L, "default", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .discardListener((key, value) -> discarded.set(value)) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete( + "old".equals(value) ? 512L : 1024L)) + .build()); + entry.put("key", "old"); + + entry.put("key", "new"); + Assertions.assertEquals("new", discarded.get()); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void weightedRefreshRejectionInvalidatesOldValue() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + AtomicInteger loads = new AtomicInteger(); + AtomicReference discarded = new AtomicReference<>(); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 15L, "default", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("schema", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .loader(key -> "v" + loads.incrementAndGet()) + .discardListener((key, value) -> discarded.set(value)) + .refreshAfterWrite(Duration.ofNanos(1L), Runnable::run) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete( + "v1".equals(value) ? 512L : 1024L)) + .build()); + + Assertions.assertEquals("v1", entry.get("key")); + Assertions.assertEquals("v1", entry.get("key")); + Assertions.assertEquals("v2", discarded.get()); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void globalOrCatalogWeightLimitRequiresEstimator() { + MetaCacheBudgetManager globalManager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + globalManager, 16L, "default", Collections.emptyMap())) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> owner.create(unestimatedStringDefinition("global"))); + } + + MetaCacheBudgetManager catalogManager = new MetaCacheBudgetManager(OptionalLong.empty()); + try (CatalogMetaCache owner = new CatalogMetaCache(catalogManager, 17L, "default", + Collections.singletonMap(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"))) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> owner.create(unestimatedStringDefinition("catalog"))); + } + } + + private static MetaCache weightedStringEntry( + CatalogMetaCache owner, String name, long estimatedBytes) { + return owner.create(MetaCacheDefinition + .builder(name, CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(estimatedBytes)) + .build()); + } + + private static MetaCacheDefinition unestimatedStringDefinition(String name) { + return MetaCacheDefinition.builder( + name, CacheSpec.of(true, -1L, 100L), ignored -> ScopePath.catalog()).build(); + } + + private static void awaitSize(MetaCache cache, long expectedSize) { + long deadline = System.nanoTime() + Duration.ofSeconds(10L).toNanos(); + while (cache.size() != expectedSize && System.nanoTime() < deadline) { + Thread.yield(); + } + Assertions.assertEquals(expectedSize, cache.size()); + } + + @SuppressWarnings("unchecked") + private static Map> catalogCacheRegistry() throws ReflectiveOperationException { + Field field = MetaCacheGovernance.class.getDeclaredField("CATALOG_CACHES"); + field.setAccessible(true); + return (Map>) field.get(null); + } + + private static final class BlockingEmptySet extends AbstractSet { + private final Set delegate = java.util.concurrent.ConcurrentHashMap.newKeySet(); + private final CountDownLatch emptyObserved = new CountDownLatch(1); + private final CountDownLatch releaseEmpty = new CountDownLatch(1); + private final CountDownLatch addObserved = new CountDownLatch(1); + private volatile boolean armed; + + void arm() { + armed = true; + } + + boolean awaitEmptyObservation() throws InterruptedException { + return emptyObserved.await(10L, TimeUnit.SECONDS); + } + + boolean awaitAdd() throws InterruptedException { + return addObserved.await(250L, TimeUnit.MILLISECONDS); + } + + void releaseEmptyObservation() { + releaseEmpty.countDown(); + } + + @Override + public Iterator iterator() { + return delegate.iterator(); + } + + @Override + public int size() { + return delegate.size(); + } + + @Override + public boolean add(E value) { + boolean added = delegate.add(value); + if (armed) { + addObserved.countDown(); + } + return added; + } + + @Override + public boolean remove(Object value) { + return delegate.remove(value); + } + + @Override + public boolean isEmpty() { + boolean empty = delegate.isEmpty(); + if (armed && empty) { + emptyObserved.countDown(); + try { + releaseEmpty.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while controlling registry removal", e); + } + } + return empty; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java new file mode 100644 index 00000000000000..6147ae923a6823 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.AbstractList; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.RandomAccess; +import java.util.concurrent.atomic.AtomicInteger; + +class ReflectiveObjectSizeEstimatorTest { + @Test + void randomAccessContainerTraversalIsBoundedBySampleSize() { + AtomicInteger reads = new AtomicInteger(); + AbstractList millionElements = new CountingList(reads); + + Assertions.assertTrue(ReflectiveObjectSizeEstimator.estimate(millionElements) > 0L); + Assertions.assertEquals(5, reads.get()); + } + + @Test + void typedListSamplingHasBoundedWorkAndIncludesTheTail() { + AtomicInteger reads = new AtomicInteger(); + AtomicInteger largestIndex = new AtomicInteger(); + AbstractList millionElements = new CountingList(reads); + + long estimated = JvmSizeUtils.sampledListPayload(millionElements, 16, value -> { + int index = Integer.parseInt(value.substring("value-".length())); + largestIndex.accumulateAndGet(index, Math::max); + return 1L; + }); + + Assertions.assertEquals(16, reads.get()); + Assertions.assertEquals(999_999, largestIndex.get()); + Assertions.assertEquals(1_000_000L, estimated); + } + + @Test + void completeAdmissionEstimateIncludesAValueOutsideTheFiveElementSample() { + Map skewed = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + skewed.put("small-" + i, "x"); + } + String largeTail = "x".repeat(10_000_000); + skewed.put("large-tail", largeTail); + + long sampled = ReflectiveObjectSizeEstimator.estimate(skewed); + MetaCacheSizeEstimate complete = MetaCacheSizeEstimators + .>reflective().estimate("key", skewed); + + Assertions.assertTrue(complete.isComplete()); + Assertions.assertTrue(complete.getBytes() >= JvmSizeUtils.stringSize(largeTail)); + Assertions.assertTrue(complete.getBytes() > sampled * 100L, + "weighted admission must not accept the fixed five-element sample as complete"); + } + + @Test + void completeAdmissionEstimateRejectsGraphsBeyondItsVisitBudget() { + AtomicInteger reads = new AtomicInteger(); + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "bounded_complete_estimate", + () -> MetaCacheSizeEstimate.complete( + ReflectiveObjectSizeEstimator.estimateComplete(new CountingList(reads)))); + + Assertions.assertFalse(estimate.isComplete()); + Assertions.assertTrue(reads.get() <= 10_001, + "complete fallback must stop when its work budget is exhausted"); + } + + @Test + void inaccessibleJdkReferenceFieldMakesTheEstimateIncomplete() { + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "reflection_failure", + () -> MetaCacheSizeEstimate.complete( + ReflectiveObjectSizeEstimator.estimate(URI.create("s3://bucket/path")))); + + Assertions.assertFalse(estimate.isComplete()); + Assertions.assertTrue(estimate.getIncompleteReason().startsWith("reflection_failure:")); + } + + @Test + void depthTruncationMakesTheEstimateIncomplete() { + Node root = new Node(); + Node current = root; + for (int i = 0; i < 25; i++) { + current.next = new Node(); + current = current.next; + } + + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "depth_limit", + () -> MetaCacheSizeEstimate.complete(ReflectiveObjectSizeEstimator.estimate(root))); + + Assertions.assertFalse(estimate.isComplete()); + Assertions.assertTrue(estimate.getIncompleteReason().startsWith("depth_limit:")); + } + + private static final class CountingList extends AbstractList implements RandomAccess { + private final AtomicInteger reads; + + private CountingList(AtomicInteger reads) { + this.reads = reads; + } + + @Override + public String get(int index) { + reads.incrementAndGet(); + return "value-" + index; + } + + @Override + public int size() { + return 1_000_000; + } + } + + private static final class Node { + private Node next; + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java index 983e9368b5f707..5093e47b8936bf 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java @@ -24,7 +24,9 @@ import java.time.Duration; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.OptionalLong; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -1187,6 +1189,47 @@ public void concurrentPutEvictAndInvalidationFinishWithoutDeadlock() throws Exce } } + @Test + public void concurrentWeightedReplacementsDoNotEvictAcrossKeyLocks() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch replacementsEstimating = new CountDownLatch(2); + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(2048L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 41L, "iceberg", Collections.emptyMap())) { + MetaCache cache = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> { + if (value.startsWith("new")) { + replacementsEstimating.countDown(); + await(replacementsEstimating); + return MetaCacheSizeEstimate.complete(1024L); + } + return MetaCacheSizeEstimate.complete(512L); + }) + .build()); + cache.put("a", "old-a"); + cache.put("b", "old-b"); + + Future first = executor.submit(() -> { + await(start); + cache.put("a", "new-a"); + }); + Future second = executor.submit(() -> { + await(start); + cache.put("b", "new-b"); + }); + start.countDown(); + + first.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + second.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assertions.assertTrue(manager.getGlobalUsedWeight() <= 2048L); + } finally { + executor.shutdownNow(); + } + } + @Test public void concurrentSameKeyPublicationsLeaveOneExactRegistration() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(4); diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java index d38571a11f2503..06a8a15acbae7b 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java @@ -179,6 +179,10 @@ public static HiveCatalogProperties of(Map properties) { * @return this, so the provider's door reads as one statement */ public HiveCatalogProperties checkCreateTimeOnlyRules() { + CacheSpec.checkWeightProperties(raw, "hive", + "table", "partition_names", "partition", "column_stats", "file", "partition_view"); + CacheSpec.checkWeightProperties(raw, "iceberg", + "table", "partition", "manifest", "partition_view"); // Restores the legacy HMSExternalCatalog.checkProperties fail-fast for the two meta-cache TTL // knobs: after the hms cutover an "hms" catalog is created via the SPI provider (not // HMSExternalCatalog), so the old per-property validation no longer ran and an invalid ttl (e.g. diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index cb4f31bf40506d..6d8f5ffa8b63fd 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -87,7 +87,7 @@ public class HiveConnector implements Connector { // every (re)build of the connector, including the lazy one an FE does after replaying the edit log. private final HiveCatalogProperties props; private final ConnectorContext context; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; private volatile HmsClient hmsClient; // Lazily-built plugin-side Kerberos authenticator (single-owner auth), null for a non-Kerberos catalog. @@ -133,17 +133,24 @@ public class HiveConnector implements Connector { // NEVER cast (a cast would CCE across the loader split). private volatile Connector hudiSibling; + // Writes are guarded by this; the volatile read rejects post-close fast paths. Closing and lazy sibling + // publication share the same monitor so a sibling can never be published after the gateway has released its + // managed metadata-cache owner. + private volatile boolean closed; + public HiveConnector(Map properties, ConnectorContext context) { HmsConfHelper.initializeHadoopConfigDir(context); this.props = HiveCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hive", this.properties); this.fileListingCache = new HiveFileListingCache(metaCache, props); // Reads its own meta.cache.hive.partition_view.(enable|ttl-second|capacity) from the catalog properties // via the framework's CacheSpec (default ON / 24h / 1000). this.partitionViewCache = new ConnectorMetadataCache<>(metaCache, "hive-partition-view", "hive", "partition_view", this.properties, - key -> ScopePath.partitionCollection(key.getDb(), key.getTable())); + key -> ScopePath.partitionCollection(key.getDb(), key.getTable()), + HivePartitionViewSizeEstimator::estimateEntry); } @Override @@ -483,36 +490,40 @@ private HmsClient getOrCreateClient() { * memoized (a null sibling leaves the field unset), so a later-available plugin recovers on the next access. */ Connector getOrCreateIcebergSibling() { - if (icebergSibling == null) { - synchronized (this) { - if (icebergSibling == null) { - Connector sibling = context.createSiblingConnector( - ICEBERG_CONNECTOR_TYPE, IcebergSiblingProperties.synthesize(properties)); - if (sibling == null) { - throw new DorisConnectorException( - "Cannot serve iceberg-on-HMS tables in catalog '" + context.getCatalogName() - + "': the iceberg connector plugin is not available"); - } - // Fail-loud invariant guard for the cache-isolation security track: the hive gateway FRONT - // DOOR never declares SUPPORTS_USER_SESSION, so fe-core keys its per-user schema/name cache - // bypass off THIS (front-door) connector's capabilities and would NOT bypass for a delegated - // sibling. The iceberg sibling is forced iceberg.catalog.type=hms (IcebergSiblingProperties - // .synthesize) and can never be REST session=user, so this must hold today. If a future change - // ever let the sibling be session=user, the front-door-only bypass would silently leak - // cross-user metadata — fail here instead. - if (sibling.getCapabilities().contains(ConnectorCapability.SUPPORTS_USER_SESSION)) { - throw new DorisConnectorException( - "iceberg-on-HMS sibling in catalog '" + context.getCatalogName() - + "' unexpectedly declares SUPPORTS_USER_SESSION: the hive gateway front " - + "door is not session=user, so fe-core's per-user schema/name cache bypass " - + "would not trigger and cross-user metadata would leak. The sibling must " - + "stay iceberg.catalog.type=hms (never REST session=user)."); - } - icebergSibling = sibling; + checkOpenForSiblingCreation(); + Connector current = icebergSibling; + if (current != null) { + return current; + } + synchronized (this) { + checkOpenForSiblingCreation(); + if (icebergSibling == null) { + Connector sibling = context.createSiblingConnector( + ICEBERG_CONNECTOR_TYPE, IcebergSiblingProperties.synthesize(properties)); + if (sibling == null) { + throw new DorisConnectorException( + "Cannot serve iceberg-on-HMS tables in catalog '" + context.getCatalogName() + + "': the iceberg connector plugin is not available"); + } + // Fail-loud invariant guard for the cache-isolation security track: the hive gateway FRONT + // DOOR never declares SUPPORTS_USER_SESSION, so fe-core keys its per-user schema/name cache + // bypass off THIS (front-door) connector's capabilities and would NOT bypass for a delegated + // sibling. The iceberg sibling is forced iceberg.catalog.type=hms (IcebergSiblingProperties + // .synthesize) and can never be REST session=user, so this must hold today. If a future change + // ever let the sibling be session=user, the front-door-only bypass would silently leak + // cross-user metadata — fail here instead. + if (sibling.getCapabilities().contains(ConnectorCapability.SUPPORTS_USER_SESSION)) { + throw new DorisConnectorException( + "iceberg-on-HMS sibling in catalog '" + context.getCatalogName() + + "' unexpectedly declares SUPPORTS_USER_SESSION: the hive gateway front " + + "door is not session=user, so fe-core's per-user schema/name cache bypass " + + "would not trigger and cross-user metadata would leak. The sibling must " + + "stay iceberg.catalog.type=hms (never REST session=user)."); } + icebergSibling = sibling; } + return icebergSibling; } - return icebergSibling; } /** @@ -527,21 +538,33 @@ Connector getOrCreateIcebergSibling() { * memoized (a null sibling leaves the field unset), so a later-available plugin recovers on the next access. */ Connector getOrCreateHudiSibling() { - if (hudiSibling == null) { - synchronized (this) { - if (hudiSibling == null) { - Connector sibling = context.createSiblingConnector( - HUDI_CONNECTOR_TYPE, HudiSiblingProperties.synthesize(properties)); - if (sibling == null) { - throw new DorisConnectorException( - "Cannot serve hudi-on-HMS tables in catalog '" + context.getCatalogName() - + "': the hudi connector plugin is not available"); - } - hudiSibling = sibling; + checkOpenForSiblingCreation(); + Connector current = hudiSibling; + if (current != null) { + return current; + } + synchronized (this) { + checkOpenForSiblingCreation(); + if (hudiSibling == null) { + Connector sibling = context.createSiblingConnector( + HUDI_CONNECTOR_TYPE, HudiSiblingProperties.synthesize(properties)); + if (sibling == null) { + throw new DorisConnectorException( + "Cannot serve hudi-on-HMS tables in catalog '" + context.getCatalogName() + + "': the hudi connector plugin is not available"); } + hudiSibling = sibling; } + return hudiSibling; + } + } + + private void checkOpenForSiblingCreation() { + if (closed) { + throw new DorisConnectorException( + "Cannot create a sibling connector after Hive catalog '" + context.getCatalogName() + + "' has been closed"); } - return hudiSibling; } private HmsClient createClient() { @@ -697,24 +720,57 @@ private static Configuration buildHmsConf(AbstractHmsMetaStoreProperties hms) { @Override public void close() throws IOException { - metaCache.close(); - HmsClient c = hmsClient; - if (c != null) { - c.close(); + HmsClient client; + Connector sibling; + Connector hudi; + synchronized (this) { + if (closed) { + return; + } + closed = true; + client = hmsClient; hmsClient = null; + sibling = icebergSibling; + icebergSibling = null; + hudi = hudiSibling; + hudiSibling = null; + } + metaCache.close(); + IOException closeFailure = null; + try { + if (client != null) { + client.close(); + } + } catch (IOException e) { + closeFailure = e; } // Forward close to the embedded iceberg sibling: the engine closes only a catalog's PRIMARY connector, // so the gateway owns the sibling's lifecycle. No-op when the sibling was never built (dormant path). - Connector sibling = icebergSibling; - if (sibling != null) { - sibling.close(); - icebergSibling = null; + try { + if (sibling != null) { + sibling.close(); + } + } catch (IOException e) { + if (closeFailure == null) { + closeFailure = e; + } else { + closeFailure.addSuppressed(e); + } } // Same for the embedded hudi sibling — the gateway owns its lifecycle too. No-op when never built. - Connector hudi = hudiSibling; - if (hudi != null) { - hudi.close(); - hudiSibling = null; + try { + if (hudi != null) { + hudi.close(); + } + } catch (IOException e) { + if (closeFailure == null) { + closeFailure = e; + } else { + closeFailure.addSuppressed(e); + } + } + if (closeFailure != null) { + throw closeFailure; } } } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java index 57cc3c0c342971..0981c94f781435 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java @@ -21,6 +21,8 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.filesystem.FileEntry; @@ -107,7 +109,7 @@ interface DirectoryLister { } private final CatalogMetaCache owner; - private final MetaCache> cache; + private final MetaCache cache; private final DirectoryLister lister; public HiveFileListingCache(HiveCatalogProperties properties) { @@ -149,10 +151,11 @@ private static DirectoryLister defaultLister(HiveCatalogProperties properties) { CacheSpec spec = CacheSpec.fromProperties(props, ENGINE, ENTRY_FILE, CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_FILE_CAPACITY)); this.cache = owner.create(MetaCacheDefinition - .>builder("hive-file", spec, + .builder("hive-file", spec, key -> key.partitionValues.isEmpty() ? ScopePath.table(key.dbName, key.tableName) : ScopePath.partition(key.dbName, key.tableName, key.partitionValues)) + .sizeEstimator(HiveFileListingSizeEstimator::estimateEntry) .build()); this.lister = Objects.requireNonNull(lister, "lister can not be null"); } @@ -180,7 +183,7 @@ public List listDataFiles(String dbName, String tableName, Strin public List listDataFiles(String dbName, String tableName, String location, List partitionValues, FileSystem fs) { return cache.get(new FileListingKey(dbName, tableName, location, partitionValues), - key -> lister.list(key.location, fs)); + key -> new FileListingValue(lister.list(key.location, fs), cache.isWeightBounded())).files; } /** Drops every cached listing for one table. Backs {@code REFRESH TABLE}. */ @@ -345,10 +348,10 @@ private static boolean isSystemicResolutionFailure(Throwable t) { * size-estimate paths sharing the same entry while making per-partition invalidation possible. */ static final class FileListingKey { - private final String dbName; - private final String tableName; - private final String location; - private final List partitionValues; + final String dbName; + final String tableName; + final String location; + final List partitionValues; FileListingKey(String dbName, String tableName, String location, List partitionValues) { this.dbName = dbName; @@ -379,4 +382,20 @@ public int hashCode() { return Objects.hash(dbName, tableName, location, partitionValues); } } + + static final class FileListingValue { + final List files; + final MetaCacheSizeEstimate sizeEstimate; + + FileListingValue(List files, boolean estimateWeight) { + this.files = estimateWeight + ? Collections.unmodifiableList(new ArrayList<>(files)) + : files; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("hive_file_listing_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + HiveFileListingSizeEstimator.estimateValue(this))) + : MetaCacheSizeEstimate.complete(0L); + } + } } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java new file mode 100644 index 00000000000000..9c7ab5e2c7f0dd --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingKey; +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingValue; + +/** Type-specific retained-heap estimator for one Hive directory-listing cache entry. */ +final class HiveFileListingSizeEstimator { + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(FileListingKey.class); + private static final long VALUE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(FileListingValue.class); + private static final long FILE_STATUS_SHALLOW_BYTES = JvmSizeUtils.instanceSize(HiveFileStatus.class); + + private HiveFileListingSizeEstimator() { + } + + /** Admission callback: the large value was sized once during construction. */ + static MetaCacheSizeEstimate estimateEntry(FileListingKey key, FileListingValue value) { + return value.sizeEstimate.isComplete() + ? MetaCacheSizeEstimate.complete(add(estimateKey(key), value.sizeEstimate.getBytes())) + : value.sizeEstimate; + } + + static long estimateKey(FileListingKey key) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.dbName)); + bytes = add(bytes, JvmSizeUtils.stringSize(key.tableName)); + bytes = add(bytes, JvmSizeUtils.stringSize(key.location)); + bytes = add(bytes, estimateOwnedStringList(key.partitionValues)); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(key)); + } + + static long estimateValue(FileListingValue value) { + long bytes = VALUE_SHALLOW_BYTES; + bytes = add(bytes, estimateArrayBackedList(value.files)); + for (HiveFileStatus file : value.files) { + bytes = add(bytes, FILE_STATUS_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(file.getPath())); + } + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(value)); + } + + private static long estimateOwnedStringList(java.util.List values) { + long bytes = estimateArrayBackedList(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateArrayBackedList(java.util.List values) { + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + return add(bytes, JvmSizeUtils.arrayListSize(values.size())); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HivePartitionViewSizeEstimator.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HivePartitionViewSizeEstimator.java new file mode 100644 index 00000000000000..0284e23023cf90 --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HivePartitionViewSizeEstimator.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Bounded type-specific estimator for Hive's large derived partition view. */ +final class HivePartitionViewSizeEstimator { + private static final int PARTITION_SAMPLE_SIZE = 16; + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorTableKey.class); + private static final long PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class); + private static final long ARRAY_LIST_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ArrayList.class); + private static final long UNMODIFIABLE_LIST_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableList(Collections.emptyList()).getClass()); + private static final long UNMODIFIABLE_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableMap(Collections.emptyMap()).getClass()); + private static final long LINKED_HASH_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize(LinkedHashMap.class); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + + private HivePartitionViewSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateEntry(ConnectorTableKey key, List partitions) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb())); + bytes = add(bytes, JvmSizeUtils.stringSize(key.getTable())); + bytes = add(bytes, ARRAY_LIST_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.objectArraySize(partitions.size())); + bytes = add(bytes, JvmSizeUtils.sampledListPayload( + partitions, PARTITION_SAMPLE_SIZE, HivePartitionViewSizeEstimator::estimatePartition)); + return MetaCacheSizeEstimate.complete( + Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(partitions))); + } + + private static long estimatePartition(ConnectorPartitionInfo partition) { + long bytes = PARTITION_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getPartitionName())); + bytes = add(bytes, estimateStringMap(partition.getPartitionValues())); + bytes = add(bytes, estimateStringMap(partition.getProperties())); + bytes = add(bytes, estimateStringList(partition.getOrderedPartitionValues())); + return add(bytes, estimateReferenceList(partition.getPartitionValueNullFlags())); + } + + private static long estimateStringMap(Map values) { + long bytes = UNMODIFIABLE_MAP_SHALLOW_BYTES; + if (values.isEmpty()) { + return bytes; + } + bytes = add(bytes, LINKED_HASH_MAP_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(values.size()))); + bytes = add(bytes, JvmSizeUtils.saturatedMultiply( + values.size(), LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES)); + for (Map.Entry entry : values.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getValue())); + } + return bytes; + } + + private static long estimateStringList(List values) { + long bytes = estimateReferenceList(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateReferenceList(List values) { + return add(UNMODIFIABLE_LIST_SHALLOW_BYTES, JvmSizeUtils.arrayListSize(values.size())); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java index 220261a5673e7a..b68ca5fa80393e 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java @@ -215,6 +215,19 @@ void badPartitionCacheTtlFailsTheCreateTimeDoor() { () -> HiveCatalogProperties.of(m).checkCreateTimeOnlyRules()); } + @Test + void icebergSiblingWeightsAreValidatedAtTheHmsCreateTimeDoor() { + Map invalidWeight = HiveTestProperties.mapWith( + "meta.cache.iceberg.manifest.max-weight", "invalid"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> HiveCatalogProperties.of(invalidWeight).checkCreateTimeOnlyRules()); + + Map unknownEntry = HiveTestProperties.mapWith( + "meta.cache.iceberg.manfiest.max-weight", "64MB"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> HiveCatalogProperties.of(unknownEntry).checkCreateTimeOnlyRules()); + } + @Test void validCatalogPassesBothDoors() { Assertions.assertDoesNotThrow(() -> HiveTestProperties.minimal().checkCreateTimeOnlyRules()); diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java index d2438edbd9fe7e..33cce0703d201e 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.hive; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsDatabaseInfo; import org.apache.doris.connector.hms.HmsPartitionInfo; @@ -32,6 +33,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -94,6 +96,20 @@ private static List names(List infos) { return infos.stream().map(ConnectorPartitionInfo::getPartitionName).collect(Collectors.toList()); } + @Test + public void largePartitionViewCanBeEstimatedWithoutTheReflectiveVisitLimit() { + List partitions = new ArrayList<>(); + for (int index = 0; index < 20_000; index++) { + String value = Integer.toString(index); + partitions.add(new ConnectorPartitionInfo("p=" + value, + Collections.singletonMap("p", value), Collections.emptyMap(), + Collections.singletonList(value), Collections.emptyList())); + } + + Assertions.assertTrue(HivePartitionViewSizeEstimator.estimateEntry( + new ConnectorTableKey("db", "table", -1L, -1L), partitions).isComplete()); + } + @Test public void listPartitionsCachesDerivedListAcrossQueries() { // WHY: cache A must memoize the BUILT List keyed by (db, table, -1, -1), so a diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java index 39727fe101bd41..8b7916c700fbe5 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java @@ -136,6 +136,19 @@ public void closeIsNoOpWhenSiblingNeverBuilt() throws Exception { Assertions.assertEquals(0, context.buildCount, "close must not trigger a sibling build"); } + @Test + public void closePreventsLateSiblingCreation() throws Exception { + RecordingSiblingContext context = new RecordingSiblingContext(new FakeSibling()); + HiveConnector connector = new HiveConnector(HiveTestProperties.minimalMap(), context); + + connector.close(); + + Assertions.assertThrows(DorisConnectorException.class, connector::getOrCreateIcebergSibling); + Assertions.assertThrows(DorisConnectorException.class, connector::getOrCreateHudiSibling); + Assertions.assertEquals(0, context.buildCount, + "a closed gateway must not publish a new managed sibling cache owner"); + } + // ---- hudi sibling holder (mirrors the iceberg cases above; hudi synthesizes props verbatim, no flavor) ---- @Test diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java index 326beba4d61f86..d89c375322ba39 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.hive; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.hms.HmsPartitionInfo; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.DorisConnectorException; @@ -91,6 +92,22 @@ public void listingIsCachedPerLocation() { Assertions.assertEquals(2, lister.totalCalls); } + @Test + public void weightBoundedListingIsEstimatedAndCached() { + CountingLister lister = new CountingLister(); + Map properties = props("meta.cache.hive.file.max-weight", "1MB"); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + HiveFileListingCache cache = new HiveFileListingCache( + owner, HiveCatalogProperties.of(properties), lister); + + List first = cache.listDataFiles("db", "t", "loc", FS); + List second = cache.listDataFiles("db", "t", "loc", FS); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, lister.totalCalls); + } + } + @Test public void keyIsScopedByDbTableAndLocation() { CountingLister lister = new CountingLister(); diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java index d7b5edfc0d6aad..e9dab8e3bb1059 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; import org.apache.hadoop.hive.common.FileUtils; @@ -123,23 +124,28 @@ public CachingHmsClient(CatalogMetaCache owner, HmsClient delegate, Map props = applyLegacyTtlCompatibility( properties == null ? Collections.emptyMap() : properties); this.tableCache = newEntry(owner, "hive-table", props, ENTRY_TABLE, DEFAULT_TABLE_CAPACITY, - key -> ScopePath.table(key.dbName, key.tableName)); + key -> ScopePath.table(key.dbName, key.tableName), HmsCacheSizeEstimator::estimateTable); this.partitionNamesCache = newEntry(owner, "hive-partition-names", props, ENTRY_PARTITION_NAMES, DEFAULT_PARTITION_NAMES_CAPACITY, - key -> ScopePath.partitionCollection(key.dbName, key.tableName)); + key -> ScopePath.partitionCollection(key.dbName, key.tableName), + HmsCacheSizeEstimator::estimatePartitionNames); this.partitionsCache = newEntry(owner, "hive-partition", props, ENTRY_PARTITION, DEFAULT_PARTITION_CAPACITY, - key -> ScopePath.partition(key.dbName, key.tableName, key.values)); + key -> ScopePath.partition(key.dbName, key.tableName, key.values), + HmsCacheSizeEstimator::estimatePartition); this.columnStatsCache = newEntry(owner, "hive-column-stats", props, ENTRY_COLUMN_STATS, DEFAULT_COLUMN_STATS_CAPACITY, - key -> ScopePath.table(key.dbName, key.tableName)); + key -> ScopePath.table(key.dbName, key.tableName), HmsCacheSizeEstimator::estimateColumnStats); } private static MetaCache newEntry(CatalogMetaCache owner, String name, - Map props, String entry, long defaultCapacity, Function scopeResolver) { + Map props, String entry, long defaultCapacity, Function scopeResolver, + MetaCacheSizeEstimator sizeEstimator) { CacheSpec spec = CacheSpec.fromProperties(props, ENGINE, entry, CacheSpec.of(true, DEFAULT_TTL_SECOND, defaultCapacity)); - return owner.create(MetaCacheDefinition.builder(name, spec, scopeResolver).build()); + return owner.create(MetaCacheDefinition.builder(name, spec, scopeResolver) + .sizeEstimator(sizeEstimator) + .build()); } /** Legacy fe-core catalog knob ({@code ExternalCatalog.SCHEMA_CACHE_TTL_SECOND}) for the table/schema cache. */ diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsCacheSizeEstimator.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsCacheSizeEstimator.java new file mode 100644 index 00000000000000..59dfd6596e26c4 --- /dev/null +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsCacheSizeEstimator.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hms; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; + +import java.util.List; + +/** Construction-time retained-size formulas for HMS cache entries. */ +final class HmsCacheSizeEstimator { + private static final long COLUMN_STATS_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(HmsColumnStatistics.class); + + private HmsCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateTable(Object key, HmsTableInfo value) { + return complete(key, value); + } + + static MetaCacheSizeEstimate estimatePartition(Object key, HmsPartitionInfo value) { + return complete(key, value); + } + + static MetaCacheSizeEstimate estimatePartitionNames(Object key, List value) { + return MetaCacheSizeEstimate.complete(add( + ReflectiveObjectSizeEstimator.estimateComplete(key), estimateStringList(value))); + } + + static MetaCacheSizeEstimate estimateColumnStats(Object key, List value) { + long bytes = add(JvmSizeUtils.instanceSize(value.getClass()), + JvmSizeUtils.objectArraySize(value.size())); + for (HmsColumnStatistics stats : value) { + bytes = add(bytes, COLUMN_STATS_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(stats.getColumnName())); + } + return MetaCacheSizeEstimate.complete(add( + ReflectiveObjectSizeEstimator.estimateComplete(key), bytes)); + } + + private static MetaCacheSizeEstimate complete(Object key, Object value) { + return MetaCacheSizeEstimate.complete(add( + ReflectiveObjectSizeEstimator.estimateComplete(key), + ReflectiveObjectSizeEstimator.estimateComplete(value))); + } + + private static long estimateStringList(List values) { + long bytes = add(JvmSizeUtils.instanceSize(values.getClass()), + JvmSizeUtils.objectArraySize(values.size())); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java index 36e2d007f7eadd..9e95fde84352b7 100644 --- a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java +++ b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -82,6 +83,47 @@ public void getTableCachesByDbAndTable() { Assertions.assertEquals(2, delegate.getTableCalls); } + @Test + public void weightedTableRejectsALargePropertyOutsideTheOldFiveElementSample() { + RecordingHmsClient delegate = new RecordingHmsClient(); + Map parameters = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + parameters.put("small-" + i, "x"); + } + parameters.put("large-tail", "x".repeat(2 * 1024 * 1024)); + delegate.tableResult = HmsTableInfo.builder() + .dbName("db") + .tableName("t") + .parameters(parameters) + .build(); + CachingHmsClient cache = cache(delegate, + props("meta.cache.hive.table.max-weight", "1MB")); + + cache.getTable("db", "t"); + cache.getTable("db", "t"); + + Assertions.assertEquals(2, delegate.getTableCalls, + "the 2MB tail must reject weighted admission instead of being hidden by five small samples"); + } + + @Test + public void weightedTableCachesSmallCompleteMetadata() { + RecordingHmsClient delegate = new RecordingHmsClient(); + delegate.tableResult = HmsTableInfo.builder() + .dbName("db") + .tableName("t") + .parameters(Map.of("format", "parquet")) + .build(); + CachingHmsClient cache = cache(delegate, + props("meta.cache.hive.table.max-weight", "1MB")); + + cache.getTable("db", "t"); + cache.getTable("db", "t"); + + Assertions.assertEquals(1, delegate.getTableCalls, + "a complete small HMS graph must still be admitted to the weighted cache"); + } + @Test public void cacheKeysAreScopedByDatabase() { RecordingHmsClient delegate = new RecordingHmsClient(); @@ -608,6 +650,7 @@ private static final class RecordingHmsClient implements HmsClient { int dropTableCalls; int closeCalls; RuntimeException getTableError; + HmsTableInfo tableResult; // Partition names the fake has NO partition for (mirrors HMS omitting non-existent partitions). final Set absentPartitionNames = new HashSet<>(); // When set, every returned partition carries these exact values regardless of the requested name @@ -626,7 +669,9 @@ public HmsTableInfo getTable(String dbName, String tableName) { if (getTableError != null) { throw getTableError; } - return HmsTableInfo.builder().dbName(dbName).tableName(tableName).build(); + return tableResult == null + ? HmsTableInfo.builder().dbName(dbName).tableName(tableName).build() + : tableResult; } @Override diff --git a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsCacheSizeEstimatorBenchmark.java b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsCacheSizeEstimatorBenchmark.java new file mode 100644 index 00000000000000..b3a6a41578caa1 --- /dev/null +++ b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsCacheSizeEstimatorBenchmark.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hms; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Dependency-free microbenchmark for the production HMS admission estimators. */ +public final class HmsCacheSizeEstimatorBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static volatile long blackhole; + + private HmsCacheSizeEstimatorBenchmark() { + } + + public static void main(String[] args) { + for (int size : new int[] {1_000, 100_000}) { + List names = partitionNames(size); + int operationsPerWindow = size <= 1_000 ? 1_000 : 20; + Result result = measure( + () -> HmsCacheSizeEstimator.estimatePartitionNames("db.t", names).getBytes(), + operationsPerWindow); + System.out.printf("hms_partition_names=%d estimated_bytes=%d ns_op=%d operations=%d%n", + size, HmsCacheSizeEstimator.estimatePartitionNames("db.t", names).getBytes(), + result.medianNanos, result.operations); + } + + HmsTableInfo table = skewedTable(); + Result tableResult = measure( + () -> HmsCacheSizeEstimator.estimateTable("db.t", table).getBytes(), 1_000); + System.out.printf("hms_table_properties=%d estimated_bytes=%d ns_op=%d operations=%d%n", + table.getParameters().size(), HmsCacheSizeEstimator.estimateTable("db.t", table).getBytes(), + tableResult.medianNanos, tableResult.operations); + } + + private static List partitionNames(int size) { + List values = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + values.add("dt=2026-09-01/hour=" + (i % 24) + "/bucket=" + i); + } + return values; + } + + private static HmsTableInfo skewedTable() { + Map parameters = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + parameters.put("small-" + i, "x"); + } + parameters.put("large-tail", "x".repeat(2 * 1024 * 1024)); + return HmsTableInfo.builder() + .dbName("db") + .tableName("t") + .parameters(parameters) + .build(); + } + + private static Result measure(LongOperation operation, int operationsPerWindow) { + for (int i = 0; i < WARMUP_WINDOWS; i++) { + runWindow(operation, operationsPerWindow); + } + long[] nanosPerOperation = new long[MEASURE_WINDOWS]; + for (int i = 0; i < MEASURE_WINDOWS; i++) { + long start = System.nanoTime(); + runWindow(operation, operationsPerWindow); + nanosPerOperation[i] = (System.nanoTime() - start) / operationsPerWindow; + } + Arrays.sort(nanosPerOperation); + return new Result(nanosPerOperation[MEASURE_WINDOWS / 2], + (long) operationsPerWindow * MEASURE_WINDOWS); + } + + private static void runWindow(LongOperation operation, int operations) { + long value = 0L; + for (int i = 0; i < operations; i++) { + value ^= operation.run(); + } + blackhole = value; + } + + @FunctionalInterface + private interface LongOperation { + long run(); + } + + private static final class Result { + private final long medianNanos; + private final long operations; + + private Result(long medianNanos, long operations) { + this.medianNanos = medianNanos; + this.operations = operations; + } + } +} diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java index 3832f57bd7c894..25f46cc58875cf 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java @@ -68,7 +68,7 @@ public class HudiConnector implements Connector { private final HudiCatalogProperties props; private final Map properties; private final ConnectorContext context; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; private volatile HmsClient hmsClient; // HMS and storage deliberately have separate authenticators: hive.metastore.username must affect set_ugi @@ -83,6 +83,7 @@ public HudiConnector(Map properties, ConnectorContext context) { this.props = HudiCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hudi", this.properties); } @Override diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java index f06b3d081fce61..34ea89ad499b31 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java @@ -17,6 +17,9 @@ package org.apache.doris.connector.hudi; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheGovernance; import org.apache.doris.connector.hms.CachingHmsClient; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsDatabaseInfo; @@ -24,11 +27,16 @@ import org.apache.doris.connector.hms.HmsTableInfo; import org.apache.doris.connector.spi.ConnectorContext; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -57,6 +65,73 @@ public class HudiConnectorHmsCacheTest { private static final List YEAR_MONTH = Arrays.asList("year", "month"); private static final List ONE_PARTITION = Collections.singletonList("year=2024/month=01"); + private final List connectors = new ArrayList<>(); + + @AfterEach + void closeConnectors() throws Exception { + for (HudiConnector connector : connectors) { + connector.close(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"meta.cache.max-weight", "meta.cache.hive.partition_names.max-weight"}) + void weightLimitAccountsHmsCacheAndRejectsOversizedValues(String property) throws Exception { + long catalogId = Long.MIN_VALUE + 71L; + long previousGlobalWeight = MetaCacheGovernance.globalEstimatedWeight(); + Map properties = new HashMap<>(HudiTestProperties.minimalMap()); + properties.put(property, "4KB"); + HudiConnector connector = connector(properties, catalogId); + List owners = MetaCacheGovernance.catalogCaches(catalogId); + Assertions.assertEquals(1, owners.size()); + CatalogMetaCache owner = owners.get(0); + Assertions.assertEquals("hudi", owner.engine()); + + FakeHmsClient delegate = new FakeHmsClient(ONE_PARTITION); + HmsClient cache = connector.wrapWithCache(delegate); + Assertions.assertEquals(4, owner.entries().size()); + MetaCache entry = owner.entries().get("hive-partition-names"); + Assertions.assertTrue(entry.isWeightBounded()); + Assertions.assertEquals(4096L, entry.metrics().getMaxWeight()); + Assertions.assertEquals("meta.cache.max-weight".equals(property), + owner.entries().get("hive-table").isWeightBounded()); + + Assertions.assertEquals(ONE_PARTITION, cache.listPartitionNames("db", "t", -1)); + Assertions.assertEquals(ONE_PARTITION, cache.listPartitionNames("db", "t", -1)); + Assertions.assertEquals(1, delegate.cachedCalls); + long retainedWeight = entry.metrics().getEstimatedWeight(); + Assertions.assertTrue(retainedWeight > 0L && retainedWeight <= 4096L); + Assertions.assertEquals(previousGlobalWeight + retainedWeight, + MetaCacheGovernance.globalEstimatedWeight()); + + delegate.names = Collections.singletonList("partition=" + "x".repeat(8192)); + Assertions.assertEquals(delegate.names, cache.listPartitionNames("db", "large", -1)); + Assertions.assertEquals(delegate.names, cache.listPartitionNames("db", "large", -1)); + Assertions.assertEquals(3, delegate.cachedCalls, "oversized values must be returned but not cached"); + Assertions.assertEquals(2L, entry.metrics().getWeightRejectCount()); + Assertions.assertEquals("entry_too_large", entry.metrics().getLastWeightRejectReason()); + Assertions.assertEquals(retainedWeight, entry.metrics().getEstimatedWeight()); + + connector.close(); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); + Assertions.assertEquals(0L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(previousGlobalWeight, MetaCacheGovernance.globalEstimatedWeight()); + } + + @Test + void noWeightLimitPreservesCountBasedCaching() { + long catalogId = Long.MIN_VALUE + 72L; + HudiConnector connector = connector(HudiTestProperties.minimalMap(), catalogId); + FakeHmsClient delegate = new FakeHmsClient(ONE_PARTITION); + HmsClient cache = connector.wrapWithCache(delegate); + CatalogMetaCache owner = MetaCacheGovernance.catalogCaches(catalogId).get(0); + MetaCache entry = owner.entries().get("hive-partition-names"); + Assertions.assertFalse(entry.isWeightBounded()); + cache.listPartitionNames("db", "t", -1); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(1, delegate.cachedCalls); + Assertions.assertEquals(0L, entry.metrics().getEstimatedWeight()); + } // ── wrap ───────────────────────────────────────────────────────────────────────────────────────────── @@ -153,8 +228,12 @@ public void invalidateOnUnbuiltClientIsNoOp() { // ── helpers ──────────────────────────────────────────────────────────────────────────────────────────── - private static HudiConnector connector() { - return new HudiConnector(HudiTestProperties.minimalMap(), new ConnectorContext() { + private HudiConnector connector() { + return connector(HudiTestProperties.minimalMap(), 1L); + } + + private HudiConnector connector(Map properties, long catalogId) { + HudiConnector connector = new HudiConnector(properties, new ConnectorContext() { @Override public String getCatalogName() { return "test_catalog"; @@ -162,9 +241,11 @@ public String getCatalogName() { @Override public long getCatalogId() { - return 1L; + return catalogId; } }); + connectors.add(connector); + return connector; } private static HudiTableHandle partitioned() { @@ -198,7 +279,7 @@ public T execute(Callable action) { private static final class FakeHmsClient implements HmsClient { int cachedCalls; int freshCalls; - private final List names; + private List names; FakeHmsClient(List names) { this.names = names; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java new file mode 100644 index 00000000000000..d9e9a63359e3f6 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java @@ -0,0 +1,702 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; +import org.apache.doris.connector.iceberg.IcebergPartitionCache.CachedPartitions; +import org.apache.doris.connector.iceberg.IcebergPartitionCache.Key; +import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; +import org.apache.doris.connector.iceberg.IcebergTableCache.TableOwner; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartition; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; + +import org.apache.iceberg.BlobMetadata; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.HistoryEntry; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortField; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.UnboundPartitionSpec; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Type-specific retained-heap estimators for the large Iceberg connector cache values. */ +final class IcebergCacheSizeEstimator { + private static final int PARTITION_VIEW_SAMPLE_SIZE = 16; + private static final String[] CONTENT_FILE_FIELD_NAMES = { + "content", "file_path", "file_format", "partition", "record_count", "file_size_in_bytes", + "column_sizes", "value_counts", "null_value_counts", "nan_value_counts", "lower_bounds", + "upper_bounds", "key_metadata", "split_offsets", "equality_ids", "sort_order_id", "first_row_id", + "referenced_data_file", "content_offset", "content_size_in_bytes" + }; + private static final long TABLE_IDENTIFIER_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableIdentifier.class); + private static final long CACHED_TABLE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableOwner.class); + private static final long TABLE_METADATA_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableMetadata.class); + private static final long PARTITION_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Key.class); + private static final long CACHED_PARTITIONS_SHALLOW_BYTES = JvmSizeUtils.instanceSize(CachedPartitions.class); + private static final long RAW_PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(IcebergRawPartition.class); + private static final long CONNECTOR_TABLE_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorTableKey.class); + private static final long CONNECTOR_PARTITION_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class); + private static final long MVCC_PARTITION_VIEW_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(ConnectorMvccPartitionView.class); + private static final long MVCC_PARTITION_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(ConnectorMvccPartition.class); + private static final long MANIFEST_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(IcebergManifestEntryKey.class); + private static final long MANIFEST_VALUE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ManifestCacheValue.class); + private static final long INTEGER_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Integer.class); + private static final long LONG_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Long.class); + private static final long HASH_MAP_NODE_SHALLOW_BYTES = classSize("java.util.HashMap$Node"); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + private static final long CONTENT_FILE_SCHEMA_BYTES = estimateContentFileSchema(); + + private IcebergCacheSizeEstimator() { + } + + /** Admission callback: the expensive table graph was sized once when {@link TableOwner} was constructed. */ + static MetaCacheSizeEstimate estimateTableEntry(TableIdentifier key, TableOwner value) { + return value.sizeEstimate.isComplete() + ? MetaCacheSizeEstimate.complete(add(estimateTableIdentifier(key), value.sizeEstimate.getBytes())) + : value.sizeEstimate; + } + + static long estimateTable(Table table) { + long bytes = add(CACHED_TABLE_SHALLOW_BYTES, JvmSizeUtils.instanceSize(table.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(table.name())); + if (!(table instanceof HasTableOperations)) { + bytes = add(bytes, JvmSizeUtils.stringSize(table.location())); + return add(bytes, estimateStringMap(table.properties())); + } + + TableOperations operations = ((HasTableOperations) table).operations(); + bytes = add(bytes, JvmSizeUtils.instanceSize(operations.getClass())); + return add(bytes, estimateTableMetadata(operations.current())); + } + + static long estimateSerializedTableMetadata(String metadataJson) { + return JvmSizeUtils.stringSize(metadataJson); + } + + static long estimatePartitionKey(Key key) { + return Math.max(add(PARTITION_KEY_SHALLOW_BYTES, estimateTableIdentifier(key.id)), + ReflectiveObjectSizeEstimator.estimate(key)); + } + + static long estimatePartitions(List partitions) { + long bytes = CACHED_PARTITIONS_SHALLOW_BYTES; + // CachedPartitions owns an unmodifiable wrapper around an exact-size ArrayList copy. + bytes = add(bytes, JvmSizeUtils.instanceSize(partitions.getClass())); + bytes = add(bytes, JvmSizeUtils.arrayListSize(partitions.size())); + for (IcebergRawPartition partition : partitions) { + bytes = add(bytes, RAW_PARTITION_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(partition.nameForWeight())); + bytes = add(bytes, estimateStringList(partition.columnNamesForWeight())); + bytes = add(bytes, estimateStringList(partition.valuesForWeight())); + bytes = add(bytes, estimateStringList(partition.transformsForWeight())); + } + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(partitions)); + } + + /** Admission callback: the partition list was sized once when {@link CachedPartitions} was constructed. */ + static MetaCacheSizeEstimate estimatePartitionEntry(Key key, CachedPartitions value) { + return value.sizeEstimate.isComplete() + ? MetaCacheSizeEstimate.complete(add(estimatePartitionKey(key), value.sizeEstimate.getBytes())) + : value.sizeEstimate; + } + + static MetaCacheSizeEstimate estimatePartitionInfoViewEntry( + ConnectorTableKey key, List partitions) { + long bytes = estimateConnectorTableKey(key); + bytes = add(bytes, estimateListStructure(partitions)); + bytes = add(bytes, JvmSizeUtils.sampledListPayload( + partitions, PARTITION_VIEW_SAMPLE_SIZE, + IcebergCacheSizeEstimator::estimateConnectorPartition)); + return MetaCacheSizeEstimate.complete( + Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(partitions))); + } + + static MetaCacheSizeEstimate estimateMvccPartitionViewEntry( + ConnectorTableKey key, ConnectorMvccPartitionView view) { + long bytes = add(estimateConnectorTableKey(key), MVCC_PARTITION_VIEW_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.instanceSize(ArrayList.class)); + bytes = add(bytes, estimateListStructure(view.getPartitions())); + bytes = add(bytes, JvmSizeUtils.sampledListPayload( + view.getPartitions(), PARTITION_VIEW_SAMPLE_SIZE, + IcebergCacheSizeEstimator::estimateMvccPartition)); + return MetaCacheSizeEstimate.complete( + Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(view))); + } + + private static long estimateMvccPartition(ConnectorMvccPartition partition) { + long bytes = MVCC_PARTITION_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getName())); + bytes = add(bytes, estimateWrappedStringList(partition.getLowerBound())); + return add(bytes, estimateWrappedStringList(partition.getUpperBound())); + } + + private static long estimateConnectorTableKey(ConnectorTableKey key) { + long bytes = CONNECTOR_TABLE_KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb())); + return add(bytes, JvmSizeUtils.stringSize(key.getTable())); + } + + private static long estimateConnectorPartition(ConnectorPartitionInfo partition) { + long bytes = CONNECTOR_PARTITION_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getPartitionName())); + bytes = add(bytes, estimateMapStructure(partition.getPartitionValues())); + bytes = add(bytes, JvmSizeUtils.instanceSize(LinkedHashMap.class)); + bytes = add(bytes, estimateMapStructure(partition.getProperties())); + bytes = add(bytes, estimateWrappedStringList(partition.getOrderedPartitionValues())); + return add(bytes, estimateWrappedReferenceList(partition.getPartitionValueNullFlags())); + } + + private static long estimateWrappedStringList(List values) { + long bytes = estimateWrappedReferenceList(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateWrappedReferenceList(List values) { + return add(JvmSizeUtils.instanceSize(ArrayList.class), estimateListStructure(values)); + } + + static long estimateManifestKey(IcebergManifestEntryKey key) { + return add(MANIFEST_KEY_SHALLOW_BYTES, JvmSizeUtils.stringSize(key.getManifestPath())); + } + + static long estimateManifestValue(ManifestCacheValue value) { + long bytes = MANIFEST_VALUE_SHALLOW_BYTES; + bytes = add(bytes, estimateContentFileList(value.getDataFiles())); + bytes = add(bytes, estimateContentFileList(value.getDeleteFiles())); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(value)); + } + + /** Admission callback: the manifest payload size is precomputed during construction. */ + static MetaCacheSizeEstimate estimateManifestEntry(IcebergManifestEntryKey key, ManifestCacheValue value) { + return value.getSizeEstimate().isComplete() + ? MetaCacheSizeEstimate.complete( + add(estimateManifestKey(key), value.getSizeEstimate().getBytes())) + : value.getSizeEstimate(); + } + + private static long estimateTableIdentifier(TableIdentifier identifier) { + long bytes = TABLE_IDENTIFIER_SHALLOW_BYTES; + Namespace namespace = identifier.namespace(); + bytes = add(bytes, JvmSizeUtils.instanceSize(namespace.getClass())); + String[] levels = namespace.levels(); + bytes = add(bytes, JvmSizeUtils.objectArraySize(levels.length)); + for (String level : levels) { + bytes = add(bytes, JvmSizeUtils.stringSize(level)); + } + return add(bytes, JvmSizeUtils.stringSize(identifier.name())); + } + + private static long estimateTableMetadata(TableMetadata metadata) { + long bytes = TABLE_METADATA_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.metadataFileLocation())); + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.uuid())); + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.location())); + bytes = add(bytes, estimateStringMap(metadata.properties())); + + List schemas = metadata.schemas(); + bytes = add(bytes, estimateListStructure(schemas)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.schemasById())); + for (Schema schema : schemas) { + bytes = add(bytes, estimateSchema(schema)); + } + + List specs = metadata.specs(); + bytes = add(bytes, estimateListStructure(specs)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.specsById())); + for (PartitionSpec spec : specs) { + bytes = add(bytes, estimatePartitionSpec(spec)); + } + + List sortOrders = metadata.sortOrders(); + bytes = add(bytes, estimateListStructure(sortOrders)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.sortOrdersById())); + for (SortOrder sortOrder : sortOrders) { + bytes = add(bytes, estimateSortOrder(sortOrder)); + } + + List snapshots = metadata.snapshots(); + bytes = add(bytes, estimateListStructure(snapshots)); + bytes = add(bytes, estimateLongIndexMap(snapshots)); + for (Snapshot snapshot : snapshots) { + bytes = add(bytes, estimateSnapshot(snapshot)); + } + + bytes = add(bytes, estimateHistory(metadata.snapshotLog())); + bytes = add(bytes, estimateMetadataLog(metadata.previousFiles())); + bytes = add(bytes, estimateSnapshotRefs(metadata.refs())); + bytes = add(bytes, estimateStatisticsFiles(metadata.statisticsFiles())); + bytes = add(bytes, estimatePartitionStatisticsFiles(metadata.partitionStatisticsFiles())); + bytes = add(bytes, estimateMetadataUpdates(metadata.changes())); + bytes = add(bytes, estimateShallowList(metadata.encryptionKeys())); + // TableMetadata retains a serializable snapshot supplier after the immutable snapshot list is loaded. + bytes = add(bytes, JvmSizeUtils.objectArraySize(1)); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(metadata)); + } + + private static long estimateSchema(Schema schema) { + List columns = schema.columns(); + long bytes = JvmSizeUtils.instanceSize(schema.getClass()); + bytes = add(bytes, JvmSizeUtils.instanceSize(schema.asStruct().getClass())); + bytes = add(bytes, estimateListStructure(columns)); + for (Types.NestedField field : columns) { + bytes = add(bytes, estimateNestedField(field)); + } + bytes = add(bytes, JvmSizeUtils.objectArraySize(schema.identifierFieldIds().size())); + bytes = add(bytes, estimateMapStructure(schema.getAliases())); + return add(bytes, estimateSchemaIndexes(columns.size())); + } + + private static long estimateNestedField(Types.NestedField field) { + long bytes = JvmSizeUtils.instanceSize(field.getClass()); + bytes = add(bytes, JvmSizeUtils.stringSize(field.name())); + bytes = add(bytes, JvmSizeUtils.stringSize(field.doc())); + return add(bytes, estimateIcebergType(field.type())); + } + + private static long estimateIcebergType(Type type) { + long bytes = JvmSizeUtils.instanceSize(type.getClass()); + if (type.isStructType()) { + List fields = type.asStructType().fields(); + bytes = add(bytes, estimateListStructure(fields)); + for (Types.NestedField field : fields) { + bytes = add(bytes, estimateNestedField(field)); + } + } else if (type.isListType()) { + bytes = add(bytes, estimateNestedField(type.asListType().fields().get(0))); + } else if (type.isMapType()) { + for (Types.NestedField field : type.asMapType().fields()) { + bytes = add(bytes, estimateNestedField(field)); + } + } + return bytes; + } + + private static long estimatePartitionSpec(PartitionSpec spec) { + List fields = spec.fields(); + long bytes = add(JvmSizeUtils.instanceSize(spec.getClass()), JvmSizeUtils.objectArraySize(fields.size())); + for (PartitionField field : fields) { + bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(field.name())); + bytes = add(bytes, JvmSizeUtils.instanceSize(field.transform().getClass())); + } + return bytes; + } + + private static long estimateSortOrder(SortOrder sortOrder) { + List fields = sortOrder.fields(); + long bytes = add(JvmSizeUtils.instanceSize(sortOrder.getClass()), JvmSizeUtils.objectArraySize(fields.size())); + for (SortField field : fields) { + bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass())); + bytes = add(bytes, JvmSizeUtils.instanceSize(field.transform().getClass())); + } + return bytes; + } + + private static long estimateSnapshot(Snapshot snapshot) { + long bytes = JvmSizeUtils.instanceSize(snapshot.getClass()); + bytes = add(bytes, estimateBoxed(snapshot.parentId(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.schemaId(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.firstRowId(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.addedRows(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.operation())); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.manifestListLocation())); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.keyId())); + return add(bytes, estimateStringMap(snapshot.summary())); + } + + private static long estimateHistory(List history) { + long bytes = estimateListStructure(history); + for (HistoryEntry entry : history) { + bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass())); + } + return bytes; + } + + private static long estimateMetadataLog(List entries) { + long bytes = estimateListStructure(entries); + for (TableMetadata.MetadataLogEntry entry : entries) { + bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(entry.file())); + } + return bytes; + } + + private static long estimateSnapshotRefs(Map refs) { + long bytes = estimateMapStructure(refs); + for (Map.Entry entry : refs.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getKey())); + SnapshotRef ref = entry.getValue(); + bytes = add(bytes, JvmSizeUtils.instanceSize(ref.getClass())); + bytes = add(bytes, estimateBoxed(ref.minSnapshotsToKeep(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(ref.maxSnapshotAgeMs(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(ref.maxRefAgeMs(), LONG_SHALLOW_BYTES)); + } + return bytes; + } + + private static long estimateStatisticsFiles(List files) { + long bytes = estimateListStructure(files); + for (StatisticsFile file : files) { + bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(file.path())); + List blobs = file.blobMetadata(); + bytes = add(bytes, estimateListStructure(blobs)); + for (BlobMetadata blob : blobs) { + bytes = add(bytes, JvmSizeUtils.instanceSize(blob.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(blob.type())); + bytes = add(bytes, estimateBoxedList(blob.fields(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateStringMap(blob.properties())); + } + } + return bytes; + } + + private static long estimatePartitionStatisticsFiles(List files) { + long bytes = estimateListStructure(files); + for (PartitionStatisticsFile file : files) { + bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(file.path())); + } + return bytes; + } + + private static long estimateShallowList(List values) { + long bytes = estimateListStructure(values); + for (Object value : values) { + bytes = add(bytes, JvmSizeUtils.instanceSize(value.getClass())); + } + return bytes; + } + + private static long estimateMetadataUpdates(List updates) { + long bytes = estimateListStructure(updates); + for (MetadataUpdate update : updates) { + bytes = add(bytes, JvmSizeUtils.instanceSize(update.getClass())); + if (update instanceof MetadataUpdate.SetProperties) { + bytes = add(bytes, estimateMapStructure(((MetadataUpdate.SetProperties) update).updated())); + } else if (update instanceof MetadataUpdate.AddPartitionSpec) { + UnboundPartitionSpec spec = ((MetadataUpdate.AddPartitionSpec) update).spec(); + bytes = add(bytes, JvmSizeUtils.instanceSize(spec.getClass())); + bytes = add(bytes, estimateListStructure(spec.fields())); + } else if (update instanceof MetadataUpdate.AddSortOrder) { + bytes = add(bytes, JvmSizeUtils.instanceSize( + ((MetadataUpdate.AddSortOrder) update).sortOrder().getClass())); + } + } + return bytes; + } + + private static long estimateContentFileList(List> files) { + if (files.isEmpty()) { + return 0L; + } + long bytes = add(estimateListStructure(files), CONTENT_FILE_SCHEMA_BYTES); + Set ownedObjects = java.util.Collections.newSetFromMap(new IdentityHashMap<>()); + for (ContentFile file : files) { + bytes = add(bytes, estimateContentFile(file, ownedObjects)); + } + return bytes; + } + + private static long estimateContentFileSchema() { + long bytes = JvmSizeUtils.instanceSize(Types.StructType.class); + bytes = add(bytes, JvmSizeUtils.arrayListSize(CONTENT_FILE_FIELD_NAMES.length)); + for (String name : CONTENT_FILE_FIELD_NAMES) { + bytes = add(bytes, JvmSizeUtils.instanceSize(Types.NestedField.class)); + bytes = add(bytes, JvmSizeUtils.stringSize(name)); + } + return bytes; + } + + private static long estimateContentFile(ContentFile file, Set ownedObjects) { + long bytes = JvmSizeUtils.instanceSize(file.getClass()); + if (file instanceof StructLike) { + bytes = add(bytes, JvmSizeUtils.intArraySize(((StructLike) file).size())); + bytes = add(bytes, LONG_SHALLOW_BYTES); + } + bytes = add(bytes, estimateOwnedCharSequence(file.path(), ownedObjects)); + bytes = add(bytes, estimateOwnedString(file.manifestLocation(), ownedObjects)); + bytes = add(bytes, estimatePartition(file.partition(), ownedObjects)); + bytes = add(bytes, estimateLongMap(file.columnSizes())); + bytes = add(bytes, estimateLongMap(file.valueCounts())); + bytes = add(bytes, estimateLongMap(file.nullValueCounts())); + bytes = add(bytes, estimateLongMap(file.nanValueCounts())); + bytes = add(bytes, estimateByteBufferMap(file.lowerBounds())); + bytes = add(bytes, estimateByteBufferMap(file.upperBounds())); + ByteBuffer keyMetadata = file.keyMetadata(); + if (keyMetadata != null) { + bytes = add(bytes, JvmSizeUtils.byteArraySize(keyMetadata.remaining())); + } + List splitOffsets = file.splitOffsets(); + if (splitOffsets != null) { + bytes = add(bytes, JvmSizeUtils.longArraySize(splitOffsets.size())); + } + List equalityFieldIds = file.equalityFieldIds(); + if (equalityFieldIds != null) { + bytes = add(bytes, JvmSizeUtils.intArraySize(equalityFieldIds.size())); + } + bytes = add(bytes, estimateBoxed(file.pos(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.sortOrderId(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.dataSequenceNumber(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.fileSequenceNumber(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.firstRowId(), LONG_SHALLOW_BYTES)); + if (file instanceof DeleteFile) { + DeleteFile deleteFile = (DeleteFile) file; + bytes = add(bytes, estimateOwnedString(deleteFile.referencedDataFile(), ownedObjects)); + bytes = add(bytes, estimateBoxed(deleteFile.contentOffset(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(deleteFile.contentSizeInBytes(), LONG_SHALLOW_BYTES)); + } + return bytes; + } + + private static long estimatePartition(StructLike partition, Set ownedObjects) { + if (partition == null || !ownedObjects.add(partition)) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(partition.getClass()); + bytes = add(bytes, JvmSizeUtils.objectArraySize(partition.size())); + for (int i = 0; i < partition.size(); i++) { + bytes = add(bytes, estimateOwnedScalar(partition.get(i, Object.class), ownedObjects)); + } + return bytes; + } + + private static long estimateOwnedScalar(Object value, Set ownedObjects) { + if (value == null || !ownedObjects.add(value)) { + return 0L; + } + if (value instanceof CharSequence) { + return estimateCharSequence((CharSequence) value); + } + if (value instanceof ByteBuffer) { + return estimateByteBuffer((ByteBuffer) value); + } + return JvmSizeUtils.instanceSize(value.getClass()); + } + + private static long estimateOwnedCharSequence(CharSequence value, Set ownedObjects) { + return value == null || !ownedObjects.add(value) ? 0L : estimateCharSequence(value); + } + + private static long estimateOwnedString(String value, Set ownedStrings) { + return value == null || !ownedStrings.add(value) ? 0L : JvmSizeUtils.stringSize(value); + } + + private static long estimateCharSequence(CharSequence value) { + if (value instanceof String) { + return JvmSizeUtils.stringSize((String) value); + } + return add(JvmSizeUtils.instanceSize(value.getClass()), JvmSizeUtils.stringSize(value.toString())); + } + + private static long estimateLongMap(Map values) { + if (values == null || values.isEmpty()) { + return 0L; + } + return add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES + LONG_SHALLOW_BYTES)); + } + + private static long estimateByteBufferMap(Map values) { + if (values == null || values.isEmpty()) { + return 0L; + } + long bytes = add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES)); + for (ByteBuffer value : values.values()) { + bytes = add(bytes, estimateByteBuffer(value)); + } + return bytes; + } + + private static long estimateByteBuffer(ByteBuffer value) { + if (value == null) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + return value.hasArray() ? add(bytes, JvmSizeUtils.byteArraySize(value.capacity())) : bytes; + } + + private static long estimateSchemaIndexes(int fieldCount) { + if (fieldCount == 0) { + return 0L; + } + long oneIndex = JvmSizeUtils.instanceSize(HashMap.class); + oneIndex = add(oneIndex, JvmSizeUtils.objectArraySize(hashCapacity(fieldCount))); + oneIndex = add(oneIndex, multiply(fieldCount, HASH_MAP_NODE_SHALLOW_BYTES)); + long bytes = multiply(3L, oneIndex); + return add(bytes, multiply(2L * fieldCount, INTEGER_SHALLOW_BYTES)); + } + + private static long estimateIntegerIndexMap(Map values) { + return add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES)); + } + + private static long estimateLongIndexMap(List values) { + if (values.isEmpty()) { + return JvmSizeUtils.instanceSize(HashMap.class); + } + long bytes = JvmSizeUtils.instanceSize(HashMap.class); + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(values.size()))); + bytes = add(bytes, multiply(values.size(), HASH_MAP_NODE_SHALLOW_BYTES)); + return add(bytes, multiply(values.size(), LONG_SHALLOW_BYTES)); + } + + private static long estimateStringMap(Map values) { + if (values == null) { + return 0L; + } + long bytes = estimateMapStructure(values); + for (Map.Entry entry : values.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getKey())); + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getValue())); + } + return bytes; + } + + private static long estimateStringList(List values) { + if (values == null || values.isEmpty()) { + return 0L; + } + long bytes = estimateListStructure(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateBoxedList(List values, long elementBytes) { + if (values == null || values.isEmpty()) { + return 0L; + } + return add(estimateListStructure(values), multiply(values.size(), elementBytes)); + } + + private static long estimateBoxed(Object value, long bytes) { + return value == null ? 0L : bytes; + } + + private static long estimateListStructure(List values) { + int capacity = values instanceof ArrayList ? arrayListCapacity(values.size()) : values.size(); + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + if (values.getClass().getName().equals("java.util.ImmutableCollections$List12")) { + return bytes; + } + return add(bytes, JvmSizeUtils.objectArraySize(capacity)); + } + + private static long estimateMapStructure(Map values) { + if (values == null) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + if (values.isEmpty()) { + return bytes; + } + if (values instanceof HashMap) { + int capacity = hashCapacity(values.size()); + long nodeBytes = values instanceof LinkedHashMap + ? LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES + : HASH_MAP_NODE_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.objectArraySize(capacity)); + return add(bytes, multiply(values.size(), nodeBytes)); + } + bytes = add(bytes, JvmSizeUtils.objectArraySize(saturatedDouble(values.size()))); + return add(bytes, multiply(values.size(), HASH_MAP_NODE_SHALLOW_BYTES)); + } + + private static int arrayListCapacity(int size) { + if (size == 0) { + return 0; + } + int capacity = 10; + while (capacity < size) { + int grown = capacity + (capacity >> 1); + if (grown < 0) { + return Integer.MAX_VALUE; + } + capacity = grown; + } + return capacity; + } + + private static int hashCapacity(int size) { + if (size == 0) { + return 0; + } + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static int saturatedDouble(int value) { + return value > Integer.MAX_VALUE / 2 ? Integer.MAX_VALUE : value * 2; + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java index 8947743c5484d4..aea432ee88cff3 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java @@ -157,6 +157,8 @@ public IcebergCatalogProperties checkCreateTimeOnlyRules() { * sentinel), {@code capacity} must be a long ≥ 0. Absent keys are skipped. */ private static void checkMetaCacheProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, "iceberg", + "table", "partition", "manifest", "partition_view"); CacheSpec.checkBooleanProperty(properties.get(IcebergConnector.TABLE_CACHE_ENABLE), IcebergConnector.TABLE_CACHE_ENABLE); CacheSpec.checkLongProperty(properties.get(IcebergConnector.TABLE_CACHE_TTL_SECOND), diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java index 49b54f94470395..0ab14f07b0f835 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.catalog.TableIdentifier; @@ -68,6 +69,7 @@ final class IcebergCommentCache { CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition .builder("iceberg-comment", spec, IcebergCommentCache::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index abeb947ff7b460..5123c6cf866e59 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -17,8 +17,10 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.iceberg.dlf.DLFCatalog; import org.apache.doris.connector.metastore.DlfMetaStoreProperties; import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; @@ -233,10 +235,10 @@ public class IcebergConnector implements Connector { mvccPartitionViewCache; private final ConnectorMetadataCache> // null under session=user listPartitionsViewCache; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; // Manifest content cache — pure metadata, default-off (meta.cache.iceberg.manifest.enable), and consumed // ONLY after a per-user resolveTable(ForRead) -- exempt: no read path without a per-user load. - private final IcebergManifestCache manifestCache = new IcebergManifestCache(metaCache); + private final IcebergManifestCache manifestCache; // Lazily-built plugin-side Kerberos authenticator (single-owner auth; see TcclPinningConnectorContext). // null for a non-Kerberos catalog. Its doAs acts on the PLUGIN's UserGroupInformation copy — the one the @@ -260,6 +262,8 @@ public IcebergConnector(Map properties, ConnectorContext context // authenticator never logs in — so without this the DDL/read hits secured HDFS as SIMPLE auth. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "iceberg", this.properties); + this.manifestCache = new IcebergManifestCache(metaCache, this.properties); // Authorization-sensitive projection (snapshotId/schemaId). Under iceberg.rest.session=user the value is // per-user AUTHORIZED metadata that a "can-list-cannot-load" principal must not see. beginQuerySnapshot // reads this cache WITHOUT a preceding per-user loadTable, so a shared (table-keyed, no user dimension) @@ -283,7 +287,7 @@ public IcebergConnector(Map properties, ConnectorContext context || IcebergScanPlanProvider.restVendedCredentialsEnabled(this.properties)) ? null : new IcebergTableCache( - metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY, + metaCache, cacheSpec("table"), this::cachedTableCleanup, catalogResourceTracker); // PERF-02: partition-view cache. Authorization-sensitive projection: a shared (table+snapshot-keyed, no // user dimension) hit would disclose one user's partition list. Its readers are all downstream of a @@ -294,7 +298,7 @@ metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPA this.partitionCache = isUserSessionEnabled() ? null : new IcebergPartitionCache( - metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); + metaCache, cacheSpec("partition")); // PERF-03: inferred-file-format cache. Same authorization-sensitive treatment as partitionCache (disabled // under session=user, kept otherwise); readers already tolerate a null cache (resolveFileFormatName). this.formatCache = isUserSessionEnabled() @@ -319,11 +323,15 @@ metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPA this.mvccPartitionViewCache = isUserSessionEnabled() ? null : new ConnectorMetadataCache<>(metaCache, "iceberg.mvcc-partition-view", - "iceberg", "partition_view", this.properties); + "iceberg", "partition_view", this.properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + IcebergCacheSizeEstimator::estimateMvccPartitionViewEntry); this.listPartitionsViewCache = isUserSessionEnabled() ? null : new ConnectorMetadataCache<>(metaCache, "iceberg.list-partitions-view", - "iceberg", "partition_view", this.properties); + "iceberg", "partition_view", this.properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + IcebergCacheSizeEstimator::estimatePartitionInfoViewEntry); } /** @@ -345,6 +353,12 @@ static long resolveTableCacheTtlSecond(Map properties) { } } + private CacheSpec cacheSpec(String entryName) { + return CacheSpec.fromProperties(this.properties, "iceberg", entryName, + CacheSpec.ofConnectorTtl( + resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY)); + } + @Override public ConnectorMetadata getMetadata(ConnectorSession session) { return new IcebergConnectorMetadata(newCatalogBackedOps(session), catalogProps, context, diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java index 2c9ce0c2ed7fb3..8af2d28cfcdbb2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.catalog.TableIdentifier; @@ -102,6 +103,7 @@ public int hashCode() { CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition .builder("iceberg-format", spec, IcebergFormatCache::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java index a20ab5fd2ecf4d..81f8229069f75a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; @@ -87,6 +88,7 @@ static final class CachedSnapshot { this.entry = owner.create(MetaCacheDefinition .builder( "iceberg-latest-snapshot", spec, IcebergLatestSnapshotCache::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java index 6bfead8166070d..b68ce2b5e43dc7 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.DataFile; @@ -135,7 +136,12 @@ private static final class ScanStats { } IcebergManifestCache(CatalogMetaCache owner) { - this(owner, DEFAULT_MANIFEST_CACHE_CAPACITY); + this(owner, Collections.emptyMap()); + } + + IcebergManifestCache(CatalogMetaCache owner, Map properties) { + this(owner, DEFAULT_MANIFEST_CACHE_CAPACITY, DEFAULT_STATS_TTL_SECONDS, + System::nanoTime, properties); } IcebergManifestCache(int maxSize) { @@ -143,7 +149,7 @@ private static final class ScanStats { } private IcebergManifestCache(CatalogMetaCache owner, int maxSize) { - this(owner, maxSize, DEFAULT_STATS_TTL_SECONDS, System::nanoTime); + this(owner, maxSize, DEFAULT_STATS_TTL_SECONDS, System::nanoTime, Collections.emptyMap()); } /** Visible for testing: injectable stats TTL + clock so the leak sweep is deterministic without sleeping. */ @@ -153,16 +159,25 @@ private IcebergManifestCache(CatalogMetaCache owner, int maxSize) { private IcebergManifestCache( CatalogMetaCache owner, int maxSize, long statsTtlSeconds, LongSupplier nanoClock) { + this(owner, maxSize, statsTtlSeconds, nanoClock, Collections.emptyMap()); + } + + private IcebergManifestCache( + CatalogMetaCache owner, int maxSize, long statsTtlSeconds, LongSupplier nanoClock, + Map properties) { this.owner = owner; // Always enabled, no expiry, capacity-bounded (CACHE_NO_TTL == -1 means "no expiration", enabled). - CacheSpec spec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Math.max(1, maxSize)); + CacheSpec defaultSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Math.max(1, maxSize)); + CacheSpec spec = CacheSpec.fromProperties(properties, "iceberg", "manifest", defaultSpec); this.entry = owner.create(MetaCacheDefinition .builder( "iceberg-manifest", spec, ignored -> ScopePath.catalog()) + .sizeEstimator(IcebergCacheSizeEstimator::estimateManifestEntry) .build()); this.equalityDeleteFieldIds = owner.create(MetaCacheDefinition .>builder( - "iceberg-equality-delete-field-ids", spec, ignored -> ScopePath.catalog()) + "iceberg-equality-delete-field-ids", defaultSpec, ignored -> ScopePath.catalog()) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); this.statsTtlNanos = TimeUnit.SECONDS.toNanos(Math.max(1L, statsTtlSeconds)); this.nanoClock = nanoClock; @@ -187,7 +202,8 @@ Set getOrLoadEqualityDeleteFieldIds( */ ManifestCacheValue getManifestCacheValue(ManifestFile manifest, Table table) { IcebergManifestEntryKey key = IcebergManifestEntryKey.of(manifest); - return entry.get(key, k -> loadManifestCacheValue(manifest, table, k.getContent())); + return entry.get(key, k -> loadManifestCacheValue( + manifest, table, k.getContent(), entry.isWeightBounded())); } /** @@ -207,7 +223,8 @@ ManifestCacheValue getManifestCacheValue(ManifestFile manifest, Table table, Str stats.misses++; } } - return entry.get(key, k -> loadManifestCacheValue(manifest, table, k.getContent())); + return entry.get(key, k -> loadManifestCacheValue( + manifest, table, k.getContent(), entry.isWeightBounded())); } /** @@ -254,12 +271,12 @@ private void sweepExpiredStats(long nowNanos) { } private static ManifestCacheValue loadManifestCacheValue(ManifestFile manifest, Table table, - ManifestContent content) { + ManifestContent content, boolean estimateWeight) { try { if (content == ManifestContent.DELETES) { - return ManifestCacheValue.forDeleteFiles(loadDeleteFiles(manifest, table)); + return ManifestCacheValue.forDeleteFiles(loadDeleteFiles(manifest, table), estimateWeight); } - return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, table)); + return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, table), estimateWeight); } catch (IOException e) { throw new RuntimeException("Failed to read iceberg manifest " + manifest.path(), e); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java index 451aa7ddd819d1..59a47b75f4cd55 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java @@ -21,11 +21,15 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; import org.apache.iceberg.catalog.TableIdentifier; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Supplier; @@ -86,19 +90,38 @@ public int hashCode() { } } + static final class CachedPartitions { + final List partitions; + final MetaCacheSizeEstimate sizeEstimate; + + CachedPartitions(List partitions, boolean estimateWeight) { + this.partitions = estimateWeight + ? Collections.unmodifiableList(new ArrayList<>(partitions)) + : partitions; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("iceberg_partition_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + IcebergCacheSizeEstimator.estimatePartitions(this.partitions))) + : MetaCacheSizeEstimate.complete(0L); + } + } + private final CatalogMetaCache owner; - private final MetaCache> entry; + private final MetaCache entry; IcebergPartitionCache(long ttlSeconds, int maxSize) { this(new CatalogMetaCache(), ttlSeconds, maxSize); } IcebergPartitionCache(CatalogMetaCache owner, long ttlSeconds, int maxSize) { + this(owner, CacheSpec.ofConnectorTtl(ttlSeconds, maxSize)); + } + + IcebergPartitionCache(CatalogMetaCache owner, CacheSpec spec) { this.owner = owner; - // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). - CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition - .>builder("iceberg-partition", spec, IcebergPartitionCache::scope) + .builder("iceberg-partition", spec, IcebergPartitionCache::scope) + .sizeEstimator(IcebergCacheSizeEstimator::estimatePartitionEntry) .build()); } @@ -113,7 +136,7 @@ boolean isEnabled() { * loader runs OUTSIDE Caffeine's compute lock (single-flight per key) and its exception propagates unwrapped. */ List getOrLoad(Key key, Supplier> loader) { - return entry.get(key, ignored -> loader.get()); + return entry.get(key, ignored -> new CachedPartitions(loader.get(), entry.isWeightBounded())).partitions; } /** Drops every cached snapshot entry for one table so the next read scans live (REFRESH TABLE). */ diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index a5ff4562e915cd..ddc8cbde915b5e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -990,6 +990,22 @@ static final class IcebergRawPartition { this.lastUpdateTime = lastUpdateTime; this.lastSnapshotId = lastSnapshotId; } + + String nameForWeight() { + return name; + } + + List columnNamesForWeight() { + return columnNames; + } + + List valuesForWeight() { + return values; + } + + List transformsForWeight() { + return transforms; + } } /** A single physical partition's computed range: time interval (for the overlap merge) + pre-rendered bounds. */ diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java index 573623c0063261..d64ded6f039a5d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java @@ -110,7 +110,7 @@ static T withBorrowedTable(ConnectorSession session, String dbName, String t Function action) { if (session == null || session.getStatementScope() == ConnectorStatementScope.NONE) { try (IcebergTableCache.TableLease lease = loader.get()) { - return action.apply(snapshotReadTable(lease.table())); + return action.apply(lease.snapshotReadTable()); } } return action.apply(sharedBorrowedTable(session, dbName, tableName, loader, unscopedLoader)); @@ -148,7 +148,7 @@ private static final class ScopedBorrow implements AutoCloseable { private ScopedBorrow(IcebergTableCache.TableLease lease) { this.lease = lease; - this.table = snapshotReadTable(lease.table()); + this.table = lease.snapshotReadTable(); } @Override diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index 614df9aee6aa0b..9f130039a196d2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -19,11 +19,18 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.JvmSizeUtils; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.TableIdentifier; import java.util.concurrent.atomic.AtomicBoolean; @@ -50,9 +57,11 @@ * {@code expireAfterAccess} with a {@code maxSize} capacity. Lives on the long-lived per-catalog * {@link IcebergConnector}; a REFRESH CATALOG rebuilds the connector and thus the cache. * - *

Values are RAW tables. The scan provider applies {@code wrapTableForScan} (the Kerberos + *

Values own RAW tables. The scan provider applies {@code wrapTableForScan} (the Kerberos * {@code doAs} FileIO wrap) per call on the way out, so no per-request authenticator is ever frozen into a - * shared entry. + * shared entry. A weight-bounded owner also retains and accounts for one serialized metadata generation; + * each statement parses that generation into a private read table so Iceberg's lazy manifest fields cannot + * grow the already-admitted cache value. * *

Credential isolation. A raw table carries its FileIO's credentials, so this cross-query layer is * built ONLY when the connector's credentials are query-independent — it is left disabled (the connector @@ -83,15 +92,19 @@ final class IcebergTableCache { IcebergTableCache(CatalogMetaCache owner, long ttlSeconds, int maxSize, Function cleanupFactory, IcebergCatalogResourceTracker resourceTracker) { + this(owner, CacheSpec.ofConnectorTtl(ttlSeconds, maxSize), cleanupFactory, resourceTracker); + } + + IcebergTableCache(CatalogMetaCache owner, CacheSpec spec, + Function cleanupFactory, IcebergCatalogResourceTracker resourceTracker) { this.owner = owner; this.cleanupFactory = cleanupFactory; this.resourceTracker = resourceTracker; - // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). - CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition .builder("iceberg-table", spec, IcebergTableCache::scope) .removalListener((identifier, tableOwner, reason) -> tableOwner.release()) .discardListener((identifier, tableOwner) -> tableOwner.release()) + .sizeEstimator(IcebergCacheSizeEstimator::estimateTableEntry) .build()); } @@ -130,7 +143,7 @@ TableLease borrow(TableIdentifier identifier, Supplier loader) { } } }; - TableOwner loaded = new TableOwner(table, cleanup, true); + TableOwner loaded = new TableOwner(table, cleanup, true, entry.isWeightBounded()); loadedHere[0] = loaded; return loaded; } finally { @@ -214,6 +227,10 @@ Table table() { return owner.table; } + Table snapshotReadTable() { + return owner.snapshotReadTable(); + } + @Override public void close() { if (closed.compareAndSet(false, true)) { @@ -222,19 +239,58 @@ public void close() { } } - private static final class TableOwner { + static final class TableOwner { private final Table table; private final Runnable cleanup; + private final String snapshotMetadataJson; + private final String snapshotMetadataLocation; + final MetaCacheSizeEstimate sizeEstimate; // A newly loaded value starts with a cache reference and a temporary loader reference. The temporary // reference bridges publication/discard to the first borrow, including invalidation-before-publication. private final AtomicInteger references; - private TableOwner(Table table, Runnable cleanup, boolean loading) { + private TableOwner(Table table, Runnable cleanup, boolean loading, boolean estimateWeight) { this.table = table; this.cleanup = cleanup; + String[] metadataJson = {null}; + String[] metadataLocation = {null}; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("iceberg_table_estimator_failure", + () -> { + long serializedMetadataBytes = 0L; + if (table instanceof BaseTable) { + TableMetadata metadata = ((BaseTable) table).operations().current(); + String json = TableMetadataParser.toJson(metadata); + metadataJson[0] = json; + metadataLocation[0] = metadata.metadataFileLocation(); + serializedMetadataBytes = + IcebergCacheSizeEstimator.estimateSerializedTableMetadata(json); + } + // Iceberg v1 serialization materializes lazy embedded-manifest state on the + // retained BaseSnapshot, so weigh the table only after serialization completes. + long bytes = JvmSizeUtils.saturatedAdd( + IcebergCacheSizeEstimator.estimateTable(table), serializedMetadataBytes); + return MetaCacheSizeEstimate.complete(bytes); + }) + : MetaCacheSizeEstimate.complete(0L); + this.snapshotMetadataJson = sizeEstimate.isComplete() ? metadataJson[0] : null; + this.snapshotMetadataLocation = sizeEstimate.isComplete() ? metadataLocation[0] : null; this.references = new AtomicInteger(loading ? 2 : 1); } + private Table snapshotReadTable() { + if (!(table instanceof BaseTable)) { + return table; + } + BaseTable baseTable = (BaseTable) table; + TableOperations operations = baseTable.operations(); + TableMetadata metadata = snapshotMetadataJson == null + ? operations.current() + : TableMetadataParser.fromJson(snapshotMetadataLocation, snapshotMetadataJson); + return new BaseTable(new IcebergSnapshotTableOperations(operations, metadata), + table.name(), baseTable.reporter()); + } + private TableLease tryBorrow() { int current = references.get(); while (current != 0) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java index b9b9da68dbd3c5..2c2b2d60925216 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java @@ -17,6 +17,9 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; + import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; @@ -33,18 +36,32 @@ public class ManifestCacheValue { private final List dataFiles; private final List deleteFiles; + private final MetaCacheSizeEstimate sizeEstimate; - private ManifestCacheValue(List dataFiles, List deleteFiles) { + private ManifestCacheValue(List dataFiles, List deleteFiles, boolean estimateWeight) { this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles; this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("iceberg_manifest_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + IcebergCacheSizeEstimator.estimateManifestValue(this))) + : MetaCacheSizeEstimate.complete(0L); } public static ManifestCacheValue forDataFiles(List dataFiles) { - return new ManifestCacheValue(dataFiles, Collections.emptyList()); + return forDataFiles(dataFiles, false); + } + + static ManifestCacheValue forDataFiles(List dataFiles, boolean estimateWeight) { + return new ManifestCacheValue(dataFiles, Collections.emptyList(), estimateWeight); } public static ManifestCacheValue forDeleteFiles(List deleteFiles) { - return new ManifestCacheValue(Collections.emptyList(), deleteFiles); + return forDeleteFiles(deleteFiles, false); + } + + static ManifestCacheValue forDeleteFiles(List deleteFiles, boolean estimateWeight) { + return new ManifestCacheValue(Collections.emptyList(), deleteFiles, estimateWeight); } public List getDataFiles() { @@ -54,4 +71,8 @@ public List getDataFiles() { public List getDeleteFiles() { return deleteFiles; } + + MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate; + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java index 8bec2551e770b8..cd7a5f5838d7d1 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.spi.ConnectorPartitionInfo; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartition; @@ -37,6 +38,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -125,6 +127,30 @@ private static List mvccNames(Optional view) .map(ConnectorMvccPartition::getName).collect(Collectors.toList()); } + @Test + public void largePartitionViewsCanBeEstimatedWithoutTheReflectiveVisitLimit() { + List listView = new ArrayList<>(); + List mvccPartitions = new ArrayList<>(); + for (int index = 0; index < 20_000; index++) { + String value = Integer.toString(index); + listView.add(new ConnectorPartitionInfo("p=" + value, + Collections.singletonMap("p", value), Collections.emptyMap(), + Collections.singletonList(value), Collections.emptyList())); + mvccPartitions.add(new ConnectorMvccPartition( + "p=" + value, Collections.singletonList(value), + Collections.singletonList(value), index)); + } + ConnectorTableKey key = new ConnectorTableKey("db", "table", 1L, 1L); + ConnectorMvccPartitionView mvccView = new ConnectorMvccPartitionView( + ConnectorMvccPartitionView.Style.RANGE, + ConnectorMvccPartitionView.Freshness.SNAPSHOT_ID, mvccPartitions, 1L); + + Assertions.assertTrue(IcebergCacheSizeEstimator.estimatePartitionInfoViewEntry( + key, listView).isComplete()); + Assertions.assertTrue(IcebergCacheSizeEstimator.estimateMvccPartitionViewEntry( + key, mvccView).isComplete()); + } + // --------------------------------------------------------------------- // getMvccPartitionView // --------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java index 920e5ae0c6d646..6ead48d4d7798b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CatalogMetaCache; + import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.ManifestFile; @@ -80,6 +82,22 @@ public void loadsDataFilesAndCachesByManifestPath() { Assertions.assertEquals(1, cache.size()); } + @Test + public void weightBoundedManifestIsEstimatedAndCached() { + Table table = tableWithTwoDataFiles(); + ManifestFile manifest = table.currentSnapshot().dataManifests(table.io()).get(0); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergManifestCache cache = new IcebergManifestCache(owner, + Collections.singletonMap("meta.cache.iceberg.manifest.max-weight", "1MB")); + + ManifestCacheValue first = cache.getManifestCacheValue(manifest, table); + ManifestCacheValue second = cache.getManifestCacheValue(manifest, table); + + Assertions.assertSame(first, second); + Assertions.assertEquals(2, first.getDataFiles().size()); + } + } + @Test public void equalityDeleteFieldIdsLoadOncePerSnapshot() { IcebergManifestCache cache = new IcebergManifestCache(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java index 38c46ebaf7b008..2f832abd4d957d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; import org.apache.iceberg.catalog.TableIdentifier; @@ -73,6 +75,29 @@ public void cachesWithinTtlAndServesTheSameList() { Assertions.assertTrue(c.isEnabled()); } + @Test + public void weightBoundedPartitionsAreEstimatedAndCached() { + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergPartitionCache cache = new IcebergPartitionCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 1024L * 1024L)); + + List first = cache.getOrLoad(key("db", "t", 5L), () -> { + loads.incrementAndGet(); + return raws(3); + }); + List second = cache.getOrLoad(key("db", "t", 5L), () -> { + loads.incrementAndGet(); + return raws(7); + }); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, loads.get()); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> first.add(raws(1).get(0))); + } + } + @Test public void differentSnapshotIdIsADifferentKey() { AtomicInteger loads = new AtomicInteger(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheBenchmark.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheBenchmark.java new file mode 100644 index 00000000000000..3bb118d9b897f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheBenchmark.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; + +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.LocationProvider; +import org.apache.iceberg.types.Types; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Dependency-free microbenchmark for weighted Iceberg admission and per-statement metadata isolation. */ +public final class IcebergTableCacheBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static volatile long blackhole; + + private IcebergTableCacheBenchmark() { + } + + public static void main(String[] args) { + Table table = tableFixture(); + TableMetadata metadata = ((BaseTable) table).operations().current(); + String json = TableMetadataParser.toJson(metadata); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 100L * 1024L * 1024L), + ignored -> () -> { }, new IcebergCatalogResourceTracker()); + try (IcebergTableCache.TableLease lease = cache.borrow( + TableIdentifier.of("db", "table"), () -> table)) { + Result result = measure(() -> { + Table statementTable = lease.snapshotReadTable(); + return ((BaseTable) statementTable).operations().current().properties().size(); + }, 100); + System.out.printf( + "iceberg_metadata_json_chars=%d properties=%d statement_copy_ns_op=%d operations=%d%n", + json.length(), metadata.properties().size(), result.medianNanos, result.operations); + } + } + } + + private static Table tableFixture() { + Schema schema = new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "payload", Types.StringType.get())); + Map properties = new LinkedHashMap<>(); + for (int i = 0; i < 1_000; i++) { + properties.put("property-" + i, "value-" + i + "-" + "x".repeat(48)); + } + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/weighted-table", properties); + metadata = TableMetadata.buildFrom(metadata) + .withMetadataLocation("file:///tmp/weighted-table/metadata/v1.metadata.json") + .discardChanges() + .build(); + return new BaseTable(new StaticTableOperations(metadata), "weighted"); + } + + private static Result measure(LongOperation operation, int operationsPerWindow) { + for (int i = 0; i < WARMUP_WINDOWS; i++) { + runWindow(operation, operationsPerWindow); + } + long[] nanosPerOperation = new long[MEASURE_WINDOWS]; + for (int i = 0; i < MEASURE_WINDOWS; i++) { + long start = System.nanoTime(); + runWindow(operation, operationsPerWindow); + nanosPerOperation[i] = (System.nanoTime() - start) / operationsPerWindow; + } + Arrays.sort(nanosPerOperation); + return new Result(nanosPerOperation[MEASURE_WINDOWS / 2], + (long) operationsPerWindow * MEASURE_WINDOWS); + } + + private static void runWindow(LongOperation operation, int operations) { + long value = 0L; + for (int i = 0; i < operations; i++) { + value ^= operation.run(); + } + blackhole = value; + } + + @FunctionalInterface + private interface LongOperation { + long run(); + } + + private static final class Result { + private final long medianNanos; + private final long operations; + + private Result(long medianNanos, long operations) { + this.medianNanos = medianNanos; + this.operations = operations; + } + } + + private static final class StaticTableOperations implements TableOperations { + private final TableMetadata metadata; + + private StaticTableOperations(TableMetadata metadata) { + this.metadata = metadata; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; + } + + @Override + public void commit(TableMetadata base, TableMetadata newMetadata) { + throw new UnsupportedOperationException(); + } + + @Override + public FileIO io() { + return null; + } + + @Override + public EncryptionManager encryption() { + return null; + } + + @Override + public String metadataFileLocation(String fileName) { + return "file:///tmp/weighted-table/metadata/" + fileName; + } + + @Override + public LocationProvider locationProvider() { + return null; + } + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java index db13f08f669c4f..36ca2a94c210f6 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java @@ -17,11 +17,24 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; + +import org.apache.iceberg.BaseTable; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -76,6 +89,110 @@ public void cachesWithinTtlAndServesTheSameTable() { Assertions.assertTrue(c.isEnabled()); } + @Test + public void weightBoundedTableIsEstimatedAndCached() { + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 1024L * 1024L), + ignored -> () -> { + }, new IcebergCatalogResourceTracker()); + + Table first = cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return table("first"); + }); + Table second = cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return table("second"); + }); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, loads.get()); + } + } + + @Test + public void weightedBorrowUsesAnIndependentSnapshotGenerationPerStatement() { + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 10L * 1024L * 1024L), + ignored -> () -> { + }, new IcebergCatalogResourceTracker()); + + try (IcebergTableCache.TableLease lease = cache.borrow(id(), + IcebergTableCacheTest::tableWithSnapshot)) { + BaseTable cached = (BaseTable) lease.table(); + BaseTable firstStatement = (BaseTable) lease.snapshotReadTable(); + BaseTable secondStatement = (BaseTable) lease.snapshotReadTable(); + + Snapshot cachedSnapshot = cached.operations().current().currentSnapshot(); + Snapshot firstSnapshot = firstStatement.operations().current().currentSnapshot(); + Snapshot secondSnapshot = secondStatement.operations().current().currentSnapshot(); + Assertions.assertNotSame(cachedSnapshot, firstSnapshot, + "manifest lazy fields must not be written into the weighted cache generation"); + Assertions.assertNotSame(firstSnapshot, secondSnapshot, + "each statement must own its snapshot lazy-loading state"); + Assertions.assertEquals(cachedSnapshot.snapshotId(), firstSnapshot.snapshotId()); + } + Assertions.assertEquals(1, cache.size()); + } + } + + @Test + public void weightedV1TableIsMeasuredAfterSerializationMaterializesEmbeddedManifests() { + Table probe = tableWithV1EmbeddedManifests(256); + long beforeSerialization = IcebergCacheSizeEstimator.estimateTable(probe); + String metadataJson = TableMetadataParser.toJson(((BaseTable) probe).operations().current()); + long afterSerialization = IcebergCacheSizeEstimator.estimateTable(probe); + long materializedBytes = afterSerialization - beforeSerialization; + Assertions.assertTrue(materializedBytes > 8192L, + "the fixture must materialize enough v1 manifest state to distinguish the two weigh orders"); + + long oldOrderPayload = beforeSerialization + + IcebergCacheSizeEstimator.estimateSerializedTableMetadata(metadataJson); + long maxWeight = oldOrderPayload + materializedBytes / 2L; + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, maxWeight), + ignored -> () -> { }, new IcebergCatalogResourceTracker()); + + cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return tableWithV1EmbeddedManifests(256); + }); + cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return tableWithV1EmbeddedManifests(256); + }); + + Assertions.assertEquals(2, loads.get(), + "the post-serialization retained graph exceeds the budget and must not be cached"); + Assertions.assertEquals(0, cache.size()); + } + } + + @Test + public void weightedBorrowSupportsMetadataWithoutAFileLocation() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/no-metadata-location", Collections.emptyMap()); + Table table = new BaseTable(new StaticTableOperations(metadata), "no-metadata-location"); + + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 10L * 1024L * 1024L), + ignored -> () -> { }, new IcebergCatalogResourceTracker()); + try (IcebergTableCache.TableLease lease = cache.borrow(id(), () -> table)) { + BaseTable statement = (BaseTable) lease.snapshotReadTable(); + Assertions.assertNull(statement.operations().current().metadataFileLocation()); + Assertions.assertEquals(schema.asStruct(), statement.schema().asStruct()); + } + Assertions.assertEquals(1, cache.size()); + } + } + @Test public void ttlZeroDisablesCachingAlwaysLive() { AtomicInteger loads = new AtomicInteger(); @@ -281,4 +398,91 @@ public void loaderExceptionPropagatesUnwrapped() { throw new NoSuchTableException("simulated concurrent drop"); })); } + + private static Table tableWithSnapshot() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata base = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/weighted-table", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{" + + "\"sequence-number\":1," + + "\"snapshot-id\":101," + + "\"timestamp-ms\":1000," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"file:///tmp/snap-101.avro\"," + + "\"schema-id\":0}"); + TableMetadata metadata = TableMetadata.buildFrom(base) + .upgradeFormatVersion(2) + .withMetadataLocation("file:///tmp/v2.metadata.json") + .setBranchSnapshot(snapshot, "main") + .discardChanges() + .build(); + return new BaseTable(new StaticTableOperations(metadata), "weighted"); + } + + private static Table tableWithV1EmbeddedManifests(int manifestCount) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata base = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/weighted-v1-table", + Collections.singletonMap(TableProperties.FORMAT_VERSION, "1")); + StringBuilder snapshotJson = new StringBuilder() + .append("{\"snapshot-id\":101,\"timestamp-ms\":1000,") + .append("\"summary\":{\"operation\":\"append\"},\"schema-id\":0,\"manifests\":["); + for (int i = 0; i < manifestCount; i++) { + if (i > 0) { + snapshotJson.append(','); + } + snapshotJson.append("\"file:///tmp/manifest-").append(i).append(".avro\""); + } + snapshotJson.append("]}"); + Snapshot snapshot = SnapshotParser.fromJson(snapshotJson.toString()); + TableMetadata metadata = TableMetadata.buildFrom(base) + .withMetadataLocation("file:///tmp/v1.metadata.json") + .setBranchSnapshot(snapshot, "main") + .discardChanges() + .build(); + return new BaseTable(new StaticTableOperations(metadata), "weighted-v1"); + } + + private static final class StaticTableOperations implements TableOperations { + private final TableMetadata metadata; + + private StaticTableOperations(TableMetadata metadata) { + this.metadata = metadata; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; + } + + @Override + public void commit(TableMetadata base, TableMetadata updated) { + throw new UnsupportedOperationException("read only"); + } + + @Override + public FileIO io() { + return null; + } + + @Override + public EncryptionManager encryption() { + return null; + } + + @Override + public String metadataFileLocation(String fileName) { + return "file:///tmp/" + fileName; + } + + @Override + public LocationProvider locationProvider() { + return null; + } + } } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java index 9b9225178d7826..3e52564feec94c 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java @@ -103,6 +103,11 @@ private PaimonCatalogFactory() { * plus each flavor's {@code appendCustomCatalogOptions()}. */ public static Options buildCatalogOptions(PaimonCatalogProperties catalogProperties) { + return buildCatalogOptions(catalogProperties, false); + } + + static Options buildCatalogOptions( + PaimonCatalogProperties catalogProperties, boolean hasEnclosingMetaCacheWeightLimit) { Options options = new Options(); Map props = catalogProperties.getRaw(); String flavor = catalogProperties.getFlavor(); @@ -136,6 +141,9 @@ public static Options buildCatalogOptions(PaimonCatalogProperties catalogPropert // filesystem: nothing custom. break; } + if (hasEnclosingMetaCacheWeightLimit && !options.contains(CatalogOptions.CACHE_ENABLED)) { + options.set(CatalogOptions.CACHE_ENABLED, false); + } return options; } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java index f2220daff2da1e..d2ef06f23df231 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java @@ -160,6 +160,7 @@ public PaimonCatalogProperties checkCreateTimeOnlyRules() { * {@code table.capacity} must be a long ≥ 0. Absent keys are skipped. */ private static void checkMetaCacheProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, "paimon", "partition_view"); CacheSpec.checkBooleanProperty(properties.get(PaimonConnector.TABLE_CACHE_ENABLE), PaimonConnector.TABLE_CACHE_ENABLE); CacheSpec.checkLongProperty(properties.get(PaimonConnector.TABLE_CACHE_TTL_SECOND), diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 761b6252f60357..b00efbcb57712c 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -140,14 +140,13 @@ public class PaimonConnector implements Connector { // returns a fresh metadata per query, so this lives on the connector and is injected into the metadata so // beginQuerySnapshot pins a stable id across queries. Cleared wholesale on REFRESH CATALOG (connector rebuilt). private final PaimonLatestSnapshotCache latestSnapshotCache; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; // FIX-B-MC2: connector-level (per-catalog, long-lived) second-level memo for the time-travel // schema-at-snapshot read. getMetadata() returns a FRESH metadata per query, so this must live on the // connector (not the metadata) to give the cross-query hit the legacy PaimonExternalMetaCache provided. // Cleared wholesale on REFRESH CATALOG (the connector is rebuilt). See PaimonSchemaAtMemo. - private final PaimonSchemaAtMemo schemaAtMemo = - new PaimonSchemaAtMemo(metaCache, PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); + private final PaimonSchemaAtMemo schemaAtMemo; // PERF-06: cross-query DERIVED partition-view cache ("cache A", the generic ConnectorMetadataCache from // fe-connector-cache), layered ABOVE the raw remote catalog.listPartitions call (PaimonCatalogOps#listPartitions): @@ -180,13 +179,17 @@ public PaimonConnector(Map properties, ConnectorContext context) // this a DDL/read against secured HDFS negotiates SIMPLE auth. See TcclPinningConnectorContext. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "paimon", properties); + this.schemaAtMemo = new PaimonSchemaAtMemo(metaCache, PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); this.latestSnapshotCache = new PaimonLatestSnapshotCache( metaCache, resolveTableCacheTtlSecond(properties), DEFAULT_TABLE_CACHE_CAPACITY); // Reads its own meta.cache.paimon.partition_view.(enable|ttl-second|capacity) from the catalog // properties via the framework's CacheSpec (default ON / 24h / 1000). this.partitionViewCache = new ConnectorMetadataCache<>( - metaCache, "paimon.partition-view", "paimon", "partition_view", properties); + metaCache, "paimon.partition-view", "paimon", "partition_view", properties, + key -> org.apache.doris.connector.cache.ScopePath.table(key.getDb(), key.getTable()), + PaimonPartitionViewSizeEstimator::estimateEntry); } /** @@ -408,7 +411,7 @@ private Catalog ensureCatalog() { } private Catalog createCatalog() { - Options options = PaimonCatalogFactory.buildCatalogOptions(catalogProps); + Options options = buildCatalogOptions(); String flavor = catalogProps.getFlavor(); // Canonical storage config from the FE-bound fe-filesystem StorageProperties (P1-T03), replacing // the legacy buildObjectStorageHadoopConfig path: object stores contribute their fs.s3a.*/fs.oss.* @@ -495,6 +498,10 @@ static boolean hasDlfCompatibleStorage(List storageProperties || "OSS_HDFS".equals(storage.providerName())); } + Options buildCatalogOptions() { + return PaimonCatalogFactory.buildCatalogOptions(catalogProps, metaCache.hasEnclosingWeightLimit()); + } + /** * Assembles the canonical storage Hadoop config from the FE-bound storage properties (P1-T03). * fe-core binds the catalog's raw property map to fe-filesystem {@link StorageProperties} and hands diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java index 62113194c76895..f382ca22d37ead 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java @@ -1338,7 +1338,12 @@ private List cachedPartitions(PaimonTableHandle paimonHa return collectPartitions(paimonHandle); } ConnectorTableKey key = partitionViewCacheKey(paimonHandle); - return partitionViewCache.get(key, () -> collectPartitions(paimonHandle)); + return partitionViewCache.get(key, () -> { + List partitions = collectPartitions(paimonHandle); + return partitionViewCache.isWeightBounded() + ? new PaimonPartitionView(key, partitions) + : partitions; + }); } /** diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java index e42ba1f5917462..0f241158c5997f 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.paimon.catalog.Identifier; @@ -63,6 +64,7 @@ final class PaimonLatestSnapshotCache { this.entry = owner.create(MetaCacheDefinition .builder("paimon-latest-snapshot", spec, id -> ScopePath.table(id.getDatabaseName(), id.getObjectName())) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java new file mode 100644 index 00000000000000..3701d60029cb7f --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.RandomAccess; + +/** Immutable partition-view value whose retained size is computed once at construction. */ +final class PaimonPartitionView extends AbstractList implements RandomAccess { + private final List partitions; + private final MetaCacheSizeEstimate sizeEstimate; + + PaimonPartitionView(ConnectorTableKey key, List partitions) { + this.partitions = Collections.unmodifiableList(new ArrayList<>(partitions)); + this.sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_partition_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + PaimonPartitionViewSizeEstimator.estimateEntryOnConstruction(key, this))); + } + + @Override + public ConnectorPartitionInfo get(int index) { + return partitions.get(index); + } + + @Override + public int size() { + return partitions.size(); + } + + MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate; + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java new file mode 100644 index 00000000000000..62354404dcd0eb --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; + +/** + * Publication-time estimate of the projection built by PaimonConnectorMetadata.collectPartitions. + * Counts every partition's strings; sampling variable-length payload can both miss and amplify a large tail. + */ +final class PaimonPartitionViewSizeEstimator { + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorTableKey.class); + private static final long VIEW_SHALLOW_BYTES = JvmSizeUtils.instanceSize(PaimonPartitionView.class); + private static final long ESTIMATE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(MetaCacheSizeEstimate.class); + private static final long PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class); + private static final long UNMODIFIABLE_LIST_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableList(Collections.emptyList()).getClass()); + private static final long UNMODIFIABLE_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableMap(Collections.emptyMap()).getClass()); + private static final long LINKED_HASH_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize(LinkedHashMap.class); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + + private PaimonPartitionViewSizeEstimator() { + } + + /** Admission callback: the complete key/value weight was computed when the immutable view was built. */ + static MetaCacheSizeEstimate estimateEntry(ConnectorTableKey key, List value) { + return ((PaimonPartitionView) value).getSizeEstimate(); + } + + static long estimateEntryOnConstruction(ConnectorTableKey key, PaimonPartitionView value) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb())); + bytes = add(bytes, JvmSizeUtils.stringSize(key.getTable())); + bytes = add(bytes, VIEW_SHALLOW_BYTES); + bytes = add(bytes, ESTIMATE_SHALLOW_BYTES); + bytes = add(bytes, UNMODIFIABLE_LIST_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.arrayListSize(value.size())); + if (!value.isEmpty()) { + // collectPartitions builds every map/list from the same partitionKeys, with shared column names. + // Compute that fixed shape once; only the name/value String payload varies between partitions. + bytes = add(bytes, multiply(value.size(), estimatePartitionStructure(value.get(0)))); + for (String column : value.get(0).getPartitionValues().keySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(column)); + } + } + for (int index = 0; index < value.size(); index++) { + ConnectorPartitionInfo partition = value.get(index); + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getPartitionName())); + List orderedValues = partition.getOrderedPartitionValues(); + for (int column = 0; column < orderedValues.size(); column++) { + // The same rendered String is retained by both partitionValues and orderedPartitionValues. + bytes = add(bytes, JvmSizeUtils.stringSize(orderedValues.get(column))); + } + } + return bytes; + } + + private static long estimatePartitionStructure(ConnectorPartitionInfo partition) { + long bytes = PARTITION_SHALLOW_BYTES; + // The collector supplies an empty properties map, wrapped by ConnectorPartitionInfo per partition. + bytes = add(bytes, UNMODIFIABLE_MAP_SHALLOW_BYTES); + + int valueCount = partition.getPartitionValues().size(); + bytes = add(bytes, UNMODIFIABLE_MAP_SHALLOW_BYTES); + bytes = add(bytes, LINKED_HASH_MAP_SHALLOW_BYTES); + if (valueCount > 0) { + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(valueCount))); + bytes = add(bytes, multiply(valueCount, LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES)); + } + + bytes = add(bytes, estimateCopiedList(partition.getOrderedPartitionValues())); + return add(bytes, estimateCopiedList(partition.getPartitionValueNullFlags())); + } + + private static long estimateCopiedList(List values) { + return add(UNMODIFIABLE_LIST_SHALLOW_BYTES, JvmSizeUtils.arrayListSize(values.size())); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java index 9c5dce4d97ef49..7406ec36533310 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import java.util.Objects; @@ -69,6 +70,7 @@ final class PaimonSchemaAtMemo { .builder( "paimon-schema-at", spec, key -> ScopePath.table(key.databaseName, key.tableName)) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmark.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmark.java new file mode 100644 index 00000000000000..e083d8bdef99fb --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmark.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import org.apache.paimon.partition.Partition; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +/** + * Offline microbenchmark, not JMH. Uses the production partition collector with recording catalog fakes: + * no RPC, filesystem access, cache-hit timing, or data-file scans. Compares collection alone against collection + * plus weighted-view construction; publication-only timing includes its list copy. Prepared-weight timing is + * just the provider callback, including loop/volatile-sink overhead, not the complete cache-admission path. + * + *

Compile via run-fe-ut.sh; run this main with the module's test classpath and the FE's java.lang/java.util + * opens. Run multiple fresh JVMs before drawing conclusions. Raw estimates are reported, not asserted to be + * exact retained heap. TAIL_END hits the former 16/5-element sampling positions; TAIL_INTERIOR misses both. + */ +public final class PaimonCacheSizeBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static final long MIN_WINDOW_NANOS = TimeUnit.MILLISECONDS.toNanos(50); + private static volatile Object blackhole; + + private PaimonCacheSizeBenchmark() { + } + + enum Distribution { + UNIFORM, + TAIL_END, + TAIL_INTERIOR + } + + public static void main(String[] args) { + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1_000, 10_000}) { + for (Distribution distribution : Distribution.values()) { + Fixture fixture = new Fixture(fieldCount, partitionCount, distribution); + List partitions = fixture.load(false); + PaimonPartitionView prepared = new PaimonPartitionView(fixture.key, partitions); + if (!prepared.getSizeEstimate().isComplete()) { + throw new IllegalStateException("Incomplete benchmark estimate: " + + prepared.getSizeEstimate().getIncompleteReason()); + } + String label = "fields=" + fieldCount + " partitions=" + partitionCount + + " distribution=" + distribution; + System.out.printf("%s estimated_bytes=%d large_value_chars=%d%n", label, + prepared.getSizeEstimate().getBytes(), distribution == Distribution.UNIFORM + ? 0 : Fixture.LARGE_VALUE_CHARS); + Map> cases = new LinkedHashMap<>(); + cases.put("collect", () -> fixture.load(false)); + cases.put("collect_weighted", () -> fixture.load(true)); + cases.put("publish_only", () -> new PaimonPartitionView(fixture.key, partitions)); + cases.put("prepared_weight", () -> PaimonPartitionViewSizeEstimator + .estimateEntry(fixture.key, prepared)); + measure(label, cases); + } + } + } + } + + static final class Fixture { + static final int LARGE_VALUE_CHARS = 1024 * 1024; + final ConnectorTableKey key = new ConnectorTableKey("db1", "t1", 1L, -1L); + private final RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + private final PaimonConnectorMetadata metadata; + private final PaimonTableHandle handle; + + Fixture(int fieldCount, int partitionCount, Distribution distribution) { + RowType.Builder schema = RowType.builder().field("region", DataTypes.STRING()); + for (int field = 1; field < fieldCount; field++) { + schema.field("column_" + field, DataTypes.STRING()); + } + FakePaimonTable table = new FakePaimonTable("t1", schema.build(), + Collections.singletonList("region"), Collections.emptyList()); + table.setOptions(Collections.singletonMap("partition.legacy-name", "true")); + ops.table = table; + ops.partitions = new ArrayList<>(partitionCount); + int largeIndex = distribution == Distribution.TAIL_END ? partitionCount - 1 + : distribution == Distribution.TAIL_INTERIOR ? partitionCount - 2 : -1; + for (int index = 0; index < partitionCount; index++) { + String value = "region-" + index; + if (index == largeIndex) { + value += "x".repeat(LARGE_VALUE_CHARS); + } + ops.partitions.add(new Partition(Collections.singletonMap("region", value), + 1L, 1L, 1, 1L, true)); + } + metadata = new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), + new RecordingConnectorContext()); + handle = new PaimonTableHandle("db1", "t1", Collections.singletonList("region"), + Collections.emptyList()); + handle.setPaimonTable(table); + } + + List load(boolean weighted) { + // Bound the recording fake's log in BOTH paths; never measure an ever-growing fixture. + ops.log.clear(); + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + // Mirrors cachedPartitions' post-collection branch, without timing cache lookup or admission locks. + return weighted ? new PaimonPartitionView(key, partitions) : partitions; + } + } + + private static void measure(String label, Map> cases) { + List names = new ArrayList<>(cases.keySet()); + List> operations = new ArrayList<>(cases.values()); + int[] batchSizes = new int[operations.size()]; + long[][] samples = new long[operations.size()][MEASURE_WINDOWS]; + for (int index = 0; index < operations.size(); index++) { + int batchSize = 1; + while (runWindow(operations.get(index), batchSize) < MIN_WINDOW_NANOS && batchSize < 1 << 20) { + batchSize *= 2; + } + batchSizes[index] = batchSize; + } + for (int window = 0; window < WARMUP_WINDOWS + MEASURE_WINDOWS; window++) { + // Rotate case order to avoid always measuring the weighted path after the baseline's allocations. + for (int offset = 0; offset < operations.size(); offset++) { + int index = (window + offset) % operations.size(); + long elapsed = runWindow(operations.get(index), batchSizes[index]); + if (window >= WARMUP_WINDOWS) { + samples[index][window - WARMUP_WINDOWS] = elapsed / batchSizes[index]; + } + } + } + for (int index = 0; index < operations.size(); index++) { + Arrays.sort(samples[index]); + System.out.printf("%s operation=%s median_ns_op=%d min_ns_op=%d max_ns_op=%d operations=%d%n", + label, names.get(index), samples[index][MEASURE_WINDOWS / 2], samples[index][0], + samples[index][MEASURE_WINDOWS - 1], (long) batchSizes[index] * MEASURE_WINDOWS); + } + } + + private static long runWindow(Supplier operation, int count) { + long start = System.nanoTime(); + for (int index = 0; index < count; index++) { + // Make allocated results escape; prepared-weight results intentionally retain the same estimate. + blackhole = operation.get(); + } + return System.nanoTime() - start; + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmarkTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmarkTest.java new file mode 100644 index 00000000000000..a2bd18845d6066 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmarkTest.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +class PaimonCacheSizeBenchmarkTest { + @Test + void weightedFixturePreservesProductionProjectionAndPreparedEstimate() { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, 32, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + List baseline = fixture.load(false); + PaimonPartitionView weighted = (PaimonPartitionView) fixture.load(true); + Assertions.assertEquals(32, baseline.size()); + Assertions.assertEquals(baseline.size(), weighted.size()); + for (int index = 0; index < baseline.size(); index++) { + Assertions.assertEquals(baseline.get(index).getPartitionName(), weighted.get(index).getPartitionName()); + Assertions.assertEquals(baseline.get(index).getPartitionValues(), weighted.get(index).getPartitionValues()); + } + Assertions.assertEquals("region=region-0", weighted.get(0).getPartitionName()); + Assertions.assertTrue(weighted.getSizeEstimate().isComplete()); + Assertions.assertTrue(weighted.getSizeEstimate().getBytes() > 0); + Assertions.assertSame(weighted.getSizeEstimate(), + PaimonPartitionViewSizeEstimator.estimateEntry(fixture.key, weighted)); + Assertions.assertNotSame(baseline, fixture.load(false), "baseline must rebuild rather than hit a cache"); + } + + @Test + void longTailFixturesCoverSampledAndUnsampledPositions() { + for (PaimonCacheSizeBenchmark.Distribution distribution : new PaimonCacheSizeBenchmark.Distribution[] { + PaimonCacheSizeBenchmark.Distribution.TAIL_END, + PaimonCacheSizeBenchmark.Distribution.TAIL_INTERIOR}) { + List partitions = new PaimonCacheSizeBenchmark.Fixture( + 100, 1_000, distribution).load(false); + int largeIndex = distribution == PaimonCacheSizeBenchmark.Distribution.TAIL_END ? 999 : 998; + Assertions.assertTrue(partitions.get(largeIndex).getPartitionValues().get("region").length() + >= PaimonCacheSizeBenchmark.Fixture.LARGE_VALUE_CHARS); + for (int samples : new int[] {5, 16}) { + boolean sampled = false; + for (int sample = 0; sample < samples; sample++) { + sampled |= sample * (partitions.size() - 1) / (samples - 1) == largeIndex; + } + Assertions.assertEquals(distribution == PaimonCacheSizeBenchmark.Distribution.TAIL_END, sampled); + } + } + } + + @Test + void longTailWeightIsIndependentOfPositionAndNotExtrapolated() { + for (int count : new int[] {1_000, 10_000}) { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + PaimonPartitionView uniform = (PaimonPartitionView) fixture.load(true); + PaimonPartitionView interior = (PaimonPartitionView) new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.TAIL_INTERIOR).load(true); + PaimonPartitionView end = (PaimonPartitionView) new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.TAIL_END).load(true); + Assertions.assertTrue(interior.getSizeEstimate().isComplete()); + Assertions.assertTrue(end.getSizeEstimate().isComplete()); + // One value String shared by the list/map, plus a separately materialized partition-name String. + ConnectorPartitionInfo small = uniform.get(count - 2); + ConnectorPartitionInfo large = interior.get(count - 2); + long expectedGrowth = JvmSizeUtils.stringSize(large.getPartitionName()) + - JvmSizeUtils.stringSize(small.getPartitionName()) + + JvmSizeUtils.stringSize(large.getOrderedPartitionValues().get(0)) + - JvmSizeUtils.stringSize(small.getOrderedPartitionValues().get(0)); + Assertions.assertTrue(expectedGrowth >= 2L * PaimonCacheSizeBenchmark.Fixture.LARGE_VALUE_CHARS); + Assertions.assertEquals(expectedGrowth, + interior.getSizeEstimate().getBytes() - uniform.getSizeEstimate().getBytes()); + Assertions.assertEquals(interior.getSizeEstimate().getBytes(), end.getSizeEstimate().getBytes()); + List reordered = new ArrayList<>(interior); + Collections.swap(reordered, 0, count - 2); + Assertions.assertEquals(interior.getSizeEstimate().getBytes(), + new PaimonPartitionView(fixture.key, reordered).getSizeEstimate().getBytes()); + } + } + + @Test + void unicodeAndMultipleColumnsAreCountedWithoutDoubleChargingMapValues() { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, 0, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + for (int columns : new int[] {1, 4, 16}) { + Map values = new LinkedHashMap<>(); + for (int column = 0; column < columns; column++) { + values.put("column-" + column, "small-" + column); + } + ConnectorPartitionInfo small = partition(values); + Map largeValues = new LinkedHashMap<>(values); + largeValues.put("column-0", "中".repeat(65_536)); + ConnectorPartitionInfo large = partition(largeValues); + long before = new PaimonPartitionView(fixture.key, List.of(small)).getSizeEstimate().getBytes(); + long after = new PaimonPartitionView(fixture.key, List.of(large)).getSizeEstimate().getBytes(); + Assertions.assertEquals(JvmSizeUtils.stringSize(largeValues.get("column-0")) + - JvmSizeUtils.stringSize(values.get("column-0")), after - before); + } + } + + @Test + void projectionRemainsImmutableWithAPreparedWeight() { + for (int count : new int[] {0, 1, 32, 1_000}) { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + List input = fixture.load(false); + PaimonPartitionView view = new PaimonPartitionView(fixture.key, input); + Assertions.assertTrue(view.getSizeEstimate().isComplete()); + Assertions.assertTrue(view.getSizeEstimate().getBytes() > 0); + Assertions.assertSame(view.getSizeEstimate(), + PaimonPartitionViewSizeEstimator.estimateEntry(fixture.key, view)); + input.clear(); + Assertions.assertEquals(count, view.size()); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> view.add(partition(Collections.emptyMap()))); + } + } + + private static ConnectorPartitionInfo partition(Map values) { + return new ConnectorPartitionInfo("fixed-name", values, Collections.emptyMap(), + new ArrayList<>(values.values()), new ArrayList<>(Collections.nCopies(values.size(), false))); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java index 7a0a4cef186e3a..c1c13f67e54186 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -91,6 +92,44 @@ public void filesystemSetsMetastoreFilesystemAndWarehouse() { Assertions.assertEquals("/wh", opts.get("warehouse")); } + @Test + public void enclosingDorisWeightLimitDisablesPaimonSdkCacheOnlyByDefault() throws Exception { + Map defaults = props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh"); + + try (PaimonConnector ungoverned = new PaimonConnector(defaults, new RecordingConnectorContext())) { + Options options = ungoverned.buildCatalogOptions(); + Assertions.assertFalse(options.contains(CatalogOptions.CACHE_ENABLED), + "without a global/catalog total, Doris must preserve the Paimon SDK default"); + } + + Map catalogLimited = new HashMap<>(defaults); + catalogLimited.put("meta.cache.max-weight", "1MB"); + try (PaimonConnector governed = new PaimonConnector(catalogLimited, new RecordingConnectorContext())) { + Options options = governed.buildCatalogOptions(); + Assertions.assertTrue(options.contains(CatalogOptions.CACHE_ENABLED)); + Assertions.assertFalse(options.get(CatalogOptions.CACHE_ENABLED), + "an enclosing Doris hard limit must not be bypassed by an unaccounted SDK cache"); + } + + Map entryLimited = new HashMap<>(defaults); + entryLimited.put("meta.cache.paimon.partition_view.max-weight", "1MB"); + try (PaimonConnector entryOnly = new PaimonConnector(entryLimited, new RecordingConnectorContext())) { + Assertions.assertFalse(entryOnly.buildCatalogOptions().contains(CatalogOptions.CACHE_ENABLED), + "an entry-only limit must preserve the Paimon SDK default"); + } + + PaimonCatalogProperties explicitTrue = PaimonCatalogProperties.of(props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh", "paimon.cache-enabled", "true")); + Assertions.assertTrue(PaimonCatalogFactory.buildCatalogOptions(explicitTrue, true) + .get(CatalogOptions.CACHE_ENABLED), "an explicit Paimon setting must win"); + + PaimonCatalogProperties explicitFalse = PaimonCatalogProperties.of(props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh", "paimon.cache-enabled", "false")); + Assertions.assertFalse(PaimonCatalogFactory.buildCatalogOptions(explicitFalse, true) + .get(CatalogOptions.CACHE_ENABLED), "an explicit Paimon setting must win"); + } + @Test public void hmsSetsHiveMetastoreUriPoolAndLocation() { Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java index 831a5d59601bc1..5302d9a750811e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java @@ -17,7 +17,10 @@ package org.apache.doris.connector.paimon; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.ConnectorPartitionInfo; import org.apache.doris.connector.spi.pushdown.ConnectorExpression; @@ -27,6 +30,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; @@ -103,6 +107,21 @@ private static List names(List infos) { return infos.stream().map(ConnectorPartitionInfo::getPartitionName).collect(Collectors.toList()); } + @Test + public void largePartitionViewCanBeEstimatedWithoutTheReflectiveVisitLimit() { + List partitions = new ArrayList<>(); + for (int index = 0; index < 20_000; index++) { + String value = Integer.toString(index); + partitions.add(new ConnectorPartitionInfo("p=" + value, + Collections.singletonMap("p", value), Collections.emptyMap(), + Collections.singletonList(value), Collections.emptyList())); + } + + PaimonPartitionView view = new PaimonPartitionView( + new ConnectorTableKey("db", "table", 1L, 1L), partitions); + Assertions.assertTrue(view.getSizeEstimate().isComplete()); + } + @Test public void listPartitionsCachesDerivedListAcrossQueries() { // WHY: cache A must memoize the BUILT List keyed by (db, table, snapshotId, @@ -126,6 +145,60 @@ public void listPartitionsCachesDerivedListAcrossQueries() { Assertions.assertEquals(1, loadCount(ops), "a cache hit must not re-enumerate (listPartitions once)"); } + @Test + public void weightBoundedPartitionViewIsEstimatedAndCached() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = regionTable(); + ops.table = table; + ops.latestSnapshotId = OptionalLong.of(100L); + ops.partitions = Arrays.asList(partition("cn"), partition("us")); + Map properties = Collections.singletonMap( + "meta.cache.paimon.partition_view.max-weight", "1MB"); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + ConnectorMetadataCache> cache = new ConnectorMetadataCache<>( + owner, "paimon.partition-view", "paimon", "partition_view", properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + PaimonPartitionViewSizeEstimator::estimateEntry); + PaimonConnectorMetadata metadata = metadataWithCache(ops, cache); + PaimonTableHandle handle = handle(table); + + List first = metadata.listPartitions(null, handle, Optional.empty()); + List second = metadata.listPartitions(null, handle, Optional.empty()); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, loadCount(ops)); + } + } + + @Test + public void unsampledLongTailRejectsAdmissionWithoutFailingPartitionListing() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = regionTable(); + ops.table = table; + ops.latestSnapshotId = OptionalLong.of(100L); + ops.partitions = new ArrayList<>(); + for (int index = 0; index < 1_000; index++) { + ops.partitions.add(partition("region-" + index)); + } + ops.partitions.set(998, partition("x".repeat(1024 * 1024))); + Map properties = Collections.singletonMap( + "meta.cache.paimon.partition_view.max-weight", "1MB"); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + ConnectorMetadataCache> cache = new ConnectorMetadataCache<>( + owner, "paimon.partition-view", "paimon", "partition_view", properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + PaimonPartitionViewSizeEstimator::estimateEntry); + PaimonConnectorMetadata metadata = metadataWithCache(ops, cache); + PaimonTableHandle handle = handle(table); + List first = metadata.listPartitions(null, handle, Optional.empty()); + List second = metadata.listPartitions(null, handle, Optional.empty()); + Assertions.assertEquals(1_000, first.size()); + Assertions.assertEquals(first, second); + Assertions.assertNotSame(first, second, "oversized views are returned, not cached"); + Assertions.assertEquals(2, loadCount(ops)); + } + } + @Test public void listPartitionsDifferentSnapshotReEnumerates() { // WHY: the underlying remote enumeration always reflects the CURRENT catalog state (it is diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 91dd81e5af5952..cb91fa2ef2dab2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -651,6 +651,10 @@ public class SchemaTable extends Table { .column("LAST_LOAD_SUCCESS_TIME", ScalarType.createStringType()) .column("LAST_LOAD_FAILURE_TIME", ScalarType.createStringType()) .column("LAST_ERROR", ScalarType.createStringType()) + .column("MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ESTIMATED_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("WEIGHT_REJECT_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("LAST_WEIGHT_REJECT_REASON", ScalarType.createStringType()) .build()) ) .put("backend_kerberos_ticket_cache", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 8a8e2dcb45bb0f..8256a365ec8442 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -39,6 +39,7 @@ import org.apache.doris.common.Version; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.MetaCacheBudgetManager; import org.apache.doris.datasource.doris.RemoteDorisExternalDatabase; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; @@ -436,6 +437,11 @@ public void checkProperties() throws DdlException { protected void checkProperties(CatalogProperty property) throws DdlException { // check refresh parameter of catalog Map properties = property.getProperties(); + try { + checkMetaCacheWeightProperties(properties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) { try { int metadataRefreshIntervalSec = Integer.parseInt( @@ -457,6 +463,11 @@ protected void checkProperties(CatalogProperty property) throws DdlException { } } + /** Strict CREATE/ALTER validation for the core metadata-cache namespace owned by this catalog. */ + protected void checkMetaCacheWeightProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, "default", "schema"); + } + /** * Validate an ALTER candidate without publishing it to this catalog. A true return value * declares that the connector performed complete detached validation; false retains the @@ -867,9 +878,27 @@ public void tryModifyCatalogProps(Map props) { private void invalidateCachesAfterPropertyUpdate(Map updatedProps) { ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - if (updatedProps.get(SCHEMA_CACHE_TTL_SECOND) != null) { + if (updatedProps.get(SCHEMA_CACHE_TTL_SECOND) != null + || updatedProps.containsKey(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { cacheMgr.removeCatalog(id); - } else { + return; + } + Set removedEngines = Sets.newHashSet(); + for (String key : updatedProps.keySet()) { + if (key == null || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + if (separator <= 0) { + continue; + } + String engine = remainder.substring(0, separator); + if (cacheMgr.isEngineRegistered(engine) && removedEngines.add(engine)) { + cacheMgr.removeCatalogByEngine(id, engine); + } + } + if (removedEngines.isEmpty()) { cacheMgr.invalidateCatalog(id); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index d22ed858b4ef87..900b880f88a724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -21,6 +21,10 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheGovernance; import org.apache.doris.datasource.doris.DorisExternalMetaCache; import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; import org.apache.doris.datasource.metacache.ExternalCatalogMetaCache; @@ -43,6 +47,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -91,6 +96,7 @@ public class ExternalMetaCacheMgr { private ExternalRowCountCache rowCountCache; public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { + MetaCacheGovernance.configureGlobalMaxWeight(configuredGlobalMaxWeight()); rowCountRefreshExecutor = newThreadPool(isCheckpointCatalog, Config.max_external_cache_loader_thread_pool_size, Config.max_external_cache_loader_thread_pool_size * 1000, @@ -118,6 +124,17 @@ public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { registerBuiltinEngineCaches(); } + private static OptionalLong configuredGlobalMaxWeight() { + String configured = Config.external_meta_cache_max_weight; + long parsed = CacheSpec.parseWeight(configured, "external_meta_cache_max_weight", + true, Runtime.getRuntime().maxMemory()); + if (configured.trim().endsWith("%") && parsed == 0L) { + throw new IllegalArgumentException( + "external_meta_cache_max_weight percentage must be greater than 0%"); + } + return parsed == 0L ? OptionalLong.empty() : OptionalLong.of(parsed); + } + private ExecutorService newThreadPool(boolean isCheckpointCatalog, int numThread, int queueSize, String poolName, int timeoutSeconds, boolean needRegisterMetric) { @@ -220,6 +237,10 @@ public void removeCatalogByEngine(long catalogId, String engine) { () -> cache.invalidateCatalog(catalogId))); } + public boolean isEngineRegistered(String engine) { + return cacheTypes.containsKey(engine); + } + /** * Invalidates database metadata without evicting row counts. Passive object-cache resets use this directly; * mutation paths must add their row-count barrier after upstream metadata has been invalidated. @@ -331,6 +352,15 @@ public List getCatalogCacheStats(long catalogId) { allCacheTypes().forEach(externalMetaCache -> externalMetaCache.stats(catalogId) .forEach((entryName, entryStats) -> stats.add( new CatalogMetaCacheStats(externalMetaCache.engine(), entryName, entryStats)))); + for (CatalogMetaCache cache : MetaCacheGovernance.catalogCaches(catalogId)) { + if (ENGINE_DEFAULT.equals(cache.engine()) || ENGINE_DORIS.equals(cache.engine())) { + continue; + } + for (Map.Entry> entry : cache.entries().entrySet()) { + stats.add(new CatalogMetaCacheStats( + cache.engine(), entry.getKey(), MetaCacheEntryStats.from(entry.getValue()))); + } + } stats.sort(Comparator.comparing(CatalogMetaCacheStats::getEngineName) .thenComparing(CatalogMetaCacheStats::getEntryName)); return stats; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java index e48c07303728e2..0c2397d40f8c59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.SessionContext; @@ -74,6 +75,12 @@ public void checkProperties() throws DdlException { } } + @Override + protected void checkMetaCacheWeightProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, DorisExternalMetaCache.ENGINE, + DorisExternalMetaCache.ENTRY_SCHEMA, DorisExternalMetaCache.ENTRY_BACKENDS); + } + public List getFeNodes() { return parseHttpHosts(catalogProperty.getOrDefault(RemoteDorisProperties.FE_HTTP_HOSTS, "")); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java index b86797ba0dceab..9f9dbfe674086a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java @@ -17,10 +17,8 @@ package org.apache.doris.datasource.metacache; -import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; -import org.apache.doris.connector.cache.ScopedMetaCache.CacheMetrics; import com.google.common.collect.Maps; @@ -32,9 +30,13 @@ * Catalog scoped entry container. */ final class CatalogMetaCacheRuntime { - private final CatalogMetaCache owner = new CatalogMetaCache(); + private final CatalogMetaCache owner; private final Map> entries = new ConcurrentHashMap<>(); + CatalogMetaCacheRuntime(long catalogId, String engine, Map catalogProperties) { + owner = CatalogMetaCache.managed(catalogId, engine, catalogProperties); + } + CatalogMetaCache owner() { return owner; } @@ -62,29 +64,6 @@ void close() { } private MetaCacheEntryStats stats(MetaCache entry) { - CacheSpec spec = entry.cacheSpec(); - CacheMetrics metrics = entry.metrics(); - long requests = metrics.getRequestCount(); - long loads = metrics.getLoadSuccessCount() + metrics.getLoadFailureCount(); - return new MetaCacheEntryStats( - spec.isEnable(), - entry.isEnabled(), - entry.isAutoRefresh(), - spec.getTtlSecond(), - spec.getCapacity(), - entry.size(), - requests, - metrics.getHitCount(), - metrics.getMissCount(), - requests == 0L ? 0D : (double) metrics.getHitCount() / requests, - metrics.getLoadSuccessCount(), - metrics.getLoadFailureCount(), - metrics.getTotalLoadTimeNanos(), - loads == 0L ? 0D : (double) metrics.getTotalLoadTimeNanos() / loads, - metrics.getEvictionCount(), - metrics.getInvalidateCount(), - metrics.getLastLoadSuccessTimeMs(), - metrics.getLastLoadFailureTimeMs(), - metrics.getLastError()); + return MetaCacheEntryStats.from(entry); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java index 751667904c25dd..b7f9809e264934 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java @@ -22,6 +22,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -87,7 +88,8 @@ public Collection aliases() { public void initCatalog(long catalogId, Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, id -> buildCatalogRuntime(safeCatalogProperties)); + catalogEntries.computeIfAbsent(catalogId, + id -> buildCatalogRuntime(id, safeCatalogProperties)); } @Override @@ -271,8 +273,8 @@ private void validateRegisteredMetaCacheEntryDef(MetaCacheEntryDef ensureTypeCompatible(registered, entryDef.getKeyType(), entryDef.getValueType()); } - private CatalogMetaCacheRuntime buildCatalogRuntime(Map catalogProperties) { - CatalogMetaCacheRuntime group = new CatalogMetaCacheRuntime(); + private CatalogMetaCacheRuntime buildCatalogRuntime(long catalogId, Map catalogProperties) { + CatalogMetaCacheRuntime group = new CatalogMetaCacheRuntime(catalogId, engine, catalogProperties); metaCacheEntryDefs.values() .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(group, entryDef, catalogProperties))); @@ -286,8 +288,9 @@ private MetaCache newMetaCacheEntry( CacheSpec cacheSpec = CacheSpec.fromProperties( catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); Function loader = wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()); - MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( - entryDef.getName(), cacheSpec, entryDef.getInvalidation()::scope); + MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( + entryDef.getName(), cacheSpec, entryDef.getInvalidation()::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()); if (loader != null) { builder.loader(loader); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java index b2d503b82ad087..5c6ea2cc968958 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java @@ -23,8 +23,8 @@ import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; import org.apache.doris.connector.cache.MetaCacheRemovalReason; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; -import org.apache.doris.connector.cache.ScopedMetaCache.CacheMetrics; import com.github.benmanes.caffeine.cache.RemovalListener; @@ -155,8 +155,9 @@ public static FeMetaCacheEntry withSyncRemovalListener(String name, } effectiveEnabled = CacheSpec.isCacheEnabled( cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); - MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( - name, cacheSpec, ignored -> ScopePath.catalog()); + MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( + name, cacheSpec, ignored -> ScopePath.catalog()) + .sizeEstimator(MetaCacheSizeEstimators.reflective()); if (loader != null) { builder.loader(key -> loadAndPause(key, loader)); } @@ -377,17 +378,7 @@ public void forEach(BiConsumer consumer) { } public MetaCacheEntryStats stats() { - CacheMetrics metrics = data.metrics(); - long requests = metrics.getRequestCount(); - long loads = metrics.getLoadSuccessCount() + metrics.getLoadFailureCount(); - return new MetaCacheEntryStats( - cacheSpec.isEnable(), effectiveEnabled, autoRefresh, cacheSpec.getTtlSecond(), cacheSpec.getCapacity(), - metrics.getPhysicalEntryCount(), requests, metrics.getHitCount(), metrics.getMissCount(), - requests == 0L ? 0D : (double) metrics.getHitCount() / requests, - metrics.getLoadSuccessCount(), metrics.getLoadFailureCount(), metrics.getTotalLoadTimeNanos(), - loads == 0L ? 0D : (double) metrics.getTotalLoadTimeNanos() / loads, - metrics.getEvictionCount(), metrics.getInvalidateCount(), metrics.getLastLoadSuccessTimeMs(), - metrics.getLastLoadFailureTimeMs(), metrics.getLastError()); + return MetaCacheEntryStats.from(data); } public static int defaultObjectStripeCount() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java index 8320622e08e698..05d71700600f8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.ScopedMetaCache.CacheMetrics; import java.util.Objects; @@ -53,6 +55,10 @@ public final class MetaCacheEntryStats { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final long maxWeight; + private final long estimatedWeight; + private final long weightAdmissionRejectedCount; + private final String lastWeightRejectReason; /** * Build an immutable stats snapshot. @@ -76,7 +82,11 @@ public MetaCacheEntryStats( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + long maxWeight, + long estimatedWeight, + long weightAdmissionRejectedCount, + String lastWeightRejectReason) { this.configEnabled = configEnabled; this.effectiveEnabled = effectiveEnabled; this.autoRefresh = autoRefresh; @@ -96,6 +106,28 @@ public MetaCacheEntryStats( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = Objects.requireNonNull(lastError, "lastError"); + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.weightAdmissionRejectedCount = weightAdmissionRejectedCount; + this.lastWeightRejectReason = Objects.requireNonNull(lastWeightRejectReason, "lastWeightRejectReason"); + } + + /** Creates the FE system-table view of a cache owned by the shared connector framework. */ + public static MetaCacheEntryStats from(MetaCache entry) { + CacheSpec spec = entry.cacheSpec(); + CacheMetrics metrics = entry.metrics(); + long requests = metrics.getRequestCount(); + long loads = metrics.getLoadSuccessCount() + metrics.getLoadFailureCount(); + return new MetaCacheEntryStats( + spec.isEnable(), entry.isEnabled(), entry.isAutoRefresh(), spec.getTtlSecond(), spec.getCapacity(), + entry.size(), requests, metrics.getHitCount(), metrics.getMissCount(), + requests == 0L ? 0D : (double) metrics.getHitCount() / requests, + metrics.getLoadSuccessCount(), metrics.getLoadFailureCount(), metrics.getTotalLoadTimeNanos(), + loads == 0L ? 0D : (double) metrics.getTotalLoadTimeNanos() / loads, + metrics.getEvictionCount(), metrics.getInvalidateCount(), metrics.getLastLoadSuccessTimeMs(), + metrics.getLastLoadFailureTimeMs(), metrics.getLastError(), metrics.getMaxWeight(), + metrics.getEstimatedWeight(), metrics.getWeightRejectCount(), + metrics.getLastWeightRejectReason()); } public boolean isConfigEnabled() { @@ -195,4 +227,20 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getWeightAdmissionRejectedCount() { + return weightAdmissionRejectedCount; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6313f3bdf6fd7c..8bdc9386654e11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -1967,6 +1967,10 @@ private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaT trow.addToColumnValue(new TCell().setStringVal( formatMetaCacheTime(entryStats.getLastLoadFailureTimeMs(), timeZone))); trow.addToColumnValue(new TCell().setStringVal(entryStats.getLastError())); // LAST_ERROR + trow.addToColumnValue(new TCell().setLongVal(entryStats.getMaxWeight())); + trow.addToColumnValue(new TCell().setLongVal(entryStats.getEstimatedWeight())); + trow.addToColumnValue(new TCell().setLongVal(entryStats.getWeightAdmissionRejectedCount())); + trow.addToColumnValue(new TCell().setStringVal(entryStats.getLastWeightRejectReason())); dataBatch.add(trow); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogMetaCacheWeightTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogMetaCacheWeightTest.java new file mode 100644 index 00000000000000..1aa5689e9e5ae7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogMetaCacheWeightTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; +import org.apache.doris.datasource.property.constants.RemoteDorisProperties; +import org.apache.doris.datasource.test.TestExternalCatalog; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +public class ExternalCatalogMetaCacheWeightTest { + + @Test + public void coreSchemaWeightIsValidatedAtTheDdlDoor() { + Map properties = testCatalogProperties(); + properties.put("meta.cache.default.schema.max-weight", "invalid"); + TestExternalCatalog invalidWeight = new TestExternalCatalog(1L, "test", "", properties, ""); + Assertions.assertThrows(DdlException.class, invalidWeight::checkProperties); + + properties = testCatalogProperties(); + properties.put("meta.cache.default.table.max-weight", "64MB"); + TestExternalCatalog unknownEntry = new TestExternalCatalog(2L, "test", "", properties, ""); + Assertions.assertThrows(DdlException.class, unknownEntry::checkProperties); + } + + @Test + public void remoteDorisValidatesItsOwnConsumedEntries() { + Map properties = remoteDorisProperties(); + properties.put("meta.cache.doris.backends.max-weight", "invalid"); + RemoteDorisExternalCatalog invalidWeight = new RemoteDorisExternalCatalog( + 1L, "remote", "", properties, ""); + Assertions.assertThrows(DdlException.class, invalidWeight::checkProperties); + + properties = remoteDorisProperties(); + properties.put("meta.cache.doris.unknown.max-weight", "64MB"); + RemoteDorisExternalCatalog unknownEntry = new RemoteDorisExternalCatalog( + 2L, "remote", "", properties, ""); + Assertions.assertThrows(DdlException.class, unknownEntry::checkProperties); + } + + private static Map testCatalogProperties() { + Map properties = new HashMap<>(); + properties.put("catalog_provider.class", RefreshCatalogTest.RefreshCatalogProvider.class.getName()); + return properties; + } + + private static Map remoteDorisProperties() { + Map properties = new HashMap<>(); + properties.put(RemoteDorisProperties.FE_THRIFT_HOSTS, "127.0.0.1:9020"); + properties.put(RemoteDorisProperties.FE_HTTP_HOSTS, "127.0.0.1:8030"); + properties.put(RemoteDorisProperties.FE_ARROW_HOSTS, "127.0.0.1:8070"); + properties.put(RemoteDorisProperties.USER, "root"); + properties.put(RemoteDorisProperties.PASSWORD, ""); + properties.put(RemoteDorisProperties.USE_ARROW_FLIGHT, "true"); + return properties; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java index e76b9d35752ae6..22bbc80ad788f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java @@ -64,6 +64,29 @@ public void testLoadMutationAndStatsUseSharedRuntime() { } } + @Test + public void testRejectedWeightedMutationReturnsWithoutRetryingForever() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService worker = Executors.newSingleThreadExecutor(); + String oversized = "x".repeat(2_048); + try { + FeMetaCacheEntry entry = new FeMetaCacheEntry<>( + "objects", ignored -> "old", + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 1_024L), + refreshExecutor, false); + Assertions.assertEquals("old", entry.get("key")); + + Future mutation = worker.submit( + () -> entry.compute("key", (key, value) -> oversized)); + + Assertions.assertSame(oversized, mutation.get(3L, TimeUnit.SECONDS)); + Assertions.assertNull(entry.getIfPresent("key")); + } finally { + worker.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testContextualOnlyAndDisabledEntry() { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java index b476c50ba5d9ff..a050b1f4d36237 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.FileFormatConstants; +import org.apache.doris.connector.cache.MetaCacheBudgetManager; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; @@ -128,6 +129,46 @@ public void testSchemaCachePropertyUsesSingleCatalogInvalidation() throws Except Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); } + @Test + public void testCatalogWeightPropertyRebuildsAllCacheBudgets() throws Exception { + TestablePluginCatalog catalog = new TestablePluginCatalog( + mockConnector("old", new ConcurrentLinkedQueue<>())); + + catalog.notifyPropertiesUpdated(Collections.singletonMap( + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB")); + + Mockito.verify(cacheMgr).removeCatalog(1L); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); + } + + @Test + public void testPluginEntryWeightPropertyReliesOnConnectorRebuild() throws Exception { + TestablePluginCatalog catalog = new TestablePluginCatalog( + mockConnector("old", new ConcurrentLinkedQueue<>())); + + catalog.notifyPropertiesUpdated(Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "1KB")); + + Mockito.verify(cacheMgr).isEngineRegistered("iceberg"); + Mockito.verify(cacheMgr, Mockito.never()).removeCatalogByEngine(Mockito.anyLong(), Mockito.anyString()); + Mockito.verify(cacheMgr).invalidateCatalog(1L); + Mockito.verify(cacheMgr, Mockito.never()).removeCatalog(Mockito.anyLong()); + } + + @Test + public void testCoreEntryWeightPropertyRebuildsRegisteredEngineCache() throws Exception { + Mockito.when(cacheMgr.isEngineRegistered("default")).thenReturn(true); + TestablePluginCatalog catalog = new TestablePluginCatalog( + mockConnector("old", new ConcurrentLinkedQueue<>())); + + catalog.notifyPropertiesUpdated(Collections.singletonMap( + "meta.cache.default.schema.max-weight", "1KB")); + + Mockito.verify(cacheMgr, Mockito.times(1)).removeCatalogByEngine(1L, "default"); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); + Mockito.verify(cacheMgr, Mockito.never()).removeCatalog(Mockito.anyLong()); + } + /** * Verify that notifyPropertiesUpdated() closes the old connector via * resetToUninitialized → onClose, and that lazy re-initialization diff --git a/regression-test/suites/external_table_p0/test_catalog_ddl.groovy b/regression-test/suites/external_table_p0/test_catalog_ddl.groovy index 29480259ed1ced..687b8fc1473d3e 100644 --- a/regression-test/suites/external_table_p0/test_catalog_ddl.groovy +++ b/regression-test/suites/external_table_p0/test_catalog_ddl.groovy @@ -46,4 +46,106 @@ suite("test_catalog_ddl", "p0,external") { assertTrue(result[0][1].contains("COMMENT \"alter_comment\"")) sql """drop catalog ${catalog1}""" + + String weightedCatalog = "test_ddl_weighted_meta_cache" + sql """drop catalog if exists ${weightedCatalog}""" + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.max-weight" = "128MB", + "meta.cache.hive.file.max-weight" = "64MB" + ) + """ + result = sql """show create catalog ${weightedCatalog}""" + assertEquals(result.size(), 1) + assertTrue(result[0][1].contains("\"meta.cache.max-weight\" = \"128MB\"")) + assertTrue(result[0][1].contains("\"meta.cache.hive.file.max-weight\" = \"64MB\"")) + + sql """ + alter catalog ${weightedCatalog} set properties( + "meta.cache.max-weight" = "96MB", + "meta.cache.hive.file.max-weight" = "48MB" + ) + """ + result = sql """show create catalog ${weightedCatalog}""" + assertTrue(result[0][1].contains("\"meta.cache.max-weight\" = \"96MB\"")) + assertTrue(result[0][1].contains("\"meta.cache.hive.file.max-weight\" = \"48MB\"")) + + test { + sql """ + alter catalog ${weightedCatalog} set properties( + "meta.cache.default.schema.max-weight" = "invalid" + ) + """ + exception "Invalid cache weight for 'meta.cache.default.schema.max-weight': invalid" + } + sql """drop catalog ${weightedCatalog}""" + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.max-weight" = "64MB", + "meta.cache.hive.file.max-weight" = "128MB" + ) + """ + exception "meta.cache.hive.file.max-weight can not exceed meta.cache.max-weight" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.max-weight" = "10%" + ) + """ + exception "Invalid cache weight for 'meta.cache.max-weight': 10%" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "es", + "hosts" = "http://10.10.10.10:8888", + "meta.cache.max-weight" = "invalid" + ) + """ + exception "Invalid cache weight for 'meta.cache.max-weight': invalid" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.hive.flie.max-weight" = "64MB" + ) + """ + exception "Unknown metadata cache weight property: meta.cache.hive.flie.max-weight" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.iceberg.manifest.max-weight" = "invalid" + ) + """ + exception "Invalid cache weight for 'meta.cache.iceberg.manifest.max-weight': invalid" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "es", + "hosts" = "http://10.10.10.10:8888", + "meta.cache.default.table.max-weight" = "64MB" + ) + """ + exception "Unknown metadata cache weight property: meta.cache.default.table.max-weight" + } }