diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index e33590fc69e857..7dde0c92e0d8bb 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -53,8 +53,14 @@ std::vector SchemaCatalogMetaCacheStatsScanner::_s_tb {"LAST_LOAD_SUCCESS_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_LOAD_FAILURE_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_ERROR", TYPE_STRING, sizeof(StringRef), true}, + {"MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"WEIGHT_REJECT_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"LAST_WEIGHT_REJECT_REASON", TYPE_STRING, sizeof(StringRef), true}, }; +static constexpr size_t kLegacyMetaCacheStatsColumnCount = 24; + SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS) {} @@ -67,9 +73,12 @@ Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { +Status SchemaCatalogMetaCacheStatsScanner::_fetch_from_fe(size_t column_count, + TFetchSchemaTableDataResult* result, + bool* fe_rejected) { + *fe_rejected = false; TSchemaTableRequestParams schema_table_request_params; - for (int i = 0; i < _s_tbls_columns.size(); i++) { + for (size_t i = 0; i < column_count; i++) { schema_table_request_params.__isset.columns_name = true; schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); } @@ -79,20 +88,36 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); request.__set_schema_table_params(schema_table_request_params); - TFetchSchemaTableDataResult result; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( _fe_addr.hostname, _fe_addr.port, - [&request, &result](FrontendServiceConnection& client) { - client->fetchSchemaTableData(result, request); + [&request, result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(*result, request); }, _rpc_timeout)); - Status status(Status::create(result.status)); + Status fe_status = Status::create(result->status); + *fe_rejected = !fe_status.ok(); + return fe_status; +} + +Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { + TFetchSchemaTableDataResult result; + bool fe_rejected = false; + Status status = _fetch_from_fe(_s_tbls_columns.size(), &result, &fe_rejected); if (!status.ok()) { - LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname - << ") failed, errmsg=" << status; - return status; + if (!fe_rejected) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << status; + return status; + } + Status first_status = status; + result = TFetchSchemaTableDataResult(); + status = _fetch_from_fe(kLegacyMetaCacheStatsColumnCount, &result, &fe_rejected); + if (!status.ok()) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << first_status; + return first_status; + } } std::vector result_data = result.data_batch; @@ -106,19 +131,28 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { _block->reserve(_block_rows_limit); - if (result_data.size() > 0) { - auto col_size = result_data[0].column_value.size(); - if (col_size != _s_tbls_columns.size()) { + size_t col_size = _s_tbls_columns.size(); + if (!result_data.empty()) { + col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size() && col_size != kLegacyMetaCacheStatsColumnCount) { return Status::InternalError( "catalog meta cache stats schema is not match for FE and BE"); } } + int available_columns = static_cast(col_size); + int total_columns = static_cast(_s_tbls_columns.size()); for (int i = 0; i < result_data.size(); i++) { TRow row = result_data[i]; - for (int j = 0; j < _s_tbls_columns.size(); j++) { - RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), - _s_tbls_columns[j].type)); + for (int j = 0; j < total_columns; j++) { + if (j < available_columns) { + RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), + _s_tbls_columns[j].type)); + } else { + auto column_guard = _block->mutate_column_scoped(j); + column_guard.mutable_column()->insert_default(); + column_guard.restore(); + } } } return Status::OK(); diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h index 836500fd97de85..006e1459edfa91 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h @@ -25,6 +25,7 @@ namespace doris { class RuntimeState; class Block; +class TFetchSchemaTableDataResult; class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { ENABLE_FACTORY_CREATOR(SchemaCatalogMetaCacheStatsScanner); @@ -40,6 +41,8 @@ class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { private: Status _get_meta_cache_from_fe(); + Status _fetch_from_fe(size_t column_count, TFetchSchemaTableDataResult* result, + bool* fe_rejected); TNetworkAddress _fe_addr; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 54a16d46685cae..82ab488829e1af 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2204,6 +2204,11 @@ public class Config extends ConfigBase { @ConfField(description = "The auto-refresh interval of the external meta cache.") public static long external_cache_refresh_time_minutes = 10; // 10 mins + @ConfField(mutable = false, masterOnly = false, + description = "FE-wide maximum weight for managed external metadata caches. Supports byte units " + + "or a percentage of the JVM max heap; 0 disables the global quota.") + public static String external_meta_cache_max_weight = "0"; + // Enable manual miss load for external meta cache to avoid blocking replayer on slow loaders. @ConfField(mutable = true, masterOnly = false, description = "Whether external meta cache uses manual miss load instead of Caffeine sync load.") diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java index 01ee88a1fc905b..50ca5f914c1f64 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheSpec.java @@ -17,10 +17,21 @@ package org.apache.doris.connector.cache; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Common cache specification for external metadata caches. @@ -37,29 +48,44 @@ *
    *
  • enable=false disables cache
    • *
  • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
    • - *
  • capacity=0 disables cache; capacity is count-based
    • + *
  • capacity=0 disables cache; a positive capacity remains the count safety limit
    • + *
  • max-weight is an optional estimated retained-byte admission limit
    • *
*/ public final class CacheSpec { + private static final Logger LOG = LogManager.getLogger(CacheSpec.class); public static final long CACHE_NO_TTL = -1L; public static final long CACHE_TTL_DISABLE_CACHE = 0L; private static final String META_CACHE_PREFIX = "meta.cache."; private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; + private static final Pattern DATA_VOLUME_PATTERN = Pattern.compile( + "^([0-9]+)\\s*(B|KB|MB|GB|TB|PB)?$", Pattern.CASE_INSENSITIVE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + if (maxWeight <= 0L) { + throw new IllegalArgumentException("maxWeight must be positive: " + maxWeight); + } + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); } /** @@ -98,7 +124,8 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getWeightProperty(properties, propertySpec.getMaxWeightKey()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** @@ -116,6 +143,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT) .build(); } @@ -168,10 +196,115 @@ public static void checkLongProperty(String value, long minValue, String key) { } } + /** Strict CREATE/ALTER-time validation for the catalog weight property. */ + public static OptionalLong checkCatalogWeightProperty(Map properties) { + if (properties == null) { + return OptionalLong.empty(); + } + String catalogValue = properties.get(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + if (catalogValue == null) { + return OptionalLong.empty(); + } + long parsed = parseWeight(catalogValue, + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0L) { + throw new IllegalArgumentException( + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } + + /** Strict CREATE/ALTER-time validation for catalog and known engine entry weight properties. */ + public static void checkWeightProperties( + Map properties, String engine, String... knownEntries) { + OptionalLong catalogMax = checkCatalogWeightProperty(properties); + if (properties == null) { + return; + } + Set entries = new HashSet<>(); + Collections.addAll(entries, knownEntries); + String prefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (!key.startsWith(prefix) || !key.endsWith(KEY_MAX_WEIGHT)) { + continue; + } + String entry = key.substring(prefix.length(), key.length() - KEY_MAX_WEIGHT.length()); + if (!entries.contains(entry)) { + throw new IllegalArgumentException("Unknown metadata cache weight property: " + key); + } + long parsed = parseWeight(property.getValue(), key, false, 0L); + if (parsed <= 0L) { + throw new IllegalArgumentException(key + " must be positive"); + } + if (catalogMax.isPresent() && parsed > catalogMax.getAsLong()) { + throw new IllegalArgumentException(key + " can not exceed " + + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + } + } + } + public static boolean isCacheEnabled(boolean enable, long ttlSecond, long capacity) { return enable && ttlSecond != 0 && capacity != 0; } + /** Parse bytes with an optional binary unit, or a heap percentage when explicitly allowed. */ + public static long parseWeight(String value, String key, boolean allowPercent, long maxHeapBytes) { + String normalized = Objects.requireNonNull(value, "value").trim(); + if (normalized.isEmpty()) { + throw invalidWeight(key, value); + } + if (normalized.endsWith("%")) { + if (!allowPercent || maxHeapBytes <= 0L) { + throw invalidWeight(key, value); + } + try { + BigDecimal percentage = new BigDecimal( + normalized.substring(0, normalized.length() - 1).trim()); + if (percentage.signum() < 0 || percentage.compareTo(BigDecimal.valueOf(100L)) > 0) { + throw invalidWeight(key, value); + } + return checkedLong(BigDecimal.valueOf(maxHeapBytes) + .multiply(percentage) + .divide(BigDecimal.valueOf(100L)) + .toBigInteger(), key, value); + } catch (NumberFormatException e) { + throw invalidWeight(key, value); + } + } + Matcher matcher = DATA_VOLUME_PATTERN.matcher(normalized); + if (!matcher.matches()) { + throw invalidWeight(key, value); + } + BigInteger amount = new BigInteger(matcher.group(1)); + String rawUnit = matcher.group(2); + String unit = rawUnit == null ? "B" : rawUnit.toUpperCase(Locale.ROOT); + int power; + switch (unit) { + case "B": + power = 0; + break; + case "KB": + power = 1; + break; + case "MB": + power = 2; + break; + case "GB": + power = 3; + break; + case "TB": + power = 4; + break; + case "PB": + power = 5; + break; + default: + throw invalidWeight(key, value); + } + return checkedLong(amount.multiply(BigInteger.valueOf(1024L).pow(power)), key, value); + } + /** * Build standard external meta cache key prefix for one engine. * Example: {@code meta.cache.iceberg.} @@ -229,6 +362,37 @@ private static long getLongProperty(Map properties, String key, } } + private static OptionalLong getWeightProperty(Map properties, String key) { + if (key == null) { + return OptionalLong.empty(); + } + String value = properties.get(key); + if (value == null) { + return OptionalLong.empty(); + } + try { + long parsed = parseWeight(value, key, false, 0L); + if (parsed <= 0L) { + throw invalidWeight(key, value); + } + return OptionalLong.of(parsed); + } catch (IllegalArgumentException e) { + LOG.warn("Ignoring invalid persisted metadata cache weight property {}={}", key, value); + return OptionalLong.empty(); + } + } + + private static long checkedLong(BigInteger value, String key, String rawValue) { + if (value.signum() < 0 || value.compareTo(LONG_MAX) > 0) { + throw invalidWeight(key, rawValue); + } + return value.longValue(); + } + + private static IllegalArgumentException invalidWeight(String key, String value) { + return new IllegalArgumentException("Invalid cache weight for '" + key + "': " + value); + } + public boolean isEnable() { return enable; } @@ -241,6 +405,19 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return isCacheEnabled(enable, ttlSecond, capacity) + && (!maxWeight.isPresent() || maxWeight.getAsLong() != 0L); + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -248,15 +425,17 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, String maxWeightKey) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; } public String getEnableKey() { @@ -283,6 +462,10 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -290,6 +473,7 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -309,6 +493,11 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key) { + this.maxWeightKey = Objects.requireNonNull(key, "key"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -316,7 +505,8 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey); } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java index 199f516582ebf5..f3f76361cbbf97 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -20,7 +20,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -33,15 +35,48 @@ */ public final class CatalogMetaCache implements AutoCloseable { private final ScopedMetaCacheRegistry registry; + private final MetaCacheBudgetManager budgetManager; + private final long catalogId; + private final String engine; + private final OptionalLong catalogMaxWeight; + private final boolean managed; private final Set names = ConcurrentHashMap.newKeySet(); + private final Map> entries = new ConcurrentHashMap<>(); private final AtomicBoolean closed = new AtomicBoolean(false); public CatalogMetaCache() { - this(new ScopedMetaCacheRegistry()); + this(new ScopedMetaCacheRegistry(), new MetaCacheBudgetManager(OptionalLong.empty()), + 0L, "standalone", OptionalLong.empty(), false); } CatalogMetaCache(ScopedMetaCacheRegistry registry) { + this(registry, new MetaCacheBudgetManager(OptionalLong.empty()), + 0L, "standalone", OptionalLong.empty(), false); + } + + public CatalogMetaCache(MetaCacheBudgetManager budgetManager, long catalogId, + String engine, Map catalogProperties) { + this(new ScopedMetaCacheRegistry(), budgetManager, catalogId, engine, + budgetManager.parseCatalogMaxWeight(catalogProperties), false); + } + + public static CatalogMetaCache managed(long catalogId, String engine, + Map catalogProperties) { + MetaCacheBudgetManager manager = MetaCacheGovernance.budgetManager(); + CatalogMetaCache cache = new CatalogMetaCache(new ScopedMetaCacheRegistry(), manager, + catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true); + MetaCacheGovernance.register(cache); + return cache; + } + + CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, + long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed) { this.registry = Objects.requireNonNull(registry, "registry can not be null"); + this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); + this.catalogId = catalogId; + this.engine = Objects.requireNonNull(engine, "engine can not be null"); + this.catalogMaxWeight = Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight can not be null"); + this.managed = managed; } public MetaCache create(MetaCacheDefinition definition) { @@ -51,14 +86,39 @@ public MetaCache create(MetaCacheDefinition definition) { if (!names.add(nonNullDefinition.name())) { throw new IllegalArgumentException("Duplicate meta cache name: " + nonNullDefinition.name()); } + MetaCacheBudgetManager.EntryBudget entryBudget = null; try { - return new MetaCache<>(nonNullDefinition, + boolean weightLimited = budgetManager.hasLimit( + catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); + if (weightLimited && nonNullDefinition.sizeEstimator() == null) { + throw new IllegalArgumentException("Weighted metadata cache requires a size estimator: " + + nonNullDefinition.name()); + } + if (weightLimited) { + entryBudget = budgetManager.createEntryBudget(catalogId, engine, + nonNullDefinition.name(), nonNullDefinition.budgetGroup(), catalogMaxWeight, + nonNullDefinition.cacheSpec().getMaxWeight()); + } + MetaCache created = new MetaCache<>(nonNullDefinition, registry.createCacheWithMetaRemovalListener(nonNullDefinition.name(), nonNullDefinition.cacheSpec(), nonNullDefinition.removalListener(), nonNullDefinition.discardListener(), - nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor())); + nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor(), + entryBudget == null ? null : nonNullDefinition.sizeEstimator(), entryBudget)); + entries.put(nonNullDefinition.name(), created); + return created; } catch (RuntimeException | Error throwable) { + if (entryBudget != null) { + entryBudget.close(); + } names.remove(nonNullDefinition.name()); + if (managed) { + try { + close(); + } catch (RuntimeException | Error closeFailure) { + throwable.addSuppressed(closeFailure); + } + } throw throwable; } } @@ -100,11 +160,38 @@ public ScopedMetaCacheRegistry.ScopeMetrics metrics() { return registry.metrics(); } + public Map> entries() { + return java.util.Collections.unmodifiableMap(entries); + } + + public long catalogId() { + return catalogId; + } + + public String engine() { + return engine; + } + + public OptionalLong catalogMaxWeight() { + return catalogMaxWeight; + } + + public boolean hasEnclosingWeightLimit() { + return budgetManager.hasLimit(catalogMaxWeight, OptionalLong.empty()); + } + @Override public void close() { if (closed.compareAndSet(false, true)) { - registry.close(); - names.clear(); + try { + registry.close(); + } finally { + names.clear(); + entries.clear(); + if (managed) { + MetaCacheGovernance.unregister(this); + } + } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java index da47360f862225..adaa6a1a711f83 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ConnectorMetadataCache.java @@ -73,15 +73,25 @@ public ConnectorMetadataCache( public ConnectorMetadataCache(CatalogMetaCache owner, String cacheName, String engine, String entryName, Map props, Function scopeResolver) { + this(owner, cacheName, engine, entryName, props, scopeResolver, MetaCacheSizeEstimators.reflective()); + } + + public ConnectorMetadataCache(CatalogMetaCache owner, String cacheName, String engine, String entryName, + Map props, Function scopeResolver, + MetaCacheSizeEstimator sizeEstimator) { this.owner = Objects.requireNonNull(owner, "owner can not be null"); Objects.requireNonNull(engine, "engine can not be null"); Objects.requireNonNull(entryName, "entryName can not be null"); Map properties = props == null ? Collections.emptyMap() : props; CacheSpec spec = CacheSpec.fromProperties(properties, engine, entryName, CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_CAPACITY)); - this.entry = owner.create(MetaCacheDefinition + MetaCacheDefinition.Builder builder = MetaCacheDefinition .builder(cacheName, spec, scopeResolver) - .build()); + .budgetGroup(entryName); + if (sizeEstimator != null) { + builder.sizeEstimator(sizeEstimator); + } + this.entry = owner.create(builder.build()); } /** Caching is on only when the resolved {@link CacheSpec} is effectively enabled (see {@link #entry}'s spec). */ @@ -89,6 +99,10 @@ public boolean isEnabled() { return entry.isEnabled(); } + public boolean isWeightBounded() { + return entry.isWeightBounded(); + } + /** * Returns the cached value for {@code key} if present, else runs {@code loader}, caches and returns it. * Disabled cache -> {@code loader} runs on every call. The loader runs OUTSIDE Caffeine's compute lock diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java new file mode 100644 index 00000000000000..6dd064a563b390 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/JvmSizeUtils.java @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import com.sun.management.HotSpotDiagnosticMXBean; + +import java.lang.management.ManagementFactory; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.List; +import java.util.Objects; +import java.util.function.ToLongFunction; + +/** + * Low-cost JVM heap layout formulas for type-specific connector cache estimators. + * + *

This class reflects class declarations once to calculate shallow sizes. It never reads fields from a runtime + * object and never walks an object graph. + */ +public final class JvmSizeUtils { + private static final VmLayout VM_LAYOUT = VmLayout.detect(); + private static final boolean COMPACT_STRINGS = vmBoolean("CompactStrings", true); + + private static final ClassValue SHALLOW_SIZES = new ClassValue<>() { + @Override + protected Long computeValue(Class type) { + if (type.isArray()) { + throw new IllegalArgumentException("Array size depends on its length: " + type); + } + long size = VM_LAYOUT.objectHeaderBytes(); + for (Class current = type; current != null; current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers())) { + size = saturatedAdd(size, fieldSize(field.getType())); + } + } + } + return align(size); + } + }; + private static final long STRING_SHALLOW_BYTES = instanceSize(String.class); + private static final long ARRAY_LIST_SHALLOW_BYTES = instanceSize(java.util.ArrayList.class); + + private JvmSizeUtils() { + } + + public static long instanceSize(Class type) { + return SHALLOW_SIZES.get(type); + } + + public static long objectArraySize(int length) { + return arraySize(length, VM_LAYOUT.referenceBytes()); + } + + public static long byteArraySize(int length) { + return arraySize(length, Byte.BYTES); + } + + public static long intArraySize(int length) { + return arraySize(length, Integer.BYTES); + } + + public static long longArraySize(int length) { + return arraySize(length, Long.BYTES); + } + + /** Estimate an array whose component is a primitive type. */ + public static long primitiveArraySize(Class componentType, int length) { + if (!componentType.isPrimitive() || componentType == void.class) { + throw new IllegalArgumentException("Not an array component primitive: " + componentType); + } + return arraySize(length, Math.toIntExact(fieldSize(componentType))); + } + + public static long arrayListSize(int backingArrayCapacity) { + return saturatedAdd(ARRAY_LIST_SHALLOW_BYTES, objectArraySize(backingArrayCapacity)); + } + + /** Estimates list element payload from evenly spaced samples, including both ends of the list. */ + public static long sampledListPayload( + List values, int sampleCount, ToLongFunction estimator) { + Objects.requireNonNull(values, "values"); + Objects.requireNonNull(estimator, "estimator"); + if (sampleCount <= 0) { + throw new IllegalArgumentException("sampleCount must be positive: " + sampleCount); + } + int size = values.size(); + if (size == 0) { + return 0L; + } + int samples = Math.min(size, sampleCount); + long sampledBytes = 0L; + for (int sample = 0; sample < samples; sample++) { + int index = samples == 1 + ? 0 : (int) ((long) sample * (size - 1) / (samples - 1)); + sampledBytes = saturatedAdd(sampledBytes, estimator.applyAsLong(values.get(index))); + } + if (samples == size) { + return sampledBytes; + } + double scaled = (double) sampledBytes * size / samples; + return scaled >= Long.MAX_VALUE ? Long.MAX_VALUE : (long) scaled; + } + + /** Estimate the heap retained by a Java 17 String and its compact-string byte array. */ + public static long stringSize(String value) { + if (value == null) { + return 0L; + } + int bytesPerCharacter = COMPACT_STRINGS && isLatin1(value) ? Byte.BYTES : Character.BYTES; + long valueBytes = saturatedMultiply(value.length(), bytesPerCharacter); + int arrayLength = valueBytes >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) valueBytes; + return saturatedAdd(STRING_SHALLOW_BYTES, byteArraySize(arrayLength)); + } + + public static long saturatedAdd(long left, long right) { + if (right > 0L && left > Long.MAX_VALUE - right) { + return Long.MAX_VALUE; + } + return left + right; + } + + public static long saturatedMultiply(long left, long right) { + if (left == 0L || right == 0L) { + return 0L; + } + if (left > Long.MAX_VALUE / right) { + return Long.MAX_VALUE; + } + return left * right; + } + + private static long arraySize(int length, int elementBytes) { + long elements = saturatedMultiply(length, elementBytes); + return align(saturatedAdd(VM_LAYOUT.arrayHeaderBytes(), elements)); + } + + private static long fieldSize(Class type) { + if (!type.isPrimitive()) { + return VM_LAYOUT.referenceBytes(); + } + if (type == long.class || type == double.class) { + return Long.BYTES; + } + if (type == int.class || type == float.class) { + return Integer.BYTES; + } + if (type == short.class || type == char.class) { + return Short.BYTES; + } + return Byte.BYTES; + } + + private static boolean isLatin1(String value) { + for (int i = 0; i < value.length(); i++) { + if (value.charAt(i) > 0xff) { + return false; + } + } + return true; + } + + private static long align(long value) { + long remainder = value % VM_LAYOUT.objectAlignmentBytes(); + return remainder == 0L + ? value + : saturatedAdd(value, VM_LAYOUT.objectAlignmentBytes() - remainder); + } + + private static boolean vmBoolean(String option, boolean defaultValue) { + try { + HotSpotDiagnosticMXBean diagnostic = hotSpotDiagnostic(); + return diagnostic == null + ? defaultValue + : Boolean.parseBoolean(diagnostic.getVMOption(option).getValue()); + } catch (RuntimeException | LinkageError ignored) { + return defaultValue; + } + } + + private static int vmInt(String option, int defaultValue) { + try { + HotSpotDiagnosticMXBean diagnostic = hotSpotDiagnostic(); + return diagnostic == null + ? defaultValue + : Integer.parseInt(diagnostic.getVMOption(option).getValue()); + } catch (RuntimeException | LinkageError ignored) { + return defaultValue; + } + } + + private static HotSpotDiagnosticMXBean hotSpotDiagnostic() { + return ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class); + } + + private record VmLayout( + int referenceBytes, + int objectHeaderBytes, + int arrayHeaderBytes, + int objectAlignmentBytes) { + + private static VmLayout detect() { + int referenceBytes = vmBoolean("UseCompressedOops", true) ? Integer.BYTES : Long.BYTES; + int classPointerBytes = vmBoolean("UseCompressedClassPointers", true) + ? Integer.BYTES : Long.BYTES; + int alignment = vmInt("ObjectAlignmentInBytes", Long.BYTES); + int objectHeaderBytes = Long.BYTES + classPointerBytes; + int arrayHeaderBytes = Math.toIntExact( + alignWithoutLayout(objectHeaderBytes + Integer.BYTES, alignment)); + return new VmLayout(referenceBytes, objectHeaderBytes, arrayHeaderBytes, alignment); + } + + private static long alignWithoutLayout(long value, int alignment) { + long remainder = value % alignment; + return remainder == 0L ? value : value + alignment - remainder; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java index 3b57fef2364dc0..d55b93d4bcdea4 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java @@ -109,6 +109,11 @@ public boolean isEnabled() { return metrics().isEffectiveEnabled(); } + /** Whether this entry currently participates in byte-based memory governance. */ + public boolean isWeightBounded() { + return metrics().isWeightBounded(); + } + public long size() { return metrics().getPhysicalEntryCount(); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheBudgetManager.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheBudgetManager.java new file mode 100644 index 00000000000000..40b22ec76c7e88 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheBudgetManager.java @@ -0,0 +1,605 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongUnaryOperator; + +/** + * Process-wide admission accounting shared by FE and connector metadata caches. + * Estimation happens before this class is entered; its lock covers only arithmetic and small maps. + */ +public final class MetaCacheBudgetManager { + private static final Logger LOG = LogManager.getLogger(MetaCacheBudgetManager.class); + private static final ExecutorService PEER_RECLAIM_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-peer-reclaim"); + thread.setDaemon(true); + return thread; + }); + + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; + + private final Object lock = new Object(); + private final OptionalLong globalMaxWeight; + private final Map catalogBuckets = new HashMap<>(); + private final Map entryGroupBuckets = new HashMap<>(); + private final Map entryBudgets = new HashMap<>(); + private final AtomicLong nextEntryBudgetId = new AtomicLong(); + private long globalUsedWeight; + + public MetaCacheBudgetManager(OptionalLong globalMaxWeight) { + this.globalMaxWeight = Objects.requireNonNull(globalMaxWeight, "globalMaxWeight"); + if (globalMaxWeight.isPresent() && globalMaxWeight.getAsLong() <= 0L) { + throw new IllegalArgumentException("global max weight must be positive when enabled"); + } + } + + public OptionalLong parseCatalogMaxWeight(Map catalogProperties) { + String configured = catalogProperties == null ? null + : catalogProperties.get(CATALOG_MAX_WEIGHT_PROPERTY); + if (configured == null) { + return OptionalLong.empty(); + } + try { + long parsed = CacheSpec.parseWeight(configured, CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0L) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } catch (IllegalArgumentException e) { + LOG.warn("Ignoring invalid persisted metadata cache property {}={}", + CATALOG_MAX_WEIGHT_PROPERTY, configured); + return OptionalLong.empty(); + } + } + + public boolean hasLimit(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + return globalMaxWeight.isPresent() || catalogMaxWeight.isPresent() || entryMaxWeight.isPresent(); + } + + public EntryBudget createEntryBudget(long catalogId, String engine, String entryName, String budgetGroup, + OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + Objects.requireNonNull(engine, "engine"); + Objects.requireNonNull(entryName, "entryName"); + Objects.requireNonNull(budgetGroup, "budgetGroup"); + OptionalLong effectiveMax = minimumPresent(globalMaxWeight, catalogMaxWeight, entryMaxWeight); + if (!effectiveMax.isPresent()) { + throw new IllegalArgumentException("entry budget requires at least one configured weight bound"); + } + + EntryGroupScope groupScope = new EntryGroupScope(catalogId, engine, budgetGroup); + EntryScope scope = new EntryScope(nextEntryBudgetId.incrementAndGet(), groupScope, entryName); + synchronized (lock) { + Bucket catalogBucket = catalogBuckets.get(catalogId); + long catalogLimit = minimumLimit(globalMaxWeight, catalogMaxWeight); + if (catalogBucket == null) { + catalogBucket = new Bucket(catalogLimit); + catalogBuckets.put(catalogId, catalogBucket); + } else if (catalogBucket.maxWeight != catalogLimit) { + throw new IllegalStateException("Conflicting catalog cache max weight for catalog " + catalogId); + } + Bucket groupBucket = entryGroupBuckets.get(groupScope); + if (groupBucket == null) { + groupBucket = new Bucket(effectiveMax.getAsLong()); + entryGroupBuckets.put(groupScope, groupBucket); + } else if (groupBucket.maxWeight != effectiveMax.getAsLong()) { + throw new IllegalStateException("Conflicting metadata cache entry max weight for " + groupScope); + } + Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + EntryBudget budget = new EntryBudget(this, scope, catalogBucket, groupBucket, entryBucket, + effectiveMax.getAsLong()); + entryBudgets.put(scope, budget); + catalogBucket.liveEntries++; + groupBucket.liveEntries++; + return budget; + } + } + + public long getGlobalUsedWeight() { + synchronized (lock) { + return globalUsedWeight; + } + } + + private Optional tryReserve(EntryBudget budget, long bytes) { + checkWeight(bytes); + synchronized (lock) { + if (budget.closed) { + return Optional.empty(); + } + if (!fits(limitOf(globalMaxWeight), globalUsedWeight, bytes) + || !fits(budget.catalogBucket.maxWeight, budget.catalogBucket.usedWeight, bytes) + || !fits(budget.groupBucket.maxWeight, budget.groupBucket.usedWeight, bytes)) { + return Optional.empty(); + } + addUsed(budget, bytes); + return Optional.of(new AdmissionReservation(this, budget, bytes)); + } + } + + private Optional tryReplace( + AdmissionReservation previous, long newBytes) { + checkWeight(newBytes); + synchronized (lock) { + if (!previous.active || previous.entryBudget.closed) { + return Optional.empty(); + } + EntryBudget budget = previous.entryBudget; + long heldBytes = Math.max(previous.accountedBytes, newBytes); + long delta = heldBytes - previous.accountedBytes; + if (delta > 0L && (!fits(limitOf(globalMaxWeight), globalUsedWeight, delta) + || !fits(budget.catalogBucket.maxWeight, budget.catalogBucket.usedWeight, delta) + || !fits(budget.groupBucket.maxWeight, budget.groupBucket.usedWeight, delta))) { + return Optional.empty(); + } + addUsed(budget, delta); + previous.active = false; + AdmissionReservation replacement = new AdmissionReservation( + this, budget, newBytes, heldBytes); + return Optional.of(new ReservationReplacement(this, previous, replacement)); + } + } + + private void commitReplacement(ReservationReplacement replacement) { + synchronized (lock) { + replacement.checkPending(); + AdmissionReservation current = replacement.current; + subtractUsed(current.entryBudget, current.accountedBytes - current.bytes); + current.accountedBytes = current.bytes; + replacement.finished = true; + } + } + + private void rollbackReplacement(ReservationReplacement replacement) { + synchronized (lock) { + replacement.checkPending(); + AdmissionReservation previous = replacement.previous; + AdmissionReservation current = replacement.current; + subtractUsed(current.entryBudget, current.accountedBytes - previous.accountedBytes); + current.bytes = 0L; + current.accountedBytes = 0L; + current.active = false; + previous.active = true; + replacement.finished = true; + } + } + + private void release(AdmissionReservation reservation) { + synchronized (lock) { + if (!reservation.active) { + return; + } + if (!reservation.entryBudget.closed) { + subtractUsed(reservation.entryBudget, reservation.accountedBytes); + } + reservation.bytes = 0L; + reservation.accountedBytes = 0L; + reservation.active = false; + } + } + + private void close(EntryBudget budget) { + synchronized (lock) { + if (budget.closed) { + return; + } + long leaked = budget.entryBucket.usedWeight; + if (leaked != 0L) { + LOG.error("Force-closing metadata cache budget {} with {} bytes still reserved", + budget.scope, leaked); + globalUsedWeight = Math.max(0L, globalUsedWeight - leaked); + budget.catalogBucket.usedWeight = Math.max(0L, budget.catalogBucket.usedWeight - leaked); + budget.groupBucket.usedWeight = Math.max(0L, budget.groupBucket.usedWeight - leaked); + budget.entryBucket.usedWeight = 0L; + } + budget.closed = true; + budget.reclaimer = null; + entryBudgets.remove(budget.scope, budget); + budget.catalogBucket.liveEntries--; + if (budget.catalogBucket.liveEntries == 0) { + catalogBuckets.remove(budget.scope.groupScope.catalogId, budget.catalogBucket); + } + budget.groupBucket.liveEntries--; + if (budget.groupBucket.liveEntries == 0) { + entryGroupBuckets.remove(budget.scope.groupScope, budget.groupBucket); + } + } + } + + private void addUsed(EntryBudget budget, long bytes) { + globalUsedWeight += bytes; + budget.catalogBucket.usedWeight += bytes; + budget.groupBucket.usedWeight += bytes; + budget.entryBucket.usedWeight += bytes; + } + + private void subtractUsed(EntryBudget budget, long bytes) { + if (bytes > globalUsedWeight || bytes > budget.catalogBucket.usedWeight + || bytes > budget.groupBucket.usedWeight + || bytes > budget.entryBucket.usedWeight) { + throw new IllegalStateException("metadata cache budget accounting underflow"); + } + globalUsedWeight -= bytes; + budget.catalogBucket.usedWeight -= bytes; + budget.groupBucket.usedWeight -= bytes; + budget.entryBucket.usedWeight -= bytes; + } + + private void requestPeerReclaim(EntryBudget requester, long additionalBytes) { + if (additionalBytes <= 0L || requester.closed) { + return; + } + synchronized (lock) { + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + long groupDeficit = deficit( + requester.groupBucket.maxWeight, requester.groupBucket.usedWeight, additionalBytes); + if (Math.max(groupDeficit, Math.max(globalDeficit, catalogDeficit)) == 0L) { + return; + } + } + requester.requestedAdmissionBytes.accumulateAndGet(additionalBytes, Math::max); + schedulePeerReclaim(requester); + } + + private void schedulePeerReclaim(EntryBudget requester) { + if (!requester.reclaimScheduled.compareAndSet(false, true)) { + return; + } + try { + PEER_RECLAIM_EXECUTOR.execute(() -> drainPeerReclaim(requester)); + } catch (RejectedExecutionException e) { + requester.reclaimScheduled.set(false); + LOG.warn("Failed to schedule metadata cache peer reclamation for {}", requester.scope, e); + } + } + + private void drainPeerReclaim(EntryBudget requester) { + try { + long requested = requester.requestedAdmissionBytes.getAndSet(0L); + if (requested <= 0L || requester.closed) { + return; + } + List candidates; + synchronized (lock) { + candidates = new ArrayList<>(); + for (EntryBudget candidate : entryBudgets.values()) { + if (!candidate.closed && candidate.reclaimer != null + && candidate.entryBucket.usedWeight > 0L) { + candidates.add(candidate); + } + } + candidates.sort((left, right) -> { + boolean leftSibling = left.scope.groupScope.catalogId + == requester.scope.groupScope.catalogId; + boolean rightSibling = right.scope.groupScope.catalogId + == requester.scope.groupScope.catalogId; + if (leftSibling != rightSibling) { + return leftSibling ? -1 : 1; + } + return Long.compare(right.entryBucket.usedWeight, left.entryBucket.usedWeight); + }); + } + long remaining = currentDeficit(requester, requested); + for (EntryBudget candidate : candidates) { + if (currentGroupDeficit(requester, requested) > 0L + && candidate.groupBucket != requester.groupBucket) { + continue; + } + boolean sibling = candidate.scope.groupScope.catalogId + == requester.scope.groupScope.catalogId; + if (!sibling && currentCatalogDeficit(requester, requested) > 0L) { + continue; + } + try { + candidate.reclaimer.applyAsLong(remaining); + } catch (RuntimeException e) { + LOG.warn("Failed to reclaim metadata cache budget from peer {}", candidate.scope, e); + } + remaining = currentDeficit(requester, requested); + if (remaining == 0L) { + break; + } + } + } finally { + requester.reclaimScheduled.set(false); + if (!requester.closed && requester.requestedAdmissionBytes.get() > 0L) { + schedulePeerReclaim(requester); + } + } + } + + private long currentDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + if (requester.closed) { + return 0L; + } + return Math.max( + deficit(requester.groupBucket.maxWeight, + requester.groupBucket.usedWeight, additionalBytes), + Math.max(deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes), + deficit(requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes))); + } + } + + private long currentGroupDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit(requester.groupBucket.maxWeight, + requester.groupBucket.usedWeight, additionalBytes); + } + } + + private long currentCatalogDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit(requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes); + } + } + + private static long deficit(long maxWeight, long usedWeight, long additionalBytes) { + if (maxWeight == Long.MAX_VALUE || additionalBytes <= maxWeight - Math.min(usedWeight, maxWeight)) { + return 0L; + } + return JvmSizeUtils.saturatedAdd(usedWeight, additionalBytes) - maxWeight; + } + + private static boolean fits(long maxWeight, long usedWeight, long delta) { + return delta >= 0L && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; + } + + private static long limitOf(OptionalLong configured) { + return configured.isPresent() ? configured.getAsLong() : Long.MAX_VALUE; + } + + private static long minimumLimit(OptionalLong first, OptionalLong second) { + return Math.min(limitOf(first), limitOf(second)); + } + + private static OptionalLong minimumPresent(OptionalLong first, OptionalLong second, OptionalLong third) { + if (!first.isPresent() && !second.isPresent() && !third.isPresent()) { + return OptionalLong.empty(); + } + return OptionalLong.of(Math.min(limitOf(first), Math.min(limitOf(second), limitOf(third)))); + } + + private static void checkWeight(long bytes) { + if (bytes < 0L) { + throw new IllegalArgumentException("cache reservation can not be negative: " + bytes); + } + } + + private static final class Bucket { + private final long maxWeight; + private long usedWeight; + private int liveEntries; + + private Bucket(long maxWeight) { + this.maxWeight = maxWeight; + } + } + + private static final class EntryGroupScope { + private final long catalogId; + private final String engine; + private final String groupName; + + private EntryGroupScope(long catalogId, String engine, String groupName) { + this.catalogId = catalogId; + this.engine = engine; + this.groupName = groupName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryGroupScope)) { + return false; + } + EntryGroupScope that = (EntryGroupScope) other; + return catalogId == that.catalogId && engine.equals(that.engine) && groupName.equals(that.groupName); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, engine, groupName); + } + + @Override + public String toString() { + return catalogId + "/" + engine + "/" + groupName; + } + } + + private static final class EntryScope { + private final long budgetId; + private final EntryGroupScope groupScope; + private final String entryName; + + private EntryScope(long budgetId, EntryGroupScope groupScope, String entryName) { + this.budgetId = budgetId; + this.groupScope = groupScope; + this.entryName = entryName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryScope)) { + return false; + } + EntryScope that = (EntryScope) other; + return budgetId == that.budgetId; + } + + @Override + public int hashCode() { + return Long.hashCode(budgetId); + } + + @Override + public String toString() { + return groupScope + "/" + entryName + "#" + budgetId; + } + } + + public static final class EntryBudget implements AutoCloseable { + private final MetaCacheBudgetManager manager; + private final EntryScope scope; + private final Bucket catalogBucket; + private final Bucket groupBucket; + private final Bucket entryBucket; + private final long effectiveMaxWeight; + private final AtomicLong requestedAdmissionBytes = new AtomicLong(); + private final AtomicBoolean reclaimScheduled = new AtomicBoolean(); + private volatile LongUnaryOperator reclaimer; + private volatile boolean closed; + + private EntryBudget(MetaCacheBudgetManager manager, EntryScope scope, Bucket catalogBucket, + Bucket groupBucket, Bucket entryBucket, long effectiveMaxWeight) { + this.manager = manager; + this.scope = scope; + this.catalogBucket = catalogBucket; + this.groupBucket = groupBucket; + this.entryBucket = entryBucket; + this.effectiveMaxWeight = effectiveMaxWeight; + } + + public Optional tryReserve(long bytes) { + return manager.tryReserve(this, bytes); + } + + public Optional tryReplace( + AdmissionReservation previous, long newBytes) { + Objects.requireNonNull(previous, "previous reservation"); + if (previous.entryBudget != this) { + throw new IllegalArgumentException("replacement reservation belongs to another entry"); + } + return manager.tryReplace(previous, newBytes); + } + + public void setReclaimer(LongUnaryOperator reclaimer) { + this.reclaimer = Objects.requireNonNull(reclaimer, "reclaimer"); + } + + public void requestPeerReclaim(long additionalBytes) { + manager.requestPeerReclaim(this, additionalBytes); + } + + public long getEffectiveMaxWeight() { + return effectiveMaxWeight; + } + + public long getUsedWeight() { + synchronized (manager.lock) { + return entryBucket.usedWeight; + } + } + + @Override + public void close() { + manager.close(this); + } + } + + public static final class AdmissionReservation { + private final MetaCacheBudgetManager manager; + private final EntryBudget entryBudget; + private long bytes; + private long accountedBytes; + private boolean active = true; + + private AdmissionReservation(MetaCacheBudgetManager manager, EntryBudget entryBudget, long bytes) { + this(manager, entryBudget, bytes, bytes); + } + + private AdmissionReservation(MetaCacheBudgetManager manager, EntryBudget entryBudget, + long bytes, long accountedBytes) { + this.manager = manager; + this.entryBudget = entryBudget; + this.bytes = bytes; + this.accountedBytes = accountedBytes; + } + + public void release() { + manager.release(this); + } + + public long getBytes() { + synchronized (manager.lock) { + return bytes; + } + } + } + + /** + * Atomic accounting hand-off between two generations of the same cache key. + * The manager temporarily holds max(old, new), never old + new, and keeps enough + * accounting to restore the old generation until publication commits. + */ + public static final class ReservationReplacement { + private final MetaCacheBudgetManager manager; + private final AdmissionReservation previous; + private final AdmissionReservation current; + private boolean finished; + + private ReservationReplacement(MetaCacheBudgetManager manager, + AdmissionReservation previous, AdmissionReservation current) { + this.manager = manager; + this.previous = previous; + this.current = current; + } + + public AdmissionReservation current() { + return current; + } + + public void commit() { + manager.commitReplacement(this); + } + + public void rollback() { + manager.rollbackReplacement(this); + } + + private void checkPending() { + if (finished) { + throw new IllegalStateException("reservation replacement is already finished"); + } + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java index ab8c0e3615e749..a461b8879e44a6 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java @@ -32,6 +32,7 @@ */ public final class MetaCacheDefinition { private final String name; + private final String budgetGroup; private final CacheSpec cacheSpec; private final Function scopeResolver; private final Function loader; @@ -39,9 +40,11 @@ public final class MetaCacheDefinition { private final BiConsumer discardListener; private final Duration refreshAfterWrite; private final Executor refreshExecutor; + private final MetaCacheSizeEstimator sizeEstimator; private MetaCacheDefinition(Builder builder) { name = requireName(builder.name); + budgetGroup = requireName(builder.budgetGroup == null ? builder.name : builder.budgetGroup); cacheSpec = Objects.requireNonNull(builder.cacheSpec, "cacheSpec can not be null"); scopeResolver = Objects.requireNonNull(builder.scopeResolver, "scopeResolver can not be null"); loader = builder.loader; @@ -49,6 +52,7 @@ private MetaCacheDefinition(Builder builder) { discardListener = builder.discardListener; refreshAfterWrite = builder.refreshAfterWrite; refreshExecutor = builder.refreshExecutor; + sizeEstimator = builder.sizeEstimator; if (refreshAfterWrite != null && loader == null) { throw new IllegalArgumentException("refresh-after-write requires a default loader"); } @@ -63,6 +67,10 @@ public String name() { return name; } + String budgetGroup() { + return budgetGroup; + } + CacheSpec cacheSpec() { return cacheSpec; } @@ -93,6 +101,10 @@ Executor refreshExecutor() { return refreshExecutor; } + MetaCacheSizeEstimator sizeEstimator() { + return sizeEstimator; + } + private static String requireName(String name) { String nonNullName = Objects.requireNonNull(name, "name can not be null"); if (nonNullName.isEmpty()) { @@ -110,6 +122,8 @@ public static final class Builder { private BiConsumer discardListener; private Duration refreshAfterWrite; private Executor refreshExecutor; + private MetaCacheSizeEstimator sizeEstimator; + private String budgetGroup; private Builder(String name, CacheSpec cacheSpec, Function scopeResolver) { this.name = name; @@ -146,6 +160,17 @@ public Builder refreshAfterWrite(Duration duration, Executor executor) { return this; } + public Builder sizeEstimator(MetaCacheSizeEstimator estimator) { + this.sizeEstimator = Objects.requireNonNull(estimator, "estimator can not be null"); + return this; + } + + /** Shares one configured weight limit across physical caches in the same logical entry group. */ + public Builder budgetGroup(String budgetGroup) { + this.budgetGroup = requireName(budgetGroup); + return this; + } + public MetaCacheDefinition build() { return new MetaCacheDefinition<>(this); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheGovernance.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheGovernance.java new file mode 100644 index 00000000000000..a19d2f2eafda15 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheGovernance.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** Process-wide bridge used because connector-cache is parent-first for every connector plugin. */ +public final class MetaCacheGovernance { + private static final Object CONFIG_LOCK = new Object(); + private static final ConcurrentHashMap> CATALOG_CACHES = + new ConcurrentHashMap<>(); + private static volatile MetaCacheBudgetManager budgetManager = + new MetaCacheBudgetManager(OptionalLong.empty()); + private static volatile OptionalLong configuredGlobalMaxWeight = OptionalLong.empty(); + + private MetaCacheGovernance() { + } + + public static void configureGlobalMaxWeight(OptionalLong globalMaxWeight) { + synchronized (CONFIG_LOCK) { + OptionalLong requested = globalMaxWeight == null ? OptionalLong.empty() : globalMaxWeight; + if (same(configuredGlobalMaxWeight, requested)) { + return; + } + if (!CATALOG_CACHES.isEmpty()) { + throw new IllegalStateException( + "Can not change external metadata cache global weight after catalogs are initialized"); + } + budgetManager = new MetaCacheBudgetManager(requested); + configuredGlobalMaxWeight = requested; + } + } + + static MetaCacheBudgetManager budgetManager() { + return budgetManager; + } + + static void register(CatalogMetaCache cache) { + CATALOG_CACHES.compute(cache.catalogId(), (ignored, caches) -> { + Set updated = caches == null ? ConcurrentHashMap.newKeySet() : caches; + updated.add(cache); + return updated; + }); + } + + static void unregister(CatalogMetaCache cache) { + CATALOG_CACHES.computeIfPresent(cache.catalogId(), (ignored, caches) -> { + caches.remove(cache); + return caches.isEmpty() ? null : caches; + }); + } + + public static List catalogCaches(long catalogId) { + Set caches = CATALOG_CACHES.get(catalogId); + return caches == null ? Collections.emptyList() : new ArrayList<>(caches); + } + + public static OptionalLong globalMaxWeight() { + return configuredGlobalMaxWeight; + } + + public static long globalEstimatedWeight() { + return budgetManager.getGlobalUsedWeight(); + } + + private static boolean same(OptionalLong left, OptionalLong right) { + return left.isPresent() == right.isPresent() + && (!left.isPresent() || left.getAsLong() == right.getAsLong()); + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimate.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimate.java new file mode 100644 index 00000000000000..c9f25bce80d3a6 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimate.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.Objects; + +/** Immutable result of estimating one metadata-cache entry. */ +public final class MetaCacheSizeEstimate { + private final long bytes; + private final boolean complete; + private final String incompleteReason; + + private MetaCacheSizeEstimate(long bytes, boolean complete, String incompleteReason) { + this.bytes = bytes; + this.complete = complete; + this.incompleteReason = Objects.requireNonNull(incompleteReason, "incompleteReason"); + } + + public static MetaCacheSizeEstimate complete(long bytes) { + if (bytes < 0L) { + throw new IllegalArgumentException("cache size estimate can not be negative: " + bytes); + } + return new MetaCacheSizeEstimate(bytes, true, ""); + } + + public static MetaCacheSizeEstimate incomplete(String reason) { + String safeReason = Objects.requireNonNull(reason, "reason").trim(); + if (safeReason.isEmpty()) { + throw new IllegalArgumentException("incomplete cache size estimate requires a reason"); + } + return new MetaCacheSizeEstimate(0L, false, safeReason); + } + + public long getBytes() { + return bytes; + } + + public boolean isComplete() { + return complete; + } + + public String getIncompleteReason() { + return incompleteReason; + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..e6af814d1cd049 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimator.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.Objects; +import java.util.function.Supplier; + +/** Estimates the retained bytes owned by one connector metadata-cache entry. */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + MetaCacheSizeEstimate estimate(K key, V value); + + /** Estimator incompatibility rejects weighted admission without failing the metadata load. */ + static MetaCacheSizeEstimate estimateSafely( + String failureReason, Supplier estimation) { + Objects.requireNonNull(failureReason, "failureReason"); + Objects.requireNonNull(estimation, "estimation"); + try { + return Objects.requireNonNull(estimation.get(), "size estimate"); + } catch (RuntimeException | LinkageError e) { + return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimators.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimators.java new file mode 100644 index 00000000000000..8ec78638fa0f38 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheSizeEstimators.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +/** Shared estimators for small or structurally unknown metadata-cache entries. */ +public final class MetaCacheSizeEstimators { + private MetaCacheSizeEstimators() { + } + + /** + * Returns a visit-bounded complete graph estimator. Values that exceed the visit budget are rejected from a + * weighted cache instead of admitting an arbitrarily low sample. Large connector values should use a + * type-specific construction-time estimator that counts their payload in a cheap linear pass. + */ + public static MetaCacheSizeEstimator reflective() { + return (key, value) -> MetaCacheSizeEstimate.complete(JvmSizeUtils.saturatedAdd( + ReflectiveObjectSizeEstimator.estimateComplete(key), + ReflectiveObjectSizeEstimator.estimateComplete(value))); + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java new file mode 100644 index 00000000000000..7577f83e5aa219 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimator.java @@ -0,0 +1,428 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.RandomAccess; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Bounded reflective object-graph estimator for validating type-specific cache estimators. + * + *

The expensive class discovery is cached in a {@link ClassValue}: each class hierarchy is reflected once to + * retain its shallow size and accessible reference fields. Runtime estimation still reads each new root's actual + * fields because two instances of the same class can retain very different lists, maps, arrays, and strings. + * The diagnostic API samples collections and object arrays. The admission API instead traverses every element + * up to a fixed visit budget: this catches skewed tail values while bounding CPU and temporary identity-set + * memory. Strongly encapsulated reference fields, depth truncation, and visit-budget exhaustion fail the + * estimate so admission can reject the value without turning an incomplete graph into a false low weight. + * + *

This is intentionally a construction-time safety net, not a Caffeine hit-path weigher. Cache values should + * combine it with a type-specific estimate once, store the larger result, and expose that stored number to + * admission accounting in O(1). + */ +public final class ReflectiveObjectSizeEstimator { + private static final int DEFAULT_SAMPLE_SIZE = 5; + private static final int DEFAULT_MAX_DEPTH = 20; + private static final int DEFAULT_COMPLETE_VISIT_BUDGET = 10_000; + + private static final long HASH_MAP_NODE_BYTES = classSize("java.util.HashMap$Node"); + private static final long LINKED_HASH_MAP_ENTRY_BYTES = classSize("java.util.LinkedHashMap$Entry"); + private static final long TREE_MAP_ENTRY_BYTES = classSize("java.util.TreeMap$Entry"); + private static final long LINKED_LIST_NODE_BYTES = classSize("java.util.LinkedList$Node"); + private static final long CONCURRENT_HASH_MAP_NODE_BYTES = classSize("java.util.concurrent.ConcurrentHashMap$Node"); + + private static final Set> SHALLOW_LEAF_TYPES = Set.of( + Boolean.class, + Byte.class, + Character.class, + Short.class, + Integer.class, + Float.class, + Long.class, + Double.class); + + private static final ClassValue CLASS_PLANS = new ClassValue<>() { + @Override + protected ClassPlan computeValue(Class type) { + List referenceFields = new ArrayList<>(); + boolean inaccessibleReferenceField = false; + for (Class current = type; current != null; current = current.getSuperclass()) { + for (Field field : current.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) + || field.getType().isPrimitive() + || field.getType().isEnum()) { + continue; + } + try { + if (field.trySetAccessible()) { + referenceFields.add(field); + } else { + inaccessibleReferenceField = true; + } + } catch (RuntimeException ignored) { + inaccessibleReferenceField = true; + } + } + } + return new ClassPlan( + JvmSizeUtils.instanceSize(type), + referenceFields.toArray(new Field[0]), + inaccessibleReferenceField); + } + }; + + private ReflectiveObjectSizeEstimator() { + } + + public static long estimate(Object root) { + return estimate(root, DEFAULT_SAMPLE_SIZE, DEFAULT_MAX_DEPTH); + } + + public static long estimate(Object root, int sampleSize, int maxDepth) { + if (sampleSize <= 0) { + throw new IllegalArgumentException("sampleSize must be positive: " + sampleSize); + } + if (maxDepth < 0) { + throw new IllegalArgumentException("maxDepth can not be negative: " + maxDepth); + } + return new Walker(sampleSize, Integer.MAX_VALUE).estimate(root, maxDepth); + } + + /** + * Completely traverses a small or medium object graph for cache admission. The visit budget bounds the + * temporary identity set and CPU cost; exceeding it makes the estimate incomplete so weighted admission + * rejects the value instead of treating a sample as an exact retained size. + */ + public static long estimateComplete(Object root) { + return estimateComplete(root, DEFAULT_COMPLETE_VISIT_BUDGET, DEFAULT_MAX_DEPTH); + } + + public static long estimateComplete(Object root, int visitBudget, int maxDepth) { + if (visitBudget <= 0) { + throw new IllegalArgumentException("visitBudget must be positive: " + visitBudget); + } + if (maxDepth < 0) { + throw new IllegalArgumentException("maxDepth can not be negative: " + maxDepth); + } + return new Walker(Integer.MAX_VALUE, visitBudget).estimate(root, maxDepth); + } + + private static final class Walker { + private final int sampleSize; + private final int visitBudget; + private final Map visited = new IdentityHashMap<>(); + + private Walker(int sampleSize, int visitBudget) { + this.sampleSize = sampleSize; + this.visitBudget = visitBudget; + } + + private long estimate(Object value, int depth) { + if (value == null || visited.put(value, Boolean.TRUE) != null) { + return 0L; + } + if (visited.size() > visitBudget) { + throw new IllegalStateException( + "Complete estimate exceeded visit budget " + visitBudget); + } + + Class type = value.getClass(); + if (type.isEnum() || value instanceof Class) { + return 0L; + } + if (value instanceof String) { + return JvmSizeUtils.stringSize((String) value); + } + if (SHALLOW_LEAF_TYPES.contains(type)) { + return JvmSizeUtils.instanceSize(type); + } + if (type.isHidden()) { + throw new IllegalStateException("Can not completely estimate hidden class: " + type.getName()); + } + if (type.isArray()) { + return estimateArray(value, depth); + } + if (value instanceof ByteBuffer) { + return estimateByteBuffer((ByteBuffer) value, depth); + } + if (value instanceof Optional) { + return estimateOptional((Optional) value, depth); + } + if (value instanceof Map) { + return estimateMap((Map) value, depth); + } + if (value instanceof Collection) { + return estimateCollection((Collection) value, depth); + } + + ClassPlan plan = CLASS_PLANS.get(type); + if (plan.inaccessibleReferenceField) { + throw new IllegalStateException( + "Can not completely estimate strongly encapsulated class: " + type.getName()); + } + long bytes = plan.shallowBytes; + if (depth == 0) { + if (plan.referenceFields.length > 0) { + throw incomplete(type); + } + return bytes; + } + for (Field field : plan.referenceFields) { + bytes = add(bytes, estimateField(value, field, depth - 1)); + } + return bytes; + } + + private long estimateArray(Object value, int depth) { + int length = Array.getLength(value); + Class componentType = value.getClass().getComponentType(); + if (componentType.isPrimitive()) { + return JvmSizeUtils.primitiveArraySize(componentType, length); + } + + long bytes = JvmSizeUtils.objectArraySize(length); + if (length == 0) { + return bytes; + } + if (depth == 0) { + throw incomplete(value.getClass()); + } + int samples = Math.min(length, sampleSize); + long sampledBytes = 0L; + for (int i = 0; i < samples; i++) { + int index = sampleIndex(i, samples, length); + sampledBytes = add(sampledBytes, estimate(Array.get(value, index), depth - 1)); + } + return add(bytes, scale(sampledBytes, samples, length)); + } + + private long estimateByteBuffer(ByteBuffer value, int depth) { + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + if (value.hasArray()) { + if (depth == 0) { + throw incomplete(value.getClass()); + } + bytes = add(bytes, estimate(value.array(), depth - 1)); + } + return bytes; + } + + private long estimateOptional(Optional value, int depth) { + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + if (value.isEmpty()) { + return bytes; + } + if (depth == 0) { + throw incomplete(value.getClass()); + } + return add(bytes, estimate(value.get(), depth - 1)); + } + + private long estimateCollection(Collection values, int depth) { + int size = values.size(); + long bytes = add( + JvmSizeUtils.instanceSize(values.getClass()), + collectionStorageBytes(values, size)); + if (size == 0) { + return bytes; + } + if (depth == 0) { + throw incomplete(values.getClass()); + } + + int samples = Math.min(size, sampleSize); + long sampledBytes = 0L; + if (values instanceof List && values instanceof RandomAccess) { + List list = (List) values; + for (int i = 0; i < samples; i++) { + sampledBytes = add(sampledBytes, + estimate(list.get(sampleIndex(i, samples, size)), depth - 1)); + } + } else { + sampledBytes = estimateIterableSamples(values, samples, depth - 1); + } + return add(bytes, scale(sampledBytes, samples, size)); + } + + private long estimateMap(Map values, int depth) { + int size = values.size(); + long bytes = add( + JvmSizeUtils.instanceSize(values.getClass()), + mapStorageBytes(values, size)); + if (size == 0) { + return bytes; + } + if (depth == 0) { + throw incomplete(values.getClass()); + } + + int samples = Math.min(size, sampleSize); + long sampledBytes = 0L; + Iterator> iterator = values.entrySet().iterator(); + for (int i = 0; i < samples; i++) { + Map.Entry entry = iterator.next(); + sampledBytes = add(sampledBytes, estimate(entry.getKey(), depth - 1)); + sampledBytes = add(sampledBytes, estimate(entry.getValue(), depth - 1)); + } + return add(bytes, scale(sampledBytes, samples, size)); + } + + private long estimateIterableSamples(Collection values, int samples, int depth) { + Iterator iterator = values.iterator(); + long sampledBytes = 0L; + for (int i = 0; i < samples; i++) { + sampledBytes = add(sampledBytes, estimate(iterator.next(), depth)); + } + return sampledBytes; + } + + private long estimateField(Object owner, Field field, int depth) { + try { + return estimate(field.get(owner), depth); + } catch (IllegalAccessException e) { + throw new IllegalStateException("Cached reference field is no longer accessible: " + field, e); + } + } + } + + private static long collectionStorageBytes(Collection values, int size) { + if (size == 0) { + return 0L; + } + if (values instanceof LinkedHashSet) { + return hashStorageBytes(size, LINKED_HASH_MAP_ENTRY_BYTES); + } + if (values instanceof HashSet) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + if (values instanceof TreeSet) { + return multiply(size, TREE_MAP_ENTRY_BYTES); + } + if (values instanceof LinkedList) { + return multiply(size, LINKED_LIST_NODE_BYTES); + } + if (values instanceof Set) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + return JvmSizeUtils.objectArraySize(size); + } + + private static long mapStorageBytes(Map values, int size) { + if (size == 0) { + return 0L; + } + if (values instanceof LinkedHashMap) { + return hashStorageBytes(size, LINKED_HASH_MAP_ENTRY_BYTES); + } + if (values instanceof HashMap) { + return hashStorageBytes(size, HASH_MAP_NODE_BYTES); + } + if (values instanceof ConcurrentHashMap) { + return hashStorageBytes(size, CONCURRENT_HASH_MAP_NODE_BYTES); + } + if (values instanceof TreeMap) { + return multiply(size, TREE_MAP_ENTRY_BYTES); + } + return add(JvmSizeUtils.objectArraySize(saturatedDouble(size)), + multiply(size, HASH_MAP_NODE_BYTES)); + } + + private static long hashStorageBytes(int size, long nodeBytes) { + return add( + JvmSizeUtils.objectArraySize(hashCapacity(size)), + multiply(size, nodeBytes)); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static int sampleIndex(int sample, int sampleCount, int totalCount) { + return sampleCount == 1 + ? 0 + : (int) ((long) sample * (totalCount - 1) / (sampleCount - 1)); + } + + private static long scale(long sampledBytes, int sampleCount, int totalCount) { + if (sampleCount == totalCount) { + return sampledBytes; + } + double scaled = (double) sampledBytes * totalCount / sampleCount; + return scaled >= Long.MAX_VALUE ? Long.MAX_VALUE : (long) scaled; + } + + private static int saturatedDouble(int value) { + return value > Integer.MAX_VALUE / 2 ? Integer.MAX_VALUE : value * 2; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static IllegalStateException incomplete(Class type) { + return new IllegalStateException("Object graph depth limit reached at: " + type.getName()); + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } + + private static final class ClassPlan { + private final long shallowBytes; + private final Field[] referenceFields; + private final boolean inaccessibleReferenceField; + + private ClassPlan(long shallowBytes, Field[] referenceFields, boolean inaccessibleReferenceField) { + this.shallowBytes = shallowBytes; + this.referenceFields = referenceFields; + this.inaccessibleReferenceField = inaccessibleReferenceField; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index 4f8ac0924e79c9..a4353067106ab9 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.NavigableMap; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; import java.util.TreeMap; import java.util.concurrent.CompletableFuture; @@ -48,6 +49,7 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; @@ -65,6 +67,8 @@ */ public final class ScopedMetaCache implements AutoCloseable { private static final Logger LOG = LogManager.getLogger(ScopedMetaCache.class); + private static final long FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES = 512L; + private static final long WEIGHT_REJECT_WARN_INTERVAL_NANOS = Duration.ofMinutes(1).toNanos(); private static final Runnable NO_OP = () -> { }; @@ -90,6 +94,9 @@ public final class ScopedMetaCache implements AutoCloseable { private final AtomicReference lastLoadSuccessTimeMs = new AtomicReference<>(-1L); private final AtomicReference lastLoadFailureTimeMs = new AtomicReference<>(-1L); private final AtomicReference lastError = new AtomicReference<>(""); + private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final LongAdder weightRejectCount = new LongAdder(); + private final AtomicLong lastWeightRejectWarnNanos = new AtomicLong(Long.MIN_VALUE); private final RemovalListener beforeRemoval; private final BiConsumer discardListener; private final Ticker ticker; @@ -98,8 +105,12 @@ public final class ScopedMetaCache implements AutoCloseable { private final Runnable afterLoadElection; private final Runnable afterBulkStage; private final Runnable afterRefreshRegistration; + private final MetaCacheSizeEstimator sizeEstimator; + private final MetaCacheBudgetManager.EntryBudget entryBudget; + private final boolean weightBounded; private final ThreadLocal> removalDeferrals = ThreadLocal.withInitial(RemovalDeferral::new); + private final ThreadLocal budgetEvictionRegistration = new ThreadLocal<>(); private BigInteger exactInvalidationSequence = BigInteger.ZERO; ScopedMetaCache( @@ -113,7 +124,9 @@ public final class ScopedMetaCache implements AutoCloseable { Executor refreshExecutor, Runnable afterLoadElection, Runnable afterBulkStage, - Runnable afterRefreshRegistration) { + Runnable afterRefreshRegistration, + MetaCacheSizeEstimator sizeEstimator, + MetaCacheBudgetManager.EntryBudget entryBudget) { this.registry = Objects.requireNonNull(registry, "registry can not be null"); this.name = Objects.requireNonNull(name, "name can not be null"); Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); @@ -127,8 +140,17 @@ public final class ScopedMetaCache implements AutoCloseable { this.afterBulkStage = Objects.requireNonNull(afterBulkStage, "afterBulkStage can not be null"); this.afterRefreshRegistration = Objects.requireNonNull( afterRefreshRegistration, "afterRefreshRegistration can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); + this.sizeEstimator = sizeEstimator; + this.entryBudget = entryBudget; + if ((sizeEstimator == null) != (entryBudget == null)) { + throw new IllegalArgumentException( + "Weighted metadata cache requires both estimator and budget: " + name); + } + this.weightBounded = entryBudget != null; + if (weightBounded) { + entryBudget.setReclaimer(this::reclaimForPeer); + } + this.effectiveEnabled = cacheSpec.isCacheEnabled(); Caffeine builder = Caffeine.newBuilder() .maximumSize(effectiveEnabled ? cacheSpec.getCapacity() : 0L) @@ -284,10 +306,46 @@ public void put(K key, ScopePath path, V value) { return; } try (PublicationLease lease = acquirePublicationLease(key, path, false)) { - guardedCommit(lease, () -> { - lease.keyNode.loadPublicationState.set(new Object()); - return publishCommitted(lease, key, value) != null; - }); + AtomicReference> preparedRef = new AtomicReference<>(); + AtomicReference> replacementRef = new AtomicReference<>(); + AtomicBoolean admissionRejected = new AtomicBoolean(false); + boolean retained = false; + try { + retained = guardedCommit(lease, () -> { + VersionedValue current = currentVersionedValue(key, path); + if (current != null) { + ReplacementValue replacement = + prepareReplacementVersionedValue(lease, key, value, current); + replacementRef.set(replacement); + if (replacement == null) { + admissionRejected.set(true); + lease.keyNode.loadPublicationState.set(new Object()); + removeCurrentVersion(key, current); + return false; + } + lease.keyNode.loadPublicationState.set(new Object()); + return installReplacement(replacement); + } + VersionedValue prepared = prepareVersionedValue(lease, key, value); + preparedRef.set(prepared); + if (prepared == null) { + admissionRejected.set(true); + return false; + } + lease.keyNode.loadPublicationState.set(new Object()); + install(prepared, lease); + return true; + }); + } finally { + if (!retained && replacementRef.get() != null) { + replacementRef.get().rollback(); + } else if (!retained && preparedRef.get() != null) { + releaseReservation(preparedRef.get()); + } + if (admissionRejected.get()) { + notifyDiscarded(key, value); + } + } } } @@ -306,29 +364,74 @@ public boolean compareAndSet( return true; } try (PublicationLease lease = acquirePublicationLease(key, path, false)) { - return guardedCommit(lease, () -> { - VersionedValue current = currentVersionedValue(key, path); - V currentValue = current == null ? null : current.value; - if (currentValue != expectedValue) { - return false; - } - lease.keyNode.loadPublicationState.set(new Object()); - action.run(); - if (updatedValue == currentValue) { + AtomicReference> preparedRef = new AtomicReference<>(); + AtomicReference> replacementRef = new AtomicReference<>(); + AtomicBoolean admissionRejected = new AtomicBoolean(false); + boolean committed = false; + try { + committed = guardedCommit(lease, () -> { + VersionedValue current = currentVersionedValue(key, path); + V currentValue = current == null ? null : current.value; + if (currentValue != expectedValue) { + return false; + } + if (updatedValue != null && updatedValue != currentValue) { + if (current != null) { + ReplacementValue replacement = + prepareReplacementVersionedValue(lease, key, updatedValue, current); + replacementRef.set(replacement); + if (replacement == null) { + admissionRejected.set(true); + lease.keyNode.loadPublicationState.set(new Object()); + action.run(); + removeCurrentVersion(key, current); + return true; + } + } else { + VersionedValue prepared = prepareVersionedValue(lease, key, updatedValue); + preparedRef.set(prepared); + if (prepared == null) { + admissionRejected.set(true); + lease.keyNode.loadPublicationState.set(new Object()); + action.run(); + return true; + } + } + } + lease.keyNode.loadPublicationState.set(new Object()); + action.run(); + if (updatedValue == currentValue) { + lease.keyNode.loadPublicationState.set(new Object()); + return true; + } + if (updatedValue != null) { + if (replacementRef.get() != null) { + lease.keyNode.loadPublicationState.set(new Object()); + return installReplacement(replacementRef.get()); + } + lease.keyNode.loadPublicationState.set(new Object()); + install(preparedRef.get(), lease); + return true; + } + if (updatedValue == null) { + if (current != null) { + removeCurrentVersion(key, current); + } + } lease.keyNode.loadPublicationState.set(new Object()); return true; + }); + } finally { + if (!committed && replacementRef.get() != null) { + replacementRef.get().rollback(); + } else if (!committed && preparedRef.get() != null) { + releaseReservation(preparedRef.get()); } - if (updatedValue == null) { - if (current != null) { - data.asMap().remove(key, current); - lease.keyNode.registration.compareAndSet(current, null); - } - } else { - publishCommitted(lease, key, updatedValue); + if (admissionRejected.get()) { + notifyDiscarded(key, updatedValue); } - lease.keyNode.loadPublicationState.set(new Object()); - return true; - }); + } + return committed; } } @@ -366,10 +469,14 @@ void invalidateKey( return; } afterStateReplacement.run(); - VersionedValue registered = invalidated.node.registration.get(); - if (registered != null && registered.keyState == invalidated.keyState) { - data.asMap().remove(key, registered); - invalidated.node.registration.compareAndSet(registered, null); + Registration registered = invalidated.node.registration.get(); + VersionedValue current = data.getIfPresent(key); + if (registered != null && current != null && current.registration == registered + && current.keyState == invalidated.keyState) { + data.asMap().remove(key, current); + } + if (registered != null && invalidated.node.registration.compareAndSet(registered, null)) { + releaseRegistration(registered); } tryPruneKey(key, invalidated.node); } @@ -414,9 +521,20 @@ public boolean publish( return false; } try (PublicationLease lease = acquirePublicationLease(key, actualScope, false)) { - VersionedValue staged = newVersionedValue(lease, key, value); - afterBulkStage.run(); - return handle.tryCommit(key, lease, staged); + VersionedValue staged = prepareVersionedValue(lease, key, value); + if (staged == null) { + return false; + } + boolean retained = false; + try { + afterBulkStage.run(); + retained = handle.tryCommit(key, lease, staged); + return retained; + } finally { + if (!retained) { + releaseReservation(staged); + } + } } } @@ -438,7 +556,12 @@ public CacheMetrics metrics() { invalidateCount.sum(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), - lastError.get())); + lastError.get(), + weightBounded, + entryBudget == null ? -1L : entryBudget.getEffectiveMaxWeight(), + entryBudget == null ? 0L : entryBudget.getUsedWeight(), + weightRejectCount.sum(), + lastWeightRejectReason.get())); } int refreshingCountForTest() { @@ -504,9 +627,11 @@ void closeFromRegistry() { void removeExpectedRaw(Object rawKey, Object expectedValue) { @SuppressWarnings("unchecked") K key = (K) rawKey; - @SuppressWarnings("unchecked") - VersionedValue versionedValue = (VersionedValue) expectedValue; - data.asMap().remove(key, versionedValue); + Registration expectedRegistration = (Registration) expectedValue; + VersionedValue current = data.getIfPresent(key); + if (current != null && current.registration == expectedRegistration) { + data.asMap().remove(key, current); + } } private PublicationLease acquirePublicationLease( @@ -527,29 +652,25 @@ private PublicationLease acquirePublicationLease( } } - private VersionedValue publish(PublicationLease lease, K key, V value) { - if (!lease.isCurrent()) { - return null; - } - VersionedValue versioned = newVersionedValue(lease, key, value); - install(versioned, lease); - if (!lease.isCurrent()) { - data.asMap().remove(key, versioned); - return null; - } - return versioned; - } - - private VersionedValue publishCommitted(PublicationLease lease, K key, V value) { - return publish(lease, key, value); - } - private boolean commitLoaded(PublicationLease lease, K key, V value, Runnable beforePublication) { Runnable action = Objects.requireNonNull(beforePublication, "beforePublication can not be null"); - return guardedCommit(lease, () -> { - action.run(); - return publishCommitted(lease, key, value) != null; - }); + VersionedValue prepared = prepareVersionedValue(lease, key, value); + boolean retained = false; + try { + retained = guardedCommit(lease, () -> { + action.run(); + if (prepared == null) { + return false; + } + install(prepared, lease); + return true; + }); + return retained; + } finally { + if (!retained && prepared != null) { + releaseReservation(prepared); + } + } } private boolean guardedCommit(PublicationLease lease, BooleanSupplier commitAction) { @@ -562,11 +683,167 @@ private boolean guardedCommit(PublicationLease lease, BooleanSupplier comm }))); } - private VersionedValue newVersionedValue( + private VersionedValue prepareVersionedValue( PublicationLease lease, K key, V value) { + MetaCacheBudgetManager.AdmissionReservation reservation = reserve(key, value); + if (weightBounded && reservation == null) { + return null; + } CacheAddress address = new CacheAddress(this, key); + ScopeSnapshot scopeSnapshot = lease.scopeLease.snapshot(); + Registration registration = new Registration(reservation, address, scopeSnapshot); return new VersionedValue<>( - key, value, address, lease.scopeLease.snapshot(), lease.keyNode, lease.keyState, ticker.read()); + key, value, address, scopeSnapshot, lease.keyNode, lease.keyState, + ticker.read(), registration); + } + + private MetaCacheBudgetManager.AdmissionReservation reserve(K key, V value) { + if (!weightBounded) { + return null; + } + long bytes = estimateWeight(key, value); + return bytes < 0L ? null : reserveEstimated(bytes); + } + + private long estimateWeight(K key, V value) { + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "estimator_failure", () -> sizeEstimator.estimate(key, value)); + if (!estimate.isComplete()) { + rejectWeight("incomplete_estimate:" + estimate.getIncompleteReason()); + return -1L; + } + if (estimate.getBytes() == 0L) { + rejectWeight("invalid_zero_estimate"); + return -1L; + } + long bytes = JvmSizeUtils.saturatedAdd( + estimate.getBytes(), FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES); + if (bytes > entryBudget.getEffectiveMaxWeight()) { + rejectWeight("entry_too_large"); + return -1L; + } + return bytes; + } + + private MetaCacheBudgetManager.AdmissionReservation reserveEstimated(long bytes) { + Optional reservation = entryBudget.tryReserve(bytes); + if (!reservation.isPresent()) { + entryBudget.requestPeerReclaim(bytes); + rejectWeight("budget_exceeded"); + return null; + } + return reservation.get(); + } + + private ReplacementValue prepareReplacementVersionedValue( + PublicationLease lease, K key, V value, VersionedValue previous) { + if (!weightBounded) { + VersionedValue versioned = prepareVersionedValue(lease, key, value); + return versioned == null ? null : new ReplacementValue<>(versioned, previous, null); + } + long bytes = estimateWeight(key, value); + if (bytes < 0L) { + return null; + } + Optional replacement = + entryBudget.tryReplace(previous.registration.reservation, bytes); + if (!replacement.isPresent()) { + long additionalBytes = Math.max(0L, + bytes - previous.registration.reservation.getBytes()); + entryBudget.requestPeerReclaim(additionalBytes); + rejectWeight("budget_exceeded"); + return null; + } + MetaCacheBudgetManager.ReservationReplacement accounting = replacement.get(); + if (!previous.registration.released.compareAndSet(false, true)) { + accounting.rollback(); + return null; + } + CacheAddress address = new CacheAddress(this, key); + ScopeSnapshot scopeSnapshot = lease.scopeLease.snapshot(); + Registration registration = new Registration(accounting.current(), address, scopeSnapshot); + VersionedValue versioned = new VersionedValue<>( + key, value, address, scopeSnapshot, lease.keyNode, lease.keyState, + ticker.read(), registration); + return new ReplacementValue<>(versioned, previous, accounting); + } + + private boolean installReplacement(ReplacementValue replacement) { + VersionedValue previous = replacement.previous; + VersionedValue current = replacement.current; + registry.register(current.address, current.registration, current.scopeSnapshot); + if (!current.keyNode.registration.compareAndSet( + previous.registration, current.registration)) { + registry.register(previous.address, previous.registration, previous.scopeSnapshot); + replacement.rollback(); + return false; + } + if (!data.asMap().replace(current.key, previous, current)) { + current.keyNode.registration.compareAndSet(current.registration, previous.registration); + registry.register(previous.address, previous.registration, previous.scopeSnapshot); + replacement.rollback(); + return false; + } + replacement.commit(); + return true; + } + + private long evictLocalColdest() { + if (!data.policy().eviction().isPresent()) { + return 0L; + } + Map> coldest = data.policy().eviction().get().coldest(1); + for (Map.Entry> candidate : coldest.entrySet()) { + VersionedValue current = data.getIfPresent(candidate.getKey()); + if (current == candidate.getValue()) { + long weight = current.registration.reservation.getBytes(); + budgetEvictionRegistration.set(current.registration); + try { + if (data.asMap().remove(candidate.getKey(), current)) { + return weight; + } + } finally { + budgetEvictionRegistration.remove(); + } + } + } + return 0L; + } + + private long reclaimForPeer(long targetBytes) { + if (targetBytes <= 0L || closed.get()) { + return 0L; + } + long reclaimed = 0L; + long removed; + while (reclaimed < targetBytes && (removed = evictLocalColdest()) > 0L) { + reclaimed = JvmSizeUtils.saturatedAdd(reclaimed, removed); + } + return reclaimed; + } + + private void rejectWeight(String reason) { + weightRejectCount.increment(); + lastWeightRejectReason.set(reason); + long now = System.nanoTime(); + long last = lastWeightRejectWarnNanos.get(); + if ((last == Long.MIN_VALUE || now - last >= WEIGHT_REJECT_WARN_INTERVAL_NANOS) + && lastWeightRejectWarnNanos.compareAndSet(last, now)) { + LOG.warn("Metadata cache entry '{}' rejected a value by weight: reason={}, used={}, max={}", + name, reason, entryBudget == null ? 0L : entryBudget.getUsedWeight(), + entryBudget == null ? -1L : entryBudget.getEffectiveMaxWeight()); + } + } + + private static void releaseReservation(VersionedValue versioned) { + releaseRegistration(versioned.registration); + } + + private static void releaseRegistration(Registration registration) { + if (registration != null && registration.released.compareAndSet(false, true) + && registration.reservation != null) { + registration.reservation.release(); + } } private void scheduleRefresh(K key, ScopePath path, Function loader, VersionedValue current) { @@ -620,34 +897,45 @@ private void scheduleRefresh(K key, ScopePath path, Function loader, Versi private void install( VersionedValue versioned, PublicationLease lease) { - registry.register(versioned.address, versioned, versioned.scopeSnapshot); - lease.keyNode.registration.set(versioned); + registry.register(versioned.address, versioned.registration, versioned.scopeSnapshot); + lease.keyNode.registration.set(versioned.registration); data.asMap().put(versioned.key, versioned); } private boolean replaceRefreshExpected(PublicationLease lease, K key, VersionedValue expected, V refreshed) { - return guardedCommit(lease, () -> { - if (data.getIfPresent(key) != expected || !expected.isCurrent(registry, keyNodes)) { - return false; - } - if (refreshed == expected.value) { + if (refreshed == expected.value) { + return guardedCommit(lease, () -> { + if (data.getIfPresent(key) != expected || !expected.isCurrent(registry, keyNodes)) { + return false; + } expected.writeTimeNanos = ticker.read(); return true; + }); + } + AtomicReference> replacementRef = new AtomicReference<>(); + boolean retained = false; + try { + retained = guardedCommit(lease, () -> { + if (data.getIfPresent(key) != expected || !expected.isCurrent(registry, keyNodes)) { + return false; + } + ReplacementValue replacement = + prepareReplacementVersionedValue(lease, key, refreshed, expected); + replacementRef.set(replacement); + if (replacement == null) { + lease.keyNode.loadPublicationState.set(new Object()); + removeCurrentVersion(key, expected); + return false; + } + return installReplacement(replacement); + }); + return retained; + } finally { + if (!retained && replacementRef.get() != null) { + replacementRef.get().rollback(); } - VersionedValue replacement = newVersionedValue(lease, key, refreshed); - registry.register(replacement.address, replacement, replacement.scopeSnapshot); - if (!lease.keyNode.registration.compareAndSet(expected, replacement)) { - registry.unregister(replacement.address, replacement, replacement.scopeSnapshot); - return false; - } - if (!data.asMap().replace(key, expected, replacement)) { - lease.keyNode.registration.compareAndSet(replacement, null); - registry.unregister(replacement.address, replacement, replacement.scopeSnapshot); - return false; - } - return true; - }); + } } private boolean tryCommitBulk( @@ -679,6 +967,13 @@ private boolean tryCommitBulk( })); } + private void removeCurrentVersion(K key, VersionedValue current) { + data.asMap().remove(key, current); + if (current.keyNode.registration.compareAndSet(current.registration, null)) { + releaseRegistration(current.registration); + } + } + private boolean deferRemovals(BooleanSupplier action) { RemovalDeferral removalDeferral = removalDeferrals.get(); removalDeferral.depth++; @@ -752,25 +1047,30 @@ private void closePhysicalState() { bulkInvalidationGate.write(exactInvalidations::clear); keyNodes.forEach((key, node) -> { replaceKeyState(node); - VersionedValue versioned = node.registration.get(); - if (versioned != null) { + Registration registered = node.registration.get(); + VersionedValue versioned = data.getIfPresent(key); + if (versioned != null && versioned.registration == registered) { data.asMap().remove(key, versioned); - node.registration.compareAndSet(versioned, null); + } + if (registered != null && node.registration.compareAndSet(registered, null)) { + releaseRegistration(registered); } tryPruneKey(key, node); }); data.invalidateAll(); data.cleanUp(); + if (entryBudget != null) { + entryBudget.close(); + } } private void onRemoval( Object rawKey, Object rawValue, RemovalCause cause) { - Objects.requireNonNull(rawKey, "removed cache key can not be null"); - Objects.requireNonNull(rawValue, "removed cache value can not be null"); @SuppressWarnings("unchecked") VersionedValue versioned = (VersionedValue) rawValue; RemovalDeferral removalDeferral = removalDeferrals.get(); - removalDeferral.removals.addLast(new DeferredRemoval<>(versioned, cause)); + removalDeferral.removals.addLast(new DeferredRemoval<>( + versioned, cause, budgetEvictionRegistration.get() == versioned.registration)); if (removalDeferral.depth > 0 || removalDeferral.draining) { return; } @@ -784,7 +1084,7 @@ private void drainDeferredRemovals(RemovalDeferral removalDeferral) { DeferredRemoval removal; while ((removal = removalDeferral.removals.pollFirst()) != null) { try { - completeRemoval(removal.versioned, removal.cause); + completeRemoval(removal.versioned, removal.cause, removal.budgetEviction); } catch (RuntimeException | Error e) { if (failure == null) { failure = e; @@ -806,8 +1106,9 @@ private void drainDeferredRemovals(RemovalDeferral removalDeferral) { } } - private void completeRemoval(VersionedValue versioned, RemovalCause cause) { - if (cause.wasEvicted()) { + private void completeRemoval( + VersionedValue versioned, RemovalCause cause, boolean budgetEviction) { + if (cause.wasEvicted() || budgetEviction) { evictionCount.increment(); } else if (cause == RemovalCause.EXPLICIT) { invalidateCount.increment(); @@ -820,9 +1121,12 @@ private void completeRemoval(VersionedValue versioned, RemovalCause cause) LOG.warn("Scoped metadata cache removal callback failed", t); } } - registry.unregister(versioned.address, versioned, versioned.scopeSnapshot); - versioned.keyNode.registration.compareAndSet(versioned, null); - tryPruneKey(key, versioned.keyNode); + synchronized (versioned.keyNode) { + registry.unregister(versioned.address, versioned.registration, versioned.scopeSnapshot); + versioned.keyNode.registration.compareAndSet(versioned.registration, null); + releaseRegistration(versioned.registration); + tryPruneKey(key, versioned.keyNode); + } } private void notifyDiscarded(K key, V value) { @@ -845,10 +1149,13 @@ private static final class RemovalDeferral { private static final class DeferredRemoval { private final VersionedValue versioned; private final RemovalCause cause; + private final boolean budgetEviction; - private DeferredRemoval(VersionedValue versioned, RemovalCause cause) { + private DeferredRemoval( + VersionedValue versioned, RemovalCause cause, boolean budgetEviction) { this.versioned = versioned; this.cause = cause; + this.budgetEviction = budgetEviction; } } @@ -956,6 +1263,11 @@ public static final class CacheMetrics { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final boolean weightBounded; + private final long maxWeight; + private final long estimatedWeight; + private final long weightRejectCount; + private final String lastWeightRejectReason; private CacheMetrics( long physicalEntryCount, @@ -974,7 +1286,12 @@ private CacheMetrics( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + boolean weightBounded, + long maxWeight, + long estimatedWeight, + long weightRejectCount, + String lastWeightRejectReason) { this.physicalEntryCount = physicalEntryCount; this.keyNodeCount = keyNodeCount; this.inFlightLoadCount = inFlightLoadCount; @@ -992,6 +1309,11 @@ private CacheMetrics( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = lastError; + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.weightRejectCount = weightRejectCount; + this.lastWeightRejectReason = lastWeightRejectReason; } public long getPhysicalEntryCount() { @@ -1061,6 +1383,26 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public boolean isWeightBounded() { + return weightBounded; + } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getWeightRejectCount() { + return weightRejectCount; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } private static final class PublicationLease implements AutoCloseable { @@ -1115,6 +1457,7 @@ private static final class VersionedValue { private final KeyNode keyNode; private final KeyState keyState; private volatile long writeTimeNanos; + private final Registration registration; private VersionedValue( K key, @@ -1123,7 +1466,8 @@ private VersionedValue( ScopeSnapshot scopeSnapshot, KeyNode keyNode, KeyState keyState, - long writeTimeNanos) { + long writeTimeNanos, + Registration registration) { this.key = key; this.value = value; this.address = address; @@ -1131,6 +1475,7 @@ private VersionedValue( this.keyNode = keyNode; this.keyState = keyState; this.writeTimeNanos = writeTimeNanos; + this.registration = registration; } private boolean isCurrent( @@ -1139,14 +1484,63 @@ private boolean isCurrent( return scopeSnapshot.isCurrent(registry) && currentKeyNodes.get(key) == keyNode && keyNode.current.get() == keyState - && keyNode.registration.get() == this; + && keyNode.registration.get() == registration; + } + } + + private static final class ReplacementValue { + private final VersionedValue current; + private final VersionedValue previous; + private final MetaCacheBudgetManager.ReservationReplacement accounting; + private boolean finished; + + private ReplacementValue(VersionedValue current, + VersionedValue previous, + MetaCacheBudgetManager.ReservationReplacement accounting) { + this.current = current; + this.previous = previous; + this.accounting = accounting; + } + + private void commit() { + if (accounting != null) { + accounting.commit(); + } + finished = true; + } + + private void rollback() { + if (finished) { + return; + } + if (accounting != null) { + current.registration.released.set(true); + previous.registration.released.set(false); + accounting.rollback(); + } + finished = true; + } + } + + /** Lightweight ownership token: deliberately does not retain the cached value. */ + private static final class Registration { + private final MetaCacheBudgetManager.AdmissionReservation reservation; + private final CacheAddress address; + private final ScopeSnapshot scopeSnapshot; + private final AtomicBoolean released = new AtomicBoolean(false); + + private Registration(MetaCacheBudgetManager.AdmissionReservation reservation, + CacheAddress address, ScopeSnapshot scopeSnapshot) { + this.reservation = reservation; + this.address = address; + this.scopeSnapshot = scopeSnapshot; } } private static final class KeyNode { private final AtomicReference current = new AtomicReference<>(new KeyState()); private final AtomicReference loadPublicationState = new AtomicReference<>(new Object()); - private final AtomicReference> registration = new AtomicReference<>(); + private final AtomicReference registration = new AtomicReference<>(); private final AtomicInteger activeLoads = new AtomicInteger(); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index d08fcdb97202ea..918b2829a7927b 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -19,6 +19,8 @@ import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.time.Duration; import java.util.ArrayList; @@ -46,6 +48,7 @@ * live values and in-flight loads rather than by every name ever observed. */ public final class ScopedMetaCacheRegistry implements AutoCloseable { + private static final Logger LOG = LogManager.getLogger(ScopedMetaCacheRegistry.class); private static final Runnable NO_OP = () -> { }; private static final BiConsumer NO_OP_SCOPE = (level, key) -> { @@ -81,19 +84,23 @@ ScopedMetaCache createCacheWithRemovalListener( String name, CacheSpec cacheSpec, RemovalListener removalListener, Duration refreshAfterWrite, Executor refreshExecutor) { return createCacheWithRemovalListener(name, cacheSpec, null, removalListener, - null, refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP); + null, refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP, + null, null); } ScopedMetaCache createCacheWithMetaRemovalListener( String name, CacheSpec cacheSpec, MetaCacheRemovalListener removalListener, BiConsumer discardListener, - Duration refreshAfterWrite, Executor refreshExecutor) { + Duration refreshAfterWrite, Executor refreshExecutor, + MetaCacheSizeEstimator sizeEstimator, + MetaCacheBudgetManager.EntryBudget entryBudget) { RemovalListener caffeineListener = removalListener == null ? null : (key, value, cause) -> removalListener.onRemoval( key, value, MetaCacheRemovalReason.valueOf(cause.name())); return createCacheWithRemovalListener( name, cacheSpec, null, caffeineListener, discardListener, - refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP); + refreshAfterWrite, refreshExecutor, NO_OP, NO_OP, NO_OP, + sizeEstimator, entryBudget); } ScopedMetaCache createCache( @@ -125,7 +132,7 @@ ScopedMetaCache createCache( : (key, value, cause) -> beforeRemoval.accept(key, value); return createCacheWithRemovalListener( name, cacheSpec, ticker, listener, null, null, null, - afterLoadElection, afterBulkStage, NO_OP); + afterLoadElection, afterBulkStage, NO_OP, null, null); } ScopedMetaCache createCacheWithRefresh( @@ -136,7 +143,7 @@ ScopedMetaCache createCacheWithRefresh( Runnable afterRefreshRegistration) { return createCacheWithRemovalListener( name, cacheSpec, null, null, null, refreshAfterWrite, refreshExecutor, - NO_OP, NO_OP, afterRefreshRegistration); + NO_OP, NO_OP, afterRefreshRegistration, null, null); } private ScopedMetaCache createCacheWithRemovalListener( @@ -149,7 +156,9 @@ private ScopedMetaCache createCacheWithRemovalListener( Executor refreshExecutor, Runnable afterLoadElection, Runnable afterBulkStage, - Runnable afterRefreshRegistration) { + Runnable afterRefreshRegistration, + MetaCacheSizeEstimator sizeEstimator, + MetaCacheBudgetManager.EntryBudget entryBudget) { checkOpen(); ScopedMetaCache cache = new ScopedMetaCache<>( this, @@ -162,7 +171,9 @@ private ScopedMetaCache createCacheWithRemovalListener( refreshExecutor, afterLoadElection, afterBulkStage, - afterRefreshRegistration); + afterRefreshRegistration, + sizeEstimator, + entryBudget); caches.add(cache); if (closed.get()) { caches.remove(cache); @@ -252,7 +263,25 @@ public void close() { cleanDetachedState(oldState); List> snapshot = new ArrayList<>(caches); caches.clear(); - snapshot.forEach(ScopedMetaCache::closeFromRegistry); + Throwable failure = null; + for (ScopedMetaCache cache : snapshot) { + try { + cache.closeFromRegistry(); + } catch (RuntimeException | Error throwable) { + LOG.error("Failed to close scoped metadata cache", throwable); + if (failure == null) { + failure = throwable; + } else { + failure.addSuppressed(throwable); + } + } + } + if (failure instanceof RuntimeException) { + throw (RuntimeException) failure; + } + if (failure != null) { + throw (Error) failure; + } } ScopeLease acquire(ScopePath path) { @@ -280,12 +309,12 @@ void removeCache(ScopedMetaCache cache) { caches.remove(cache); } - void register(CacheAddress address, Object versionedValue, ScopeSnapshot snapshot) { - snapshot.leafState().entries.put(address, versionedValue); + void register(CacheAddress address, Object ownershipToken, ScopeSnapshot snapshot) { + snapshot.leafState().entries.put(address, ownershipToken); } - void unregister(CacheAddress address, Object versionedValue, ScopeSnapshot snapshot) { - snapshot.leafState().entries.remove(address, versionedValue); + void unregister(CacheAddress address, Object ownershipToken, ScopeSnapshot snapshot) { + snapshot.leafState().entries.remove(address, ownershipToken); tryPrune(snapshot); } @@ -404,9 +433,9 @@ private void bumpPublicationStates(List nodes) { } private void cleanDetachedState(ScopeState state) { - state.entries.forEach((address, value) -> { - if (state.entries.remove(address, value)) { - address.removeExpected(value); + state.entries.forEach((address, ownershipToken) -> { + if (state.entries.remove(address, ownershipToken)) { + address.removeExpected(ownershipToken); } }); state.children.values().forEach(child -> cleanDetachedState(child.current.get())); @@ -859,6 +888,7 @@ private void cancelPrune() { private static final class ScopeState { private final long generation; private final ConcurrentMap children = new ConcurrentHashMap<>(); + // Ownership tokens contain no cached value, so the registry does not become a second value owner. private final ConcurrentMap entries = new ConcurrentHashMap<>(); private volatile ScopeSnapshot scopeSnapshot; diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java index c0cde92bd92229..08d5808096eb9b 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CacheSpecTest.java @@ -246,4 +246,50 @@ public void isMetaCacheKeyForEngine() { CacheSpec.isMetaCacheKeyForEngine("meta.cache.paimon.table.ttl-second", "iceberg")); Assertions.assertFalse(CacheSpec.isMetaCacheKeyForEngine(null, "iceberg")); } + + @Test + public void parsesAndValidatesWeightProperties() { + Assertions.assertEquals(1024L, CacheSpec.parseWeight("1KB", "weight", false, 0L)); + Assertions.assertEquals(512L, CacheSpec.parseWeight("25%", "weight", true, 2048L)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1.5GB", "weight", false, 0L)); + + Map properties = new HashMap<>(); + properties.put("meta.cache.max-weight", "2GB"); + properties.put("meta.cache.iceberg.table.max-weight", "1GB"); + CacheSpec.checkWeightProperties(properties, "iceberg", "table"); + + properties.put("meta.cache.iceberg.table.max-weight", "3GB"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkWeightProperties(properties, "iceberg", "table")); + + properties.put("meta.cache.iceberg.table.max-weight", "0"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkWeightProperties(properties, "iceberg", "table")); + + properties.put("meta.cache.iceberg.table.max-weight", "1GB"); + properties.put("meta.cache.iceberg.unknown.max-weight", "1MB"); + IllegalArgumentException unknownEntry = Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkWeightProperties(properties, "iceberg", "table")); + Assertions.assertTrue(unknownEntry.getMessage().contains("meta.cache.iceberg.unknown.max-weight")); + + properties.clear(); + properties.put("meta.cache.max-weight", "invalid"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.checkCatalogWeightProperty(properties)); + } + + @Test + public void invalidPersistedEntryWeightFallsBackToUnbounded() { + Map properties = new HashMap<>(); + properties.put("meta.cache.hive.file.max-weight", "invalid"); + CacheSpec spec = CacheSpec.fromProperties( + properties, "hive", "file", CacheSpec.of(true, 60L, 100L)); + Assertions.assertFalse(spec.getMaxWeight().isPresent()); + + properties.put("meta.cache.hive.file.max-weight", "0"); + spec = CacheSpec.fromProperties(properties, "hive", "file", + CacheSpec.of(true, 10L, 100L)); + Assertions.assertFalse(spec.getMaxWeight().isPresent()); + } } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEstimatorBenchmark.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEstimatorBenchmark.java new file mode 100644 index 00000000000000..44182b954b4140 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheEstimatorBenchmark.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Dependency-free microbenchmark for bounded estimators versus an exact list traversal. + * Run after test compilation with the module's target/classes and target/test-classes on the classpath. + */ +public final class MetaCacheEstimatorBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static volatile long blackhole; + + private MetaCacheEstimatorBenchmark() { + } + + public static void main(String[] args) { + for (int size : new int[] {1_000, 100_000}) { + List values = fixture(size); + Result sampled = measure(() -> ReflectiveObjectSizeEstimator.estimate(values), 10_000); + Result typedSampled = measure(() -> JvmSizeUtils.sampledListPayload( + values, 16, JvmSizeUtils::stringSize), 10_000); + int exactOperationsPerWindow = size <= 1_000 ? 1_000 : 20; + Result exact = measure(() -> exactStringListSize(values), exactOperationsPerWindow); + Result boundedAdmission = measure( + () -> completeOrRejected(values), exactOperationsPerWindow); + System.out.printf( + "uniform_size=%d reflective_sampled_ns_op=%d typed_sampled_ns_op=%d " + + "exact_ns_op=%d bounded_admission_ns_op=%d reflective_sampled_ops=%d " + + "typed_sampled_ops=%d exact_ops=%d bounded_admission_ops=%d%n", + size, sampled.medianNanos, typedSampled.medianNanos, exact.medianNanos, + boundedAdmission.medianNanos, sampled.operations, typedSampled.operations, + exact.operations, boundedAdmission.operations); + } + + Map skewed = skewedFixture(); + long sampledBytes = ReflectiveObjectSizeEstimator.estimate(skewed); + long completeBytes = ReflectiveObjectSizeEstimator.estimateComplete(skewed); + Result sampled = measure(() -> ReflectiveObjectSizeEstimator.estimate(skewed), 10_000); + Result complete = measure(() -> ReflectiveObjectSizeEstimator.estimateComplete(skewed), 1_000); + System.out.printf( + "skewed_map_size=%d sampled_bytes=%d complete_bytes=%d sampled_ns_op=%d " + + "complete_ns_op=%d sampled_ops=%d complete_ops=%d%n", + skewed.size(), sampledBytes, completeBytes, sampled.medianNanos, complete.medianNanos, + sampled.operations, complete.operations); + } + + private static List fixture(int size) { + List values = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + values.add("partition-" + i + "-value-0123456789"); + } + return values; + } + + private static long exactStringListSize(List values) { + long bytes = JvmSizeUtils.arrayListSize(values.size()); + for (String value : values) { + bytes = JvmSizeUtils.saturatedAdd(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long completeOrRejected(Object value) { + try { + return ReflectiveObjectSizeEstimator.estimateComplete(value); + } catch (IllegalStateException expected) { + return -1L; + } + } + + private static Map skewedFixture() { + Map values = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + values.put("small-" + i, "x"); + } + values.put("large-tail", "x".repeat(2 * 1024 * 1024)); + return values; + } + + private static Result measure(LongOperation operation, int operationsPerWindow) { + for (int i = 0; i < WARMUP_WINDOWS; i++) { + runWindow(operation, operationsPerWindow); + } + long[] nanosPerOperation = new long[MEASURE_WINDOWS]; + for (int i = 0; i < MEASURE_WINDOWS; i++) { + long start = System.nanoTime(); + runWindow(operation, operationsPerWindow); + nanosPerOperation[i] = (System.nanoTime() - start) / operationsPerWindow; + } + Arrays.sort(nanosPerOperation); + return new Result(nanosPerOperation[MEASURE_WINDOWS / 2], + (long) operationsPerWindow * MEASURE_WINDOWS); + } + + private static void runWindow(LongOperation operation, int operations) { + long value = 0L; + for (int i = 0; i < operations; i++) { + value ^= operation.run(); + } + blackhole = value; + } + + @FunctionalInterface + private interface LongOperation { + long run(); + } + + private static final class Result { + private final long medianNanos; + private final long operations; + + private Result(long medianNanos, long operations) { + this.medianNanos = medianNanos; + this.operations = operations; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java new file mode 100644 index 00000000000000..b7983a4a420e5d --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java @@ -0,0 +1,463 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.time.Duration; +import java.util.AbstractSet; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +class MetaCacheWeightGovernanceTest { + @Test + void admissionRemovalAndCloseKeepHierarchicalAccountingBalanced() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(4096L)); + CatalogMetaCache owner = new CatalogMetaCache(manager, 7L, "iceberg", Collections.emptyMap()); + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.ofWeight(true, -1L, 100L, 2048L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build()); + + entry.put("one", "value"); + Assertions.assertEquals(1024L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(1024L, manager.getGlobalUsedWeight()); + + entry.invalidateKey("one"); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + owner.close(); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + void oversizedValueIsReturnedButNotCached() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 8L, "paimon", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("partition", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(1024L)) + .build()); + + Assertions.assertEquals("value", entry.get("key", ignored -> "value")); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(1L, entry.metrics().getWeightRejectCount()); + Assertions.assertEquals("entry_too_large", entry.metrics().getLastWeightRejectReason()); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void sameKeyReplacementUsesOnlyTheLargerGenerationWeight() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 9L, "iceberg", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build()); + + entry.put("key", "generation-one"); + entry.put("key", "generation-two"); + + Assertions.assertEquals("generation-two", entry.getIfPresent("key")); + Assertions.assertEquals(1024L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(0L, entry.metrics().getWeightRejectCount()); + entry.invalidateKey("key"); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void budgetPressureReclaimsOneColdEntryBeforeTheNextAdmission() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(2048L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 12L, "iceberg", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("partition", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build()); + + entry.put("one", "value-one"); + entry.put("two", "value-two"); + entry.put("three", "value-three"); + + Assertions.assertNull(entry.getIfPresent("three")); + awaitSize(entry, 1L); + entry.put("three", "value-three"); + Assertions.assertEquals("value-three", entry.getIfPresent("three")); + Assertions.assertEquals(2L, entry.size()); + Assertions.assertEquals(2048L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(1L, entry.metrics().getWeightRejectCount()); + Assertions.assertEquals(1L, entry.metrics().getEvictionCount()); + } + } + + @Test + void managedOwnerClosesWhenPhysicalCacheConstructionFails() { + long catalogId = Long.MIN_VALUE + 31L; + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); + CatalogMetaCache owner = CatalogMetaCache.managed(catalogId, "iceberg", Collections.emptyMap()); + + Assertions.assertThrows(IllegalArgumentException.class, () -> owner.create(MetaCacheDefinition + .builder("invalid", CacheSpec.of(true, -1L, -1L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) + .build())); + + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); + } + + @Test + void concurrentRegistrationCannotBeDetachedByLastOwnerRemoval() throws Exception { + long catalogId = Long.MIN_VALUE + 32L; + CatalogMetaCache retiring = CatalogMetaCache.managed( + catalogId, "iceberg", Collections.emptyMap()); + CatalogMetaCache replacement = new CatalogMetaCache( + new ScopedMetaCacheRegistry(), MetaCacheGovernance.budgetManager(), catalogId, + "iceberg", OptionalLong.empty(), true); + BlockingEmptySet owners = new BlockingEmptySet<>(); + owners.add(retiring); + owners.arm(); + catalogCacheRegistry().put(catalogId, owners); + ExecutorService executor = Executors.newFixedThreadPool(2); + Future unregister = null; + Future register = null; + try { + unregister = executor.submit(retiring::close); + Assertions.assertTrue(owners.awaitEmptyObservation(), + "the retiring owner must reach its empty-set decision"); + CountDownLatch registerStarted = new CountDownLatch(1); + register = executor.submit(() -> { + registerStarted.countDown(); + MetaCacheGovernance.register(replacement); + }); + Assertions.assertTrue(registerStarted.await(10L, TimeUnit.SECONDS)); + boolean registeredBeforeRemovalCompleted = owners.awaitAdd(); + + owners.releaseEmptyObservation(); + unregister.get(10L, TimeUnit.SECONDS); + register.get(10L, TimeUnit.SECONDS); + + Assertions.assertFalse(registeredBeforeRemovalCompleted, + "registration must serialize with removal for the same catalog id"); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).contains(replacement), + "the replacement owner must remain discoverable after the retiring owner closes"); + } finally { + owners.releaseEmptyObservation(); + if (unregister != null) { + unregister.cancel(true); + } + if (register != null) { + register.cancel(true); + } + retiring.close(); + replacement.close(); + executor.shutdownNow(); + } + } + + @Test + void estimatorIsNotInvokedWhenNoWeightLimitIsConfigured() { + AtomicInteger estimates = new AtomicInteger(); + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.empty()); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 10L, "hive", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("file", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> { + estimates.incrementAndGet(); + return MetaCacheSizeEstimate.complete(512L); + }) + .build()); + + entry.put("key", "value"); + Assertions.assertEquals("value", entry.getIfPresent("key")); + Assertions.assertFalse(entry.isWeightBounded()); + Assertions.assertEquals(0, estimates.get()); + } + } + + @Test + void estimatorFailureReturnsTheLoadedValueWithoutCachingIt() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(4096L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 11L, "hive", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> { + throw new IllegalStateException("unsupported value layout"); + }) + .build()); + + Assertions.assertEquals("value", entry.get("key", ignored -> "value")); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(1L, entry.metrics().getWeightRejectCount()); + Assertions.assertTrue(entry.metrics().getLastWeightRejectReason() + .startsWith("incomplete_estimate:estimator_failure:")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void overlappingCatalogOwnersShareCatalogBudgetWithoutNameCollision() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(4096L)); + CatalogMetaCache firstOwner = new CatalogMetaCache( + manager, 12L, "iceberg", Collections.emptyMap()); + CatalogMetaCache secondOwner = new CatalogMetaCache( + manager, 12L, "iceberg", Collections.emptyMap()); + try { + MetaCache first = weightedStringEntry(firstOwner, "table", 512L); + MetaCache second = weightedStringEntry(secondOwner, "table", 512L); + + first.put("first", "value"); + second.put("second", "value"); + Assertions.assertEquals(2048L, manager.getGlobalUsedWeight()); + + firstOwner.close(); + Assertions.assertEquals(1024L, manager.getGlobalUsedWeight()); + } finally { + firstOwner.close(); + secondOwner.close(); + } + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + void physicalCachesInOneBudgetGroupShareEntryLimit() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.empty()); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 18L, "iceberg", Collections.emptyMap())) { + CacheSpec spec = CacheSpec.ofWeight(true, -1L, 100L, 1536L); + MetaCache first = owner.create(MetaCacheDefinition + .builder("mvcc-partition-view", spec, ignored -> ScopePath.catalog()) + .budgetGroup("partition_view") + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(1024L)) + .build()); + MetaCache second = owner.create(MetaCacheDefinition + .builder("list-partitions-view", spec, ignored -> ScopePath.catalog()) + .budgetGroup("partition_view") + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(1024L)) + .build()); + + first.put("first", "value"); + second.put("second", "value"); + + Assertions.assertNull(second.getIfPresent("second")); + Assertions.assertTrue(manager.getGlobalUsedWeight() <= 1536L); + } + } + + @Test + void weightedCompareAndSetRejectionCommitsAndInvalidatesOldValue() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + AtomicInteger commits = new AtomicInteger(); + AtomicReference discarded = new AtomicReference<>(); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 13L, "default", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .discardListener((key, value) -> discarded.set(value)) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete( + "old".equals(value) ? 512L : 1024L)) + .build()); + entry.put("key", "old"); + + Assertions.assertTrue(entry.compareAndSet("key", "old", "new", commits::incrementAndGet)); + Assertions.assertEquals(1, commits.get()); + Assertions.assertEquals("new", discarded.get()); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void weightedPutRejectionInvalidatesOldValue() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + AtomicReference discarded = new AtomicReference<>(); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 14L, "default", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .discardListener((key, value) -> discarded.set(value)) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete( + "old".equals(value) ? 512L : 1024L)) + .build()); + entry.put("key", "old"); + + entry.put("key", "new"); + Assertions.assertEquals("new", discarded.get()); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void weightedRefreshRejectionInvalidatesOldValue() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + AtomicInteger loads = new AtomicInteger(); + AtomicReference discarded = new AtomicReference<>(); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 15L, "default", Collections.emptyMap())) { + MetaCache entry = owner.create(MetaCacheDefinition + .builder("schema", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .loader(key -> "v" + loads.incrementAndGet()) + .discardListener((key, value) -> discarded.set(value)) + .refreshAfterWrite(Duration.ofNanos(1L), Runnable::run) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete( + "v1".equals(value) ? 512L : 1024L)) + .build()); + + Assertions.assertEquals("v1", entry.get("key")); + Assertions.assertEquals("v1", entry.get("key")); + Assertions.assertEquals("v2", discarded.get()); + Assertions.assertNull(entry.getIfPresent("key")); + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + } + + @Test + void globalOrCatalogWeightLimitRequiresEstimator() { + MetaCacheBudgetManager globalManager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + globalManager, 16L, "default", Collections.emptyMap())) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> owner.create(unestimatedStringDefinition("global"))); + } + + MetaCacheBudgetManager catalogManager = new MetaCacheBudgetManager(OptionalLong.empty()); + try (CatalogMetaCache owner = new CatalogMetaCache(catalogManager, 17L, "default", + Collections.singletonMap(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"))) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> owner.create(unestimatedStringDefinition("catalog"))); + } + } + + private static MetaCache weightedStringEntry( + CatalogMetaCache owner, String name, long estimatedBytes) { + return owner.create(MetaCacheDefinition + .builder(name, CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(estimatedBytes)) + .build()); + } + + private static MetaCacheDefinition unestimatedStringDefinition(String name) { + return MetaCacheDefinition.builder( + name, CacheSpec.of(true, -1L, 100L), ignored -> ScopePath.catalog()).build(); + } + + private static void awaitSize(MetaCache cache, long expectedSize) { + long deadline = System.nanoTime() + Duration.ofSeconds(10L).toNanos(); + while (cache.size() != expectedSize && System.nanoTime() < deadline) { + Thread.yield(); + } + Assertions.assertEquals(expectedSize, cache.size()); + } + + @SuppressWarnings("unchecked") + private static Map> catalogCacheRegistry() throws ReflectiveOperationException { + Field field = MetaCacheGovernance.class.getDeclaredField("CATALOG_CACHES"); + field.setAccessible(true); + return (Map>) field.get(null); + } + + private static final class BlockingEmptySet extends AbstractSet { + private final Set delegate = java.util.concurrent.ConcurrentHashMap.newKeySet(); + private final CountDownLatch emptyObserved = new CountDownLatch(1); + private final CountDownLatch releaseEmpty = new CountDownLatch(1); + private final CountDownLatch addObserved = new CountDownLatch(1); + private volatile boolean armed; + + void arm() { + armed = true; + } + + boolean awaitEmptyObservation() throws InterruptedException { + return emptyObserved.await(10L, TimeUnit.SECONDS); + } + + boolean awaitAdd() throws InterruptedException { + return addObserved.await(250L, TimeUnit.MILLISECONDS); + } + + void releaseEmptyObservation() { + releaseEmpty.countDown(); + } + + @Override + public Iterator iterator() { + return delegate.iterator(); + } + + @Override + public int size() { + return delegate.size(); + } + + @Override + public boolean add(E value) { + boolean added = delegate.add(value); + if (armed) { + addObserved.countDown(); + } + return added; + } + + @Override + public boolean remove(Object value) { + return delegate.remove(value); + } + + @Override + public boolean isEmpty() { + boolean empty = delegate.isEmpty(); + if (armed && empty) { + emptyObserved.countDown(); + try { + releaseEmpty.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while controlling registry removal", e); + } + } + return empty; + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java new file mode 100644 index 00000000000000..6147ae923a6823 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ReflectiveObjectSizeEstimatorTest.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.AbstractList; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.RandomAccess; +import java.util.concurrent.atomic.AtomicInteger; + +class ReflectiveObjectSizeEstimatorTest { + @Test + void randomAccessContainerTraversalIsBoundedBySampleSize() { + AtomicInteger reads = new AtomicInteger(); + AbstractList millionElements = new CountingList(reads); + + Assertions.assertTrue(ReflectiveObjectSizeEstimator.estimate(millionElements) > 0L); + Assertions.assertEquals(5, reads.get()); + } + + @Test + void typedListSamplingHasBoundedWorkAndIncludesTheTail() { + AtomicInteger reads = new AtomicInteger(); + AtomicInteger largestIndex = new AtomicInteger(); + AbstractList millionElements = new CountingList(reads); + + long estimated = JvmSizeUtils.sampledListPayload(millionElements, 16, value -> { + int index = Integer.parseInt(value.substring("value-".length())); + largestIndex.accumulateAndGet(index, Math::max); + return 1L; + }); + + Assertions.assertEquals(16, reads.get()); + Assertions.assertEquals(999_999, largestIndex.get()); + Assertions.assertEquals(1_000_000L, estimated); + } + + @Test + void completeAdmissionEstimateIncludesAValueOutsideTheFiveElementSample() { + Map skewed = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + skewed.put("small-" + i, "x"); + } + String largeTail = "x".repeat(10_000_000); + skewed.put("large-tail", largeTail); + + long sampled = ReflectiveObjectSizeEstimator.estimate(skewed); + MetaCacheSizeEstimate complete = MetaCacheSizeEstimators + .>reflective().estimate("key", skewed); + + Assertions.assertTrue(complete.isComplete()); + Assertions.assertTrue(complete.getBytes() >= JvmSizeUtils.stringSize(largeTail)); + Assertions.assertTrue(complete.getBytes() > sampled * 100L, + "weighted admission must not accept the fixed five-element sample as complete"); + } + + @Test + void completeAdmissionEstimateRejectsGraphsBeyondItsVisitBudget() { + AtomicInteger reads = new AtomicInteger(); + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "bounded_complete_estimate", + () -> MetaCacheSizeEstimate.complete( + ReflectiveObjectSizeEstimator.estimateComplete(new CountingList(reads)))); + + Assertions.assertFalse(estimate.isComplete()); + Assertions.assertTrue(reads.get() <= 10_001, + "complete fallback must stop when its work budget is exhausted"); + } + + @Test + void inaccessibleJdkReferenceFieldMakesTheEstimateIncomplete() { + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "reflection_failure", + () -> MetaCacheSizeEstimate.complete( + ReflectiveObjectSizeEstimator.estimate(URI.create("s3://bucket/path")))); + + Assertions.assertFalse(estimate.isComplete()); + Assertions.assertTrue(estimate.getIncompleteReason().startsWith("reflection_failure:")); + } + + @Test + void depthTruncationMakesTheEstimateIncomplete() { + Node root = new Node(); + Node current = root; + for (int i = 0; i < 25; i++) { + current.next = new Node(); + current = current.next; + } + + MetaCacheSizeEstimate estimate = MetaCacheSizeEstimator.estimateSafely( + "depth_limit", + () -> MetaCacheSizeEstimate.complete(ReflectiveObjectSizeEstimator.estimate(root))); + + Assertions.assertFalse(estimate.isComplete()); + Assertions.assertTrue(estimate.getIncompleteReason().startsWith("depth_limit:")); + } + + private static final class CountingList extends AbstractList implements RandomAccess { + private final AtomicInteger reads; + + private CountingList(AtomicInteger reads) { + this.reads = reads; + } + + @Override + public String get(int index) { + reads.incrementAndGet(); + return "value-" + index; + } + + @Override + public int size() { + return 1_000_000; + } + } + + private static final class Node { + private Node next; + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java index 983e9368b5f707..5093e47b8936bf 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/ScopedMetaCacheConcurrencyTest.java @@ -24,7 +24,9 @@ import java.time.Duration; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.OptionalLong; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -1187,6 +1189,47 @@ public void concurrentPutEvictAndInvalidationFinishWithoutDeadlock() throws Exce } } + @Test + public void concurrentWeightedReplacementsDoNotEvictAcrossKeyLocks() throws Exception { + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch replacementsEstimating = new CountDownLatch(2); + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(2048L)); + try (CatalogMetaCache owner = new CatalogMetaCache( + manager, 41L, "iceberg", Collections.emptyMap())) { + MetaCache cache = owner.create(MetaCacheDefinition + .builder("table", CacheSpec.of(true, -1L, 100L), + ignored -> ScopePath.catalog()) + .sizeEstimator((key, value) -> { + if (value.startsWith("new")) { + replacementsEstimating.countDown(); + await(replacementsEstimating); + return MetaCacheSizeEstimate.complete(1024L); + } + return MetaCacheSizeEstimate.complete(512L); + }) + .build()); + cache.put("a", "old-a"); + cache.put("b", "old-b"); + + Future first = executor.submit(() -> { + await(start); + cache.put("a", "new-a"); + }); + Future second = executor.submit(() -> { + await(start); + cache.put("b", "new-b"); + }); + start.countDown(); + + first.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + second.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assertions.assertTrue(manager.getGlobalUsedWeight() <= 2048L); + } finally { + executor.shutdownNow(); + } + } + @Test public void concurrentSameKeyPublicationsLeaveOneExactRegistration() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(4); diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java index d38571a11f2503..06a8a15acbae7b 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveCatalogProperties.java @@ -179,6 +179,10 @@ public static HiveCatalogProperties of(Map properties) { * @return this, so the provider's door reads as one statement */ public HiveCatalogProperties checkCreateTimeOnlyRules() { + CacheSpec.checkWeightProperties(raw, "hive", + "table", "partition_names", "partition", "column_stats", "file", "partition_view"); + CacheSpec.checkWeightProperties(raw, "iceberg", + "table", "partition", "manifest", "partition_view"); // Restores the legacy HMSExternalCatalog.checkProperties fail-fast for the two meta-cache TTL // knobs: after the hms cutover an "hms" catalog is created via the SPI provider (not // HMSExternalCatalog), so the old per-property validation no longer ran and an invalid ttl (e.g. diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index cb4f31bf40506d..6d8f5ffa8b63fd 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -87,7 +87,7 @@ public class HiveConnector implements Connector { // every (re)build of the connector, including the lazy one an FE does after replaying the edit log. private final HiveCatalogProperties props; private final ConnectorContext context; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; private volatile HmsClient hmsClient; // Lazily-built plugin-side Kerberos authenticator (single-owner auth), null for a non-Kerberos catalog. @@ -133,17 +133,24 @@ public class HiveConnector implements Connector { // NEVER cast (a cast would CCE across the loader split). private volatile Connector hudiSibling; + // Writes are guarded by this; the volatile read rejects post-close fast paths. Closing and lazy sibling + // publication share the same monitor so a sibling can never be published after the gateway has released its + // managed metadata-cache owner. + private volatile boolean closed; + public HiveConnector(Map properties, ConnectorContext context) { HmsConfHelper.initializeHadoopConfigDir(context); this.props = HiveCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hive", this.properties); this.fileListingCache = new HiveFileListingCache(metaCache, props); // Reads its own meta.cache.hive.partition_view.(enable|ttl-second|capacity) from the catalog properties // via the framework's CacheSpec (default ON / 24h / 1000). this.partitionViewCache = new ConnectorMetadataCache<>(metaCache, "hive-partition-view", "hive", "partition_view", this.properties, - key -> ScopePath.partitionCollection(key.getDb(), key.getTable())); + key -> ScopePath.partitionCollection(key.getDb(), key.getTable()), + HivePartitionViewSizeEstimator::estimateEntry); } @Override @@ -483,36 +490,40 @@ private HmsClient getOrCreateClient() { * memoized (a null sibling leaves the field unset), so a later-available plugin recovers on the next access. */ Connector getOrCreateIcebergSibling() { - if (icebergSibling == null) { - synchronized (this) { - if (icebergSibling == null) { - Connector sibling = context.createSiblingConnector( - ICEBERG_CONNECTOR_TYPE, IcebergSiblingProperties.synthesize(properties)); - if (sibling == null) { - throw new DorisConnectorException( - "Cannot serve iceberg-on-HMS tables in catalog '" + context.getCatalogName() - + "': the iceberg connector plugin is not available"); - } - // Fail-loud invariant guard for the cache-isolation security track: the hive gateway FRONT - // DOOR never declares SUPPORTS_USER_SESSION, so fe-core keys its per-user schema/name cache - // bypass off THIS (front-door) connector's capabilities and would NOT bypass for a delegated - // sibling. The iceberg sibling is forced iceberg.catalog.type=hms (IcebergSiblingProperties - // .synthesize) and can never be REST session=user, so this must hold today. If a future change - // ever let the sibling be session=user, the front-door-only bypass would silently leak - // cross-user metadata — fail here instead. - if (sibling.getCapabilities().contains(ConnectorCapability.SUPPORTS_USER_SESSION)) { - throw new DorisConnectorException( - "iceberg-on-HMS sibling in catalog '" + context.getCatalogName() - + "' unexpectedly declares SUPPORTS_USER_SESSION: the hive gateway front " - + "door is not session=user, so fe-core's per-user schema/name cache bypass " - + "would not trigger and cross-user metadata would leak. The sibling must " - + "stay iceberg.catalog.type=hms (never REST session=user)."); - } - icebergSibling = sibling; + checkOpenForSiblingCreation(); + Connector current = icebergSibling; + if (current != null) { + return current; + } + synchronized (this) { + checkOpenForSiblingCreation(); + if (icebergSibling == null) { + Connector sibling = context.createSiblingConnector( + ICEBERG_CONNECTOR_TYPE, IcebergSiblingProperties.synthesize(properties)); + if (sibling == null) { + throw new DorisConnectorException( + "Cannot serve iceberg-on-HMS tables in catalog '" + context.getCatalogName() + + "': the iceberg connector plugin is not available"); + } + // Fail-loud invariant guard for the cache-isolation security track: the hive gateway FRONT + // DOOR never declares SUPPORTS_USER_SESSION, so fe-core keys its per-user schema/name cache + // bypass off THIS (front-door) connector's capabilities and would NOT bypass for a delegated + // sibling. The iceberg sibling is forced iceberg.catalog.type=hms (IcebergSiblingProperties + // .synthesize) and can never be REST session=user, so this must hold today. If a future change + // ever let the sibling be session=user, the front-door-only bypass would silently leak + // cross-user metadata — fail here instead. + if (sibling.getCapabilities().contains(ConnectorCapability.SUPPORTS_USER_SESSION)) { + throw new DorisConnectorException( + "iceberg-on-HMS sibling in catalog '" + context.getCatalogName() + + "' unexpectedly declares SUPPORTS_USER_SESSION: the hive gateway front " + + "door is not session=user, so fe-core's per-user schema/name cache bypass " + + "would not trigger and cross-user metadata would leak. The sibling must " + + "stay iceberg.catalog.type=hms (never REST session=user)."); } + icebergSibling = sibling; } + return icebergSibling; } - return icebergSibling; } /** @@ -527,21 +538,33 @@ Connector getOrCreateIcebergSibling() { * memoized (a null sibling leaves the field unset), so a later-available plugin recovers on the next access. */ Connector getOrCreateHudiSibling() { - if (hudiSibling == null) { - synchronized (this) { - if (hudiSibling == null) { - Connector sibling = context.createSiblingConnector( - HUDI_CONNECTOR_TYPE, HudiSiblingProperties.synthesize(properties)); - if (sibling == null) { - throw new DorisConnectorException( - "Cannot serve hudi-on-HMS tables in catalog '" + context.getCatalogName() - + "': the hudi connector plugin is not available"); - } - hudiSibling = sibling; + checkOpenForSiblingCreation(); + Connector current = hudiSibling; + if (current != null) { + return current; + } + synchronized (this) { + checkOpenForSiblingCreation(); + if (hudiSibling == null) { + Connector sibling = context.createSiblingConnector( + HUDI_CONNECTOR_TYPE, HudiSiblingProperties.synthesize(properties)); + if (sibling == null) { + throw new DorisConnectorException( + "Cannot serve hudi-on-HMS tables in catalog '" + context.getCatalogName() + + "': the hudi connector plugin is not available"); } + hudiSibling = sibling; } + return hudiSibling; + } + } + + private void checkOpenForSiblingCreation() { + if (closed) { + throw new DorisConnectorException( + "Cannot create a sibling connector after Hive catalog '" + context.getCatalogName() + + "' has been closed"); } - return hudiSibling; } private HmsClient createClient() { @@ -697,24 +720,57 @@ private static Configuration buildHmsConf(AbstractHmsMetaStoreProperties hms) { @Override public void close() throws IOException { - metaCache.close(); - HmsClient c = hmsClient; - if (c != null) { - c.close(); + HmsClient client; + Connector sibling; + Connector hudi; + synchronized (this) { + if (closed) { + return; + } + closed = true; + client = hmsClient; hmsClient = null; + sibling = icebergSibling; + icebergSibling = null; + hudi = hudiSibling; + hudiSibling = null; + } + metaCache.close(); + IOException closeFailure = null; + try { + if (client != null) { + client.close(); + } + } catch (IOException e) { + closeFailure = e; } // Forward close to the embedded iceberg sibling: the engine closes only a catalog's PRIMARY connector, // so the gateway owns the sibling's lifecycle. No-op when the sibling was never built (dormant path). - Connector sibling = icebergSibling; - if (sibling != null) { - sibling.close(); - icebergSibling = null; + try { + if (sibling != null) { + sibling.close(); + } + } catch (IOException e) { + if (closeFailure == null) { + closeFailure = e; + } else { + closeFailure.addSuppressed(e); + } } // Same for the embedded hudi sibling — the gateway owns its lifecycle too. No-op when never built. - Connector hudi = hudiSibling; - if (hudi != null) { - hudi.close(); - hudiSibling = null; + try { + if (hudi != null) { + hudi.close(); + } + } catch (IOException e) { + if (closeFailure == null) { + closeFailure = e; + } else { + closeFailure.addSuppressed(e); + } + } + if (closeFailure != null) { + throw closeFailure; } } } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java index 57cc3c0c342971..0981c94f781435 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingCache.java @@ -21,6 +21,8 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.DorisConnectorException; import org.apache.doris.filesystem.FileEntry; @@ -107,7 +109,7 @@ interface DirectoryLister { } private final CatalogMetaCache owner; - private final MetaCache> cache; + private final MetaCache cache; private final DirectoryLister lister; public HiveFileListingCache(HiveCatalogProperties properties) { @@ -149,10 +151,11 @@ private static DirectoryLister defaultLister(HiveCatalogProperties properties) { CacheSpec spec = CacheSpec.fromProperties(props, ENGINE, ENTRY_FILE, CacheSpec.of(true, DEFAULT_TTL_SECOND, DEFAULT_FILE_CAPACITY)); this.cache = owner.create(MetaCacheDefinition - .>builder("hive-file", spec, + .builder("hive-file", spec, key -> key.partitionValues.isEmpty() ? ScopePath.table(key.dbName, key.tableName) : ScopePath.partition(key.dbName, key.tableName, key.partitionValues)) + .sizeEstimator(HiveFileListingSizeEstimator::estimateEntry) .build()); this.lister = Objects.requireNonNull(lister, "lister can not be null"); } @@ -180,7 +183,7 @@ public List listDataFiles(String dbName, String tableName, Strin public List listDataFiles(String dbName, String tableName, String location, List partitionValues, FileSystem fs) { return cache.get(new FileListingKey(dbName, tableName, location, partitionValues), - key -> lister.list(key.location, fs)); + key -> new FileListingValue(lister.list(key.location, fs), cache.isWeightBounded())).files; } /** Drops every cached listing for one table. Backs {@code REFRESH TABLE}. */ @@ -345,10 +348,10 @@ private static boolean isSystemicResolutionFailure(Throwable t) { * size-estimate paths sharing the same entry while making per-partition invalidation possible. */ static final class FileListingKey { - private final String dbName; - private final String tableName; - private final String location; - private final List partitionValues; + final String dbName; + final String tableName; + final String location; + final List partitionValues; FileListingKey(String dbName, String tableName, String location, List partitionValues) { this.dbName = dbName; @@ -379,4 +382,20 @@ public int hashCode() { return Objects.hash(dbName, tableName, location, partitionValues); } } + + static final class FileListingValue { + final List files; + final MetaCacheSizeEstimate sizeEstimate; + + FileListingValue(List files, boolean estimateWeight) { + this.files = estimateWeight + ? Collections.unmodifiableList(new ArrayList<>(files)) + : files; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("hive_file_listing_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + HiveFileListingSizeEstimator.estimateValue(this))) + : MetaCacheSizeEstimate.complete(0L); + } + } } diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java new file mode 100644 index 00000000000000..9c7ab5e2c7f0dd --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveFileListingSizeEstimator.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingKey; +import org.apache.doris.connector.hive.HiveFileListingCache.FileListingValue; + +/** Type-specific retained-heap estimator for one Hive directory-listing cache entry. */ +final class HiveFileListingSizeEstimator { + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(FileListingKey.class); + private static final long VALUE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(FileListingValue.class); + private static final long FILE_STATUS_SHALLOW_BYTES = JvmSizeUtils.instanceSize(HiveFileStatus.class); + + private HiveFileListingSizeEstimator() { + } + + /** Admission callback: the large value was sized once during construction. */ + static MetaCacheSizeEstimate estimateEntry(FileListingKey key, FileListingValue value) { + return value.sizeEstimate.isComplete() + ? MetaCacheSizeEstimate.complete(add(estimateKey(key), value.sizeEstimate.getBytes())) + : value.sizeEstimate; + } + + static long estimateKey(FileListingKey key) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.dbName)); + bytes = add(bytes, JvmSizeUtils.stringSize(key.tableName)); + bytes = add(bytes, JvmSizeUtils.stringSize(key.location)); + bytes = add(bytes, estimateOwnedStringList(key.partitionValues)); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(key)); + } + + static long estimateValue(FileListingValue value) { + long bytes = VALUE_SHALLOW_BYTES; + bytes = add(bytes, estimateArrayBackedList(value.files)); + for (HiveFileStatus file : value.files) { + bytes = add(bytes, FILE_STATUS_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(file.getPath())); + } + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(value)); + } + + private static long estimateOwnedStringList(java.util.List values) { + long bytes = estimateArrayBackedList(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateArrayBackedList(java.util.List values) { + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + return add(bytes, JvmSizeUtils.arrayListSize(values.size())); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HivePartitionViewSizeEstimator.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HivePartitionViewSizeEstimator.java new file mode 100644 index 00000000000000..0284e23023cf90 --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HivePartitionViewSizeEstimator.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Bounded type-specific estimator for Hive's large derived partition view. */ +final class HivePartitionViewSizeEstimator { + private static final int PARTITION_SAMPLE_SIZE = 16; + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorTableKey.class); + private static final long PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class); + private static final long ARRAY_LIST_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ArrayList.class); + private static final long UNMODIFIABLE_LIST_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableList(Collections.emptyList()).getClass()); + private static final long UNMODIFIABLE_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableMap(Collections.emptyMap()).getClass()); + private static final long LINKED_HASH_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize(LinkedHashMap.class); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + + private HivePartitionViewSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateEntry(ConnectorTableKey key, List partitions) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb())); + bytes = add(bytes, JvmSizeUtils.stringSize(key.getTable())); + bytes = add(bytes, ARRAY_LIST_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.objectArraySize(partitions.size())); + bytes = add(bytes, JvmSizeUtils.sampledListPayload( + partitions, PARTITION_SAMPLE_SIZE, HivePartitionViewSizeEstimator::estimatePartition)); + return MetaCacheSizeEstimate.complete( + Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(partitions))); + } + + private static long estimatePartition(ConnectorPartitionInfo partition) { + long bytes = PARTITION_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getPartitionName())); + bytes = add(bytes, estimateStringMap(partition.getPartitionValues())); + bytes = add(bytes, estimateStringMap(partition.getProperties())); + bytes = add(bytes, estimateStringList(partition.getOrderedPartitionValues())); + return add(bytes, estimateReferenceList(partition.getPartitionValueNullFlags())); + } + + private static long estimateStringMap(Map values) { + long bytes = UNMODIFIABLE_MAP_SHALLOW_BYTES; + if (values.isEmpty()) { + return bytes; + } + bytes = add(bytes, LINKED_HASH_MAP_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(values.size()))); + bytes = add(bytes, JvmSizeUtils.saturatedMultiply( + values.size(), LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES)); + for (Map.Entry entry : values.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getValue())); + } + return bytes; + } + + private static long estimateStringList(List values) { + long bytes = estimateReferenceList(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateReferenceList(List values) { + return add(UNMODIFIABLE_LIST_SHALLOW_BYTES, JvmSizeUtils.arrayListSize(values.size())); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java index 220261a5673e7a..b68ca5fa80393e 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveCatalogPropertiesTest.java @@ -215,6 +215,19 @@ void badPartitionCacheTtlFailsTheCreateTimeDoor() { () -> HiveCatalogProperties.of(m).checkCreateTimeOnlyRules()); } + @Test + void icebergSiblingWeightsAreValidatedAtTheHmsCreateTimeDoor() { + Map invalidWeight = HiveTestProperties.mapWith( + "meta.cache.iceberg.manifest.max-weight", "invalid"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> HiveCatalogProperties.of(invalidWeight).checkCreateTimeOnlyRules()); + + Map unknownEntry = HiveTestProperties.mapWith( + "meta.cache.iceberg.manfiest.max-weight", "64MB"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> HiveCatalogProperties.of(unknownEntry).checkCreateTimeOnlyRules()); + } + @Test void validCatalogPassesBothDoors() { Assertions.assertDoesNotThrow(() -> HiveTestProperties.minimal().checkCreateTimeOnlyRules()); diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java index d2438edbd9fe7e..33cce0703d201e 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataPartitionViewCacheTest.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.hive; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsDatabaseInfo; import org.apache.doris.connector.hms.HmsPartitionInfo; @@ -32,6 +33,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -94,6 +96,20 @@ private static List names(List infos) { return infos.stream().map(ConnectorPartitionInfo::getPartitionName).collect(Collectors.toList()); } + @Test + public void largePartitionViewCanBeEstimatedWithoutTheReflectiveVisitLimit() { + List partitions = new ArrayList<>(); + for (int index = 0; index < 20_000; index++) { + String value = Integer.toString(index); + partitions.add(new ConnectorPartitionInfo("p=" + value, + Collections.singletonMap("p", value), Collections.emptyMap(), + Collections.singletonList(value), Collections.emptyList())); + } + + Assertions.assertTrue(HivePartitionViewSizeEstimator.estimateEntry( + new ConnectorTableKey("db", "table", -1L, -1L), partitions).isComplete()); + } + @Test public void listPartitionsCachesDerivedListAcrossQueries() { // WHY: cache A must memoize the BUILT List keyed by (db, table, -1, -1), so a diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java index 39727fe101bd41..8b7916c700fbe5 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java @@ -136,6 +136,19 @@ public void closeIsNoOpWhenSiblingNeverBuilt() throws Exception { Assertions.assertEquals(0, context.buildCount, "close must not trigger a sibling build"); } + @Test + public void closePreventsLateSiblingCreation() throws Exception { + RecordingSiblingContext context = new RecordingSiblingContext(new FakeSibling()); + HiveConnector connector = new HiveConnector(HiveTestProperties.minimalMap(), context); + + connector.close(); + + Assertions.assertThrows(DorisConnectorException.class, connector::getOrCreateIcebergSibling); + Assertions.assertThrows(DorisConnectorException.class, connector::getOrCreateHudiSibling); + Assertions.assertEquals(0, context.buildCount, + "a closed gateway must not publish a new managed sibling cache owner"); + } + // ---- hudi sibling holder (mirrors the iceberg cases above; hudi synthesizes props verbatim, no flavor) ---- @Test diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java index 326beba4d61f86..d89c375322ba39 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveFileListingCacheTest.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.hive; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.hms.HmsPartitionInfo; import org.apache.doris.connector.spi.ConnectorSession; import org.apache.doris.connector.spi.DorisConnectorException; @@ -91,6 +92,22 @@ public void listingIsCachedPerLocation() { Assertions.assertEquals(2, lister.totalCalls); } + @Test + public void weightBoundedListingIsEstimatedAndCached() { + CountingLister lister = new CountingLister(); + Map properties = props("meta.cache.hive.file.max-weight", "1MB"); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + HiveFileListingCache cache = new HiveFileListingCache( + owner, HiveCatalogProperties.of(properties), lister); + + List first = cache.listDataFiles("db", "t", "loc", FS); + List second = cache.listDataFiles("db", "t", "loc", FS); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, lister.totalCalls); + } + } + @Test public void keyIsScopedByDbTableAndLocation() { CountingLister lister = new CountingLister(); diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java index d7b5edfc0d6aad..e9dab8e3bb1059 100644 --- a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/CachingHmsClient.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; import org.apache.hadoop.hive.common.FileUtils; @@ -123,23 +124,28 @@ public CachingHmsClient(CatalogMetaCache owner, HmsClient delegate, Map props = applyLegacyTtlCompatibility( properties == null ? Collections.emptyMap() : properties); this.tableCache = newEntry(owner, "hive-table", props, ENTRY_TABLE, DEFAULT_TABLE_CAPACITY, - key -> ScopePath.table(key.dbName, key.tableName)); + key -> ScopePath.table(key.dbName, key.tableName), HmsCacheSizeEstimator::estimateTable); this.partitionNamesCache = newEntry(owner, "hive-partition-names", props, ENTRY_PARTITION_NAMES, DEFAULT_PARTITION_NAMES_CAPACITY, - key -> ScopePath.partitionCollection(key.dbName, key.tableName)); + key -> ScopePath.partitionCollection(key.dbName, key.tableName), + HmsCacheSizeEstimator::estimatePartitionNames); this.partitionsCache = newEntry(owner, "hive-partition", props, ENTRY_PARTITION, DEFAULT_PARTITION_CAPACITY, - key -> ScopePath.partition(key.dbName, key.tableName, key.values)); + key -> ScopePath.partition(key.dbName, key.tableName, key.values), + HmsCacheSizeEstimator::estimatePartition); this.columnStatsCache = newEntry(owner, "hive-column-stats", props, ENTRY_COLUMN_STATS, DEFAULT_COLUMN_STATS_CAPACITY, - key -> ScopePath.table(key.dbName, key.tableName)); + key -> ScopePath.table(key.dbName, key.tableName), HmsCacheSizeEstimator::estimateColumnStats); } private static MetaCache newEntry(CatalogMetaCache owner, String name, - Map props, String entry, long defaultCapacity, Function scopeResolver) { + Map props, String entry, long defaultCapacity, Function scopeResolver, + MetaCacheSizeEstimator sizeEstimator) { CacheSpec spec = CacheSpec.fromProperties(props, ENGINE, entry, CacheSpec.of(true, DEFAULT_TTL_SECOND, defaultCapacity)); - return owner.create(MetaCacheDefinition.builder(name, spec, scopeResolver).build()); + return owner.create(MetaCacheDefinition.builder(name, spec, scopeResolver) + .sizeEstimator(sizeEstimator) + .build()); } /** Legacy fe-core catalog knob ({@code ExternalCatalog.SCHEMA_CACHE_TTL_SECOND}) for the table/schema cache. */ diff --git a/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsCacheSizeEstimator.java b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsCacheSizeEstimator.java new file mode 100644 index 00000000000000..59dfd6596e26c4 --- /dev/null +++ b/fe/fe-connector/fe-connector-hms/src/main/java/org/apache/doris/connector/hms/HmsCacheSizeEstimator.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hms; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; + +import java.util.List; + +/** Construction-time retained-size formulas for HMS cache entries. */ +final class HmsCacheSizeEstimator { + private static final long COLUMN_STATS_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(HmsColumnStatistics.class); + + private HmsCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateTable(Object key, HmsTableInfo value) { + return complete(key, value); + } + + static MetaCacheSizeEstimate estimatePartition(Object key, HmsPartitionInfo value) { + return complete(key, value); + } + + static MetaCacheSizeEstimate estimatePartitionNames(Object key, List value) { + return MetaCacheSizeEstimate.complete(add( + ReflectiveObjectSizeEstimator.estimateComplete(key), estimateStringList(value))); + } + + static MetaCacheSizeEstimate estimateColumnStats(Object key, List value) { + long bytes = add(JvmSizeUtils.instanceSize(value.getClass()), + JvmSizeUtils.objectArraySize(value.size())); + for (HmsColumnStatistics stats : value) { + bytes = add(bytes, COLUMN_STATS_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(stats.getColumnName())); + } + return MetaCacheSizeEstimate.complete(add( + ReflectiveObjectSizeEstimator.estimateComplete(key), bytes)); + } + + private static MetaCacheSizeEstimate complete(Object key, Object value) { + return MetaCacheSizeEstimate.complete(add( + ReflectiveObjectSizeEstimator.estimateComplete(key), + ReflectiveObjectSizeEstimator.estimateComplete(value))); + } + + private static long estimateStringList(List values) { + long bytes = add(JvmSizeUtils.instanceSize(values.getClass()), + JvmSizeUtils.objectArraySize(values.size())); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java index 36e2d007f7eadd..9e95fde84352b7 100644 --- a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java +++ b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/CachingHmsClientTest.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -82,6 +83,47 @@ public void getTableCachesByDbAndTable() { Assertions.assertEquals(2, delegate.getTableCalls); } + @Test + public void weightedTableRejectsALargePropertyOutsideTheOldFiveElementSample() { + RecordingHmsClient delegate = new RecordingHmsClient(); + Map parameters = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + parameters.put("small-" + i, "x"); + } + parameters.put("large-tail", "x".repeat(2 * 1024 * 1024)); + delegate.tableResult = HmsTableInfo.builder() + .dbName("db") + .tableName("t") + .parameters(parameters) + .build(); + CachingHmsClient cache = cache(delegate, + props("meta.cache.hive.table.max-weight", "1MB")); + + cache.getTable("db", "t"); + cache.getTable("db", "t"); + + Assertions.assertEquals(2, delegate.getTableCalls, + "the 2MB tail must reject weighted admission instead of being hidden by five small samples"); + } + + @Test + public void weightedTableCachesSmallCompleteMetadata() { + RecordingHmsClient delegate = new RecordingHmsClient(); + delegate.tableResult = HmsTableInfo.builder() + .dbName("db") + .tableName("t") + .parameters(Map.of("format", "parquet")) + .build(); + CachingHmsClient cache = cache(delegate, + props("meta.cache.hive.table.max-weight", "1MB")); + + cache.getTable("db", "t"); + cache.getTable("db", "t"); + + Assertions.assertEquals(1, delegate.getTableCalls, + "a complete small HMS graph must still be admitted to the weighted cache"); + } + @Test public void cacheKeysAreScopedByDatabase() { RecordingHmsClient delegate = new RecordingHmsClient(); @@ -608,6 +650,7 @@ private static final class RecordingHmsClient implements HmsClient { int dropTableCalls; int closeCalls; RuntimeException getTableError; + HmsTableInfo tableResult; // Partition names the fake has NO partition for (mirrors HMS omitting non-existent partitions). final Set absentPartitionNames = new HashSet<>(); // When set, every returned partition carries these exact values regardless of the requested name @@ -626,7 +669,9 @@ public HmsTableInfo getTable(String dbName, String tableName) { if (getTableError != null) { throw getTableError; } - return HmsTableInfo.builder().dbName(dbName).tableName(tableName).build(); + return tableResult == null + ? HmsTableInfo.builder().dbName(dbName).tableName(tableName).build() + : tableResult; } @Override diff --git a/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsCacheSizeEstimatorBenchmark.java b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsCacheSizeEstimatorBenchmark.java new file mode 100644 index 00000000000000..b3a6a41578caa1 --- /dev/null +++ b/fe/fe-connector/fe-connector-hms/src/test/java/org/apache/doris/connector/hms/HmsCacheSizeEstimatorBenchmark.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hms; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Dependency-free microbenchmark for the production HMS admission estimators. */ +public final class HmsCacheSizeEstimatorBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static volatile long blackhole; + + private HmsCacheSizeEstimatorBenchmark() { + } + + public static void main(String[] args) { + for (int size : new int[] {1_000, 100_000}) { + List names = partitionNames(size); + int operationsPerWindow = size <= 1_000 ? 1_000 : 20; + Result result = measure( + () -> HmsCacheSizeEstimator.estimatePartitionNames("db.t", names).getBytes(), + operationsPerWindow); + System.out.printf("hms_partition_names=%d estimated_bytes=%d ns_op=%d operations=%d%n", + size, HmsCacheSizeEstimator.estimatePartitionNames("db.t", names).getBytes(), + result.medianNanos, result.operations); + } + + HmsTableInfo table = skewedTable(); + Result tableResult = measure( + () -> HmsCacheSizeEstimator.estimateTable("db.t", table).getBytes(), 1_000); + System.out.printf("hms_table_properties=%d estimated_bytes=%d ns_op=%d operations=%d%n", + table.getParameters().size(), HmsCacheSizeEstimator.estimateTable("db.t", table).getBytes(), + tableResult.medianNanos, tableResult.operations); + } + + private static List partitionNames(int size) { + List values = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + values.add("dt=2026-09-01/hour=" + (i % 24) + "/bucket=" + i); + } + return values; + } + + private static HmsTableInfo skewedTable() { + Map parameters = new LinkedHashMap<>(); + for (int i = 0; i < 99; i++) { + parameters.put("small-" + i, "x"); + } + parameters.put("large-tail", "x".repeat(2 * 1024 * 1024)); + return HmsTableInfo.builder() + .dbName("db") + .tableName("t") + .parameters(parameters) + .build(); + } + + private static Result measure(LongOperation operation, int operationsPerWindow) { + for (int i = 0; i < WARMUP_WINDOWS; i++) { + runWindow(operation, operationsPerWindow); + } + long[] nanosPerOperation = new long[MEASURE_WINDOWS]; + for (int i = 0; i < MEASURE_WINDOWS; i++) { + long start = System.nanoTime(); + runWindow(operation, operationsPerWindow); + nanosPerOperation[i] = (System.nanoTime() - start) / operationsPerWindow; + } + Arrays.sort(nanosPerOperation); + return new Result(nanosPerOperation[MEASURE_WINDOWS / 2], + (long) operationsPerWindow * MEASURE_WINDOWS); + } + + private static void runWindow(LongOperation operation, int operations) { + long value = 0L; + for (int i = 0; i < operations; i++) { + value ^= operation.run(); + } + blackhole = value; + } + + @FunctionalInterface + private interface LongOperation { + long run(); + } + + private static final class Result { + private final long medianNanos; + private final long operations; + + private Result(long medianNanos, long operations) { + this.medianNanos = medianNanos; + this.operations = operations; + } + } +} diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java index 3832f57bd7c894..25f46cc58875cf 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java @@ -68,7 +68,7 @@ public class HudiConnector implements Connector { private final HudiCatalogProperties props; private final Map properties; private final ConnectorContext context; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; private volatile HmsClient hmsClient; // HMS and storage deliberately have separate authenticators: hive.metastore.username must affect set_ugi @@ -83,6 +83,7 @@ public HudiConnector(Map properties, ConnectorContext context) { this.props = HudiCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hudi", this.properties); } @Override diff --git a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java index f06b3d081fce61..34ea89ad499b31 100644 --- a/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java +++ b/fe/fe-connector/fe-connector-hudi/src/test/java/org/apache/doris/connector/hudi/HudiConnectorHmsCacheTest.java @@ -17,6 +17,9 @@ package org.apache.doris.connector.hudi; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheGovernance; import org.apache.doris.connector.hms.CachingHmsClient; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsDatabaseInfo; @@ -24,11 +27,16 @@ import org.apache.doris.connector.hms.HmsTableInfo; import org.apache.doris.connector.spi.ConnectorContext; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; @@ -57,6 +65,73 @@ public class HudiConnectorHmsCacheTest { private static final List YEAR_MONTH = Arrays.asList("year", "month"); private static final List ONE_PARTITION = Collections.singletonList("year=2024/month=01"); + private final List connectors = new ArrayList<>(); + + @AfterEach + void closeConnectors() throws Exception { + for (HudiConnector connector : connectors) { + connector.close(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"meta.cache.max-weight", "meta.cache.hive.partition_names.max-weight"}) + void weightLimitAccountsHmsCacheAndRejectsOversizedValues(String property) throws Exception { + long catalogId = Long.MIN_VALUE + 71L; + long previousGlobalWeight = MetaCacheGovernance.globalEstimatedWeight(); + Map properties = new HashMap<>(HudiTestProperties.minimalMap()); + properties.put(property, "4KB"); + HudiConnector connector = connector(properties, catalogId); + List owners = MetaCacheGovernance.catalogCaches(catalogId); + Assertions.assertEquals(1, owners.size()); + CatalogMetaCache owner = owners.get(0); + Assertions.assertEquals("hudi", owner.engine()); + + FakeHmsClient delegate = new FakeHmsClient(ONE_PARTITION); + HmsClient cache = connector.wrapWithCache(delegate); + Assertions.assertEquals(4, owner.entries().size()); + MetaCache entry = owner.entries().get("hive-partition-names"); + Assertions.assertTrue(entry.isWeightBounded()); + Assertions.assertEquals(4096L, entry.metrics().getMaxWeight()); + Assertions.assertEquals("meta.cache.max-weight".equals(property), + owner.entries().get("hive-table").isWeightBounded()); + + Assertions.assertEquals(ONE_PARTITION, cache.listPartitionNames("db", "t", -1)); + Assertions.assertEquals(ONE_PARTITION, cache.listPartitionNames("db", "t", -1)); + Assertions.assertEquals(1, delegate.cachedCalls); + long retainedWeight = entry.metrics().getEstimatedWeight(); + Assertions.assertTrue(retainedWeight > 0L && retainedWeight <= 4096L); + Assertions.assertEquals(previousGlobalWeight + retainedWeight, + MetaCacheGovernance.globalEstimatedWeight()); + + delegate.names = Collections.singletonList("partition=" + "x".repeat(8192)); + Assertions.assertEquals(delegate.names, cache.listPartitionNames("db", "large", -1)); + Assertions.assertEquals(delegate.names, cache.listPartitionNames("db", "large", -1)); + Assertions.assertEquals(3, delegate.cachedCalls, "oversized values must be returned but not cached"); + Assertions.assertEquals(2L, entry.metrics().getWeightRejectCount()); + Assertions.assertEquals("entry_too_large", entry.metrics().getLastWeightRejectReason()); + Assertions.assertEquals(retainedWeight, entry.metrics().getEstimatedWeight()); + + connector.close(); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); + Assertions.assertEquals(0L, entry.metrics().getEstimatedWeight()); + Assertions.assertEquals(previousGlobalWeight, MetaCacheGovernance.globalEstimatedWeight()); + } + + @Test + void noWeightLimitPreservesCountBasedCaching() { + long catalogId = Long.MIN_VALUE + 72L; + HudiConnector connector = connector(HudiTestProperties.minimalMap(), catalogId); + FakeHmsClient delegate = new FakeHmsClient(ONE_PARTITION); + HmsClient cache = connector.wrapWithCache(delegate); + CatalogMetaCache owner = MetaCacheGovernance.catalogCaches(catalogId).get(0); + MetaCache entry = owner.entries().get("hive-partition-names"); + Assertions.assertFalse(entry.isWeightBounded()); + cache.listPartitionNames("db", "t", -1); + cache.listPartitionNames("db", "t", -1); + Assertions.assertEquals(1, delegate.cachedCalls); + Assertions.assertEquals(0L, entry.metrics().getEstimatedWeight()); + } // ── wrap ───────────────────────────────────────────────────────────────────────────────────────────── @@ -153,8 +228,12 @@ public void invalidateOnUnbuiltClientIsNoOp() { // ── helpers ──────────────────────────────────────────────────────────────────────────────────────────── - private static HudiConnector connector() { - return new HudiConnector(HudiTestProperties.minimalMap(), new ConnectorContext() { + private HudiConnector connector() { + return connector(HudiTestProperties.minimalMap(), 1L); + } + + private HudiConnector connector(Map properties, long catalogId) { + HudiConnector connector = new HudiConnector(properties, new ConnectorContext() { @Override public String getCatalogName() { return "test_catalog"; @@ -162,9 +241,11 @@ public String getCatalogName() { @Override public long getCatalogId() { - return 1L; + return catalogId; } }); + connectors.add(connector); + return connector; } private static HudiTableHandle partitioned() { @@ -198,7 +279,7 @@ public T execute(Callable action) { private static final class FakeHmsClient implements HmsClient { int cachedCalls; int freshCalls; - private final List names; + private List names; FakeHmsClient(List names) { this.names = names; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java new file mode 100644 index 00000000000000..d9e9a63359e3f6 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCacheSizeEstimator.java @@ -0,0 +1,702 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.ReflectiveObjectSizeEstimator; +import org.apache.doris.connector.iceberg.IcebergPartitionCache.CachedPartitions; +import org.apache.doris.connector.iceberg.IcebergPartitionCache.Key; +import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; +import org.apache.doris.connector.iceberg.IcebergTableCache.TableOwner; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartition; +import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; + +import org.apache.iceberg.BlobMetadata; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.HistoryEntry; +import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortField; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.UnboundPartitionSpec; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Type-specific retained-heap estimators for the large Iceberg connector cache values. */ +final class IcebergCacheSizeEstimator { + private static final int PARTITION_VIEW_SAMPLE_SIZE = 16; + private static final String[] CONTENT_FILE_FIELD_NAMES = { + "content", "file_path", "file_format", "partition", "record_count", "file_size_in_bytes", + "column_sizes", "value_counts", "null_value_counts", "nan_value_counts", "lower_bounds", + "upper_bounds", "key_metadata", "split_offsets", "equality_ids", "sort_order_id", "first_row_id", + "referenced_data_file", "content_offset", "content_size_in_bytes" + }; + private static final long TABLE_IDENTIFIER_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableIdentifier.class); + private static final long CACHED_TABLE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableOwner.class); + private static final long TABLE_METADATA_SHALLOW_BYTES = JvmSizeUtils.instanceSize(TableMetadata.class); + private static final long PARTITION_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Key.class); + private static final long CACHED_PARTITIONS_SHALLOW_BYTES = JvmSizeUtils.instanceSize(CachedPartitions.class); + private static final long RAW_PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(IcebergRawPartition.class); + private static final long CONNECTOR_TABLE_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorTableKey.class); + private static final long CONNECTOR_PARTITION_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class); + private static final long MVCC_PARTITION_VIEW_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(ConnectorMvccPartitionView.class); + private static final long MVCC_PARTITION_SHALLOW_BYTES = + JvmSizeUtils.instanceSize(ConnectorMvccPartition.class); + private static final long MANIFEST_KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(IcebergManifestEntryKey.class); + private static final long MANIFEST_VALUE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ManifestCacheValue.class); + private static final long INTEGER_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Integer.class); + private static final long LONG_SHALLOW_BYTES = JvmSizeUtils.instanceSize(Long.class); + private static final long HASH_MAP_NODE_SHALLOW_BYTES = classSize("java.util.HashMap$Node"); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + private static final long CONTENT_FILE_SCHEMA_BYTES = estimateContentFileSchema(); + + private IcebergCacheSizeEstimator() { + } + + /** Admission callback: the expensive table graph was sized once when {@link TableOwner} was constructed. */ + static MetaCacheSizeEstimate estimateTableEntry(TableIdentifier key, TableOwner value) { + return value.sizeEstimate.isComplete() + ? MetaCacheSizeEstimate.complete(add(estimateTableIdentifier(key), value.sizeEstimate.getBytes())) + : value.sizeEstimate; + } + + static long estimateTable(Table table) { + long bytes = add(CACHED_TABLE_SHALLOW_BYTES, JvmSizeUtils.instanceSize(table.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(table.name())); + if (!(table instanceof HasTableOperations)) { + bytes = add(bytes, JvmSizeUtils.stringSize(table.location())); + return add(bytes, estimateStringMap(table.properties())); + } + + TableOperations operations = ((HasTableOperations) table).operations(); + bytes = add(bytes, JvmSizeUtils.instanceSize(operations.getClass())); + return add(bytes, estimateTableMetadata(operations.current())); + } + + static long estimateSerializedTableMetadata(String metadataJson) { + return JvmSizeUtils.stringSize(metadataJson); + } + + static long estimatePartitionKey(Key key) { + return Math.max(add(PARTITION_KEY_SHALLOW_BYTES, estimateTableIdentifier(key.id)), + ReflectiveObjectSizeEstimator.estimate(key)); + } + + static long estimatePartitions(List partitions) { + long bytes = CACHED_PARTITIONS_SHALLOW_BYTES; + // CachedPartitions owns an unmodifiable wrapper around an exact-size ArrayList copy. + bytes = add(bytes, JvmSizeUtils.instanceSize(partitions.getClass())); + bytes = add(bytes, JvmSizeUtils.arrayListSize(partitions.size())); + for (IcebergRawPartition partition : partitions) { + bytes = add(bytes, RAW_PARTITION_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.stringSize(partition.nameForWeight())); + bytes = add(bytes, estimateStringList(partition.columnNamesForWeight())); + bytes = add(bytes, estimateStringList(partition.valuesForWeight())); + bytes = add(bytes, estimateStringList(partition.transformsForWeight())); + } + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(partitions)); + } + + /** Admission callback: the partition list was sized once when {@link CachedPartitions} was constructed. */ + static MetaCacheSizeEstimate estimatePartitionEntry(Key key, CachedPartitions value) { + return value.sizeEstimate.isComplete() + ? MetaCacheSizeEstimate.complete(add(estimatePartitionKey(key), value.sizeEstimate.getBytes())) + : value.sizeEstimate; + } + + static MetaCacheSizeEstimate estimatePartitionInfoViewEntry( + ConnectorTableKey key, List partitions) { + long bytes = estimateConnectorTableKey(key); + bytes = add(bytes, estimateListStructure(partitions)); + bytes = add(bytes, JvmSizeUtils.sampledListPayload( + partitions, PARTITION_VIEW_SAMPLE_SIZE, + IcebergCacheSizeEstimator::estimateConnectorPartition)); + return MetaCacheSizeEstimate.complete( + Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(partitions))); + } + + static MetaCacheSizeEstimate estimateMvccPartitionViewEntry( + ConnectorTableKey key, ConnectorMvccPartitionView view) { + long bytes = add(estimateConnectorTableKey(key), MVCC_PARTITION_VIEW_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.instanceSize(ArrayList.class)); + bytes = add(bytes, estimateListStructure(view.getPartitions())); + bytes = add(bytes, JvmSizeUtils.sampledListPayload( + view.getPartitions(), PARTITION_VIEW_SAMPLE_SIZE, + IcebergCacheSizeEstimator::estimateMvccPartition)); + return MetaCacheSizeEstimate.complete( + Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(view))); + } + + private static long estimateMvccPartition(ConnectorMvccPartition partition) { + long bytes = MVCC_PARTITION_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getName())); + bytes = add(bytes, estimateWrappedStringList(partition.getLowerBound())); + return add(bytes, estimateWrappedStringList(partition.getUpperBound())); + } + + private static long estimateConnectorTableKey(ConnectorTableKey key) { + long bytes = CONNECTOR_TABLE_KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb())); + return add(bytes, JvmSizeUtils.stringSize(key.getTable())); + } + + private static long estimateConnectorPartition(ConnectorPartitionInfo partition) { + long bytes = CONNECTOR_PARTITION_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getPartitionName())); + bytes = add(bytes, estimateMapStructure(partition.getPartitionValues())); + bytes = add(bytes, JvmSizeUtils.instanceSize(LinkedHashMap.class)); + bytes = add(bytes, estimateMapStructure(partition.getProperties())); + bytes = add(bytes, estimateWrappedStringList(partition.getOrderedPartitionValues())); + return add(bytes, estimateWrappedReferenceList(partition.getPartitionValueNullFlags())); + } + + private static long estimateWrappedStringList(List values) { + long bytes = estimateWrappedReferenceList(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateWrappedReferenceList(List values) { + return add(JvmSizeUtils.instanceSize(ArrayList.class), estimateListStructure(values)); + } + + static long estimateManifestKey(IcebergManifestEntryKey key) { + return add(MANIFEST_KEY_SHALLOW_BYTES, JvmSizeUtils.stringSize(key.getManifestPath())); + } + + static long estimateManifestValue(ManifestCacheValue value) { + long bytes = MANIFEST_VALUE_SHALLOW_BYTES; + bytes = add(bytes, estimateContentFileList(value.getDataFiles())); + bytes = add(bytes, estimateContentFileList(value.getDeleteFiles())); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(value)); + } + + /** Admission callback: the manifest payload size is precomputed during construction. */ + static MetaCacheSizeEstimate estimateManifestEntry(IcebergManifestEntryKey key, ManifestCacheValue value) { + return value.getSizeEstimate().isComplete() + ? MetaCacheSizeEstimate.complete( + add(estimateManifestKey(key), value.getSizeEstimate().getBytes())) + : value.getSizeEstimate(); + } + + private static long estimateTableIdentifier(TableIdentifier identifier) { + long bytes = TABLE_IDENTIFIER_SHALLOW_BYTES; + Namespace namespace = identifier.namespace(); + bytes = add(bytes, JvmSizeUtils.instanceSize(namespace.getClass())); + String[] levels = namespace.levels(); + bytes = add(bytes, JvmSizeUtils.objectArraySize(levels.length)); + for (String level : levels) { + bytes = add(bytes, JvmSizeUtils.stringSize(level)); + } + return add(bytes, JvmSizeUtils.stringSize(identifier.name())); + } + + private static long estimateTableMetadata(TableMetadata metadata) { + long bytes = TABLE_METADATA_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.metadataFileLocation())); + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.uuid())); + bytes = add(bytes, JvmSizeUtils.stringSize(metadata.location())); + bytes = add(bytes, estimateStringMap(metadata.properties())); + + List schemas = metadata.schemas(); + bytes = add(bytes, estimateListStructure(schemas)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.schemasById())); + for (Schema schema : schemas) { + bytes = add(bytes, estimateSchema(schema)); + } + + List specs = metadata.specs(); + bytes = add(bytes, estimateListStructure(specs)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.specsById())); + for (PartitionSpec spec : specs) { + bytes = add(bytes, estimatePartitionSpec(spec)); + } + + List sortOrders = metadata.sortOrders(); + bytes = add(bytes, estimateListStructure(sortOrders)); + bytes = add(bytes, estimateIntegerIndexMap(metadata.sortOrdersById())); + for (SortOrder sortOrder : sortOrders) { + bytes = add(bytes, estimateSortOrder(sortOrder)); + } + + List snapshots = metadata.snapshots(); + bytes = add(bytes, estimateListStructure(snapshots)); + bytes = add(bytes, estimateLongIndexMap(snapshots)); + for (Snapshot snapshot : snapshots) { + bytes = add(bytes, estimateSnapshot(snapshot)); + } + + bytes = add(bytes, estimateHistory(metadata.snapshotLog())); + bytes = add(bytes, estimateMetadataLog(metadata.previousFiles())); + bytes = add(bytes, estimateSnapshotRefs(metadata.refs())); + bytes = add(bytes, estimateStatisticsFiles(metadata.statisticsFiles())); + bytes = add(bytes, estimatePartitionStatisticsFiles(metadata.partitionStatisticsFiles())); + bytes = add(bytes, estimateMetadataUpdates(metadata.changes())); + bytes = add(bytes, estimateShallowList(metadata.encryptionKeys())); + // TableMetadata retains a serializable snapshot supplier after the immutable snapshot list is loaded. + bytes = add(bytes, JvmSizeUtils.objectArraySize(1)); + return Math.max(bytes, ReflectiveObjectSizeEstimator.estimate(metadata)); + } + + private static long estimateSchema(Schema schema) { + List columns = schema.columns(); + long bytes = JvmSizeUtils.instanceSize(schema.getClass()); + bytes = add(bytes, JvmSizeUtils.instanceSize(schema.asStruct().getClass())); + bytes = add(bytes, estimateListStructure(columns)); + for (Types.NestedField field : columns) { + bytes = add(bytes, estimateNestedField(field)); + } + bytes = add(bytes, JvmSizeUtils.objectArraySize(schema.identifierFieldIds().size())); + bytes = add(bytes, estimateMapStructure(schema.getAliases())); + return add(bytes, estimateSchemaIndexes(columns.size())); + } + + private static long estimateNestedField(Types.NestedField field) { + long bytes = JvmSizeUtils.instanceSize(field.getClass()); + bytes = add(bytes, JvmSizeUtils.stringSize(field.name())); + bytes = add(bytes, JvmSizeUtils.stringSize(field.doc())); + return add(bytes, estimateIcebergType(field.type())); + } + + private static long estimateIcebergType(Type type) { + long bytes = JvmSizeUtils.instanceSize(type.getClass()); + if (type.isStructType()) { + List fields = type.asStructType().fields(); + bytes = add(bytes, estimateListStructure(fields)); + for (Types.NestedField field : fields) { + bytes = add(bytes, estimateNestedField(field)); + } + } else if (type.isListType()) { + bytes = add(bytes, estimateNestedField(type.asListType().fields().get(0))); + } else if (type.isMapType()) { + for (Types.NestedField field : type.asMapType().fields()) { + bytes = add(bytes, estimateNestedField(field)); + } + } + return bytes; + } + + private static long estimatePartitionSpec(PartitionSpec spec) { + List fields = spec.fields(); + long bytes = add(JvmSizeUtils.instanceSize(spec.getClass()), JvmSizeUtils.objectArraySize(fields.size())); + for (PartitionField field : fields) { + bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(field.name())); + bytes = add(bytes, JvmSizeUtils.instanceSize(field.transform().getClass())); + } + return bytes; + } + + private static long estimateSortOrder(SortOrder sortOrder) { + List fields = sortOrder.fields(); + long bytes = add(JvmSizeUtils.instanceSize(sortOrder.getClass()), JvmSizeUtils.objectArraySize(fields.size())); + for (SortField field : fields) { + bytes = add(bytes, JvmSizeUtils.instanceSize(field.getClass())); + bytes = add(bytes, JvmSizeUtils.instanceSize(field.transform().getClass())); + } + return bytes; + } + + private static long estimateSnapshot(Snapshot snapshot) { + long bytes = JvmSizeUtils.instanceSize(snapshot.getClass()); + bytes = add(bytes, estimateBoxed(snapshot.parentId(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.schemaId(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.firstRowId(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(snapshot.addedRows(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.operation())); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.manifestListLocation())); + bytes = add(bytes, JvmSizeUtils.stringSize(snapshot.keyId())); + return add(bytes, estimateStringMap(snapshot.summary())); + } + + private static long estimateHistory(List history) { + long bytes = estimateListStructure(history); + for (HistoryEntry entry : history) { + bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass())); + } + return bytes; + } + + private static long estimateMetadataLog(List entries) { + long bytes = estimateListStructure(entries); + for (TableMetadata.MetadataLogEntry entry : entries) { + bytes = add(bytes, JvmSizeUtils.instanceSize(entry.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(entry.file())); + } + return bytes; + } + + private static long estimateSnapshotRefs(Map refs) { + long bytes = estimateMapStructure(refs); + for (Map.Entry entry : refs.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getKey())); + SnapshotRef ref = entry.getValue(); + bytes = add(bytes, JvmSizeUtils.instanceSize(ref.getClass())); + bytes = add(bytes, estimateBoxed(ref.minSnapshotsToKeep(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(ref.maxSnapshotAgeMs(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(ref.maxRefAgeMs(), LONG_SHALLOW_BYTES)); + } + return bytes; + } + + private static long estimateStatisticsFiles(List files) { + long bytes = estimateListStructure(files); + for (StatisticsFile file : files) { + bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(file.path())); + List blobs = file.blobMetadata(); + bytes = add(bytes, estimateListStructure(blobs)); + for (BlobMetadata blob : blobs) { + bytes = add(bytes, JvmSizeUtils.instanceSize(blob.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(blob.type())); + bytes = add(bytes, estimateBoxedList(blob.fields(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateStringMap(blob.properties())); + } + } + return bytes; + } + + private static long estimatePartitionStatisticsFiles(List files) { + long bytes = estimateListStructure(files); + for (PartitionStatisticsFile file : files) { + bytes = add(bytes, JvmSizeUtils.instanceSize(file.getClass())); + bytes = add(bytes, JvmSizeUtils.stringSize(file.path())); + } + return bytes; + } + + private static long estimateShallowList(List values) { + long bytes = estimateListStructure(values); + for (Object value : values) { + bytes = add(bytes, JvmSizeUtils.instanceSize(value.getClass())); + } + return bytes; + } + + private static long estimateMetadataUpdates(List updates) { + long bytes = estimateListStructure(updates); + for (MetadataUpdate update : updates) { + bytes = add(bytes, JvmSizeUtils.instanceSize(update.getClass())); + if (update instanceof MetadataUpdate.SetProperties) { + bytes = add(bytes, estimateMapStructure(((MetadataUpdate.SetProperties) update).updated())); + } else if (update instanceof MetadataUpdate.AddPartitionSpec) { + UnboundPartitionSpec spec = ((MetadataUpdate.AddPartitionSpec) update).spec(); + bytes = add(bytes, JvmSizeUtils.instanceSize(spec.getClass())); + bytes = add(bytes, estimateListStructure(spec.fields())); + } else if (update instanceof MetadataUpdate.AddSortOrder) { + bytes = add(bytes, JvmSizeUtils.instanceSize( + ((MetadataUpdate.AddSortOrder) update).sortOrder().getClass())); + } + } + return bytes; + } + + private static long estimateContentFileList(List> files) { + if (files.isEmpty()) { + return 0L; + } + long bytes = add(estimateListStructure(files), CONTENT_FILE_SCHEMA_BYTES); + Set ownedObjects = java.util.Collections.newSetFromMap(new IdentityHashMap<>()); + for (ContentFile file : files) { + bytes = add(bytes, estimateContentFile(file, ownedObjects)); + } + return bytes; + } + + private static long estimateContentFileSchema() { + long bytes = JvmSizeUtils.instanceSize(Types.StructType.class); + bytes = add(bytes, JvmSizeUtils.arrayListSize(CONTENT_FILE_FIELD_NAMES.length)); + for (String name : CONTENT_FILE_FIELD_NAMES) { + bytes = add(bytes, JvmSizeUtils.instanceSize(Types.NestedField.class)); + bytes = add(bytes, JvmSizeUtils.stringSize(name)); + } + return bytes; + } + + private static long estimateContentFile(ContentFile file, Set ownedObjects) { + long bytes = JvmSizeUtils.instanceSize(file.getClass()); + if (file instanceof StructLike) { + bytes = add(bytes, JvmSizeUtils.intArraySize(((StructLike) file).size())); + bytes = add(bytes, LONG_SHALLOW_BYTES); + } + bytes = add(bytes, estimateOwnedCharSequence(file.path(), ownedObjects)); + bytes = add(bytes, estimateOwnedString(file.manifestLocation(), ownedObjects)); + bytes = add(bytes, estimatePartition(file.partition(), ownedObjects)); + bytes = add(bytes, estimateLongMap(file.columnSizes())); + bytes = add(bytes, estimateLongMap(file.valueCounts())); + bytes = add(bytes, estimateLongMap(file.nullValueCounts())); + bytes = add(bytes, estimateLongMap(file.nanValueCounts())); + bytes = add(bytes, estimateByteBufferMap(file.lowerBounds())); + bytes = add(bytes, estimateByteBufferMap(file.upperBounds())); + ByteBuffer keyMetadata = file.keyMetadata(); + if (keyMetadata != null) { + bytes = add(bytes, JvmSizeUtils.byteArraySize(keyMetadata.remaining())); + } + List splitOffsets = file.splitOffsets(); + if (splitOffsets != null) { + bytes = add(bytes, JvmSizeUtils.longArraySize(splitOffsets.size())); + } + List equalityFieldIds = file.equalityFieldIds(); + if (equalityFieldIds != null) { + bytes = add(bytes, JvmSizeUtils.intArraySize(equalityFieldIds.size())); + } + bytes = add(bytes, estimateBoxed(file.pos(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.sortOrderId(), INTEGER_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.dataSequenceNumber(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.fileSequenceNumber(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(file.firstRowId(), LONG_SHALLOW_BYTES)); + if (file instanceof DeleteFile) { + DeleteFile deleteFile = (DeleteFile) file; + bytes = add(bytes, estimateOwnedString(deleteFile.referencedDataFile(), ownedObjects)); + bytes = add(bytes, estimateBoxed(deleteFile.contentOffset(), LONG_SHALLOW_BYTES)); + bytes = add(bytes, estimateBoxed(deleteFile.contentSizeInBytes(), LONG_SHALLOW_BYTES)); + } + return bytes; + } + + private static long estimatePartition(StructLike partition, Set ownedObjects) { + if (partition == null || !ownedObjects.add(partition)) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(partition.getClass()); + bytes = add(bytes, JvmSizeUtils.objectArraySize(partition.size())); + for (int i = 0; i < partition.size(); i++) { + bytes = add(bytes, estimateOwnedScalar(partition.get(i, Object.class), ownedObjects)); + } + return bytes; + } + + private static long estimateOwnedScalar(Object value, Set ownedObjects) { + if (value == null || !ownedObjects.add(value)) { + return 0L; + } + if (value instanceof CharSequence) { + return estimateCharSequence((CharSequence) value); + } + if (value instanceof ByteBuffer) { + return estimateByteBuffer((ByteBuffer) value); + } + return JvmSizeUtils.instanceSize(value.getClass()); + } + + private static long estimateOwnedCharSequence(CharSequence value, Set ownedObjects) { + return value == null || !ownedObjects.add(value) ? 0L : estimateCharSequence(value); + } + + private static long estimateOwnedString(String value, Set ownedStrings) { + return value == null || !ownedStrings.add(value) ? 0L : JvmSizeUtils.stringSize(value); + } + + private static long estimateCharSequence(CharSequence value) { + if (value instanceof String) { + return JvmSizeUtils.stringSize((String) value); + } + return add(JvmSizeUtils.instanceSize(value.getClass()), JvmSizeUtils.stringSize(value.toString())); + } + + private static long estimateLongMap(Map values) { + if (values == null || values.isEmpty()) { + return 0L; + } + return add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES + LONG_SHALLOW_BYTES)); + } + + private static long estimateByteBufferMap(Map values) { + if (values == null || values.isEmpty()) { + return 0L; + } + long bytes = add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES)); + for (ByteBuffer value : values.values()) { + bytes = add(bytes, estimateByteBuffer(value)); + } + return bytes; + } + + private static long estimateByteBuffer(ByteBuffer value) { + if (value == null) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(value.getClass()); + return value.hasArray() ? add(bytes, JvmSizeUtils.byteArraySize(value.capacity())) : bytes; + } + + private static long estimateSchemaIndexes(int fieldCount) { + if (fieldCount == 0) { + return 0L; + } + long oneIndex = JvmSizeUtils.instanceSize(HashMap.class); + oneIndex = add(oneIndex, JvmSizeUtils.objectArraySize(hashCapacity(fieldCount))); + oneIndex = add(oneIndex, multiply(fieldCount, HASH_MAP_NODE_SHALLOW_BYTES)); + long bytes = multiply(3L, oneIndex); + return add(bytes, multiply(2L * fieldCount, INTEGER_SHALLOW_BYTES)); + } + + private static long estimateIntegerIndexMap(Map values) { + return add(estimateMapStructure(values), multiply(values.size(), INTEGER_SHALLOW_BYTES)); + } + + private static long estimateLongIndexMap(List values) { + if (values.isEmpty()) { + return JvmSizeUtils.instanceSize(HashMap.class); + } + long bytes = JvmSizeUtils.instanceSize(HashMap.class); + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(values.size()))); + bytes = add(bytes, multiply(values.size(), HASH_MAP_NODE_SHALLOW_BYTES)); + return add(bytes, multiply(values.size(), LONG_SHALLOW_BYTES)); + } + + private static long estimateStringMap(Map values) { + if (values == null) { + return 0L; + } + long bytes = estimateMapStructure(values); + for (Map.Entry entry : values.entrySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getKey())); + bytes = add(bytes, JvmSizeUtils.stringSize(entry.getValue())); + } + return bytes; + } + + private static long estimateStringList(List values) { + if (values == null || values.isEmpty()) { + return 0L; + } + long bytes = estimateListStructure(values); + for (String value : values) { + bytes = add(bytes, JvmSizeUtils.stringSize(value)); + } + return bytes; + } + + private static long estimateBoxedList(List values, long elementBytes) { + if (values == null || values.isEmpty()) { + return 0L; + } + return add(estimateListStructure(values), multiply(values.size(), elementBytes)); + } + + private static long estimateBoxed(Object value, long bytes) { + return value == null ? 0L : bytes; + } + + private static long estimateListStructure(List values) { + int capacity = values instanceof ArrayList ? arrayListCapacity(values.size()) : values.size(); + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + if (values.getClass().getName().equals("java.util.ImmutableCollections$List12")) { + return bytes; + } + return add(bytes, JvmSizeUtils.objectArraySize(capacity)); + } + + private static long estimateMapStructure(Map values) { + if (values == null) { + return 0L; + } + long bytes = JvmSizeUtils.instanceSize(values.getClass()); + if (values.isEmpty()) { + return bytes; + } + if (values instanceof HashMap) { + int capacity = hashCapacity(values.size()); + long nodeBytes = values instanceof LinkedHashMap + ? LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES + : HASH_MAP_NODE_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.objectArraySize(capacity)); + return add(bytes, multiply(values.size(), nodeBytes)); + } + bytes = add(bytes, JvmSizeUtils.objectArraySize(saturatedDouble(values.size()))); + return add(bytes, multiply(values.size(), HASH_MAP_NODE_SHALLOW_BYTES)); + } + + private static int arrayListCapacity(int size) { + if (size == 0) { + return 0; + } + int capacity = 10; + while (capacity < size) { + int grown = capacity + (capacity >> 1); + if (grown < 0) { + return Integer.MAX_VALUE; + } + capacity = grown; + } + return capacity; + } + + private static int hashCapacity(int size) { + if (size == 0) { + return 0; + } + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static int saturatedDouble(int value) { + return value > Integer.MAX_VALUE / 2 ? Integer.MAX_VALUE : value * 2; + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java index 8947743c5484d4..aea432ee88cff3 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogProperties.java @@ -157,6 +157,8 @@ public IcebergCatalogProperties checkCreateTimeOnlyRules() { * sentinel), {@code capacity} must be a long ≥ 0. Absent keys are skipped. */ private static void checkMetaCacheProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, "iceberg", + "table", "partition", "manifest", "partition_view"); CacheSpec.checkBooleanProperty(properties.get(IcebergConnector.TABLE_CACHE_ENABLE), IcebergConnector.TABLE_CACHE_ENABLE); CacheSpec.checkLongProperty(properties.get(IcebergConnector.TABLE_CACHE_TTL_SECOND), diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java index 49b54f94470395..0ab14f07b0f835 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCommentCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.catalog.TableIdentifier; @@ -68,6 +69,7 @@ final class IcebergCommentCache { CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition .builder("iceberg-comment", spec, IcebergCommentCache::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index abeb947ff7b460..5123c6cf866e59 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -17,8 +17,10 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.iceberg.dlf.DLFCatalog; import org.apache.doris.connector.metastore.DlfMetaStoreProperties; import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; @@ -233,10 +235,10 @@ public class IcebergConnector implements Connector { mvccPartitionViewCache; private final ConnectorMetadataCache> // null under session=user listPartitionsViewCache; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; // Manifest content cache — pure metadata, default-off (meta.cache.iceberg.manifest.enable), and consumed // ONLY after a per-user resolveTable(ForRead) -- exempt: no read path without a per-user load. - private final IcebergManifestCache manifestCache = new IcebergManifestCache(metaCache); + private final IcebergManifestCache manifestCache; // Lazily-built plugin-side Kerberos authenticator (single-owner auth; see TcclPinningConnectorContext). // null for a non-Kerberos catalog. Its doAs acts on the PLUGIN's UserGroupInformation copy — the one the @@ -260,6 +262,8 @@ public IcebergConnector(Map properties, ConnectorContext context // authenticator never logs in — so without this the DDL/read hits secured HDFS as SIMPLE auth. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "iceberg", this.properties); + this.manifestCache = new IcebergManifestCache(metaCache, this.properties); // Authorization-sensitive projection (snapshotId/schemaId). Under iceberg.rest.session=user the value is // per-user AUTHORIZED metadata that a "can-list-cannot-load" principal must not see. beginQuerySnapshot // reads this cache WITHOUT a preceding per-user loadTable, so a shared (table-keyed, no user dimension) @@ -283,7 +287,7 @@ public IcebergConnector(Map properties, ConnectorContext context || IcebergScanPlanProvider.restVendedCredentialsEnabled(this.properties)) ? null : new IcebergTableCache( - metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY, + metaCache, cacheSpec("table"), this::cachedTableCleanup, catalogResourceTracker); // PERF-02: partition-view cache. Authorization-sensitive projection: a shared (table+snapshot-keyed, no // user dimension) hit would disclose one user's partition list. Its readers are all downstream of a @@ -294,7 +298,7 @@ metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPA this.partitionCache = isUserSessionEnabled() ? null : new IcebergPartitionCache( - metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY); + metaCache, cacheSpec("partition")); // PERF-03: inferred-file-format cache. Same authorization-sensitive treatment as partitionCache (disabled // under session=user, kept otherwise); readers already tolerate a null cache (resolveFileFormatName). this.formatCache = isUserSessionEnabled() @@ -319,11 +323,15 @@ metaCache, resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPA this.mvccPartitionViewCache = isUserSessionEnabled() ? null : new ConnectorMetadataCache<>(metaCache, "iceberg.mvcc-partition-view", - "iceberg", "partition_view", this.properties); + "iceberg", "partition_view", this.properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + IcebergCacheSizeEstimator::estimateMvccPartitionViewEntry); this.listPartitionsViewCache = isUserSessionEnabled() ? null : new ConnectorMetadataCache<>(metaCache, "iceberg.list-partitions-view", - "iceberg", "partition_view", this.properties); + "iceberg", "partition_view", this.properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + IcebergCacheSizeEstimator::estimatePartitionInfoViewEntry); } /** @@ -345,6 +353,12 @@ static long resolveTableCacheTtlSecond(Map properties) { } } + private CacheSpec cacheSpec(String entryName) { + return CacheSpec.fromProperties(this.properties, "iceberg", entryName, + CacheSpec.ofConnectorTtl( + resolveTableCacheTtlSecond(this.properties), DEFAULT_TABLE_CACHE_CAPACITY)); + } + @Override public ConnectorMetadata getMetadata(ConnectorSession session) { return new IcebergConnectorMetadata(newCatalogBackedOps(session), catalogProps, context, diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java index 2c9ce0c2ed7fb3..8af2d28cfcdbb2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergFormatCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.catalog.TableIdentifier; @@ -102,6 +103,7 @@ public int hashCode() { CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition .builder("iceberg-format", spec, IcebergFormatCache::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java index a20ab5fd2ecf4d..81f8229069f75a 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergLatestSnapshotCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartitionView; @@ -87,6 +88,7 @@ static final class CachedSnapshot { this.entry = owner.create(MetaCacheDefinition .builder( "iceberg-latest-snapshot", spec, IcebergLatestSnapshotCache::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java index 6bfead8166070d..b68ce2b5e43dc7 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergManifestCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.iceberg.DataFile; @@ -135,7 +136,12 @@ private static final class ScanStats { } IcebergManifestCache(CatalogMetaCache owner) { - this(owner, DEFAULT_MANIFEST_CACHE_CAPACITY); + this(owner, Collections.emptyMap()); + } + + IcebergManifestCache(CatalogMetaCache owner, Map properties) { + this(owner, DEFAULT_MANIFEST_CACHE_CAPACITY, DEFAULT_STATS_TTL_SECONDS, + System::nanoTime, properties); } IcebergManifestCache(int maxSize) { @@ -143,7 +149,7 @@ private static final class ScanStats { } private IcebergManifestCache(CatalogMetaCache owner, int maxSize) { - this(owner, maxSize, DEFAULT_STATS_TTL_SECONDS, System::nanoTime); + this(owner, maxSize, DEFAULT_STATS_TTL_SECONDS, System::nanoTime, Collections.emptyMap()); } /** Visible for testing: injectable stats TTL + clock so the leak sweep is deterministic without sleeping. */ @@ -153,16 +159,25 @@ private IcebergManifestCache(CatalogMetaCache owner, int maxSize) { private IcebergManifestCache( CatalogMetaCache owner, int maxSize, long statsTtlSeconds, LongSupplier nanoClock) { + this(owner, maxSize, statsTtlSeconds, nanoClock, Collections.emptyMap()); + } + + private IcebergManifestCache( + CatalogMetaCache owner, int maxSize, long statsTtlSeconds, LongSupplier nanoClock, + Map properties) { this.owner = owner; // Always enabled, no expiry, capacity-bounded (CACHE_NO_TTL == -1 means "no expiration", enabled). - CacheSpec spec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Math.max(1, maxSize)); + CacheSpec defaultSpec = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Math.max(1, maxSize)); + CacheSpec spec = CacheSpec.fromProperties(properties, "iceberg", "manifest", defaultSpec); this.entry = owner.create(MetaCacheDefinition .builder( "iceberg-manifest", spec, ignored -> ScopePath.catalog()) + .sizeEstimator(IcebergCacheSizeEstimator::estimateManifestEntry) .build()); this.equalityDeleteFieldIds = owner.create(MetaCacheDefinition .>builder( - "iceberg-equality-delete-field-ids", spec, ignored -> ScopePath.catalog()) + "iceberg-equality-delete-field-ids", defaultSpec, ignored -> ScopePath.catalog()) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); this.statsTtlNanos = TimeUnit.SECONDS.toNanos(Math.max(1L, statsTtlSeconds)); this.nanoClock = nanoClock; @@ -187,7 +202,8 @@ Set getOrLoadEqualityDeleteFieldIds( */ ManifestCacheValue getManifestCacheValue(ManifestFile manifest, Table table) { IcebergManifestEntryKey key = IcebergManifestEntryKey.of(manifest); - return entry.get(key, k -> loadManifestCacheValue(manifest, table, k.getContent())); + return entry.get(key, k -> loadManifestCacheValue( + manifest, table, k.getContent(), entry.isWeightBounded())); } /** @@ -207,7 +223,8 @@ ManifestCacheValue getManifestCacheValue(ManifestFile manifest, Table table, Str stats.misses++; } } - return entry.get(key, k -> loadManifestCacheValue(manifest, table, k.getContent())); + return entry.get(key, k -> loadManifestCacheValue( + manifest, table, k.getContent(), entry.isWeightBounded())); } /** @@ -254,12 +271,12 @@ private void sweepExpiredStats(long nowNanos) { } private static ManifestCacheValue loadManifestCacheValue(ManifestFile manifest, Table table, - ManifestContent content) { + ManifestContent content, boolean estimateWeight) { try { if (content == ManifestContent.DELETES) { - return ManifestCacheValue.forDeleteFiles(loadDeleteFiles(manifest, table)); + return ManifestCacheValue.forDeleteFiles(loadDeleteFiles(manifest, table), estimateWeight); } - return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, table)); + return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, table), estimateWeight); } catch (IOException e) { throw new RuntimeException("Failed to read iceberg manifest " + manifest.path(), e); } diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java index 451aa7ddd819d1..59a47b75f4cd55 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionCache.java @@ -21,11 +21,15 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; import org.apache.iceberg.catalog.TableIdentifier; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Supplier; @@ -86,19 +90,38 @@ public int hashCode() { } } + static final class CachedPartitions { + final List partitions; + final MetaCacheSizeEstimate sizeEstimate; + + CachedPartitions(List partitions, boolean estimateWeight) { + this.partitions = estimateWeight + ? Collections.unmodifiableList(new ArrayList<>(partitions)) + : partitions; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("iceberg_partition_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + IcebergCacheSizeEstimator.estimatePartitions(this.partitions))) + : MetaCacheSizeEstimate.complete(0L); + } + } + private final CatalogMetaCache owner; - private final MetaCache> entry; + private final MetaCache entry; IcebergPartitionCache(long ttlSeconds, int maxSize) { this(new CatalogMetaCache(), ttlSeconds, maxSize); } IcebergPartitionCache(CatalogMetaCache owner, long ttlSeconds, int maxSize) { + this(owner, CacheSpec.ofConnectorTtl(ttlSeconds, maxSize)); + } + + IcebergPartitionCache(CatalogMetaCache owner, CacheSpec spec) { this.owner = owner; - // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). - CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition - .>builder("iceberg-partition", spec, IcebergPartitionCache::scope) + .builder("iceberg-partition", spec, IcebergPartitionCache::scope) + .sizeEstimator(IcebergCacheSizeEstimator::estimatePartitionEntry) .build()); } @@ -113,7 +136,7 @@ boolean isEnabled() { * loader runs OUTSIDE Caffeine's compute lock (single-flight per key) and its exception propagates unwrapped. */ List getOrLoad(Key key, Supplier> loader) { - return entry.get(key, ignored -> loader.get()); + return entry.get(key, ignored -> new CachedPartitions(loader.get(), entry.isWeightBounded())).partitions; } /** Drops every cached snapshot entry for one table so the next read scans live (REFRESH TABLE). */ diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java index a5ff4562e915cd..ddc8cbde915b5e 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergPartitionUtils.java @@ -990,6 +990,22 @@ static final class IcebergRawPartition { this.lastUpdateTime = lastUpdateTime; this.lastSnapshotId = lastSnapshotId; } + + String nameForWeight() { + return name; + } + + List columnNamesForWeight() { + return columnNames; + } + + List valuesForWeight() { + return values; + } + + List transformsForWeight() { + return transforms; + } } /** A single physical partition's computed range: time interval (for the overlap merge) + pre-rendered bounds. */ diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java index 573623c0063261..d64ded6f039a5d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergStatementScope.java @@ -110,7 +110,7 @@ static T withBorrowedTable(ConnectorSession session, String dbName, String t Function action) { if (session == null || session.getStatementScope() == ConnectorStatementScope.NONE) { try (IcebergTableCache.TableLease lease = loader.get()) { - return action.apply(snapshotReadTable(lease.table())); + return action.apply(lease.snapshotReadTable()); } } return action.apply(sharedBorrowedTable(session, dbName, tableName, loader, unscopedLoader)); @@ -148,7 +148,7 @@ private static final class ScopedBorrow implements AutoCloseable { private ScopedBorrow(IcebergTableCache.TableLease lease) { this.lease = lease; - this.table = snapshotReadTable(lease.table()); + this.table = lease.snapshotReadTable(); } @Override diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index 614df9aee6aa0b..9f130039a196d2 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -19,11 +19,18 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.JvmSizeUtils; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; import org.apache.doris.connector.cache.ScopePath; +import org.apache.iceberg.BaseTable; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; import org.apache.iceberg.catalog.TableIdentifier; import java.util.concurrent.atomic.AtomicBoolean; @@ -50,9 +57,11 @@ * {@code expireAfterAccess} with a {@code maxSize} capacity. Lives on the long-lived per-catalog * {@link IcebergConnector}; a REFRESH CATALOG rebuilds the connector and thus the cache. * - *

Values are RAW tables. The scan provider applies {@code wrapTableForScan} (the Kerberos + *

Values own RAW tables. The scan provider applies {@code wrapTableForScan} (the Kerberos * {@code doAs} FileIO wrap) per call on the way out, so no per-request authenticator is ever frozen into a - * shared entry. + * shared entry. A weight-bounded owner also retains and accounts for one serialized metadata generation; + * each statement parses that generation into a private read table so Iceberg's lazy manifest fields cannot + * grow the already-admitted cache value. * *

Credential isolation. A raw table carries its FileIO's credentials, so this cross-query layer is * built ONLY when the connector's credentials are query-independent — it is left disabled (the connector @@ -83,15 +92,19 @@ final class IcebergTableCache { IcebergTableCache(CatalogMetaCache owner, long ttlSeconds, int maxSize, Function cleanupFactory, IcebergCatalogResourceTracker resourceTracker) { + this(owner, CacheSpec.ofConnectorTtl(ttlSeconds, maxSize), cleanupFactory, resourceTracker); + } + + IcebergTableCache(CatalogMetaCache owner, CacheSpec spec, + Function cleanupFactory, IcebergCatalogResourceTracker resourceTracker) { this.owner = owner; this.cleanupFactory = cleanupFactory; this.resourceTracker = resourceTracker; - // "<= 0 disables" connector TTL contract, folded to CacheSpec's disable sentinel (CacheSpec.ofConnectorTtl). - CacheSpec spec = CacheSpec.ofConnectorTtl(ttlSeconds, maxSize); this.entry = owner.create(MetaCacheDefinition .builder("iceberg-table", spec, IcebergTableCache::scope) .removalListener((identifier, tableOwner, reason) -> tableOwner.release()) .discardListener((identifier, tableOwner) -> tableOwner.release()) + .sizeEstimator(IcebergCacheSizeEstimator::estimateTableEntry) .build()); } @@ -130,7 +143,7 @@ TableLease borrow(TableIdentifier identifier, Supplier loader) { } } }; - TableOwner loaded = new TableOwner(table, cleanup, true); + TableOwner loaded = new TableOwner(table, cleanup, true, entry.isWeightBounded()); loadedHere[0] = loaded; return loaded; } finally { @@ -214,6 +227,10 @@ Table table() { return owner.table; } + Table snapshotReadTable() { + return owner.snapshotReadTable(); + } + @Override public void close() { if (closed.compareAndSet(false, true)) { @@ -222,19 +239,58 @@ public void close() { } } - private static final class TableOwner { + static final class TableOwner { private final Table table; private final Runnable cleanup; + private final String snapshotMetadataJson; + private final String snapshotMetadataLocation; + final MetaCacheSizeEstimate sizeEstimate; // A newly loaded value starts with a cache reference and a temporary loader reference. The temporary // reference bridges publication/discard to the first borrow, including invalidation-before-publication. private final AtomicInteger references; - private TableOwner(Table table, Runnable cleanup, boolean loading) { + private TableOwner(Table table, Runnable cleanup, boolean loading, boolean estimateWeight) { this.table = table; this.cleanup = cleanup; + String[] metadataJson = {null}; + String[] metadataLocation = {null}; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("iceberg_table_estimator_failure", + () -> { + long serializedMetadataBytes = 0L; + if (table instanceof BaseTable) { + TableMetadata metadata = ((BaseTable) table).operations().current(); + String json = TableMetadataParser.toJson(metadata); + metadataJson[0] = json; + metadataLocation[0] = metadata.metadataFileLocation(); + serializedMetadataBytes = + IcebergCacheSizeEstimator.estimateSerializedTableMetadata(json); + } + // Iceberg v1 serialization materializes lazy embedded-manifest state on the + // retained BaseSnapshot, so weigh the table only after serialization completes. + long bytes = JvmSizeUtils.saturatedAdd( + IcebergCacheSizeEstimator.estimateTable(table), serializedMetadataBytes); + return MetaCacheSizeEstimate.complete(bytes); + }) + : MetaCacheSizeEstimate.complete(0L); + this.snapshotMetadataJson = sizeEstimate.isComplete() ? metadataJson[0] : null; + this.snapshotMetadataLocation = sizeEstimate.isComplete() ? metadataLocation[0] : null; this.references = new AtomicInteger(loading ? 2 : 1); } + private Table snapshotReadTable() { + if (!(table instanceof BaseTable)) { + return table; + } + BaseTable baseTable = (BaseTable) table; + TableOperations operations = baseTable.operations(); + TableMetadata metadata = snapshotMetadataJson == null + ? operations.current() + : TableMetadataParser.fromJson(snapshotMetadataLocation, snapshotMetadataJson); + return new BaseTable(new IcebergSnapshotTableOperations(operations, metadata), + table.name(), baseTable.reporter()); + } + private TableLease tryBorrow() { int current = references.get(); while (current != 0) { diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java index b9b9da68dbd3c5..2c2b2d60925216 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/ManifestCacheValue.java @@ -17,6 +17,9 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; + import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; @@ -33,18 +36,32 @@ public class ManifestCacheValue { private final List dataFiles; private final List deleteFiles; + private final MetaCacheSizeEstimate sizeEstimate; - private ManifestCacheValue(List dataFiles, List deleteFiles) { + private ManifestCacheValue(List dataFiles, List deleteFiles, boolean estimateWeight) { this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles; this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles; + this.sizeEstimate = estimateWeight + ? MetaCacheSizeEstimator.estimateSafely("iceberg_manifest_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + IcebergCacheSizeEstimator.estimateManifestValue(this))) + : MetaCacheSizeEstimate.complete(0L); } public static ManifestCacheValue forDataFiles(List dataFiles) { - return new ManifestCacheValue(dataFiles, Collections.emptyList()); + return forDataFiles(dataFiles, false); + } + + static ManifestCacheValue forDataFiles(List dataFiles, boolean estimateWeight) { + return new ManifestCacheValue(dataFiles, Collections.emptyList(), estimateWeight); } public static ManifestCacheValue forDeleteFiles(List deleteFiles) { - return new ManifestCacheValue(Collections.emptyList(), deleteFiles); + return forDeleteFiles(deleteFiles, false); + } + + static ManifestCacheValue forDeleteFiles(List deleteFiles, boolean estimateWeight) { + return new ManifestCacheValue(Collections.emptyList(), deleteFiles, estimateWeight); } public List getDataFiles() { @@ -54,4 +71,8 @@ public List getDataFiles() { public List getDeleteFiles() { return deleteFiles; } + + MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate; + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java index 8bec2551e770b8..cd7a5f5838d7d1 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorMetadataPartitionViewCacheTest.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.iceberg; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; import org.apache.doris.connector.spi.ConnectorPartitionInfo; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.mvcc.ConnectorMvccPartition; @@ -37,6 +38,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -125,6 +127,30 @@ private static List mvccNames(Optional view) .map(ConnectorMvccPartition::getName).collect(Collectors.toList()); } + @Test + public void largePartitionViewsCanBeEstimatedWithoutTheReflectiveVisitLimit() { + List listView = new ArrayList<>(); + List mvccPartitions = new ArrayList<>(); + for (int index = 0; index < 20_000; index++) { + String value = Integer.toString(index); + listView.add(new ConnectorPartitionInfo("p=" + value, + Collections.singletonMap("p", value), Collections.emptyMap(), + Collections.singletonList(value), Collections.emptyList())); + mvccPartitions.add(new ConnectorMvccPartition( + "p=" + value, Collections.singletonList(value), + Collections.singletonList(value), index)); + } + ConnectorTableKey key = new ConnectorTableKey("db", "table", 1L, 1L); + ConnectorMvccPartitionView mvccView = new ConnectorMvccPartitionView( + ConnectorMvccPartitionView.Style.RANGE, + ConnectorMvccPartitionView.Freshness.SNAPSHOT_ID, mvccPartitions, 1L); + + Assertions.assertTrue(IcebergCacheSizeEstimator.estimatePartitionInfoViewEntry( + key, listView).isComplete()); + Assertions.assertTrue(IcebergCacheSizeEstimator.estimateMvccPartitionViewEntry( + key, mvccView).isComplete()); + } + // --------------------------------------------------------------------- // getMvccPartitionView // --------------------------------------------------------------------- diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java index 920e5ae0c6d646..6ead48d4d7798b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergManifestCacheTest.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CatalogMetaCache; + import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.ManifestFile; @@ -80,6 +82,22 @@ public void loadsDataFilesAndCachesByManifestPath() { Assertions.assertEquals(1, cache.size()); } + @Test + public void weightBoundedManifestIsEstimatedAndCached() { + Table table = tableWithTwoDataFiles(); + ManifestFile manifest = table.currentSnapshot().dataManifests(table.io()).get(0); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergManifestCache cache = new IcebergManifestCache(owner, + Collections.singletonMap("meta.cache.iceberg.manifest.max-weight", "1MB")); + + ManifestCacheValue first = cache.getManifestCacheValue(manifest, table); + ManifestCacheValue second = cache.getManifestCacheValue(manifest, table); + + Assertions.assertSame(first, second); + Assertions.assertEquals(2, first.getDataFiles().size()); + } + } + @Test public void equalityDeleteFieldIdsLoadOncePerSnapshot() { IcebergManifestCache cache = new IcebergManifestCache(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java index 38c46ebaf7b008..2f832abd4d957d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergPartitionCacheTest.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.iceberg.IcebergPartitionUtils.IcebergRawPartition; import org.apache.iceberg.catalog.TableIdentifier; @@ -73,6 +75,29 @@ public void cachesWithinTtlAndServesTheSameList() { Assertions.assertTrue(c.isEnabled()); } + @Test + public void weightBoundedPartitionsAreEstimatedAndCached() { + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergPartitionCache cache = new IcebergPartitionCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 1024L * 1024L)); + + List first = cache.getOrLoad(key("db", "t", 5L), () -> { + loads.incrementAndGet(); + return raws(3); + }); + List second = cache.getOrLoad(key("db", "t", 5L), () -> { + loads.incrementAndGet(); + return raws(7); + }); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, loads.get()); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> first.add(raws(1).get(0))); + } + } + @Test public void differentSnapshotIdIsADifferentKey() { AtomicInteger loads = new AtomicInteger(); diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheBenchmark.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheBenchmark.java new file mode 100644 index 00000000000000..3bb118d9b897f8 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheBenchmark.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; + +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.LocationProvider; +import org.apache.iceberg.types.Types; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Dependency-free microbenchmark for weighted Iceberg admission and per-statement metadata isolation. */ +public final class IcebergTableCacheBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static volatile long blackhole; + + private IcebergTableCacheBenchmark() { + } + + public static void main(String[] args) { + Table table = tableFixture(); + TableMetadata metadata = ((BaseTable) table).operations().current(); + String json = TableMetadataParser.toJson(metadata); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 100L * 1024L * 1024L), + ignored -> () -> { }, new IcebergCatalogResourceTracker()); + try (IcebergTableCache.TableLease lease = cache.borrow( + TableIdentifier.of("db", "table"), () -> table)) { + Result result = measure(() -> { + Table statementTable = lease.snapshotReadTable(); + return ((BaseTable) statementTable).operations().current().properties().size(); + }, 100); + System.out.printf( + "iceberg_metadata_json_chars=%d properties=%d statement_copy_ns_op=%d operations=%d%n", + json.length(), metadata.properties().size(), result.medianNanos, result.operations); + } + } + } + + private static Table tableFixture() { + Schema schema = new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "payload", Types.StringType.get())); + Map properties = new LinkedHashMap<>(); + for (int i = 0; i < 1_000; i++) { + properties.put("property-" + i, "value-" + i + "-" + "x".repeat(48)); + } + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/weighted-table", properties); + metadata = TableMetadata.buildFrom(metadata) + .withMetadataLocation("file:///tmp/weighted-table/metadata/v1.metadata.json") + .discardChanges() + .build(); + return new BaseTable(new StaticTableOperations(metadata), "weighted"); + } + + private static Result measure(LongOperation operation, int operationsPerWindow) { + for (int i = 0; i < WARMUP_WINDOWS; i++) { + runWindow(operation, operationsPerWindow); + } + long[] nanosPerOperation = new long[MEASURE_WINDOWS]; + for (int i = 0; i < MEASURE_WINDOWS; i++) { + long start = System.nanoTime(); + runWindow(operation, operationsPerWindow); + nanosPerOperation[i] = (System.nanoTime() - start) / operationsPerWindow; + } + Arrays.sort(nanosPerOperation); + return new Result(nanosPerOperation[MEASURE_WINDOWS / 2], + (long) operationsPerWindow * MEASURE_WINDOWS); + } + + private static void runWindow(LongOperation operation, int operations) { + long value = 0L; + for (int i = 0; i < operations; i++) { + value ^= operation.run(); + } + blackhole = value; + } + + @FunctionalInterface + private interface LongOperation { + long run(); + } + + private static final class Result { + private final long medianNanos; + private final long operations; + + private Result(long medianNanos, long operations) { + this.medianNanos = medianNanos; + this.operations = operations; + } + } + + private static final class StaticTableOperations implements TableOperations { + private final TableMetadata metadata; + + private StaticTableOperations(TableMetadata metadata) { + this.metadata = metadata; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; + } + + @Override + public void commit(TableMetadata base, TableMetadata newMetadata) { + throw new UnsupportedOperationException(); + } + + @Override + public FileIO io() { + return null; + } + + @Override + public EncryptionManager encryption() { + return null; + } + + @Override + public String metadataFileLocation(String fileName) { + return "file:///tmp/weighted-table/metadata/" + fileName; + } + + @Override + public LocationProvider locationProvider() { + return null; + } + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java index db13f08f669c4f..36ca2a94c210f6 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTableCacheTest.java @@ -17,11 +17,24 @@ package org.apache.doris.connector.iceberg; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; + +import org.apache.iceberg.BaseTable; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -76,6 +89,110 @@ public void cachesWithinTtlAndServesTheSameTable() { Assertions.assertTrue(c.isEnabled()); } + @Test + public void weightBoundedTableIsEstimatedAndCached() { + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 1024L * 1024L), + ignored -> () -> { + }, new IcebergCatalogResourceTracker()); + + Table first = cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return table("first"); + }); + Table second = cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return table("second"); + }); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, loads.get()); + } + } + + @Test + public void weightedBorrowUsesAnIndependentSnapshotGenerationPerStatement() { + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 10L * 1024L * 1024L), + ignored -> () -> { + }, new IcebergCatalogResourceTracker()); + + try (IcebergTableCache.TableLease lease = cache.borrow(id(), + IcebergTableCacheTest::tableWithSnapshot)) { + BaseTable cached = (BaseTable) lease.table(); + BaseTable firstStatement = (BaseTable) lease.snapshotReadTable(); + BaseTable secondStatement = (BaseTable) lease.snapshotReadTable(); + + Snapshot cachedSnapshot = cached.operations().current().currentSnapshot(); + Snapshot firstSnapshot = firstStatement.operations().current().currentSnapshot(); + Snapshot secondSnapshot = secondStatement.operations().current().currentSnapshot(); + Assertions.assertNotSame(cachedSnapshot, firstSnapshot, + "manifest lazy fields must not be written into the weighted cache generation"); + Assertions.assertNotSame(firstSnapshot, secondSnapshot, + "each statement must own its snapshot lazy-loading state"); + Assertions.assertEquals(cachedSnapshot.snapshotId(), firstSnapshot.snapshotId()); + } + Assertions.assertEquals(1, cache.size()); + } + } + + @Test + public void weightedV1TableIsMeasuredAfterSerializationMaterializesEmbeddedManifests() { + Table probe = tableWithV1EmbeddedManifests(256); + long beforeSerialization = IcebergCacheSizeEstimator.estimateTable(probe); + String metadataJson = TableMetadataParser.toJson(((BaseTable) probe).operations().current()); + long afterSerialization = IcebergCacheSizeEstimator.estimateTable(probe); + long materializedBytes = afterSerialization - beforeSerialization; + Assertions.assertTrue(materializedBytes > 8192L, + "the fixture must materialize enough v1 manifest state to distinguish the two weigh orders"); + + long oldOrderPayload = beforeSerialization + + IcebergCacheSizeEstimator.estimateSerializedTableMetadata(metadataJson); + long maxWeight = oldOrderPayload + materializedBytes / 2L; + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, maxWeight), + ignored -> () -> { }, new IcebergCatalogResourceTracker()); + + cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return tableWithV1EmbeddedManifests(256); + }); + cache.getOrLoad(id(), () -> { + loads.incrementAndGet(); + return tableWithV1EmbeddedManifests(256); + }); + + Assertions.assertEquals(2, loads.get(), + "the post-serialization retained graph exceeds the budget and must not be cached"); + Assertions.assertEquals(0, cache.size()); + } + } + + @Test + public void weightedBorrowSupportsMetadataWithoutAFileLocation() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/no-metadata-location", Collections.emptyMap()); + Table table = new BaseTable(new StaticTableOperations(metadata), "no-metadata-location"); + + try (CatalogMetaCache owner = new CatalogMetaCache()) { + IcebergTableCache cache = new IcebergTableCache( + owner, CacheSpec.ofWeight(true, 100L, 1000L, 10L * 1024L * 1024L), + ignored -> () -> { }, new IcebergCatalogResourceTracker()); + try (IcebergTableCache.TableLease lease = cache.borrow(id(), () -> table)) { + BaseTable statement = (BaseTable) lease.snapshotReadTable(); + Assertions.assertNull(statement.operations().current().metadataFileLocation()); + Assertions.assertEquals(schema.asStruct(), statement.schema().asStruct()); + } + Assertions.assertEquals(1, cache.size()); + } + } + @Test public void ttlZeroDisablesCachingAlwaysLive() { AtomicInteger loads = new AtomicInteger(); @@ -281,4 +398,91 @@ public void loaderExceptionPropagatesUnwrapped() { throw new NoSuchTableException("simulated concurrent drop"); })); } + + private static Table tableWithSnapshot() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata base = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/weighted-table", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{" + + "\"sequence-number\":1," + + "\"snapshot-id\":101," + + "\"timestamp-ms\":1000," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"file:///tmp/snap-101.avro\"," + + "\"schema-id\":0}"); + TableMetadata metadata = TableMetadata.buildFrom(base) + .upgradeFormatVersion(2) + .withMetadataLocation("file:///tmp/v2.metadata.json") + .setBranchSnapshot(snapshot, "main") + .discardChanges() + .build(); + return new BaseTable(new StaticTableOperations(metadata), "weighted"); + } + + private static Table tableWithV1EmbeddedManifests(int manifestCount) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata base = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:///tmp/weighted-v1-table", + Collections.singletonMap(TableProperties.FORMAT_VERSION, "1")); + StringBuilder snapshotJson = new StringBuilder() + .append("{\"snapshot-id\":101,\"timestamp-ms\":1000,") + .append("\"summary\":{\"operation\":\"append\"},\"schema-id\":0,\"manifests\":["); + for (int i = 0; i < manifestCount; i++) { + if (i > 0) { + snapshotJson.append(','); + } + snapshotJson.append("\"file:///tmp/manifest-").append(i).append(".avro\""); + } + snapshotJson.append("]}"); + Snapshot snapshot = SnapshotParser.fromJson(snapshotJson.toString()); + TableMetadata metadata = TableMetadata.buildFrom(base) + .withMetadataLocation("file:///tmp/v1.metadata.json") + .setBranchSnapshot(snapshot, "main") + .discardChanges() + .build(); + return new BaseTable(new StaticTableOperations(metadata), "weighted-v1"); + } + + private static final class StaticTableOperations implements TableOperations { + private final TableMetadata metadata; + + private StaticTableOperations(TableMetadata metadata) { + this.metadata = metadata; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; + } + + @Override + public void commit(TableMetadata base, TableMetadata updated) { + throw new UnsupportedOperationException("read only"); + } + + @Override + public FileIO io() { + return null; + } + + @Override + public EncryptionManager encryption() { + return null; + } + + @Override + public String metadataFileLocation(String fileName) { + return "file:///tmp/" + fileName; + } + + @Override + public LocationProvider locationProvider() { + return null; + } + } } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java index 9b9225178d7826..3e52564feec94c 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogFactory.java @@ -103,6 +103,11 @@ private PaimonCatalogFactory() { * plus each flavor's {@code appendCustomCatalogOptions()}. */ public static Options buildCatalogOptions(PaimonCatalogProperties catalogProperties) { + return buildCatalogOptions(catalogProperties, false); + } + + static Options buildCatalogOptions( + PaimonCatalogProperties catalogProperties, boolean hasEnclosingMetaCacheWeightLimit) { Options options = new Options(); Map props = catalogProperties.getRaw(); String flavor = catalogProperties.getFlavor(); @@ -136,6 +141,9 @@ public static Options buildCatalogOptions(PaimonCatalogProperties catalogPropert // filesystem: nothing custom. break; } + if (hasEnclosingMetaCacheWeightLimit && !options.contains(CatalogOptions.CACHE_ENABLED)) { + options.set(CatalogOptions.CACHE_ENABLED, false); + } return options; } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java index f2220daff2da1e..d2ef06f23df231 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonCatalogProperties.java @@ -160,6 +160,7 @@ public PaimonCatalogProperties checkCreateTimeOnlyRules() { * {@code table.capacity} must be a long ≥ 0. Absent keys are skipped. */ private static void checkMetaCacheProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, "paimon", "partition_view"); CacheSpec.checkBooleanProperty(properties.get(PaimonConnector.TABLE_CACHE_ENABLE), PaimonConnector.TABLE_CACHE_ENABLE); CacheSpec.checkLongProperty(properties.get(PaimonConnector.TABLE_CACHE_TTL_SECOND), diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index 761b6252f60357..b00efbcb57712c 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -140,14 +140,13 @@ public class PaimonConnector implements Connector { // returns a fresh metadata per query, so this lives on the connector and is injected into the metadata so // beginQuerySnapshot pins a stable id across queries. Cleared wholesale on REFRESH CATALOG (connector rebuilt). private final PaimonLatestSnapshotCache latestSnapshotCache; - private final CatalogMetaCache metaCache = new CatalogMetaCache(); + private final CatalogMetaCache metaCache; // FIX-B-MC2: connector-level (per-catalog, long-lived) second-level memo for the time-travel // schema-at-snapshot read. getMetadata() returns a FRESH metadata per query, so this must live on the // connector (not the metadata) to give the cross-query hit the legacy PaimonExternalMetaCache provided. // Cleared wholesale on REFRESH CATALOG (the connector is rebuilt). See PaimonSchemaAtMemo. - private final PaimonSchemaAtMemo schemaAtMemo = - new PaimonSchemaAtMemo(metaCache, PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); + private final PaimonSchemaAtMemo schemaAtMemo; // PERF-06: cross-query DERIVED partition-view cache ("cache A", the generic ConnectorMetadataCache from // fe-connector-cache), layered ABOVE the raw remote catalog.listPartitions call (PaimonCatalogOps#listPartitions): @@ -180,13 +179,17 @@ public PaimonConnector(Map properties, ConnectorContext context) // this a DDL/read against secured HDFS negotiates SIMPLE auth. See TcclPinningConnectorContext. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); + this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "paimon", properties); + this.schemaAtMemo = new PaimonSchemaAtMemo(metaCache, PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); this.latestSnapshotCache = new PaimonLatestSnapshotCache( metaCache, resolveTableCacheTtlSecond(properties), DEFAULT_TABLE_CACHE_CAPACITY); // Reads its own meta.cache.paimon.partition_view.(enable|ttl-second|capacity) from the catalog // properties via the framework's CacheSpec (default ON / 24h / 1000). this.partitionViewCache = new ConnectorMetadataCache<>( - metaCache, "paimon.partition-view", "paimon", "partition_view", properties); + metaCache, "paimon.partition-view", "paimon", "partition_view", properties, + key -> org.apache.doris.connector.cache.ScopePath.table(key.getDb(), key.getTable()), + PaimonPartitionViewSizeEstimator::estimateEntry); } /** @@ -408,7 +411,7 @@ private Catalog ensureCatalog() { } private Catalog createCatalog() { - Options options = PaimonCatalogFactory.buildCatalogOptions(catalogProps); + Options options = buildCatalogOptions(); String flavor = catalogProps.getFlavor(); // Canonical storage config from the FE-bound fe-filesystem StorageProperties (P1-T03), replacing // the legacy buildObjectStorageHadoopConfig path: object stores contribute their fs.s3a.*/fs.oss.* @@ -495,6 +498,10 @@ static boolean hasDlfCompatibleStorage(List storageProperties || "OSS_HDFS".equals(storage.providerName())); } + Options buildCatalogOptions() { + return PaimonCatalogFactory.buildCatalogOptions(catalogProps, metaCache.hasEnclosingWeightLimit()); + } + /** * Assembles the canonical storage Hadoop config from the FE-bound storage properties (P1-T03). * fe-core binds the catalog's raw property map to fe-filesystem {@link StorageProperties} and hands diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java index 62113194c76895..f382ca22d37ead 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java @@ -1338,7 +1338,12 @@ private List cachedPartitions(PaimonTableHandle paimonHa return collectPartitions(paimonHandle); } ConnectorTableKey key = partitionViewCacheKey(paimonHandle); - return partitionViewCache.get(key, () -> collectPartitions(paimonHandle)); + return partitionViewCache.get(key, () -> { + List partitions = collectPartitions(paimonHandle); + return partitionViewCache.isWeightBounded() + ? new PaimonPartitionView(key, partitions) + : partitions; + }); } /** diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java index e42ba1f5917462..0f241158c5997f 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonLatestSnapshotCache.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import org.apache.paimon.catalog.Identifier; @@ -63,6 +64,7 @@ final class PaimonLatestSnapshotCache { this.entry = owner.create(MetaCacheDefinition .builder("paimon-latest-snapshot", spec, id -> ScopePath.table(id.getDatabaseName(), id.getObjectName())) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java new file mode 100644 index 00000000000000..3701d60029cb7f --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionView.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.cache.MetaCacheSizeEstimator; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.RandomAccess; + +/** Immutable partition-view value whose retained size is computed once at construction. */ +final class PaimonPartitionView extends AbstractList implements RandomAccess { + private final List partitions; + private final MetaCacheSizeEstimate sizeEstimate; + + PaimonPartitionView(ConnectorTableKey key, List partitions) { + this.partitions = Collections.unmodifiableList(new ArrayList<>(partitions)); + this.sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_partition_estimator_failure", + () -> MetaCacheSizeEstimate.complete( + PaimonPartitionViewSizeEstimator.estimateEntryOnConstruction(key, this))); + } + + @Override + public ConnectorPartitionInfo get(int index) { + return partitions.get(index); + } + + @Override + public int size() { + return partitions.size(); + } + + MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate; + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java new file mode 100644 index 00000000000000..62354404dcd0eb --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonPartitionViewSizeEstimator.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.cache.MetaCacheSizeEstimate; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; + +/** + * Publication-time estimate of the projection built by PaimonConnectorMetadata.collectPartitions. + * Counts every partition's strings; sampling variable-length payload can both miss and amplify a large tail. + */ +final class PaimonPartitionViewSizeEstimator { + private static final long KEY_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorTableKey.class); + private static final long VIEW_SHALLOW_BYTES = JvmSizeUtils.instanceSize(PaimonPartitionView.class); + private static final long ESTIMATE_SHALLOW_BYTES = JvmSizeUtils.instanceSize(MetaCacheSizeEstimate.class); + private static final long PARTITION_SHALLOW_BYTES = JvmSizeUtils.instanceSize(ConnectorPartitionInfo.class); + private static final long UNMODIFIABLE_LIST_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableList(Collections.emptyList()).getClass()); + private static final long UNMODIFIABLE_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize( + Collections.unmodifiableMap(Collections.emptyMap()).getClass()); + private static final long LINKED_HASH_MAP_SHALLOW_BYTES = JvmSizeUtils.instanceSize(LinkedHashMap.class); + private static final long LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES = classSize("java.util.LinkedHashMap$Entry"); + + private PaimonPartitionViewSizeEstimator() { + } + + /** Admission callback: the complete key/value weight was computed when the immutable view was built. */ + static MetaCacheSizeEstimate estimateEntry(ConnectorTableKey key, List value) { + return ((PaimonPartitionView) value).getSizeEstimate(); + } + + static long estimateEntryOnConstruction(ConnectorTableKey key, PaimonPartitionView value) { + long bytes = KEY_SHALLOW_BYTES; + bytes = add(bytes, JvmSizeUtils.stringSize(key.getDb())); + bytes = add(bytes, JvmSizeUtils.stringSize(key.getTable())); + bytes = add(bytes, VIEW_SHALLOW_BYTES); + bytes = add(bytes, ESTIMATE_SHALLOW_BYTES); + bytes = add(bytes, UNMODIFIABLE_LIST_SHALLOW_BYTES); + bytes = add(bytes, JvmSizeUtils.arrayListSize(value.size())); + if (!value.isEmpty()) { + // collectPartitions builds every map/list from the same partitionKeys, with shared column names. + // Compute that fixed shape once; only the name/value String payload varies between partitions. + bytes = add(bytes, multiply(value.size(), estimatePartitionStructure(value.get(0)))); + for (String column : value.get(0).getPartitionValues().keySet()) { + bytes = add(bytes, JvmSizeUtils.stringSize(column)); + } + } + for (int index = 0; index < value.size(); index++) { + ConnectorPartitionInfo partition = value.get(index); + bytes = add(bytes, JvmSizeUtils.stringSize(partition.getPartitionName())); + List orderedValues = partition.getOrderedPartitionValues(); + for (int column = 0; column < orderedValues.size(); column++) { + // The same rendered String is retained by both partitionValues and orderedPartitionValues. + bytes = add(bytes, JvmSizeUtils.stringSize(orderedValues.get(column))); + } + } + return bytes; + } + + private static long estimatePartitionStructure(ConnectorPartitionInfo partition) { + long bytes = PARTITION_SHALLOW_BYTES; + // The collector supplies an empty properties map, wrapped by ConnectorPartitionInfo per partition. + bytes = add(bytes, UNMODIFIABLE_MAP_SHALLOW_BYTES); + + int valueCount = partition.getPartitionValues().size(); + bytes = add(bytes, UNMODIFIABLE_MAP_SHALLOW_BYTES); + bytes = add(bytes, LINKED_HASH_MAP_SHALLOW_BYTES); + if (valueCount > 0) { + bytes = add(bytes, JvmSizeUtils.objectArraySize(hashCapacity(valueCount))); + bytes = add(bytes, multiply(valueCount, LINKED_HASH_MAP_ENTRY_SHALLOW_BYTES)); + } + + bytes = add(bytes, estimateCopiedList(partition.getOrderedPartitionValues())); + return add(bytes, estimateCopiedList(partition.getPartitionValueNullFlags())); + } + + private static long estimateCopiedList(List values) { + return add(UNMODIFIABLE_LIST_SHALLOW_BYTES, JvmSizeUtils.arrayListSize(values.size())); + } + + private static int hashCapacity(int size) { + long needed = (size * 4L + 2L) / 3L; + int capacity = 16; + while (capacity < needed && capacity < 1 << 30) { + capacity <<= 1; + } + return capacity; + } + + private static long classSize(String className) { + try { + return JvmSizeUtils.instanceSize(Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("Required JVM collection class is missing: " + className, e); + } + } + + private static long multiply(long left, long right) { + return JvmSizeUtils.saturatedMultiply(left, right); + } + + private static long add(long left, long right) { + return JvmSizeUtils.saturatedAdd(left, right); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java index 9c5dce4d97ef49..7406ec36533310 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonSchemaAtMemo.java @@ -21,6 +21,7 @@ import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; import java.util.Objects; @@ -69,6 +70,7 @@ final class PaimonSchemaAtMemo { .builder( "paimon-schema-at", spec, key -> ScopePath.table(key.databaseName, key.tableName)) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmark.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmark.java new file mode 100644 index 00000000000000..e083d8bdef99fb --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmark.java @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import org.apache.paimon.partition.Partition; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +/** + * Offline microbenchmark, not JMH. Uses the production partition collector with recording catalog fakes: + * no RPC, filesystem access, cache-hit timing, or data-file scans. Compares collection alone against collection + * plus weighted-view construction; publication-only timing includes its list copy. Prepared-weight timing is + * just the provider callback, including loop/volatile-sink overhead, not the complete cache-admission path. + * + *

Compile via run-fe-ut.sh; run this main with the module's test classpath and the FE's java.lang/java.util + * opens. Run multiple fresh JVMs before drawing conclusions. Raw estimates are reported, not asserted to be + * exact retained heap. TAIL_END hits the former 16/5-element sampling positions; TAIL_INTERIOR misses both. + */ +public final class PaimonCacheSizeBenchmark { + private static final int WARMUP_WINDOWS = 5; + private static final int MEASURE_WINDOWS = 15; + private static final long MIN_WINDOW_NANOS = TimeUnit.MILLISECONDS.toNanos(50); + private static volatile Object blackhole; + + private PaimonCacheSizeBenchmark() { + } + + enum Distribution { + UNIFORM, + TAIL_END, + TAIL_INTERIOR + } + + public static void main(String[] args) { + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1_000, 10_000}) { + for (Distribution distribution : Distribution.values()) { + Fixture fixture = new Fixture(fieldCount, partitionCount, distribution); + List partitions = fixture.load(false); + PaimonPartitionView prepared = new PaimonPartitionView(fixture.key, partitions); + if (!prepared.getSizeEstimate().isComplete()) { + throw new IllegalStateException("Incomplete benchmark estimate: " + + prepared.getSizeEstimate().getIncompleteReason()); + } + String label = "fields=" + fieldCount + " partitions=" + partitionCount + + " distribution=" + distribution; + System.out.printf("%s estimated_bytes=%d large_value_chars=%d%n", label, + prepared.getSizeEstimate().getBytes(), distribution == Distribution.UNIFORM + ? 0 : Fixture.LARGE_VALUE_CHARS); + Map> cases = new LinkedHashMap<>(); + cases.put("collect", () -> fixture.load(false)); + cases.put("collect_weighted", () -> fixture.load(true)); + cases.put("publish_only", () -> new PaimonPartitionView(fixture.key, partitions)); + cases.put("prepared_weight", () -> PaimonPartitionViewSizeEstimator + .estimateEntry(fixture.key, prepared)); + measure(label, cases); + } + } + } + } + + static final class Fixture { + static final int LARGE_VALUE_CHARS = 1024 * 1024; + final ConnectorTableKey key = new ConnectorTableKey("db1", "t1", 1L, -1L); + private final RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + private final PaimonConnectorMetadata metadata; + private final PaimonTableHandle handle; + + Fixture(int fieldCount, int partitionCount, Distribution distribution) { + RowType.Builder schema = RowType.builder().field("region", DataTypes.STRING()); + for (int field = 1; field < fieldCount; field++) { + schema.field("column_" + field, DataTypes.STRING()); + } + FakePaimonTable table = new FakePaimonTable("t1", schema.build(), + Collections.singletonList("region"), Collections.emptyList()); + table.setOptions(Collections.singletonMap("partition.legacy-name", "true")); + ops.table = table; + ops.partitions = new ArrayList<>(partitionCount); + int largeIndex = distribution == Distribution.TAIL_END ? partitionCount - 1 + : distribution == Distribution.TAIL_INTERIOR ? partitionCount - 2 : -1; + for (int index = 0; index < partitionCount; index++) { + String value = "region-" + index; + if (index == largeIndex) { + value += "x".repeat(LARGE_VALUE_CHARS); + } + ops.partitions.add(new Partition(Collections.singletonMap("region", value), + 1L, 1L, 1, 1L, true)); + } + metadata = new PaimonConnectorMetadata(ops, PaimonCatalogProperties.of(Collections.emptyMap()), + new RecordingConnectorContext()); + handle = new PaimonTableHandle("db1", "t1", Collections.singletonList("region"), + Collections.emptyList()); + handle.setPaimonTable(table); + } + + List load(boolean weighted) { + // Bound the recording fake's log in BOTH paths; never measure an ever-growing fixture. + ops.log.clear(); + List partitions = metadata.listPartitions(null, handle, Optional.empty()); + // Mirrors cachedPartitions' post-collection branch, without timing cache lookup or admission locks. + return weighted ? new PaimonPartitionView(key, partitions) : partitions; + } + } + + private static void measure(String label, Map> cases) { + List names = new ArrayList<>(cases.keySet()); + List> operations = new ArrayList<>(cases.values()); + int[] batchSizes = new int[operations.size()]; + long[][] samples = new long[operations.size()][MEASURE_WINDOWS]; + for (int index = 0; index < operations.size(); index++) { + int batchSize = 1; + while (runWindow(operations.get(index), batchSize) < MIN_WINDOW_NANOS && batchSize < 1 << 20) { + batchSize *= 2; + } + batchSizes[index] = batchSize; + } + for (int window = 0; window < WARMUP_WINDOWS + MEASURE_WINDOWS; window++) { + // Rotate case order to avoid always measuring the weighted path after the baseline's allocations. + for (int offset = 0; offset < operations.size(); offset++) { + int index = (window + offset) % operations.size(); + long elapsed = runWindow(operations.get(index), batchSizes[index]); + if (window >= WARMUP_WINDOWS) { + samples[index][window - WARMUP_WINDOWS] = elapsed / batchSizes[index]; + } + } + } + for (int index = 0; index < operations.size(); index++) { + Arrays.sort(samples[index]); + System.out.printf("%s operation=%s median_ns_op=%d min_ns_op=%d max_ns_op=%d operations=%d%n", + label, names.get(index), samples[index][MEASURE_WINDOWS / 2], samples[index][0], + samples[index][MEASURE_WINDOWS - 1], (long) batchSizes[index] * MEASURE_WINDOWS); + } + } + + private static long runWindow(Supplier operation, int count) { + long start = System.nanoTime(); + for (int index = 0; index < count; index++) { + // Make allocated results escape; prepared-weight results intentionally retain the same estimate. + blackhole = operation.get(); + } + return System.nanoTime() - start; + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmarkTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmarkTest.java new file mode 100644 index 00000000000000..a2bd18845d6066 --- /dev/null +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCacheSizeBenchmarkTest.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.paimon; + +import org.apache.doris.connector.cache.JvmSizeUtils; +import org.apache.doris.connector.spi.ConnectorPartitionInfo; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +class PaimonCacheSizeBenchmarkTest { + @Test + void weightedFixturePreservesProductionProjectionAndPreparedEstimate() { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, 32, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + List baseline = fixture.load(false); + PaimonPartitionView weighted = (PaimonPartitionView) fixture.load(true); + Assertions.assertEquals(32, baseline.size()); + Assertions.assertEquals(baseline.size(), weighted.size()); + for (int index = 0; index < baseline.size(); index++) { + Assertions.assertEquals(baseline.get(index).getPartitionName(), weighted.get(index).getPartitionName()); + Assertions.assertEquals(baseline.get(index).getPartitionValues(), weighted.get(index).getPartitionValues()); + } + Assertions.assertEquals("region=region-0", weighted.get(0).getPartitionName()); + Assertions.assertTrue(weighted.getSizeEstimate().isComplete()); + Assertions.assertTrue(weighted.getSizeEstimate().getBytes() > 0); + Assertions.assertSame(weighted.getSizeEstimate(), + PaimonPartitionViewSizeEstimator.estimateEntry(fixture.key, weighted)); + Assertions.assertNotSame(baseline, fixture.load(false), "baseline must rebuild rather than hit a cache"); + } + + @Test + void longTailFixturesCoverSampledAndUnsampledPositions() { + for (PaimonCacheSizeBenchmark.Distribution distribution : new PaimonCacheSizeBenchmark.Distribution[] { + PaimonCacheSizeBenchmark.Distribution.TAIL_END, + PaimonCacheSizeBenchmark.Distribution.TAIL_INTERIOR}) { + List partitions = new PaimonCacheSizeBenchmark.Fixture( + 100, 1_000, distribution).load(false); + int largeIndex = distribution == PaimonCacheSizeBenchmark.Distribution.TAIL_END ? 999 : 998; + Assertions.assertTrue(partitions.get(largeIndex).getPartitionValues().get("region").length() + >= PaimonCacheSizeBenchmark.Fixture.LARGE_VALUE_CHARS); + for (int samples : new int[] {5, 16}) { + boolean sampled = false; + for (int sample = 0; sample < samples; sample++) { + sampled |= sample * (partitions.size() - 1) / (samples - 1) == largeIndex; + } + Assertions.assertEquals(distribution == PaimonCacheSizeBenchmark.Distribution.TAIL_END, sampled); + } + } + } + + @Test + void longTailWeightIsIndependentOfPositionAndNotExtrapolated() { + for (int count : new int[] {1_000, 10_000}) { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + PaimonPartitionView uniform = (PaimonPartitionView) fixture.load(true); + PaimonPartitionView interior = (PaimonPartitionView) new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.TAIL_INTERIOR).load(true); + PaimonPartitionView end = (PaimonPartitionView) new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.TAIL_END).load(true); + Assertions.assertTrue(interior.getSizeEstimate().isComplete()); + Assertions.assertTrue(end.getSizeEstimate().isComplete()); + // One value String shared by the list/map, plus a separately materialized partition-name String. + ConnectorPartitionInfo small = uniform.get(count - 2); + ConnectorPartitionInfo large = interior.get(count - 2); + long expectedGrowth = JvmSizeUtils.stringSize(large.getPartitionName()) + - JvmSizeUtils.stringSize(small.getPartitionName()) + + JvmSizeUtils.stringSize(large.getOrderedPartitionValues().get(0)) + - JvmSizeUtils.stringSize(small.getOrderedPartitionValues().get(0)); + Assertions.assertTrue(expectedGrowth >= 2L * PaimonCacheSizeBenchmark.Fixture.LARGE_VALUE_CHARS); + Assertions.assertEquals(expectedGrowth, + interior.getSizeEstimate().getBytes() - uniform.getSizeEstimate().getBytes()); + Assertions.assertEquals(interior.getSizeEstimate().getBytes(), end.getSizeEstimate().getBytes()); + List reordered = new ArrayList<>(interior); + Collections.swap(reordered, 0, count - 2); + Assertions.assertEquals(interior.getSizeEstimate().getBytes(), + new PaimonPartitionView(fixture.key, reordered).getSizeEstimate().getBytes()); + } + } + + @Test + void unicodeAndMultipleColumnsAreCountedWithoutDoubleChargingMapValues() { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, 0, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + for (int columns : new int[] {1, 4, 16}) { + Map values = new LinkedHashMap<>(); + for (int column = 0; column < columns; column++) { + values.put("column-" + column, "small-" + column); + } + ConnectorPartitionInfo small = partition(values); + Map largeValues = new LinkedHashMap<>(values); + largeValues.put("column-0", "中".repeat(65_536)); + ConnectorPartitionInfo large = partition(largeValues); + long before = new PaimonPartitionView(fixture.key, List.of(small)).getSizeEstimate().getBytes(); + long after = new PaimonPartitionView(fixture.key, List.of(large)).getSizeEstimate().getBytes(); + Assertions.assertEquals(JvmSizeUtils.stringSize(largeValues.get("column-0")) + - JvmSizeUtils.stringSize(values.get("column-0")), after - before); + } + } + + @Test + void projectionRemainsImmutableWithAPreparedWeight() { + for (int count : new int[] {0, 1, 32, 1_000}) { + PaimonCacheSizeBenchmark.Fixture fixture = new PaimonCacheSizeBenchmark.Fixture( + 10, count, PaimonCacheSizeBenchmark.Distribution.UNIFORM); + List input = fixture.load(false); + PaimonPartitionView view = new PaimonPartitionView(fixture.key, input); + Assertions.assertTrue(view.getSizeEstimate().isComplete()); + Assertions.assertTrue(view.getSizeEstimate().getBytes() > 0); + Assertions.assertSame(view.getSizeEstimate(), + PaimonPartitionViewSizeEstimator.estimateEntry(fixture.key, view)); + input.clear(); + Assertions.assertEquals(count, view.size()); + Assertions.assertThrows(UnsupportedOperationException.class, + () -> view.add(partition(Collections.emptyMap()))); + } + } + + private static ConnectorPartitionInfo partition(Map values) { + return new ConnectorPartitionInfo("fixed-name", values, Collections.emptyMap(), + new ArrayList<>(values.values()), new ArrayList<>(Collections.nCopies(values.size(), false))); + } +} diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java index 7a0a4cef186e3a..c1c13f67e54186 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonCatalogFactoryTest.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -91,6 +92,44 @@ public void filesystemSetsMetastoreFilesystemAndWarehouse() { Assertions.assertEquals("/wh", opts.get("warehouse")); } + @Test + public void enclosingDorisWeightLimitDisablesPaimonSdkCacheOnlyByDefault() throws Exception { + Map defaults = props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh"); + + try (PaimonConnector ungoverned = new PaimonConnector(defaults, new RecordingConnectorContext())) { + Options options = ungoverned.buildCatalogOptions(); + Assertions.assertFalse(options.contains(CatalogOptions.CACHE_ENABLED), + "without a global/catalog total, Doris must preserve the Paimon SDK default"); + } + + Map catalogLimited = new HashMap<>(defaults); + catalogLimited.put("meta.cache.max-weight", "1MB"); + try (PaimonConnector governed = new PaimonConnector(catalogLimited, new RecordingConnectorContext())) { + Options options = governed.buildCatalogOptions(); + Assertions.assertTrue(options.contains(CatalogOptions.CACHE_ENABLED)); + Assertions.assertFalse(options.get(CatalogOptions.CACHE_ENABLED), + "an enclosing Doris hard limit must not be bypassed by an unaccounted SDK cache"); + } + + Map entryLimited = new HashMap<>(defaults); + entryLimited.put("meta.cache.paimon.partition_view.max-weight", "1MB"); + try (PaimonConnector entryOnly = new PaimonConnector(entryLimited, new RecordingConnectorContext())) { + Assertions.assertFalse(entryOnly.buildCatalogOptions().contains(CatalogOptions.CACHE_ENABLED), + "an entry-only limit must preserve the Paimon SDK default"); + } + + PaimonCatalogProperties explicitTrue = PaimonCatalogProperties.of(props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh", "paimon.cache-enabled", "true")); + Assertions.assertTrue(PaimonCatalogFactory.buildCatalogOptions(explicitTrue, true) + .get(CatalogOptions.CACHE_ENABLED), "an explicit Paimon setting must win"); + + PaimonCatalogProperties explicitFalse = PaimonCatalogProperties.of(props( + "paimon.catalog.type", "filesystem", "warehouse", "/wh", "paimon.cache-enabled", "false")); + Assertions.assertFalse(PaimonCatalogFactory.buildCatalogOptions(explicitFalse, true) + .get(CatalogOptions.CACHE_ENABLED), "an explicit Paimon setting must win"); + } + @Test public void hmsSetsHiveMetastoreUriPoolAndLocation() { Options opts = PaimonCatalogFactory.buildCatalogOptions(PaimonCatalogProperties.of(props( diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java index 831a5d59601bc1..5302d9a750811e 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataPartitionViewCacheTest.java @@ -17,7 +17,10 @@ package org.apache.doris.connector.paimon; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.ConnectorMetadataCache; +import org.apache.doris.connector.cache.ConnectorTableKey; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.ConnectorPartitionInfo; import org.apache.doris.connector.spi.pushdown.ConnectorExpression; @@ -27,6 +30,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; @@ -103,6 +107,21 @@ private static List names(List infos) { return infos.stream().map(ConnectorPartitionInfo::getPartitionName).collect(Collectors.toList()); } + @Test + public void largePartitionViewCanBeEstimatedWithoutTheReflectiveVisitLimit() { + List partitions = new ArrayList<>(); + for (int index = 0; index < 20_000; index++) { + String value = Integer.toString(index); + partitions.add(new ConnectorPartitionInfo("p=" + value, + Collections.singletonMap("p", value), Collections.emptyMap(), + Collections.singletonList(value), Collections.emptyList())); + } + + PaimonPartitionView view = new PaimonPartitionView( + new ConnectorTableKey("db", "table", 1L, 1L), partitions); + Assertions.assertTrue(view.getSizeEstimate().isComplete()); + } + @Test public void listPartitionsCachesDerivedListAcrossQueries() { // WHY: cache A must memoize the BUILT List keyed by (db, table, snapshotId, @@ -126,6 +145,60 @@ public void listPartitionsCachesDerivedListAcrossQueries() { Assertions.assertEquals(1, loadCount(ops), "a cache hit must not re-enumerate (listPartitions once)"); } + @Test + public void weightBoundedPartitionViewIsEstimatedAndCached() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = regionTable(); + ops.table = table; + ops.latestSnapshotId = OptionalLong.of(100L); + ops.partitions = Arrays.asList(partition("cn"), partition("us")); + Map properties = Collections.singletonMap( + "meta.cache.paimon.partition_view.max-weight", "1MB"); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + ConnectorMetadataCache> cache = new ConnectorMetadataCache<>( + owner, "paimon.partition-view", "paimon", "partition_view", properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + PaimonPartitionViewSizeEstimator::estimateEntry); + PaimonConnectorMetadata metadata = metadataWithCache(ops, cache); + PaimonTableHandle handle = handle(table); + + List first = metadata.listPartitions(null, handle, Optional.empty()); + List second = metadata.listPartitions(null, handle, Optional.empty()); + + Assertions.assertSame(first, second); + Assertions.assertEquals(1, loadCount(ops)); + } + } + + @Test + public void unsampledLongTailRejectsAdmissionWithoutFailingPartitionListing() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + FakePaimonTable table = regionTable(); + ops.table = table; + ops.latestSnapshotId = OptionalLong.of(100L); + ops.partitions = new ArrayList<>(); + for (int index = 0; index < 1_000; index++) { + ops.partitions.add(partition("region-" + index)); + } + ops.partitions.set(998, partition("x".repeat(1024 * 1024))); + Map properties = Collections.singletonMap( + "meta.cache.paimon.partition_view.max-weight", "1MB"); + try (CatalogMetaCache owner = new CatalogMetaCache()) { + ConnectorMetadataCache> cache = new ConnectorMetadataCache<>( + owner, "paimon.partition-view", "paimon", "partition_view", properties, + key -> ScopePath.table(key.getDb(), key.getTable()), + PaimonPartitionViewSizeEstimator::estimateEntry); + PaimonConnectorMetadata metadata = metadataWithCache(ops, cache); + PaimonTableHandle handle = handle(table); + List first = metadata.listPartitions(null, handle, Optional.empty()); + List second = metadata.listPartitions(null, handle, Optional.empty()); + Assertions.assertEquals(1_000, first.size()); + Assertions.assertEquals(first, second); + Assertions.assertNotSame(first, second, "oversized views are returned, not cached"); + Assertions.assertEquals(2, loadCount(ops)); + } + } + @Test public void listPartitionsDifferentSnapshotReEnumerates() { // WHY: the underlying remote enumeration always reflects the CURRENT catalog state (it is diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 91dd81e5af5952..cb91fa2ef2dab2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -651,6 +651,10 @@ public class SchemaTable extends Table { .column("LAST_LOAD_SUCCESS_TIME", ScalarType.createStringType()) .column("LAST_LOAD_FAILURE_TIME", ScalarType.createStringType()) .column("LAST_ERROR", ScalarType.createStringType()) + .column("MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ESTIMATED_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("WEIGHT_REJECT_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("LAST_WEIGHT_REJECT_REASON", ScalarType.createStringType()) .build()) ) .put("backend_kerberos_ticket_cache", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 8a8e2dcb45bb0f..8256a365ec8442 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -39,6 +39,7 @@ import org.apache.doris.common.Version; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.MetaCacheBudgetManager; import org.apache.doris.datasource.doris.RemoteDorisExternalDatabase; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; @@ -436,6 +437,11 @@ public void checkProperties() throws DdlException { protected void checkProperties(CatalogProperty property) throws DdlException { // check refresh parameter of catalog Map properties = property.getProperties(); + try { + checkMetaCacheWeightProperties(properties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) { try { int metadataRefreshIntervalSec = Integer.parseInt( @@ -457,6 +463,11 @@ protected void checkProperties(CatalogProperty property) throws DdlException { } } + /** Strict CREATE/ALTER validation for the core metadata-cache namespace owned by this catalog. */ + protected void checkMetaCacheWeightProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, "default", "schema"); + } + /** * Validate an ALTER candidate without publishing it to this catalog. A true return value * declares that the connector performed complete detached validation; false retains the @@ -867,9 +878,27 @@ public void tryModifyCatalogProps(Map props) { private void invalidateCachesAfterPropertyUpdate(Map updatedProps) { ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - if (updatedProps.get(SCHEMA_CACHE_TTL_SECOND) != null) { + if (updatedProps.get(SCHEMA_CACHE_TTL_SECOND) != null + || updatedProps.containsKey(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { cacheMgr.removeCatalog(id); - } else { + return; + } + Set removedEngines = Sets.newHashSet(); + for (String key : updatedProps.keySet()) { + if (key == null || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + if (separator <= 0) { + continue; + } + String engine = remainder.substring(0, separator); + if (cacheMgr.isEngineRegistered(engine) && removedEngines.add(engine)) { + cacheMgr.removeCatalogByEngine(id, engine); + } + } + if (removedEngines.isEmpty()) { cacheMgr.invalidateCatalog(id); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index d22ed858b4ef87..900b880f88a724 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -21,6 +21,10 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheGovernance; import org.apache.doris.datasource.doris.DorisExternalMetaCache; import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; import org.apache.doris.datasource.metacache.ExternalCatalogMetaCache; @@ -43,6 +47,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -91,6 +96,7 @@ public class ExternalMetaCacheMgr { private ExternalRowCountCache rowCountCache; public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { + MetaCacheGovernance.configureGlobalMaxWeight(configuredGlobalMaxWeight()); rowCountRefreshExecutor = newThreadPool(isCheckpointCatalog, Config.max_external_cache_loader_thread_pool_size, Config.max_external_cache_loader_thread_pool_size * 1000, @@ -118,6 +124,17 @@ public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { registerBuiltinEngineCaches(); } + private static OptionalLong configuredGlobalMaxWeight() { + String configured = Config.external_meta_cache_max_weight; + long parsed = CacheSpec.parseWeight(configured, "external_meta_cache_max_weight", + true, Runtime.getRuntime().maxMemory()); + if (configured.trim().endsWith("%") && parsed == 0L) { + throw new IllegalArgumentException( + "external_meta_cache_max_weight percentage must be greater than 0%"); + } + return parsed == 0L ? OptionalLong.empty() : OptionalLong.of(parsed); + } + private ExecutorService newThreadPool(boolean isCheckpointCatalog, int numThread, int queueSize, String poolName, int timeoutSeconds, boolean needRegisterMetric) { @@ -220,6 +237,10 @@ public void removeCatalogByEngine(long catalogId, String engine) { () -> cache.invalidateCatalog(catalogId))); } + public boolean isEngineRegistered(String engine) { + return cacheTypes.containsKey(engine); + } + /** * Invalidates database metadata without evicting row counts. Passive object-cache resets use this directly; * mutation paths must add their row-count barrier after upstream metadata has been invalidated. @@ -331,6 +352,15 @@ public List getCatalogCacheStats(long catalogId) { allCacheTypes().forEach(externalMetaCache -> externalMetaCache.stats(catalogId) .forEach((entryName, entryStats) -> stats.add( new CatalogMetaCacheStats(externalMetaCache.engine(), entryName, entryStats)))); + for (CatalogMetaCache cache : MetaCacheGovernance.catalogCaches(catalogId)) { + if (ENGINE_DEFAULT.equals(cache.engine()) || ENGINE_DORIS.equals(cache.engine())) { + continue; + } + for (Map.Entry> entry : cache.entries().entrySet()) { + stats.add(new CatalogMetaCacheStats( + cache.engine(), entry.getKey(), MetaCacheEntryStats.from(entry.getValue()))); + } + } stats.sort(Comparator.comparing(CatalogMetaCacheStats::getEngineName) .thenComparing(CatalogMetaCacheStats::getEntryName)); return stats; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java index e48c07303728e2..0c2397d40f8c59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.SessionContext; @@ -74,6 +75,12 @@ public void checkProperties() throws DdlException { } } + @Override + protected void checkMetaCacheWeightProperties(Map properties) { + CacheSpec.checkWeightProperties(properties, DorisExternalMetaCache.ENGINE, + DorisExternalMetaCache.ENTRY_SCHEMA, DorisExternalMetaCache.ENTRY_BACKENDS); + } + public List getFeNodes() { return parseHttpHosts(catalogProperty.getOrDefault(RemoteDorisProperties.FE_HTTP_HOSTS, "")); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java index b86797ba0dceab..9f9dbfe674086a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java @@ -17,10 +17,8 @@ package org.apache.doris.datasource.metacache; -import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; -import org.apache.doris.connector.cache.ScopedMetaCache.CacheMetrics; import com.google.common.collect.Maps; @@ -32,9 +30,13 @@ * Catalog scoped entry container. */ final class CatalogMetaCacheRuntime { - private final CatalogMetaCache owner = new CatalogMetaCache(); + private final CatalogMetaCache owner; private final Map> entries = new ConcurrentHashMap<>(); + CatalogMetaCacheRuntime(long catalogId, String engine, Map catalogProperties) { + owner = CatalogMetaCache.managed(catalogId, engine, catalogProperties); + } + CatalogMetaCache owner() { return owner; } @@ -62,29 +64,6 @@ void close() { } private MetaCacheEntryStats stats(MetaCache entry) { - CacheSpec spec = entry.cacheSpec(); - CacheMetrics metrics = entry.metrics(); - long requests = metrics.getRequestCount(); - long loads = metrics.getLoadSuccessCount() + metrics.getLoadFailureCount(); - return new MetaCacheEntryStats( - spec.isEnable(), - entry.isEnabled(), - entry.isAutoRefresh(), - spec.getTtlSecond(), - spec.getCapacity(), - entry.size(), - requests, - metrics.getHitCount(), - metrics.getMissCount(), - requests == 0L ? 0D : (double) metrics.getHitCount() / requests, - metrics.getLoadSuccessCount(), - metrics.getLoadFailureCount(), - metrics.getTotalLoadTimeNanos(), - loads == 0L ? 0D : (double) metrics.getTotalLoadTimeNanos() / loads, - metrics.getEvictionCount(), - metrics.getInvalidateCount(), - metrics.getLastLoadSuccessTimeMs(), - metrics.getLastLoadFailureTimeMs(), - metrics.getLastError()); + return MetaCacheEntryStats.from(entry); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java index 751667904c25dd..b7f9809e264934 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java @@ -22,6 +22,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -87,7 +88,8 @@ public Collection aliases() { public void initCatalog(long catalogId, Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, id -> buildCatalogRuntime(safeCatalogProperties)); + catalogEntries.computeIfAbsent(catalogId, + id -> buildCatalogRuntime(id, safeCatalogProperties)); } @Override @@ -271,8 +273,8 @@ private void validateRegisteredMetaCacheEntryDef(MetaCacheEntryDef ensureTypeCompatible(registered, entryDef.getKeyType(), entryDef.getValueType()); } - private CatalogMetaCacheRuntime buildCatalogRuntime(Map catalogProperties) { - CatalogMetaCacheRuntime group = new CatalogMetaCacheRuntime(); + private CatalogMetaCacheRuntime buildCatalogRuntime(long catalogId, Map catalogProperties) { + CatalogMetaCacheRuntime group = new CatalogMetaCacheRuntime(catalogId, engine, catalogProperties); metaCacheEntryDefs.values() .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(group, entryDef, catalogProperties))); @@ -286,8 +288,9 @@ private MetaCache newMetaCacheEntry( CacheSpec cacheSpec = CacheSpec.fromProperties( catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); Function loader = wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()); - MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( - entryDef.getName(), cacheSpec, entryDef.getInvalidation()::scope); + MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( + entryDef.getName(), cacheSpec, entryDef.getInvalidation()::scope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()); if (loader != null) { builder.loader(loader); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java index b2d503b82ad087..5c6ea2cc968958 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java @@ -23,8 +23,8 @@ import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; import org.apache.doris.connector.cache.MetaCacheRemovalReason; +import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; -import org.apache.doris.connector.cache.ScopedMetaCache.CacheMetrics; import com.github.benmanes.caffeine.cache.RemovalListener; @@ -155,8 +155,9 @@ public static FeMetaCacheEntry withSyncRemovalListener(String name, } effectiveEnabled = CacheSpec.isCacheEnabled( cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); - MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( - name, cacheSpec, ignored -> ScopePath.catalog()); + MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( + name, cacheSpec, ignored -> ScopePath.catalog()) + .sizeEstimator(MetaCacheSizeEstimators.reflective()); if (loader != null) { builder.loader(key -> loadAndPause(key, loader)); } @@ -377,17 +378,7 @@ public void forEach(BiConsumer consumer) { } public MetaCacheEntryStats stats() { - CacheMetrics metrics = data.metrics(); - long requests = metrics.getRequestCount(); - long loads = metrics.getLoadSuccessCount() + metrics.getLoadFailureCount(); - return new MetaCacheEntryStats( - cacheSpec.isEnable(), effectiveEnabled, autoRefresh, cacheSpec.getTtlSecond(), cacheSpec.getCapacity(), - metrics.getPhysicalEntryCount(), requests, metrics.getHitCount(), metrics.getMissCount(), - requests == 0L ? 0D : (double) metrics.getHitCount() / requests, - metrics.getLoadSuccessCount(), metrics.getLoadFailureCount(), metrics.getTotalLoadTimeNanos(), - loads == 0L ? 0D : (double) metrics.getTotalLoadTimeNanos() / loads, - metrics.getEvictionCount(), metrics.getInvalidateCount(), metrics.getLastLoadSuccessTimeMs(), - metrics.getLastLoadFailureTimeMs(), metrics.getLastError()); + return MetaCacheEntryStats.from(data); } public static int defaultObjectStripeCount() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java index 8320622e08e698..05d71700600f8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.ScopedMetaCache.CacheMetrics; import java.util.Objects; @@ -53,6 +55,10 @@ public final class MetaCacheEntryStats { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final long maxWeight; + private final long estimatedWeight; + private final long weightAdmissionRejectedCount; + private final String lastWeightRejectReason; /** * Build an immutable stats snapshot. @@ -76,7 +82,11 @@ public MetaCacheEntryStats( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + long maxWeight, + long estimatedWeight, + long weightAdmissionRejectedCount, + String lastWeightRejectReason) { this.configEnabled = configEnabled; this.effectiveEnabled = effectiveEnabled; this.autoRefresh = autoRefresh; @@ -96,6 +106,28 @@ public MetaCacheEntryStats( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = Objects.requireNonNull(lastError, "lastError"); + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.weightAdmissionRejectedCount = weightAdmissionRejectedCount; + this.lastWeightRejectReason = Objects.requireNonNull(lastWeightRejectReason, "lastWeightRejectReason"); + } + + /** Creates the FE system-table view of a cache owned by the shared connector framework. */ + public static MetaCacheEntryStats from(MetaCache entry) { + CacheSpec spec = entry.cacheSpec(); + CacheMetrics metrics = entry.metrics(); + long requests = metrics.getRequestCount(); + long loads = metrics.getLoadSuccessCount() + metrics.getLoadFailureCount(); + return new MetaCacheEntryStats( + spec.isEnable(), entry.isEnabled(), entry.isAutoRefresh(), spec.getTtlSecond(), spec.getCapacity(), + entry.size(), requests, metrics.getHitCount(), metrics.getMissCount(), + requests == 0L ? 0D : (double) metrics.getHitCount() / requests, + metrics.getLoadSuccessCount(), metrics.getLoadFailureCount(), metrics.getTotalLoadTimeNanos(), + loads == 0L ? 0D : (double) metrics.getTotalLoadTimeNanos() / loads, + metrics.getEvictionCount(), metrics.getInvalidateCount(), metrics.getLastLoadSuccessTimeMs(), + metrics.getLastLoadFailureTimeMs(), metrics.getLastError(), metrics.getMaxWeight(), + metrics.getEstimatedWeight(), metrics.getWeightRejectCount(), + metrics.getLastWeightRejectReason()); } public boolean isConfigEnabled() { @@ -195,4 +227,20 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getWeightAdmissionRejectedCount() { + return weightAdmissionRejectedCount; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6313f3bdf6fd7c..8bdc9386654e11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -1967,6 +1967,10 @@ private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaT trow.addToColumnValue(new TCell().setStringVal( formatMetaCacheTime(entryStats.getLastLoadFailureTimeMs(), timeZone))); trow.addToColumnValue(new TCell().setStringVal(entryStats.getLastError())); // LAST_ERROR + trow.addToColumnValue(new TCell().setLongVal(entryStats.getMaxWeight())); + trow.addToColumnValue(new TCell().setLongVal(entryStats.getEstimatedWeight())); + trow.addToColumnValue(new TCell().setLongVal(entryStats.getWeightAdmissionRejectedCount())); + trow.addToColumnValue(new TCell().setStringVal(entryStats.getLastWeightRejectReason())); dataBatch.add(trow); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogMetaCacheWeightTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogMetaCacheWeightTest.java new file mode 100644 index 00000000000000..1aa5689e9e5ae7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogMetaCacheWeightTest.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; +import org.apache.doris.datasource.property.constants.RemoteDorisProperties; +import org.apache.doris.datasource.test.TestExternalCatalog; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +public class ExternalCatalogMetaCacheWeightTest { + + @Test + public void coreSchemaWeightIsValidatedAtTheDdlDoor() { + Map properties = testCatalogProperties(); + properties.put("meta.cache.default.schema.max-weight", "invalid"); + TestExternalCatalog invalidWeight = new TestExternalCatalog(1L, "test", "", properties, ""); + Assertions.assertThrows(DdlException.class, invalidWeight::checkProperties); + + properties = testCatalogProperties(); + properties.put("meta.cache.default.table.max-weight", "64MB"); + TestExternalCatalog unknownEntry = new TestExternalCatalog(2L, "test", "", properties, ""); + Assertions.assertThrows(DdlException.class, unknownEntry::checkProperties); + } + + @Test + public void remoteDorisValidatesItsOwnConsumedEntries() { + Map properties = remoteDorisProperties(); + properties.put("meta.cache.doris.backends.max-weight", "invalid"); + RemoteDorisExternalCatalog invalidWeight = new RemoteDorisExternalCatalog( + 1L, "remote", "", properties, ""); + Assertions.assertThrows(DdlException.class, invalidWeight::checkProperties); + + properties = remoteDorisProperties(); + properties.put("meta.cache.doris.unknown.max-weight", "64MB"); + RemoteDorisExternalCatalog unknownEntry = new RemoteDorisExternalCatalog( + 2L, "remote", "", properties, ""); + Assertions.assertThrows(DdlException.class, unknownEntry::checkProperties); + } + + private static Map testCatalogProperties() { + Map properties = new HashMap<>(); + properties.put("catalog_provider.class", RefreshCatalogTest.RefreshCatalogProvider.class.getName()); + return properties; + } + + private static Map remoteDorisProperties() { + Map properties = new HashMap<>(); + properties.put(RemoteDorisProperties.FE_THRIFT_HOSTS, "127.0.0.1:9020"); + properties.put(RemoteDorisProperties.FE_HTTP_HOSTS, "127.0.0.1:8030"); + properties.put(RemoteDorisProperties.FE_ARROW_HOSTS, "127.0.0.1:8070"); + properties.put(RemoteDorisProperties.USER, "root"); + properties.put(RemoteDorisProperties.PASSWORD, ""); + properties.put(RemoteDorisProperties.USE_ARROW_FLIGHT, "true"); + return properties; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java index e76b9d35752ae6..22bbc80ad788f0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java @@ -64,6 +64,29 @@ public void testLoadMutationAndStatsUseSharedRuntime() { } } + @Test + public void testRejectedWeightedMutationReturnsWithoutRetryingForever() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService worker = Executors.newSingleThreadExecutor(); + String oversized = "x".repeat(2_048); + try { + FeMetaCacheEntry entry = new FeMetaCacheEntry<>( + "objects", ignored -> "old", + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 1_024L), + refreshExecutor, false); + Assertions.assertEquals("old", entry.get("key")); + + Future mutation = worker.submit( + () -> entry.compute("key", (key, value) -> oversized)); + + Assertions.assertSame(oversized, mutation.get(3L, TimeUnit.SECONDS)); + Assertions.assertNull(entry.getIfPresent("key")); + } finally { + worker.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testContextualOnlyAndDisabledEntry() { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java index b476c50ba5d9ff..a050b1f4d36237 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.FileFormatConstants; +import org.apache.doris.connector.cache.MetaCacheBudgetManager; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorMetadata; import org.apache.doris.connector.spi.ConnectorSession; @@ -128,6 +129,46 @@ public void testSchemaCachePropertyUsesSingleCatalogInvalidation() throws Except Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); } + @Test + public void testCatalogWeightPropertyRebuildsAllCacheBudgets() throws Exception { + TestablePluginCatalog catalog = new TestablePluginCatalog( + mockConnector("old", new ConcurrentLinkedQueue<>())); + + catalog.notifyPropertiesUpdated(Collections.singletonMap( + MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB")); + + Mockito.verify(cacheMgr).removeCatalog(1L); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); + } + + @Test + public void testPluginEntryWeightPropertyReliesOnConnectorRebuild() throws Exception { + TestablePluginCatalog catalog = new TestablePluginCatalog( + mockConnector("old", new ConcurrentLinkedQueue<>())); + + catalog.notifyPropertiesUpdated(Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "1KB")); + + Mockito.verify(cacheMgr).isEngineRegistered("iceberg"); + Mockito.verify(cacheMgr, Mockito.never()).removeCatalogByEngine(Mockito.anyLong(), Mockito.anyString()); + Mockito.verify(cacheMgr).invalidateCatalog(1L); + Mockito.verify(cacheMgr, Mockito.never()).removeCatalog(Mockito.anyLong()); + } + + @Test + public void testCoreEntryWeightPropertyRebuildsRegisteredEngineCache() throws Exception { + Mockito.when(cacheMgr.isEngineRegistered("default")).thenReturn(true); + TestablePluginCatalog catalog = new TestablePluginCatalog( + mockConnector("old", new ConcurrentLinkedQueue<>())); + + catalog.notifyPropertiesUpdated(Collections.singletonMap( + "meta.cache.default.schema.max-weight", "1KB")); + + Mockito.verify(cacheMgr, Mockito.times(1)).removeCatalogByEngine(1L, "default"); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); + Mockito.verify(cacheMgr, Mockito.never()).removeCatalog(Mockito.anyLong()); + } + /** * Verify that notifyPropertiesUpdated() closes the old connector via * resetToUninitialized → onClose, and that lazy re-initialization diff --git a/regression-test/suites/external_table_p0/test_catalog_ddl.groovy b/regression-test/suites/external_table_p0/test_catalog_ddl.groovy index 29480259ed1ced..687b8fc1473d3e 100644 --- a/regression-test/suites/external_table_p0/test_catalog_ddl.groovy +++ b/regression-test/suites/external_table_p0/test_catalog_ddl.groovy @@ -46,4 +46,106 @@ suite("test_catalog_ddl", "p0,external") { assertTrue(result[0][1].contains("COMMENT \"alter_comment\"")) sql """drop catalog ${catalog1}""" + + String weightedCatalog = "test_ddl_weighted_meta_cache" + sql """drop catalog if exists ${weightedCatalog}""" + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.max-weight" = "128MB", + "meta.cache.hive.file.max-weight" = "64MB" + ) + """ + result = sql """show create catalog ${weightedCatalog}""" + assertEquals(result.size(), 1) + assertTrue(result[0][1].contains("\"meta.cache.max-weight\" = \"128MB\"")) + assertTrue(result[0][1].contains("\"meta.cache.hive.file.max-weight\" = \"64MB\"")) + + sql """ + alter catalog ${weightedCatalog} set properties( + "meta.cache.max-weight" = "96MB", + "meta.cache.hive.file.max-weight" = "48MB" + ) + """ + result = sql """show create catalog ${weightedCatalog}""" + assertTrue(result[0][1].contains("\"meta.cache.max-weight\" = \"96MB\"")) + assertTrue(result[0][1].contains("\"meta.cache.hive.file.max-weight\" = \"48MB\"")) + + test { + sql """ + alter catalog ${weightedCatalog} set properties( + "meta.cache.default.schema.max-weight" = "invalid" + ) + """ + exception "Invalid cache weight for 'meta.cache.default.schema.max-weight': invalid" + } + sql """drop catalog ${weightedCatalog}""" + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.max-weight" = "64MB", + "meta.cache.hive.file.max-weight" = "128MB" + ) + """ + exception "meta.cache.hive.file.max-weight can not exceed meta.cache.max-weight" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.max-weight" = "10%" + ) + """ + exception "Invalid cache weight for 'meta.cache.max-weight': 10%" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "es", + "hosts" = "http://10.10.10.10:8888", + "meta.cache.max-weight" = "invalid" + ) + """ + exception "Invalid cache weight for 'meta.cache.max-weight': invalid" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.hive.flie.max-weight" = "64MB" + ) + """ + exception "Unknown metadata cache weight property: meta.cache.hive.flie.max-weight" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "hms", + "hive.metastore.uris" = "thrift://127.0.0.1:9083", + "meta.cache.iceberg.manifest.max-weight" = "invalid" + ) + """ + exception "Invalid cache weight for 'meta.cache.iceberg.manifest.max-weight': invalid" + } + + test { + sql """ + create catalog ${weightedCatalog} properties( + "type" = "es", + "hosts" = "http://10.10.10.10:8888", + "meta.cache.default.table.max-weight" = "64MB" + ) + """ + exception "Unknown metadata cache weight property: meta.cache.default.table.max-weight" + } }