From 5b6d523f11abe44649d5bc9c37062a8848af50cb Mon Sep 17 00:00:00 2001 From: dmkozh Date: Thu, 21 May 2026 12:58:44 -0400 Subject: [PATCH 1/7] Add DISABLE_TX_META_FOR_TESTING config flag. BUILD_TESTS builds unconditionally populate mLastLedgerTxMeta and force-enable tx meta so tests can inspect it, but since the apply-load benchmarks are built with BUILD_TESTS as well we need a way to unconditionally disable meta. --- docs/apply-load-benchmark-sac.cfg | 2 ++ docs/apply-load-benchmark-token.cfg | 2 ++ .../aws/apply-load-aws-benchmark.cfg | 2 ++ .../aws/apply-load-aws-ledger-limits.cfg | 2 ++ .../aws/apply-load-aws-max-sac-tps.cfg | 2 ++ src/ledger/LedgerManagerImpl.cpp | 24 +++++++++++++------ src/main/Config.cpp | 3 +++ src/main/Config.h | 5 ++++ 8 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/apply-load-benchmark-sac.cfg b/docs/apply-load-benchmark-sac.cfg index 7473130a40..39ac81d44c 100644 --- a/docs/apply-load-benchmark-sac.cfg +++ b/docs/apply-load-benchmark-sac.cfg @@ -16,6 +16,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/docs/apply-load-benchmark-token.cfg b/docs/apply-load-benchmark-token.cfg index 14dc7b3091..0c6560e812 100644 --- a/docs/apply-load-benchmark-token.cfg +++ b/docs/apply-load-benchmark-token.cfg @@ -16,6 +16,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/scripts/apply_load/aws/apply-load-aws-benchmark.cfg b/scripts/apply_load/aws/apply-load-aws-benchmark.cfg index 0778b4556a..4c335330d6 100644 --- a/scripts/apply_load/aws/apply-load-aws-benchmark.cfg +++ b/scripts/apply_load/aws/apply-load-aws-benchmark.cfg @@ -17,6 +17,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg b/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg index f890fd19d8..fd1eff16b4 100644 --- a/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg +++ b/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg @@ -13,6 +13,8 @@ LOG_FILE_PATH="{log_file_path}" # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = false +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg b/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg index 8cc287670f..0c8c921264 100644 --- a/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg +++ b/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg @@ -17,6 +17,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index aa45c7f16d..00beec33cc 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -1606,8 +1606,9 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, } #ifdef BUILD_TESTS - // We always store the ledgerCloseMeta in tests so we can inspect it. - if (!ledgerCloseMeta) + // We always store the ledgerCloseMeta in tests so we can inspect it, + // unless explicitly disabled for benchmarking. + if (!ledgerCloseMeta && !mApp.getConfig().DISABLE_TX_META_FOR_TESTING) { ledgerCloseMeta = std::make_unique( header.current().ledgerVersion); @@ -2612,7 +2613,10 @@ LedgerManagerImpl::processResultAndMeta( { auto metaXDR = txMetaBuilder.finalize(result.isSuccess()); #ifdef BUILD_TESTS - mLastLedgerTxMeta.emplace_back(metaXDR); + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + mLastLedgerTxMeta.emplace_back(metaXDR); + } #endif ledgerCloseMeta->setTxProcessingMetaAndResultPair( @@ -2621,8 +2625,11 @@ LedgerManagerImpl::processResultAndMeta( else { #ifdef BUILD_TESTS - mLastLedgerTxMeta.emplace_back( - txMetaBuilder.finalize(result.isSuccess())); + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + mLastLedgerTxMeta.emplace_back( + txMetaBuilder.finalize(result.isSuccess())); + } #endif } } @@ -2668,8 +2675,11 @@ LedgerManagerImpl::applyTransactions( bool enableTxMeta = ledgerCloseMeta != nullptr; #ifdef BUILD_TESTS // In tests we want to always enable tx meta because we store it in - // mLastLedgerTxMeta. - enableTxMeta = true; + // mLastLedgerTxMeta, unless explicitly disabled for benchmarking. + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + enableTxMeta = true; + } #endif std::optional sorobanConfig; if (protocolVersionStartsFrom(ltx.loadHeader().current().ledgerVersion, diff --git a/src/main/Config.cpp b/src/main/Config.cpp index 36c9f79bd7..b2ec4068c8 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -172,6 +172,7 @@ Config::Config() : NODE_SEED(SecretKey::random()) BACKGROUND_OVERLAY_PROCESSING = true; PARALLEL_LEDGER_APPLY = true; DISABLE_SOROBAN_METRICS_FOR_TESTING = false; + DISABLE_TX_META_FOR_TESTING = false; BACKGROUND_TX_SIG_VERIFICATION = true; BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 14; // 2^14 == 16 kb BUCKETLIST_DB_INDEX_CUTOFF = 20; // 20 mb @@ -1188,6 +1189,8 @@ Config::processConfig(std::shared_ptr t) [&]() { DISABLE_SOROBAN_METRICS_FOR_TESTING = readBool(item); }}, + {"DISABLE_TX_META_FOR_TESTING", + [&]() { DISABLE_TX_META_FOR_TESTING = readBool(item); }}, {"EXPERIMENTAL_BACKGROUND_TX_SIG_VERIFICATION", [&]() { CLOG_WARNING(Overlay, diff --git a/src/main/Config.h b/src/main/Config.h index 92de868722..99e0e20265 100644 --- a/src/main/Config.h +++ b/src/main/Config.h @@ -525,6 +525,11 @@ class Config : public std::enable_shared_from_this // Disable expensive Soroban metrics for performance testing bool DISABLE_SOROBAN_METRICS_FOR_TESTING; + // Disable transaction metadata collection in test builds. + // This is useful for benchmarking, which is typically done in BUILD_TESTS + // builds. + bool DISABLE_TX_META_FOR_TESTING; + // Batch transactions for flooding purposes (experimental). // Has no effect on non-test builds. size_t EXPERIMENTAL_TX_BATCH_MAX_SIZE; From 0b4922c9bbcda339f3c96359bede86cc7c0001e7 Mon Sep 17 00:00:00 2001 From: dmkozh Date: Thu, 21 May 2026 12:58:09 -0400 Subject: [PATCH 2/7] Parallelize in-memory state update with bucket list operations. During ledger close, run addHotArchiveBatch, addLiveBatch and updateInMemorySorobanState concurrently. They modify independent data structures and so need no synchronization. --- src/ledger/LedgerManagerImpl.cpp | 45 +++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index 00beec33cc..9767db7193 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -75,6 +75,7 @@ #include "LedgerManagerImpl.h" #include +#include #include #include #include @@ -2981,6 +2982,11 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( // `ledgerApplied` protects this call with a mutex std::vector initEntries, liveEntries; std::vector deadEntries; + + std::future hotArchiveBatchFuture; + EvictedStateVectors evictedState; + std::vector restoredHotArchiveKeys; + // Any V20 features must be behind initialLedgerVers check, see comment // in LedgerManagerImpl::ledgerApplied if (protocolVersionStartsFrom(initialLedgerVers, SOROBAN_PROTOCOL_VERSION)) @@ -2990,16 +2996,13 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( // `getAllTTLKeysWithoutSealing` must be called at the right time // _after_ all operations have been applied, but _before_ evictions. auto sorobanConfig = SorobanNetworkConfig::loadFromLedger(ltx); - auto evictedState = - mApp.getBucketManager().resolveBackgroundEvictionScan( - lclApplyView, ltx, ltx.getAllKeysWithoutSealing()); + evictedState = mApp.getBucketManager().resolveBackgroundEvictionScan( + lclApplyView, ltx, ltx.getAllKeysWithoutSealing()); if (protocolVersionStartsFrom( initialLedgerVers, LiveBucket::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION)) { - std::vector restoredHotArchiveKeys; - auto const& restoredHotArchiveKeyMap = ltx.getRestoredHotArchiveKeys(); for (auto const& [key, entry] : restoredHotArchiveKeyMap) @@ -3029,9 +3032,14 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( } else { - mApp.getBucketManager().addHotArchiveBatch( - mApp, lh, evictedState.archivedEntries, - restoredHotArchiveKeys); + hotArchiveBatchFuture = + std::async(std::launch::async, [this, lh, &evictedState, + &restoredHotArchiveKeys]() { + mApp.getBucketManager().addHotArchiveBatch( + mApp, lh, evictedState.archivedEntries, + restoredHotArchiveKeys); + }); + // Validate evicted entries against Protocol 23 corruption // data if configured if (mApp.getProtocol23CorruptionDataVerifier()) @@ -3071,12 +3079,29 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( } // NB: getAllEntries seals the ltx. ltx.getAllEntries(initEntries, liveEntries, deadEntries); + + // Launch async task to update in-memory Soroban state. This is independent + // from both addHotArchiveBatch and addLiveBatch, so all can run in + // parallel. + std::future inMemoryStateUpdateFuture; + + inMemoryStateUpdateFuture = std::async( + std::launch::async, [this, &initEntries, &liveEntries, &deadEntries, + &lh, &finalSorobanConfig]() { + mApplyState.updateInMemorySorobanState( + initEntries, liveEntries, deadEntries, lh, finalSorobanConfig); + }); + mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion, initEntries); mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion, liveEntries); mApp.getBucketManager().addLiveBatch(mApp, lh, initEntries, liveEntries, deadEntries); - mApplyState.updateInMemorySorobanState(initEntries, liveEntries, - deadEntries, lh, finalSorobanConfig); + // Wait for all async operations to complete before returning. + if (hotArchiveBatchFuture.valid()) + { + hotArchiveBatchFuture.get(); + } + inMemoryStateUpdateFuture.get(); return finalSorobanConfig; } From 3c8046d910ae2830c18f24bc0b089fe124b06562 Mon Sep 17 00:00:00 2001 From: dmkozh Date: Thu, 21 May 2026 12:58:20 -0400 Subject: [PATCH 3/7] Shard verifySig cache to reduce mutex contention. Replace the single global mutex + RandomEvictionCache with 16 sharded caches (each with its own mutex) so concurrent signature verification threads no longer block on a shared lock. Also use the single-lookup maybeGet() in place of the exists()/get() double-lookup. --- src/crypto/SecretKey.cpp | 99 ++++++++++++++++---------------- src/crypto/SecretKey.h | 7 --- src/crypto/test/CryptoTests.cpp | 1 - src/herder/Upgrades.cpp | 1 - src/ledger/LedgerManagerImpl.cpp | 4 -- src/main/ApplicationImpl.cpp | 8 --- 6 files changed, 50 insertions(+), 70 deletions(-) diff --git a/src/crypto/SecretKey.cpp b/src/crypto/SecretKey.cpp index 1c92d1c090..6e4851152e 100644 --- a/src/crypto/SecretKey.cpp +++ b/src/crypto/SecretKey.cpp @@ -18,6 +18,8 @@ #include "util/Math.h" #include "util/RandomEvictionCache.h" #include +#include +#include #include #include #include @@ -41,16 +43,29 @@ namespace stellar // to the state of the process; caching its results centrally // makes all signature-verification in the program faster and // has no effect on correctness. +// +// The cache is sharded across NUM_VERIFY_CACHE_SHARDS shards to +// reduce mutex contention when multiple threads verify signatures +// in parallel. Each shard has its own mutex and cache partition. constexpr size_t VERIFY_SIG_CACHE_SIZE = 250'000; -static std::mutex gVerifySigCacheMutex; -static RandomEvictionCache gVerifySigCache(VERIFY_SIG_CACHE_SIZE); -static uint64_t gVerifyCacheHit = 0; -static uint64_t gVerifyCacheMiss = 0; +constexpr size_t NUM_VERIFY_CACHE_SHARDS = 16; +constexpr size_t VERIFY_SIG_CACHE_SHARD_SIZE = + VERIFY_SIG_CACHE_SIZE / NUM_VERIFY_CACHE_SHARDS; + +struct VerifySigCacheShard +{ + std::mutex mMutex; + RandomEvictionCache mCache; + VerifySigCacheShard() : mCache(VERIFY_SIG_CACHE_SHARD_SIZE) + { + } +}; -// Global flag to use Rust ed25519-dalek for signature verification -// Protected by gVerifySigCacheMutex -static bool gUseRustDalekVerify = false; +static std::array + gVerifySigCacheShards; +static std::atomic gVerifyCacheHit{0}; +static std::atomic gVerifyCacheMiss{0}; static Hash verifySigCacheKey(PublicKey const& key, Signature const& signature, @@ -322,32 +337,29 @@ SecretKey::fromStrKeySeed(std::string const& strKeySeed) void PubKeyUtils::clearVerifySigCache() { - std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.clear(); -} - -void -PubKeyUtils::enableRustDalekVerify() -{ - std::lock_guard guard(gVerifySigCacheMutex); - gUseRustDalekVerify = true; + for (auto& shard : gVerifySigCacheShards) + { + std::lock_guard guard(shard.mMutex); + shard.mCache.clear(); + } } void PubKeyUtils::seedVerifySigCache(unsigned int seed) { - std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.seed(seed); + for (size_t i = 0; i < NUM_VERIFY_CACHE_SHARDS; ++i) + { + std::lock_guard guard(gVerifySigCacheShards[i].mMutex); + gVerifySigCacheShards[i].mCache.seed(seed + + static_cast(i)); + } } void PubKeyUtils::flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses) { - std::lock_guard guard(gVerifySigCacheMutex); - hits = gVerifyCacheHit; - misses = gVerifyCacheMiss; - gVerifyCacheHit = 0; - gVerifyCacheMiss = 0; + hits = gVerifyCacheHit.exchange(0, std::memory_order_relaxed); + misses = gVerifyCacheMiss.exchange(0, std::memory_order_relaxed); } std::string @@ -456,41 +468,30 @@ PubKeyUtils::verifySig(PublicKey const& key, Signature const& signature, } auto cacheKey = verifySigCacheKey(key, signature, bin); - bool shouldUseRustDalekVerify; + + // Select shard based on cache key hash to distribute lock contention + auto shardIdx = std::hash{}(cacheKey) % NUM_VERIFY_CACHE_SHARDS; + auto& shard = gVerifySigCacheShards[shardIdx]; { - std::lock_guard guard(gVerifySigCacheMutex); - if (gVerifySigCache.exists(cacheKey)) + std::lock_guard guard(shard.mMutex); + if (auto* cached = shard.mCache.maybeGet(cacheKey)) { - ++gVerifyCacheHit; - std::string hitStr("hit"); - ZoneText(hitStr.c_str(), hitStr.size()); - return {gVerifySigCache.get(cacheKey), - VerifySigCacheLookupResult::HIT}; + gVerifyCacheHit.fetch_add(1, std::memory_order_relaxed); + ZoneText("hit", 3); + return {*cached, VerifySigCacheLookupResult::HIT}; } - - shouldUseRustDalekVerify = gUseRustDalekVerify; } - std::string missStr("miss"); - ZoneText(missStr.c_str(), missStr.size()); + ZoneText("miss", 4); - bool ok; - if (shouldUseRustDalekVerify) - { - ok = stellar::rust_bridge::verify_ed25519_signature_dalek( - key.ed25519().data(), signature.data(), bin.data(), bin.size()); - } - else + bool ok = stellar::rust_bridge::verify_ed25519_signature_dalek( + key.ed25519().data(), signature.data(), bin.data(), bin.size()); { - ok = (crypto_sign_verify_detached(signature.data(), bin.data(), - bin.size(), - key.ed25519().data()) == 0); + std::lock_guard guard(shard.mMutex); + gVerifyCacheMiss.fetch_add(1, std::memory_order_relaxed); + shard.mCache.put(cacheKey, ok); } - - std::lock_guard guard(gVerifySigCacheMutex); - ++gVerifyCacheMiss; - gVerifySigCache.put(cacheKey, ok); return {ok, VerifySigCacheLookupResult::MISS}; } diff --git a/src/crypto/SecretKey.h b/src/crypto/SecretKey.h index 9c3c96aeba..c30286e4d0 100644 --- a/src/crypto/SecretKey.h +++ b/src/crypto/SecretKey.h @@ -166,13 +166,6 @@ void clearVerifySigCache(); void seedVerifySigCache(unsigned int seed); void flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses); -// Enable Rust ed25519-dalek for signature verification -// Once enabled, it cannot be disabled. It should be enabled at the protocol 24 -// boundary. -// Note: This should be removed following the protocol 24 upgrade, rust ed25519 -// can be used unconditionally after upgrade, even for replay. -void enableRustDalekVerify(); - PublicKey random(); #ifdef BUILD_TESTS PublicKey pseudoRandomForTesting(); diff --git a/src/crypto/test/CryptoTests.cpp b/src/crypto/test/CryptoTests.cpp index 100a37163f..858c10a406 100644 --- a/src/crypto/test/CryptoTests.cpp +++ b/src/crypto/test/CryptoTests.cpp @@ -1630,7 +1630,6 @@ ZcashTestVector const ZCASH_TEST_VECTORS[196] = { TEST_CASE("Ed25519 test vectors from Zcash", "[crypto]") { - PubKeyUtils::enableRustDalekVerify(); for (auto const& tv : ZCASH_TEST_VECTORS) { PublicKey pk; diff --git a/src/herder/Upgrades.cpp b/src/herder/Upgrades.cpp index e8b1423f64..a2b232979a 100644 --- a/src/herder/Upgrades.cpp +++ b/src/herder/Upgrades.cpp @@ -1249,7 +1249,6 @@ Upgrades::applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx, } if (needUpgradeToVersion(ProtocolVersion::V_25, prevVersion, newVersion)) { - PubKeyUtils::enableRustDalekVerify(); SorobanNetworkConfig::createCostTypesForV25(ltx, app); } if (needUpgradeToVersion(ProtocolVersion::V_26, prevVersion, newVersion)) diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index 9767db7193..736fbbc80d 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -1930,10 +1930,6 @@ LedgerManagerImpl::setLastClosedLedger( advanceLastClosedLedgerState(output); auto ledgerVersion = lastClosed.header.ledgerVersion; - if (protocolVersionStartsFrom(ledgerVersion, ProtocolVersion::V_25)) - { - PubKeyUtils::enableRustDalekVerify(); - } if (rebuildInMemoryState) { diff --git a/src/main/ApplicationImpl.cpp b/src/main/ApplicationImpl.cpp index 810522d4df..c0c9ab48c1 100644 --- a/src/main/ApplicationImpl.cpp +++ b/src/main/ApplicationImpl.cpp @@ -801,14 +801,6 @@ ApplicationImpl::start() // LCL is now loaded; unblock HTTP endpoints that were gated during boot. mCommandHandler->setReady(); - // Check if we're already on protocol V_24 or later and enable Rust Dalek - auto const& lcl = mLedgerManager->getLastClosedLedgerHeader(); - if (protocolVersionStartsFrom(lcl.header.ledgerVersion, - ProtocolVersion::V_25)) - { - PubKeyUtils::enableRustDalekVerify(); - } - startServices(); } From 84d01c9e5a121856a30dccac555d204926520c9f Mon Sep 17 00:00:00 2001 From: dmkozh Date: Thu, 21 May 2026 12:58:26 -0400 Subject: [PATCH 4/7] Parallelize InMemoryIndex construction with bucket put loop. Build the LiveBucketIndex on an async worker thread while the put loop in mergeInMemory runs on the main thread. Both only read mergedEntries as const, so they need no synchronization. --- src/bucket/BucketOutputIterator.cpp | 9 ++++++-- src/bucket/BucketOutputIterator.h | 2 ++ src/bucket/LiveBucket.cpp | 32 ++++++++++++++++++++++++----- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/bucket/BucketOutputIterator.cpp b/src/bucket/BucketOutputIterator.cpp index 6645f51143..43fd611cd9 100644 --- a/src/bucket/BucketOutputIterator.cpp +++ b/src/bucket/BucketOutputIterator.cpp @@ -168,7 +168,8 @@ template std::shared_ptr BucketOutputIterator::getBucket( BucketManager& bucketManager, MergeKey* mergeKey, - std::unique_ptr> inMemoryState) + std::unique_ptr> inMemoryState, + std::shared_ptr preBuiltIndex) { ZoneScoped; if (mBuf) @@ -219,7 +220,11 @@ BucketOutputIterator::getBucket( if (!index) { - if constexpr (std::is_same_v) + if (preBuiltIndex) + { + index = std::move(preBuiltIndex); + } + else if constexpr (std::is_same_v) { if (inMemoryState) { diff --git a/src/bucket/BucketOutputIterator.h b/src/bucket/BucketOutputIterator.h index a76e1c6bb7..99b42ec2d0 100644 --- a/src/bucket/BucketOutputIterator.h +++ b/src/bucket/BucketOutputIterator.h @@ -55,6 +55,8 @@ template class BucketOutputIterator std::shared_ptr getBucket( BucketManager& bucketManager, MergeKey* mergeKey = nullptr, std::unique_ptr> inMemoryState = + nullptr, + std::shared_ptr preBuiltIndex = nullptr); }; } diff --git a/src/bucket/LiveBucket.cpp b/src/bucket/LiveBucket.cpp index 8101c9d183..6c1b0bbdb5 100644 --- a/src/bucket/LiveBucket.cpp +++ b/src/bucket/LiveBucket.cpp @@ -10,6 +10,7 @@ #include "bucket/BucketOutputIterator.h" #include "bucket/BucketUtils.h" #include "bucket/LedgerCmp.h" +#include #include namespace stellar @@ -587,29 +588,50 @@ LiveBucket::mergeInMemory(BucketManager& bucketManager, mergedEntries.emplace_back(entry); }; - mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, mc, - shadowIterators, keepShadowedLifecycleEntries); + { + ZoneNamedN(zoneMerge, "mergeInMemory merge", true); + mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, + mc, shadowIterators, keepShadowedLifecycleEntries); + } if (countMergeEvents) { bucketManager.incrMergeCounters(mc); } + // Start index construction on worker thread — reads mergedEntries (const), + // completely independent of the put loop's serialize/hash/write work. + auto indexFuture = std::async(std::launch::async, [&]() { + return std::make_shared(bucketManager, mergedEntries, + meta); + }); + // Write merge output to a bucket and save to disk LiveBucketOutputIterator out(bucketManager.getTmpDir(), /*keepTombstoneEntries=*/true, meta, mc, ctx, doFsync); - for (auto const& e : mergedEntries) { - out.put(e); + ZoneNamedN(zonePut, "mergeInMemory put loop", true); + for (auto const& e : mergedEntries) + { + out.put(e); + } + } + + // Collect the pre-built index + std::shared_ptr preBuiltIndex; + { + ZoneNamedN(zoneWait, "mergeInMemory index future wait", true); + preBuiltIndex = indexFuture.get(); } // Store the merged entries in memory in the new bucket in case this // bucket sees another incoming merge as level 0 curr. return out.getBucket( bucketManager, nullptr, - std::make_unique>(std::move(mergedEntries))); + std::make_unique>(std::move(mergedEntries)), + std::move(preBuiltIndex)); } BucketEntryCounters const& From b0a244353baf9527e5d22949a51e0e57a991b1a8 Mon Sep 17 00:00:00 2001 From: dmkozh Date: Thu, 21 May 2026 12:58:39 -0400 Subject: [PATCH 5/7] Skip building invariant delta when no invariants are enabled. The delta is only consumed by checkOnOperationApply, which is a no-op when no invariants are registered. --- src/ledger/LedgerManagerImpl.cpp | 8 +++++--- src/transactions/TransactionFrame.cpp | 10 ++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index 736fbbc80d..5f081a06b2 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -2496,10 +2496,13 @@ LedgerManagerImpl::checkAllTxBundleInvariants( AppConnector& app, ApplyStage const& stage, Config const& config, ParallelLedgerInfo const& ledgerInfo, LedgerHeader const& header) { + bool const hasInvariants = !config.INVARIANT_CHECKS.empty(); for (auto const& txBundle : stage) { - // First check the invariants - if (txBundle.getResPayload().isSuccess()) + // Only run invariant checks if any invariants are enabled. + // The delta is not built when invariants are disabled (see + // parallelApply), so we must not call getDelta() in that case. + if (hasInvariants && txBundle.getResPayload().isSuccess()) { try { @@ -2527,7 +2530,6 @@ LedgerManagerImpl::checkAllTxBundleInvariants( // We don't call processPostApply for post v23 transactions at the // moment because processPostApply is currently a no-op for those - // transactions. txBundle.getEffects().getMeta().maybeSetRefundableFeeMeta( txBundle.getResPayload().getRefundableFeeTracker()); diff --git a/src/transactions/TransactionFrame.cpp b/src/transactions/TransactionFrame.cpp index cb495142e6..199c319f03 100644 --- a/src/transactions/TransactionFrame.cpp +++ b/src/transactions/TransactionFrame.cpp @@ -2277,8 +2277,14 @@ TransactionFrame::parallelApply( if (res) { - threadState.setEffectsDeltaFromSuccessfulTx(*res, ledgerInfo, - effects); + // Only build the LedgerTxnDelta when invariant checks are + // enabled — the delta is consumed exclusively by + // checkOnOperationApply which is a no-op otherwise. + if (!config.INVARIANT_CHECKS.empty()) + { + threadState.setEffectsDeltaFromSuccessfulTx(*res, ledgerInfo, + effects); + } opMeta.setLedgerChangesFromSuccessfulOp(threadState, *res, ledgerInfo.getLedgerSeq()); } From f3881f2fb55b1881ab9402dcaa12ea4caf8cf44c Mon Sep 17 00:00:00 2001 From: dmkozh Date: Thu, 21 May 2026 12:58:50 -0400 Subject: [PATCH 6/7] Cache XDR size in InMemorySorobanState entries. --- src/invariant/test/InvariantTests.cpp | 13 ++++++++---- src/ledger/InMemorySorobanState.cpp | 19 ++++++++++-------- src/ledger/InMemorySorobanState.h | 29 +++++++++++++++++---------- src/ledger/LedgerTxn.cpp | 1 + 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/src/invariant/test/InvariantTests.cpp b/src/invariant/test/InvariantTests.cpp index 5dfd9078bd..4e45b7176b 100644 --- a/src/invariant/test/InvariantTests.cpp +++ b/src/invariant/test/InvariantTests.cpp @@ -670,7 +670,9 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") auto ttlData = entryData.ttlData; modifiedState.mContractDataEntries.erase(it); modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(modifiedEntry, ttlData)); + InternalContractDataMapEntry( + modifiedEntry, ttlData, + static_cast(xdr::xdr_size(modifiedEntry)))); } auto result = @@ -711,7 +713,9 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") createContractDataWithTTL(PERSISTENT, 1000); TTLData ttlData(extraTTL.data.ttl().liveUntilLedgerSeq, 1); modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(extraEntry, ttlData)); + InternalContractDataMapEntry( + extraEntry, ttlData, + static_cast(xdr::xdr_size(extraEntry)))); } auto result = @@ -741,8 +745,9 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") TTLData wrongTTL(42, 1); modifiedState.mContractDataEntries.erase(it); - modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(entryCopy, wrongTTL)); + modifiedState.mContractDataEntries.emplace(InternalContractDataMapEntry( + entryCopy, wrongTTL, + static_cast(xdr::xdr_size(entryCopy)))); auto result = invariant.checkSnapshot(makeSnap(), modifiedState, noopIsStopping); diff --git a/src/ledger/InMemorySorobanState.cpp b/src/ledger/InMemorySorobanState.cpp index 9a71cd52f5..3d7624e4c8 100644 --- a/src/ledger/InMemorySorobanState.cpp +++ b/src/ledger/InMemorySorobanState.cpp @@ -8,6 +8,7 @@ #include "ledger/LedgerTypeUtils.h" #include "ledger/SorobanMetrics.h" #include "util/GlobalChecks.h" +#include #include #include @@ -57,9 +58,10 @@ InMemorySorobanState::updateContractDataTTL( { // Since entries are immutable, we must erase and re-insert auto ledgerEntryPtr = dataIt->get().ledgerEntry; + auto sizeBytes = dataIt->get().sizeBytes; mContractDataEntries.erase(dataIt); - mContractDataEntries.emplace( - InternalContractDataMapEntry(std::move(ledgerEntryPtr), newTtlData)); + mContractDataEntries.emplace(InternalContractDataMapEntry( + std::move(ledgerEntryPtr), newTtlData, sizeBytes)); } void @@ -99,7 +101,7 @@ InMemorySorobanState::updateContractData(LedgerEntry const& ledgerEntry) releaseAssertOrThrow(dataIt != mContractDataEntries.end()); releaseAssertOrThrow(dataIt->get().ledgerEntry != nullptr); - uint32_t oldSize = xdr::xdr_size(*dataIt->get().ledgerEntry); + uint32_t oldSize = dataIt->get().sizeBytes; uint32_t newSize = xdr::xdr_size(ledgerEntry); updateStateSizeOnEntryUpdate(oldSize, newSize, /*isContractCode=*/false); @@ -107,7 +109,7 @@ InMemorySorobanState::updateContractData(LedgerEntry const& ledgerEntry) auto preservedTTL = dataIt->get().ttlData; mContractDataEntries.erase(dataIt); mContractDataEntries.emplace( - InternalContractDataMapEntry(ledgerEntry, preservedTTL)); + InternalContractDataMapEntry(ledgerEntry, preservedTTL, newSize)); } void @@ -135,10 +137,10 @@ InMemorySorobanState::createContractDataEntry(LedgerEntry const& ledgerEntry) } // else: TTL hasn't arrived yet, initialize to 0 (will be updated later) - updateStateSizeOnEntryUpdate(0, xdr::xdr_size(ledgerEntry), - /*isContractCode=*/false); + uint32_t sizeBytes = xdr::xdr_size(ledgerEntry); + updateStateSizeOnEntryUpdate(0, sizeBytes, /*isContractCode=*/false); mContractDataEntries.emplace( - InternalContractDataMapEntry(ledgerEntry, ttlData)); + InternalContractDataMapEntry(ledgerEntry, ttlData, sizeBytes)); } bool @@ -196,7 +198,7 @@ InMemorySorobanState::deleteContractData(LedgerKey const& ledgerKey) mContractDataEntries.find(InternalContractDataMapEntry(ledgerKey)); releaseAssertOrThrow(it != mContractDataEntries.end()); releaseAssertOrThrow(it->get().ledgerEntry != nullptr); - updateStateSizeOnEntryUpdate(xdr::xdr_size(*it->get().ledgerEntry), 0, + updateStateSizeOnEntryUpdate(it->get().sizeBytes, 0, /*isContractCode=*/false); mContractDataEntries.erase(it); } @@ -539,6 +541,7 @@ InMemorySorobanState::updateState( std::optional const& sorobanConfig, SorobanMetrics& metrics) { + ZoneScoped; // After initialization, we must apply every ledger in order to the // in-memory state with no gaps. releaseAssertOrThrow(mLastClosedLedgerSeq + 1 == lh.ledgerSeq); diff --git a/src/ledger/InMemorySorobanState.h b/src/ledger/InMemorySorobanState.h index 17a67a3454..83a1c3807b 100644 --- a/src/ledger/InMemorySorobanState.h +++ b/src/ledger/InMemorySorobanState.h @@ -45,14 +45,20 @@ struct TTLData // ContractDataMapEntryT stores a ContractData LedgerEntry and its TTL. TTL is // stored directly with the data to avoid an additional lookup and save memory. +// We also cache the XDR size to avoid repeated xdr_size() calls during updates. struct ContractDataMapEntryT { std::shared_ptr const ledgerEntry; TTLData const ttlData; + // Cached XDR serialized size to avoid repeated xdr_size() calls + uint32_t const sizeBytes; explicit ContractDataMapEntryT( - std::shared_ptr&& ledgerEntry, TTLData ttlData) - : ledgerEntry(std::move(ledgerEntry)), ttlData(ttlData) + std::shared_ptr&& ledgerEntry, TTLData ttlData, + uint32_t sizeBytes) + : ledgerEntry(std::move(ledgerEntry)) + , ttlData(ttlData) + , sizeBytes(sizeBytes) { } }; @@ -131,8 +137,6 @@ class InternalContractDataMapEntry } }; - // ValueEntry stores actual ContractData entries in the map. - // Contains both the LedgerEntry and its TTL information. struct ValueEntry : public AbstractEntry { private: @@ -140,8 +144,8 @@ class InternalContractDataMapEntry public: ValueEntry(std::shared_ptr&& ledgerEntry, - TTLData ttlData) - : entry(std::move(ledgerEntry), ttlData) + TTLData ttlData, uint32_t sizeBytes) + : entry(std::move(ledgerEntry), ttlData, sizeBytes) { } @@ -169,7 +173,7 @@ class InternalContractDataMapEntry { return std::make_unique( std::make_shared(*entry.ledgerEntry), - entry.ttlData); + entry.ttlData, entry.sizeBytes); } }; @@ -223,16 +227,19 @@ class InternalContractDataMapEntry // Creates a ValueEntry from a LedgerEntry (copies the entry) InternalContractDataMapEntry(LedgerEntry const& ledgerEntry, - TTLData ttlData) + TTLData ttlData, uint32_t sizeBytes) : impl(std::make_unique( - std::make_shared(ledgerEntry), ttlData)) + std::make_shared(ledgerEntry), ttlData, + sizeBytes)) { } // Creates a ValueEntry from a shared_ptr (avoids copying) InternalContractDataMapEntry( - std::shared_ptr&& ledgerEntry, TTLData ttlData) - : impl(std::make_unique(std::move(ledgerEntry), ttlData)) + std::shared_ptr&& ledgerEntry, TTLData ttlData, + uint32_t sizeBytes) + : impl(std::make_unique(std::move(ledgerEntry), ttlData, + sizeBytes)) { } diff --git a/src/ledger/LedgerTxn.cpp b/src/ledger/LedgerTxn.cpp index 8b16e34831..613be9cf1d 100644 --- a/src/ledger/LedgerTxn.cpp +++ b/src/ledger/LedgerTxn.cpp @@ -1628,6 +1628,7 @@ LedgerTxn::Impl::getAllEntries(std::vector& initEntries, std::vector& liveEntries, std::vector& deadEntries) { + ZoneScoped; abortIfWrongThread("getAllEntries"); std::vector resInit, resLive; std::vector resDead; From 15f5451831daa208af728e7ef6af57f5ba37889a Mon Sep 17 00:00:00 2001 From: dmkozh Date: Thu, 21 May 2026 12:58:59 -0400 Subject: [PATCH 7/7] Use BitSet in recordStorageChanges instead of maps. This relaxes the TTL invariant check a bit, instead of checking that created TTL keys belong to the specific entries, we just ensure that the expected entry counts match. This should be a pretty reasonable tradeoff - it seems like it would be pretty hard to create a mismatched TTL entry (vs missing it entirely). --- .../InvokeHostFunctionOpFrame.cpp | 76 +++++++++++-------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/src/transactions/InvokeHostFunctionOpFrame.cpp b/src/transactions/InvokeHostFunctionOpFrame.cpp index 6814b7df43..dcbc2b7060 100644 --- a/src/transactions/InvokeHostFunctionOpFrame.cpp +++ b/src/transactions/InvokeHostFunctionOpFrame.cpp @@ -18,6 +18,7 @@ #include "ledger/LedgerTxnImpl.h" #include "rust/CppShims.h" +#include "util/BitSet.h" #include "xdr/Stellar-transaction.h" #include #include @@ -609,9 +610,16 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper recordStorageChanges(InvokeHostFunctionOutput const& out) { ZoneScoped; - // Create or update every entry returned. - UnorderedSet createdAndModifiedKeys; - UnorderedSet createdKeys; + // Track which RW footprint keys appear in the host output without + // hashing LedgerKeys. Footprints are small, so a linear scan over a + // BitSet-backed coverage map is cheaper than maintaining hash sets. + auto const& rwKeys = mResources.footprint.readWrite; + BitSet rwKeyCovered(rwKeys.size()); + size_t numCreatedSorobanEntries = 0; + size_t numCreatedTTLEntries = 0; + bool const allowClassicCreations = protocolVersionStartsFrom( + getLedgerVersion(), ProtocolVersion::V_26); + for (auto const& buf : out.modified_ledger_entries) { LedgerEntry le; @@ -626,15 +634,22 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper return false; } - createdAndModifiedKeys.insert(lk); - - uint32_t keySize = static_cast(xdr::xdr_size(lk)); uint32_t entrySize = static_cast(buf.data.size()); + for (size_t j = 0; j < rwKeys.size(); ++j) + { + if (!rwKeyCovered.get(j) && rwKeys[j] == lk) + { + rwKeyCovered.set(j); + break; + } + } + // ttlEntry write fees come out of refundableFee, already // accounted for by the host if (lk.type() != TTL) { + uint32_t keySize = static_cast(xdr::xdr_size(lk)); mMetrics.noteWriteEntry(isContractCodeEntry(lk), keySize, entrySize); if (mResources.writeBytes < mMetrics.mLedgerWriteByte) @@ -653,42 +668,39 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper if (upsertLedgerEntry(lk, le)) { - createdKeys.insert(lk); + if (isSorobanEntry(lk)) + { + ++numCreatedSorobanEntries; + } + else if (lk.type() == TTL) + { + ++numCreatedTTLEntries; + } + else if (allowClassicCreations) + { + releaseAssertOrThrow(lk.type() == ACCOUNT || + lk.type() == TRUSTLINE); + } + else + { + releaseAssertOrThrow(false); + } } } - // Check that each newly created ContractCode or ContractData entry also - // creates a ttlEntry. Starting from protocol 26 (CAP-73), the Stellar - // Asset Contract can also create classic entries (ACCOUNT, TRUSTLINE). - for (auto const& key : createdKeys) - { - if (isSorobanEntry(key)) - { - auto ttlKey = getTTLKey(key); - releaseAssertOrThrow(createdKeys.find(ttlKey) != - createdKeys.end()); - } - else if (protocolVersionStartsFrom(getLedgerVersion(), - ProtocolVersion::V_26)) - { - releaseAssertOrThrow(key.type() == TTL || - key.type() == ACCOUNT || - key.type() == TRUSTLINE); - } - else - { - releaseAssertOrThrow(key.type() == TTL); - } - } + // Verify that each newly created Soroban entry has a corresponding + // newly created TTL entry (1:1 pairing guaranteed by the host). + releaseAssertOrThrow(numCreatedSorobanEntries == numCreatedTTLEntries); // Erase every entry not returned. // NB: The entries that haven't been touched are passed through // from host, so this should never result in removing an entry // that hasn't been removed by host explicitly. - for (auto const& lk : mResources.footprint.readWrite) + for (size_t j = 0; j < rwKeys.size(); ++j) { - if (createdAndModifiedKeys.find(lk) == createdAndModifiedKeys.end()) + if (!rwKeyCovered.get(j)) { + auto const& lk = rwKeys[j]; if (eraseLedgerEntryIfExists(lk)) { releaseAssertOrThrow(isSorobanEntry(lk));