diff --git a/docs/apply-load-benchmark-sac.cfg b/docs/apply-load-benchmark-sac.cfg index 7473130a40..39ac81d44c 100644 --- a/docs/apply-load-benchmark-sac.cfg +++ b/docs/apply-load-benchmark-sac.cfg @@ -16,6 +16,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/docs/apply-load-benchmark-token.cfg b/docs/apply-load-benchmark-token.cfg index 14dc7b3091..0c6560e812 100644 --- a/docs/apply-load-benchmark-token.cfg +++ b/docs/apply-load-benchmark-token.cfg @@ -16,6 +16,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/scripts/apply_load/aws/apply-load-aws-benchmark.cfg b/scripts/apply_load/aws/apply-load-aws-benchmark.cfg index 0778b4556a..4c335330d6 100644 --- a/scripts/apply_load/aws/apply-load-aws-benchmark.cfg +++ b/scripts/apply_load/aws/apply-load-aws-benchmark.cfg @@ -17,6 +17,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg b/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg index f890fd19d8..fd1eff16b4 100644 --- a/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg +++ b/scripts/apply_load/aws/apply-load-aws-ledger-limits.cfg @@ -13,6 +13,8 @@ LOG_FILE_PATH="{log_file_path}" # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = false +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg b/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg index 8cc287670f..0c8c921264 100644 --- a/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg +++ b/scripts/apply_load/aws/apply-load-aws-max-sac-tps.cfg @@ -17,6 +17,8 @@ APPLY_LOAD_TIME_WRITES = true # eventually, it is useful to disable these when optimizing anything besides # the metrics. DISABLE_SOROBAN_METRICS_FOR_TESTING = true +# Disable transaction metadata collection (BUILD_TESTS forces it otherwise) +DISABLE_TX_META_FOR_TESTING = true # Disable metadata output METADATA_OUTPUT_STREAM = "" # Disable metadata debug diff --git a/src/bucket/BucketOutputIterator.cpp b/src/bucket/BucketOutputIterator.cpp index 6645f51143..43fd611cd9 100644 --- a/src/bucket/BucketOutputIterator.cpp +++ b/src/bucket/BucketOutputIterator.cpp @@ -168,7 +168,8 @@ template std::shared_ptr BucketOutputIterator::getBucket( BucketManager& bucketManager, MergeKey* mergeKey, - std::unique_ptr> inMemoryState) + std::unique_ptr> inMemoryState, + std::shared_ptr preBuiltIndex) { ZoneScoped; if (mBuf) @@ -219,7 +220,11 @@ BucketOutputIterator::getBucket( if (!index) { - if constexpr (std::is_same_v) + if (preBuiltIndex) + { + index = std::move(preBuiltIndex); + } + else if constexpr (std::is_same_v) { if (inMemoryState) { diff --git a/src/bucket/BucketOutputIterator.h b/src/bucket/BucketOutputIterator.h index a76e1c6bb7..99b42ec2d0 100644 --- a/src/bucket/BucketOutputIterator.h +++ b/src/bucket/BucketOutputIterator.h @@ -55,6 +55,8 @@ template class BucketOutputIterator std::shared_ptr getBucket( BucketManager& bucketManager, MergeKey* mergeKey = nullptr, std::unique_ptr> inMemoryState = + nullptr, + std::shared_ptr preBuiltIndex = nullptr); }; } diff --git a/src/bucket/LiveBucket.cpp b/src/bucket/LiveBucket.cpp index 8101c9d183..6c1b0bbdb5 100644 --- a/src/bucket/LiveBucket.cpp +++ b/src/bucket/LiveBucket.cpp @@ -10,6 +10,7 @@ #include "bucket/BucketOutputIterator.h" #include "bucket/BucketUtils.h" #include "bucket/LedgerCmp.h" +#include #include namespace stellar @@ -587,29 +588,50 @@ LiveBucket::mergeInMemory(BucketManager& bucketManager, mergedEntries.emplace_back(entry); }; - mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, mc, - shadowIterators, keepShadowedLifecycleEntries); + { + ZoneNamedN(zoneMerge, "mergeInMemory merge", true); + mergeInternal(bucketManager, inputSource, putFunc, maxProtocolVersion, + mc, shadowIterators, keepShadowedLifecycleEntries); + } if (countMergeEvents) { bucketManager.incrMergeCounters(mc); } + // Start index construction on worker thread — reads mergedEntries (const), + // completely independent of the put loop's serialize/hash/write work. + auto indexFuture = std::async(std::launch::async, [&]() { + return std::make_shared(bucketManager, mergedEntries, + meta); + }); + // Write merge output to a bucket and save to disk LiveBucketOutputIterator out(bucketManager.getTmpDir(), /*keepTombstoneEntries=*/true, meta, mc, ctx, doFsync); - for (auto const& e : mergedEntries) { - out.put(e); + ZoneNamedN(zonePut, "mergeInMemory put loop", true); + for (auto const& e : mergedEntries) + { + out.put(e); + } + } + + // Collect the pre-built index + std::shared_ptr preBuiltIndex; + { + ZoneNamedN(zoneWait, "mergeInMemory index future wait", true); + preBuiltIndex = indexFuture.get(); } // Store the merged entries in memory in the new bucket in case this // bucket sees another incoming merge as level 0 curr. return out.getBucket( bucketManager, nullptr, - std::make_unique>(std::move(mergedEntries))); + std::make_unique>(std::move(mergedEntries)), + std::move(preBuiltIndex)); } BucketEntryCounters const& diff --git a/src/crypto/SecretKey.cpp b/src/crypto/SecretKey.cpp index 1c92d1c090..6e4851152e 100644 --- a/src/crypto/SecretKey.cpp +++ b/src/crypto/SecretKey.cpp @@ -18,6 +18,8 @@ #include "util/Math.h" #include "util/RandomEvictionCache.h" #include +#include +#include #include #include #include @@ -41,16 +43,29 @@ namespace stellar // to the state of the process; caching its results centrally // makes all signature-verification in the program faster and // has no effect on correctness. +// +// The cache is sharded across NUM_VERIFY_CACHE_SHARDS shards to +// reduce mutex contention when multiple threads verify signatures +// in parallel. Each shard has its own mutex and cache partition. constexpr size_t VERIFY_SIG_CACHE_SIZE = 250'000; -static std::mutex gVerifySigCacheMutex; -static RandomEvictionCache gVerifySigCache(VERIFY_SIG_CACHE_SIZE); -static uint64_t gVerifyCacheHit = 0; -static uint64_t gVerifyCacheMiss = 0; +constexpr size_t NUM_VERIFY_CACHE_SHARDS = 16; +constexpr size_t VERIFY_SIG_CACHE_SHARD_SIZE = + VERIFY_SIG_CACHE_SIZE / NUM_VERIFY_CACHE_SHARDS; + +struct VerifySigCacheShard +{ + std::mutex mMutex; + RandomEvictionCache mCache; + VerifySigCacheShard() : mCache(VERIFY_SIG_CACHE_SHARD_SIZE) + { + } +}; -// Global flag to use Rust ed25519-dalek for signature verification -// Protected by gVerifySigCacheMutex -static bool gUseRustDalekVerify = false; +static std::array + gVerifySigCacheShards; +static std::atomic gVerifyCacheHit{0}; +static std::atomic gVerifyCacheMiss{0}; static Hash verifySigCacheKey(PublicKey const& key, Signature const& signature, @@ -322,32 +337,29 @@ SecretKey::fromStrKeySeed(std::string const& strKeySeed) void PubKeyUtils::clearVerifySigCache() { - std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.clear(); -} - -void -PubKeyUtils::enableRustDalekVerify() -{ - std::lock_guard guard(gVerifySigCacheMutex); - gUseRustDalekVerify = true; + for (auto& shard : gVerifySigCacheShards) + { + std::lock_guard guard(shard.mMutex); + shard.mCache.clear(); + } } void PubKeyUtils::seedVerifySigCache(unsigned int seed) { - std::lock_guard guard(gVerifySigCacheMutex); - gVerifySigCache.seed(seed); + for (size_t i = 0; i < NUM_VERIFY_CACHE_SHARDS; ++i) + { + std::lock_guard guard(gVerifySigCacheShards[i].mMutex); + gVerifySigCacheShards[i].mCache.seed(seed + + static_cast(i)); + } } void PubKeyUtils::flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses) { - std::lock_guard guard(gVerifySigCacheMutex); - hits = gVerifyCacheHit; - misses = gVerifyCacheMiss; - gVerifyCacheHit = 0; - gVerifyCacheMiss = 0; + hits = gVerifyCacheHit.exchange(0, std::memory_order_relaxed); + misses = gVerifyCacheMiss.exchange(0, std::memory_order_relaxed); } std::string @@ -456,41 +468,30 @@ PubKeyUtils::verifySig(PublicKey const& key, Signature const& signature, } auto cacheKey = verifySigCacheKey(key, signature, bin); - bool shouldUseRustDalekVerify; + + // Select shard based on cache key hash to distribute lock contention + auto shardIdx = std::hash{}(cacheKey) % NUM_VERIFY_CACHE_SHARDS; + auto& shard = gVerifySigCacheShards[shardIdx]; { - std::lock_guard guard(gVerifySigCacheMutex); - if (gVerifySigCache.exists(cacheKey)) + std::lock_guard guard(shard.mMutex); + if (auto* cached = shard.mCache.maybeGet(cacheKey)) { - ++gVerifyCacheHit; - std::string hitStr("hit"); - ZoneText(hitStr.c_str(), hitStr.size()); - return {gVerifySigCache.get(cacheKey), - VerifySigCacheLookupResult::HIT}; + gVerifyCacheHit.fetch_add(1, std::memory_order_relaxed); + ZoneText("hit", 3); + return {*cached, VerifySigCacheLookupResult::HIT}; } - - shouldUseRustDalekVerify = gUseRustDalekVerify; } - std::string missStr("miss"); - ZoneText(missStr.c_str(), missStr.size()); + ZoneText("miss", 4); - bool ok; - if (shouldUseRustDalekVerify) - { - ok = stellar::rust_bridge::verify_ed25519_signature_dalek( - key.ed25519().data(), signature.data(), bin.data(), bin.size()); - } - else + bool ok = stellar::rust_bridge::verify_ed25519_signature_dalek( + key.ed25519().data(), signature.data(), bin.data(), bin.size()); { - ok = (crypto_sign_verify_detached(signature.data(), bin.data(), - bin.size(), - key.ed25519().data()) == 0); + std::lock_guard guard(shard.mMutex); + gVerifyCacheMiss.fetch_add(1, std::memory_order_relaxed); + shard.mCache.put(cacheKey, ok); } - - std::lock_guard guard(gVerifySigCacheMutex); - ++gVerifyCacheMiss; - gVerifySigCache.put(cacheKey, ok); return {ok, VerifySigCacheLookupResult::MISS}; } diff --git a/src/crypto/SecretKey.h b/src/crypto/SecretKey.h index 9c3c96aeba..c30286e4d0 100644 --- a/src/crypto/SecretKey.h +++ b/src/crypto/SecretKey.h @@ -166,13 +166,6 @@ void clearVerifySigCache(); void seedVerifySigCache(unsigned int seed); void flushVerifySigCacheCounts(uint64_t& hits, uint64_t& misses); -// Enable Rust ed25519-dalek for signature verification -// Once enabled, it cannot be disabled. It should be enabled at the protocol 24 -// boundary. -// Note: This should be removed following the protocol 24 upgrade, rust ed25519 -// can be used unconditionally after upgrade, even for replay. -void enableRustDalekVerify(); - PublicKey random(); #ifdef BUILD_TESTS PublicKey pseudoRandomForTesting(); diff --git a/src/crypto/test/CryptoTests.cpp b/src/crypto/test/CryptoTests.cpp index 100a37163f..858c10a406 100644 --- a/src/crypto/test/CryptoTests.cpp +++ b/src/crypto/test/CryptoTests.cpp @@ -1630,7 +1630,6 @@ ZcashTestVector const ZCASH_TEST_VECTORS[196] = { TEST_CASE("Ed25519 test vectors from Zcash", "[crypto]") { - PubKeyUtils::enableRustDalekVerify(); for (auto const& tv : ZCASH_TEST_VECTORS) { PublicKey pk; diff --git a/src/herder/Upgrades.cpp b/src/herder/Upgrades.cpp index e8b1423f64..a2b232979a 100644 --- a/src/herder/Upgrades.cpp +++ b/src/herder/Upgrades.cpp @@ -1249,7 +1249,6 @@ Upgrades::applyVersionUpgrade(Application& app, AbstractLedgerTxn& ltx, } if (needUpgradeToVersion(ProtocolVersion::V_25, prevVersion, newVersion)) { - PubKeyUtils::enableRustDalekVerify(); SorobanNetworkConfig::createCostTypesForV25(ltx, app); } if (needUpgradeToVersion(ProtocolVersion::V_26, prevVersion, newVersion)) diff --git a/src/invariant/test/InvariantTests.cpp b/src/invariant/test/InvariantTests.cpp index 5dfd9078bd..4e45b7176b 100644 --- a/src/invariant/test/InvariantTests.cpp +++ b/src/invariant/test/InvariantTests.cpp @@ -670,7 +670,9 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") auto ttlData = entryData.ttlData; modifiedState.mContractDataEntries.erase(it); modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(modifiedEntry, ttlData)); + InternalContractDataMapEntry( + modifiedEntry, ttlData, + static_cast(xdr::xdr_size(modifiedEntry)))); } auto result = @@ -711,7 +713,9 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") createContractDataWithTTL(PERSISTENT, 1000); TTLData ttlData(extraTTL.data.ttl().liveUntilLedgerSeq, 1); modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(extraEntry, ttlData)); + InternalContractDataMapEntry( + extraEntry, ttlData, + static_cast(xdr::xdr_size(extraEntry)))); } auto result = @@ -741,8 +745,9 @@ TEST_CASE("BucketList state consistency invariant", "[invariant]") TTLData wrongTTL(42, 1); modifiedState.mContractDataEntries.erase(it); - modifiedState.mContractDataEntries.emplace( - InternalContractDataMapEntry(entryCopy, wrongTTL)); + modifiedState.mContractDataEntries.emplace(InternalContractDataMapEntry( + entryCopy, wrongTTL, + static_cast(xdr::xdr_size(entryCopy)))); auto result = invariant.checkSnapshot(makeSnap(), modifiedState, noopIsStopping); diff --git a/src/ledger/InMemorySorobanState.cpp b/src/ledger/InMemorySorobanState.cpp index 9a71cd52f5..3d7624e4c8 100644 --- a/src/ledger/InMemorySorobanState.cpp +++ b/src/ledger/InMemorySorobanState.cpp @@ -8,6 +8,7 @@ #include "ledger/LedgerTypeUtils.h" #include "ledger/SorobanMetrics.h" #include "util/GlobalChecks.h" +#include #include #include @@ -57,9 +58,10 @@ InMemorySorobanState::updateContractDataTTL( { // Since entries are immutable, we must erase and re-insert auto ledgerEntryPtr = dataIt->get().ledgerEntry; + auto sizeBytes = dataIt->get().sizeBytes; mContractDataEntries.erase(dataIt); - mContractDataEntries.emplace( - InternalContractDataMapEntry(std::move(ledgerEntryPtr), newTtlData)); + mContractDataEntries.emplace(InternalContractDataMapEntry( + std::move(ledgerEntryPtr), newTtlData, sizeBytes)); } void @@ -99,7 +101,7 @@ InMemorySorobanState::updateContractData(LedgerEntry const& ledgerEntry) releaseAssertOrThrow(dataIt != mContractDataEntries.end()); releaseAssertOrThrow(dataIt->get().ledgerEntry != nullptr); - uint32_t oldSize = xdr::xdr_size(*dataIt->get().ledgerEntry); + uint32_t oldSize = dataIt->get().sizeBytes; uint32_t newSize = xdr::xdr_size(ledgerEntry); updateStateSizeOnEntryUpdate(oldSize, newSize, /*isContractCode=*/false); @@ -107,7 +109,7 @@ InMemorySorobanState::updateContractData(LedgerEntry const& ledgerEntry) auto preservedTTL = dataIt->get().ttlData; mContractDataEntries.erase(dataIt); mContractDataEntries.emplace( - InternalContractDataMapEntry(ledgerEntry, preservedTTL)); + InternalContractDataMapEntry(ledgerEntry, preservedTTL, newSize)); } void @@ -135,10 +137,10 @@ InMemorySorobanState::createContractDataEntry(LedgerEntry const& ledgerEntry) } // else: TTL hasn't arrived yet, initialize to 0 (will be updated later) - updateStateSizeOnEntryUpdate(0, xdr::xdr_size(ledgerEntry), - /*isContractCode=*/false); + uint32_t sizeBytes = xdr::xdr_size(ledgerEntry); + updateStateSizeOnEntryUpdate(0, sizeBytes, /*isContractCode=*/false); mContractDataEntries.emplace( - InternalContractDataMapEntry(ledgerEntry, ttlData)); + InternalContractDataMapEntry(ledgerEntry, ttlData, sizeBytes)); } bool @@ -196,7 +198,7 @@ InMemorySorobanState::deleteContractData(LedgerKey const& ledgerKey) mContractDataEntries.find(InternalContractDataMapEntry(ledgerKey)); releaseAssertOrThrow(it != mContractDataEntries.end()); releaseAssertOrThrow(it->get().ledgerEntry != nullptr); - updateStateSizeOnEntryUpdate(xdr::xdr_size(*it->get().ledgerEntry), 0, + updateStateSizeOnEntryUpdate(it->get().sizeBytes, 0, /*isContractCode=*/false); mContractDataEntries.erase(it); } @@ -539,6 +541,7 @@ InMemorySorobanState::updateState( std::optional const& sorobanConfig, SorobanMetrics& metrics) { + ZoneScoped; // After initialization, we must apply every ledger in order to the // in-memory state with no gaps. releaseAssertOrThrow(mLastClosedLedgerSeq + 1 == lh.ledgerSeq); diff --git a/src/ledger/InMemorySorobanState.h b/src/ledger/InMemorySorobanState.h index 17a67a3454..83a1c3807b 100644 --- a/src/ledger/InMemorySorobanState.h +++ b/src/ledger/InMemorySorobanState.h @@ -45,14 +45,20 @@ struct TTLData // ContractDataMapEntryT stores a ContractData LedgerEntry and its TTL. TTL is // stored directly with the data to avoid an additional lookup and save memory. +// We also cache the XDR size to avoid repeated xdr_size() calls during updates. struct ContractDataMapEntryT { std::shared_ptr const ledgerEntry; TTLData const ttlData; + // Cached XDR serialized size to avoid repeated xdr_size() calls + uint32_t const sizeBytes; explicit ContractDataMapEntryT( - std::shared_ptr&& ledgerEntry, TTLData ttlData) - : ledgerEntry(std::move(ledgerEntry)), ttlData(ttlData) + std::shared_ptr&& ledgerEntry, TTLData ttlData, + uint32_t sizeBytes) + : ledgerEntry(std::move(ledgerEntry)) + , ttlData(ttlData) + , sizeBytes(sizeBytes) { } }; @@ -131,8 +137,6 @@ class InternalContractDataMapEntry } }; - // ValueEntry stores actual ContractData entries in the map. - // Contains both the LedgerEntry and its TTL information. struct ValueEntry : public AbstractEntry { private: @@ -140,8 +144,8 @@ class InternalContractDataMapEntry public: ValueEntry(std::shared_ptr&& ledgerEntry, - TTLData ttlData) - : entry(std::move(ledgerEntry), ttlData) + TTLData ttlData, uint32_t sizeBytes) + : entry(std::move(ledgerEntry), ttlData, sizeBytes) { } @@ -169,7 +173,7 @@ class InternalContractDataMapEntry { return std::make_unique( std::make_shared(*entry.ledgerEntry), - entry.ttlData); + entry.ttlData, entry.sizeBytes); } }; @@ -223,16 +227,19 @@ class InternalContractDataMapEntry // Creates a ValueEntry from a LedgerEntry (copies the entry) InternalContractDataMapEntry(LedgerEntry const& ledgerEntry, - TTLData ttlData) + TTLData ttlData, uint32_t sizeBytes) : impl(std::make_unique( - std::make_shared(ledgerEntry), ttlData)) + std::make_shared(ledgerEntry), ttlData, + sizeBytes)) { } // Creates a ValueEntry from a shared_ptr (avoids copying) InternalContractDataMapEntry( - std::shared_ptr&& ledgerEntry, TTLData ttlData) - : impl(std::make_unique(std::move(ledgerEntry), ttlData)) + std::shared_ptr&& ledgerEntry, TTLData ttlData, + uint32_t sizeBytes) + : impl(std::make_unique(std::move(ledgerEntry), ttlData, + sizeBytes)) { } diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index aa45c7f16d..5f081a06b2 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -75,6 +75,7 @@ #include "LedgerManagerImpl.h" #include +#include #include #include #include @@ -1606,8 +1607,9 @@ LedgerManagerImpl::applyLedger(LedgerCloseData const& ledgerData, } #ifdef BUILD_TESTS - // We always store the ledgerCloseMeta in tests so we can inspect it. - if (!ledgerCloseMeta) + // We always store the ledgerCloseMeta in tests so we can inspect it, + // unless explicitly disabled for benchmarking. + if (!ledgerCloseMeta && !mApp.getConfig().DISABLE_TX_META_FOR_TESTING) { ledgerCloseMeta = std::make_unique( header.current().ledgerVersion); @@ -1928,10 +1930,6 @@ LedgerManagerImpl::setLastClosedLedger( advanceLastClosedLedgerState(output); auto ledgerVersion = lastClosed.header.ledgerVersion; - if (protocolVersionStartsFrom(ledgerVersion, ProtocolVersion::V_25)) - { - PubKeyUtils::enableRustDalekVerify(); - } if (rebuildInMemoryState) { @@ -2498,10 +2496,13 @@ LedgerManagerImpl::checkAllTxBundleInvariants( AppConnector& app, ApplyStage const& stage, Config const& config, ParallelLedgerInfo const& ledgerInfo, LedgerHeader const& header) { + bool const hasInvariants = !config.INVARIANT_CHECKS.empty(); for (auto const& txBundle : stage) { - // First check the invariants - if (txBundle.getResPayload().isSuccess()) + // Only run invariant checks if any invariants are enabled. + // The delta is not built when invariants are disabled (see + // parallelApply), so we must not call getDelta() in that case. + if (hasInvariants && txBundle.getResPayload().isSuccess()) { try { @@ -2529,7 +2530,6 @@ LedgerManagerImpl::checkAllTxBundleInvariants( // We don't call processPostApply for post v23 transactions at the // moment because processPostApply is currently a no-op for those - // transactions. txBundle.getEffects().getMeta().maybeSetRefundableFeeMeta( txBundle.getResPayload().getRefundableFeeTracker()); @@ -2612,7 +2612,10 @@ LedgerManagerImpl::processResultAndMeta( { auto metaXDR = txMetaBuilder.finalize(result.isSuccess()); #ifdef BUILD_TESTS - mLastLedgerTxMeta.emplace_back(metaXDR); + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + mLastLedgerTxMeta.emplace_back(metaXDR); + } #endif ledgerCloseMeta->setTxProcessingMetaAndResultPair( @@ -2621,8 +2624,11 @@ LedgerManagerImpl::processResultAndMeta( else { #ifdef BUILD_TESTS - mLastLedgerTxMeta.emplace_back( - txMetaBuilder.finalize(result.isSuccess())); + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + mLastLedgerTxMeta.emplace_back( + txMetaBuilder.finalize(result.isSuccess())); + } #endif } } @@ -2668,8 +2674,11 @@ LedgerManagerImpl::applyTransactions( bool enableTxMeta = ledgerCloseMeta != nullptr; #ifdef BUILD_TESTS // In tests we want to always enable tx meta because we store it in - // mLastLedgerTxMeta. - enableTxMeta = true; + // mLastLedgerTxMeta, unless explicitly disabled for benchmarking. + if (!mApp.getConfig().DISABLE_TX_META_FOR_TESTING) + { + enableTxMeta = true; + } #endif std::optional sorobanConfig; if (protocolVersionStartsFrom(ltx.loadHeader().current().ledgerVersion, @@ -2971,6 +2980,11 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( // `ledgerApplied` protects this call with a mutex std::vector initEntries, liveEntries; std::vector deadEntries; + + std::future hotArchiveBatchFuture; + EvictedStateVectors evictedState; + std::vector restoredHotArchiveKeys; + // Any V20 features must be behind initialLedgerVers check, see comment // in LedgerManagerImpl::ledgerApplied if (protocolVersionStartsFrom(initialLedgerVers, SOROBAN_PROTOCOL_VERSION)) @@ -2980,16 +2994,13 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( // `getAllTTLKeysWithoutSealing` must be called at the right time // _after_ all operations have been applied, but _before_ evictions. auto sorobanConfig = SorobanNetworkConfig::loadFromLedger(ltx); - auto evictedState = - mApp.getBucketManager().resolveBackgroundEvictionScan( - lclApplyView, ltx, ltx.getAllKeysWithoutSealing()); + evictedState = mApp.getBucketManager().resolveBackgroundEvictionScan( + lclApplyView, ltx, ltx.getAllKeysWithoutSealing()); if (protocolVersionStartsFrom( initialLedgerVers, LiveBucket::FIRST_PROTOCOL_SUPPORTING_PERSISTENT_EVICTION)) { - std::vector restoredHotArchiveKeys; - auto const& restoredHotArchiveKeyMap = ltx.getRestoredHotArchiveKeys(); for (auto const& [key, entry] : restoredHotArchiveKeyMap) @@ -3019,9 +3030,14 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( } else { - mApp.getBucketManager().addHotArchiveBatch( - mApp, lh, evictedState.archivedEntries, - restoredHotArchiveKeys); + hotArchiveBatchFuture = + std::async(std::launch::async, [this, lh, &evictedState, + &restoredHotArchiveKeys]() { + mApp.getBucketManager().addHotArchiveBatch( + mApp, lh, evictedState.archivedEntries, + restoredHotArchiveKeys); + }); + // Validate evicted entries against Protocol 23 corruption // data if configured if (mApp.getProtocol23CorruptionDataVerifier()) @@ -3061,12 +3077,29 @@ LedgerManagerImpl::finalizeLedgerTxnChanges( } // NB: getAllEntries seals the ltx. ltx.getAllEntries(initEntries, liveEntries, deadEntries); + + // Launch async task to update in-memory Soroban state. This is independent + // from both addHotArchiveBatch and addLiveBatch, so all can run in + // parallel. + std::future inMemoryStateUpdateFuture; + + inMemoryStateUpdateFuture = std::async( + std::launch::async, [this, &initEntries, &liveEntries, &deadEntries, + &lh, &finalSorobanConfig]() { + mApplyState.updateInMemorySorobanState( + initEntries, liveEntries, deadEntries, lh, finalSorobanConfig); + }); + mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion, initEntries); mApplyState.addAnyContractsToModuleCache(lh.ledgerVersion, liveEntries); mApp.getBucketManager().addLiveBatch(mApp, lh, initEntries, liveEntries, deadEntries); - mApplyState.updateInMemorySorobanState(initEntries, liveEntries, - deadEntries, lh, finalSorobanConfig); + // Wait for all async operations to complete before returning. + if (hotArchiveBatchFuture.valid()) + { + hotArchiveBatchFuture.get(); + } + inMemoryStateUpdateFuture.get(); return finalSorobanConfig; } diff --git a/src/ledger/LedgerTxn.cpp b/src/ledger/LedgerTxn.cpp index 8b16e34831..613be9cf1d 100644 --- a/src/ledger/LedgerTxn.cpp +++ b/src/ledger/LedgerTxn.cpp @@ -1628,6 +1628,7 @@ LedgerTxn::Impl::getAllEntries(std::vector& initEntries, std::vector& liveEntries, std::vector& deadEntries) { + ZoneScoped; abortIfWrongThread("getAllEntries"); std::vector resInit, resLive; std::vector resDead; diff --git a/src/main/ApplicationImpl.cpp b/src/main/ApplicationImpl.cpp index 810522d4df..c0c9ab48c1 100644 --- a/src/main/ApplicationImpl.cpp +++ b/src/main/ApplicationImpl.cpp @@ -801,14 +801,6 @@ ApplicationImpl::start() // LCL is now loaded; unblock HTTP endpoints that were gated during boot. mCommandHandler->setReady(); - // Check if we're already on protocol V_24 or later and enable Rust Dalek - auto const& lcl = mLedgerManager->getLastClosedLedgerHeader(); - if (protocolVersionStartsFrom(lcl.header.ledgerVersion, - ProtocolVersion::V_25)) - { - PubKeyUtils::enableRustDalekVerify(); - } - startServices(); } diff --git a/src/main/Config.cpp b/src/main/Config.cpp index 36c9f79bd7..b2ec4068c8 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -172,6 +172,7 @@ Config::Config() : NODE_SEED(SecretKey::random()) BACKGROUND_OVERLAY_PROCESSING = true; PARALLEL_LEDGER_APPLY = true; DISABLE_SOROBAN_METRICS_FOR_TESTING = false; + DISABLE_TX_META_FOR_TESTING = false; BACKGROUND_TX_SIG_VERIFICATION = true; BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 14; // 2^14 == 16 kb BUCKETLIST_DB_INDEX_CUTOFF = 20; // 20 mb @@ -1188,6 +1189,8 @@ Config::processConfig(std::shared_ptr t) [&]() { DISABLE_SOROBAN_METRICS_FOR_TESTING = readBool(item); }}, + {"DISABLE_TX_META_FOR_TESTING", + [&]() { DISABLE_TX_META_FOR_TESTING = readBool(item); }}, {"EXPERIMENTAL_BACKGROUND_TX_SIG_VERIFICATION", [&]() { CLOG_WARNING(Overlay, diff --git a/src/main/Config.h b/src/main/Config.h index 92de868722..99e0e20265 100644 --- a/src/main/Config.h +++ b/src/main/Config.h @@ -525,6 +525,11 @@ class Config : public std::enable_shared_from_this // Disable expensive Soroban metrics for performance testing bool DISABLE_SOROBAN_METRICS_FOR_TESTING; + // Disable transaction metadata collection in test builds. + // This is useful for benchmarking, which is typically done in BUILD_TESTS + // builds. + bool DISABLE_TX_META_FOR_TESTING; + // Batch transactions for flooding purposes (experimental). // Has no effect on non-test builds. size_t EXPERIMENTAL_TX_BATCH_MAX_SIZE; diff --git a/src/transactions/InvokeHostFunctionOpFrame.cpp b/src/transactions/InvokeHostFunctionOpFrame.cpp index 6814b7df43..dcbc2b7060 100644 --- a/src/transactions/InvokeHostFunctionOpFrame.cpp +++ b/src/transactions/InvokeHostFunctionOpFrame.cpp @@ -18,6 +18,7 @@ #include "ledger/LedgerTxnImpl.h" #include "rust/CppShims.h" +#include "util/BitSet.h" #include "xdr/Stellar-transaction.h" #include #include @@ -609,9 +610,16 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper recordStorageChanges(InvokeHostFunctionOutput const& out) { ZoneScoped; - // Create or update every entry returned. - UnorderedSet createdAndModifiedKeys; - UnorderedSet createdKeys; + // Track which RW footprint keys appear in the host output without + // hashing LedgerKeys. Footprints are small, so a linear scan over a + // BitSet-backed coverage map is cheaper than maintaining hash sets. + auto const& rwKeys = mResources.footprint.readWrite; + BitSet rwKeyCovered(rwKeys.size()); + size_t numCreatedSorobanEntries = 0; + size_t numCreatedTTLEntries = 0; + bool const allowClassicCreations = protocolVersionStartsFrom( + getLedgerVersion(), ProtocolVersion::V_26); + for (auto const& buf : out.modified_ledger_entries) { LedgerEntry le; @@ -626,15 +634,22 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper return false; } - createdAndModifiedKeys.insert(lk); - - uint32_t keySize = static_cast(xdr::xdr_size(lk)); uint32_t entrySize = static_cast(buf.data.size()); + for (size_t j = 0; j < rwKeys.size(); ++j) + { + if (!rwKeyCovered.get(j) && rwKeys[j] == lk) + { + rwKeyCovered.set(j); + break; + } + } + // ttlEntry write fees come out of refundableFee, already // accounted for by the host if (lk.type() != TTL) { + uint32_t keySize = static_cast(xdr::xdr_size(lk)); mMetrics.noteWriteEntry(isContractCodeEntry(lk), keySize, entrySize); if (mResources.writeBytes < mMetrics.mLedgerWriteByte) @@ -653,42 +668,39 @@ class InvokeHostFunctionApplyHelper : virtual LedgerAccessHelper if (upsertLedgerEntry(lk, le)) { - createdKeys.insert(lk); + if (isSorobanEntry(lk)) + { + ++numCreatedSorobanEntries; + } + else if (lk.type() == TTL) + { + ++numCreatedTTLEntries; + } + else if (allowClassicCreations) + { + releaseAssertOrThrow(lk.type() == ACCOUNT || + lk.type() == TRUSTLINE); + } + else + { + releaseAssertOrThrow(false); + } } } - // Check that each newly created ContractCode or ContractData entry also - // creates a ttlEntry. Starting from protocol 26 (CAP-73), the Stellar - // Asset Contract can also create classic entries (ACCOUNT, TRUSTLINE). - for (auto const& key : createdKeys) - { - if (isSorobanEntry(key)) - { - auto ttlKey = getTTLKey(key); - releaseAssertOrThrow(createdKeys.find(ttlKey) != - createdKeys.end()); - } - else if (protocolVersionStartsFrom(getLedgerVersion(), - ProtocolVersion::V_26)) - { - releaseAssertOrThrow(key.type() == TTL || - key.type() == ACCOUNT || - key.type() == TRUSTLINE); - } - else - { - releaseAssertOrThrow(key.type() == TTL); - } - } + // Verify that each newly created Soroban entry has a corresponding + // newly created TTL entry (1:1 pairing guaranteed by the host). + releaseAssertOrThrow(numCreatedSorobanEntries == numCreatedTTLEntries); // Erase every entry not returned. // NB: The entries that haven't been touched are passed through // from host, so this should never result in removing an entry // that hasn't been removed by host explicitly. - for (auto const& lk : mResources.footprint.readWrite) + for (size_t j = 0; j < rwKeys.size(); ++j) { - if (createdAndModifiedKeys.find(lk) == createdAndModifiedKeys.end()) + if (!rwKeyCovered.get(j)) { + auto const& lk = rwKeys[j]; if (eraseLedgerEntryIfExists(lk)) { releaseAssertOrThrow(isSorobanEntry(lk)); diff --git a/src/transactions/TransactionFrame.cpp b/src/transactions/TransactionFrame.cpp index cb495142e6..199c319f03 100644 --- a/src/transactions/TransactionFrame.cpp +++ b/src/transactions/TransactionFrame.cpp @@ -2277,8 +2277,14 @@ TransactionFrame::parallelApply( if (res) { - threadState.setEffectsDeltaFromSuccessfulTx(*res, ledgerInfo, - effects); + // Only build the LedgerTxnDelta when invariant checks are + // enabled — the delta is consumed exclusively by + // checkOnOperationApply which is a no-op otherwise. + if (!config.INVARIANT_CHECKS.empty()) + { + threadState.setEffectsDeltaFromSuccessfulTx(*res, ledgerInfo, + effects); + } opMeta.setLedgerChangesFromSuccessfulOp(threadState, *res, ledgerInfo.getLedgerSeq()); }