backport: bitcoin#19169, #22621, #25344, #25685, #25730, #25734, #25942, #26462, #26532, #26668

doc/release-notes-25730.md

@@ -0,0 +1,6 @@
+RPC Wallet
+----------
+
+- RPC `listunspent` now has a new argument `include_immature_coinbase`
+  to include coinbase UTXOs that don't meet the minimum spendability
+  depth requirement (which before were silently skipped). (#25730)

src/Makefile.bench.include

@@ -94,6 +94,7 @@ bench_bench_dash_SOURCES += bench/coin_selection.cpp
 bench_bench_dash_SOURCES += bench/wallet_balance.cpp
 bench_bench_dash_SOURCES += bench/wallet_create.cpp
 bench_bench_dash_SOURCES += bench/wallet_loading.cpp
+bench_bench_dash_SOURCES += bench/wallet_create_tx.cpp
 bench_bench_dash_LDADD += $(BDB_LIBS) $(SQLITE_LIBS)
 endif
 

src/Makefile.test.include

@@ -259,8 +259,6 @@ FUZZ_WALLET_SRC += \
 endif # USE_SQLITE
 
 BITCOIN_TEST_SUITE += \
-  wallet/test/util.cpp \
-  wallet/test/util.h \
   wallet/test/wallet_test_fixture.cpp \
   wallet/test/wallet_test_fixture.h \
   wallet/test/init_test_fixture.cpp \

src/Makefile.test_util.include

@@ -25,10 +25,13 @@ TEST_UTIL_H = \
   test/util/str.h \
   test/util/transaction_utils.h \
   test/util/txmempool.h \
-  test/util/wallet.h \
   test/util/validation.h \
   test/util/xoroshiro128plusplus.h
 
+if ENABLE_WALLET
+TEST_UTIL_H += wallet/test/util.h
+endif # ENABLE_WALLET
+
 libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS)
 libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libtest_util_a_SOURCES = \
@@ -45,6 +48,10 @@ libtest_util_a_SOURCES = \
   test/util/str.cpp \
   test/util/transaction_utils.cpp \
   test/util/txmempool.cpp \
-  test/util/validation.cpp \
-  test/util/wallet.cpp \
-  $(TEST_UTIL_H)
+  test/util/validation.cpp
+
+if ENABLE_WALLET
+libtest_util_a_SOURCES += wallet/test/util.cpp
+endif # ENABLE_WALLET
+
+libtest_util_a_SOURCES += $(TEST_UTIL_H)

src/bench/block_assemble.cpp

@@ -10,7 +10,6 @@
 #include <test/util/mining.h>
 #include <test/util/script.h>
 #include <test/util/setup_common.h>
-#include <test/util/wallet.h>
 #include <txmempool.h>
 #include <validation.h>
 

src/bench/coin_selection.cpp

@@ -58,7 +58,7 @@ static void CoinSelection(benchmark::Bench& bench)
     wallet::CoinsResult available_coins;
     for (const auto& wtx : wtxs) {
         const auto txout = wtx->tx->vout.at(0);
-        available_coins.legacy.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0);
+        available_coins.coins[OutputType::LEGACY].emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0);
     }
     const CoinEligibilityFilter filter_standard(1, 6, 0);
     FastRandomContext rand{};

src/bench/wallet_balance.cpp

@@ -7,7 +7,7 @@
 #include <node/context.h>
 #include <test/util/mining.h>
 #include <test/util/setup_common.h>
-#include <test/util/wallet.h>
+#include <wallet/test/util.h>
 #include <validationinterface.h>
 #include <wallet/receive.h>
 #include <wallet/wallet.h>
@@ -19,6 +19,8 @@ using wallet::CWallet;
 using wallet::DBErrors;
 using wallet::WALLET_FLAG_DESCRIPTORS;
 
+const std::string ADDRESS_B58T_UNSPENDABLE = "yXXXXXXXXXXXXXXXXXXXXXXXXXXXVd2rXU";
+
 static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine, const uint32_t epoch_iters)
 {
     const auto test_setup = MakeNoLogFileContext<const TestingSetup>();

src/bench/wallet_create_tx.cpp

@@ -0,0 +1,144 @@
+// Copyright (c) 2022 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or https://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <chainparams.h>
+#include <wallet/coincontrol.h>
+#include <consensus/merkle.h>
+#include <kernel/chain.h>
+#include <node/context.h>
+#include <test/util/setup_common.h>
+#include <validation.h>
+#include <wallet/spend.h>
+#include <wallet/test/util.h>
+#include <wallet/wallet.h>
+
+using wallet::CWallet;
+using wallet::CreateMockWalletDatabase;
+using wallet::DBErrors;
+using wallet::WALLET_FLAG_DESCRIPTORS;
+
+struct TipBlock
+{
+    uint256 prev_block_hash;
+    int64_t prev_block_time;
+    int tip_height;
+};
+
+TipBlock getTip(const CChainParams& params, const node::NodeContext& context)
+{
+    auto tip = WITH_LOCK(::cs_main, return context.chainman->ActiveTip());
+    return (tip) ? TipBlock{tip->GetBlockHash(), tip->GetBlockTime(), tip->nHeight} :
+           TipBlock{params.GenesisBlock().GetHash(), params.GenesisBlock().GetBlockTime(), 0};
+}
+
+void generateFakeBlock(const CChainParams& params,
+                       const node::NodeContext& context,
+                       CWallet& wallet,
+                       const CScript& coinbase_out_script)
+{
+    TipBlock tip{getTip(params, context)};
+
+    // Create block
+    CBlock block;
+    CMutableTransaction coinbase_tx;
+    coinbase_tx.vin.resize(1);
+    coinbase_tx.vin[0].prevout.SetNull();
+    coinbase_tx.vout.resize(2);
+    coinbase_tx.vout[0].scriptPubKey = coinbase_out_script;
+    coinbase_tx.vout[0].nValue = 49 * COIN;
+    coinbase_tx.vin[0].scriptSig = CScript() << ++tip.tip_height << OP_0;
+    coinbase_tx.vout[1].scriptPubKey = coinbase_out_script; // extra output
+    coinbase_tx.vout[1].nValue = 1 * COIN;
+    block.vtx = {MakeTransactionRef(std::move(coinbase_tx))};
+
+    block.nVersion = VERSIONBITS_LAST_OLD_BLOCK_VERSION;
+    block.hashPrevBlock = tip.prev_block_hash;
+    block.hashMerkleRoot = BlockMerkleRoot(block);
+    block.nTime = ++tip.prev_block_time;
+    block.nBits = params.GenesisBlock().nBits;
+    block.nNonce = 0;
+
+    {
+        LOCK(::cs_main);
+        // Add it to the index
+        const uint256 hash{block.GetHash()};
+        CBlockIndex* pindex{context.chainman->m_blockman.AddToBlockIndex(block, hash, context.chainman->m_best_header)};
+        // add it to the chain
+        context.chainman->ActiveChain().SetTip(*pindex);
+    }
+
+    // notify wallet
+    const auto& pindex = WITH_LOCK(::cs_main, return context.chainman->ActiveChain().Tip());
+    wallet.blockConnected(kernel::MakeBlockInfo(pindex, &block));
+}
+
+struct PreSelectInputs {
+    // How many coins from the wallet the process should select
+    int num_of_internal_inputs;
+    // future: this could have external inputs as well.
+};
+
+static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type, bool allow_other_inputs, std::optional<PreSelectInputs> preset_inputs)
+{
+    const auto test_setup = MakeNoLogFileContext<const TestingSetup>();
+
+    CWallet wallet{test_setup->m_node.chain.get(), test_setup->m_node.coinjoin_loader.get(), "", gArgs, CreateMockWalletDatabase()};
+    {
+        LOCK(wallet.cs_wallet);
+        wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS);
+        wallet.SetupDescriptorScriptPubKeyMans("", "");
+        if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false);
+    }
+
+    // Generate destinations
+    CScript dest = GetScriptForDestination(getNewDestination(wallet));
+
+    // Generate chain; each coinbase will have two outputs to fill-up the wallet
+    const auto& params = Params();
+    unsigned int chain_size = 5000; // 5k blocks means 10k UTXO for the wallet (minus 200 due COINBASE_MATURITY)
+    for (unsigned int i = 0; i < chain_size; ++i) {
+        generateFakeBlock(params, test_setup->m_node, wallet, dest);
+    }
+
+    // Check available balance
+    auto bal = WITH_LOCK(wallet.cs_wallet, return wallet::AvailableCoins(wallet).total_amount); // Cache
+    assert(bal == 50 * COIN * (chain_size - COINBASE_MATURITY));
+
+    wallet::CCoinControl coin_control;
+    coin_control.m_allow_other_inputs = allow_other_inputs;
+
+    CAmount target = 0;
+    if (preset_inputs) {
+        // Select inputs, each has 49 BTC
+        wallet::CoinFilterParams filter_coins;
+        filter_coins.max_count = preset_inputs->num_of_internal_inputs;
+        const auto& res = WITH_LOCK(wallet.cs_wallet,
+                                    return wallet::AvailableCoins(wallet, /*coinControl=*/nullptr, /*feerate=*/std::nullopt, filter_coins));
+        for (int i=0; i < preset_inputs->num_of_internal_inputs; i++) {
+            const auto& coin{res.coins.at(output_type)[i]};
+            target += coin.txout.nValue;
+            coin_control.Select(coin.outpoint);
+        }
+    }
+
+    // If automatic coin selection is enabled, add the value of another UTXO to the target
+    if (coin_control.m_allow_other_inputs) target += 50 * COIN;
+    std::vector<wallet::CRecipient> recipients = {{dest, target, true}};
+
+    bench.epochIterations(5).run([&] {
+        LOCK(wallet.cs_wallet);
+        const auto& tx_res = CreateTransaction(wallet, recipients, -1, coin_control);
+        assert(tx_res);
+    });
+}
+
+static void WalletCreateTxUseOnlyPresetInputs(benchmark::Bench& bench) { WalletCreateTx(bench, OutputType::LEGACY, /*allow_other_inputs=*/false,
+                                                                                        {{/*num_of_internal_inputs=*/4}}); }
+
+static void WalletCreateTxUsePresetInputsAndCoinSelection(benchmark::Bench& bench) { WalletCreateTx(bench, OutputType::LEGACY, /*allow_other_inputs=*/true,
+                                                                                                    {{/*num_of_internal_inputs=*/4}}); }
+
+BENCHMARK(WalletCreateTxUseOnlyPresetInputs, benchmark::PriorityLevel::LOW)
+BENCHMARK(WalletCreateTxUsePresetInputsAndCoinSelection, benchmark::PriorityLevel::LOW)

src/bench/wallet_loading.cpp

@@ -7,7 +7,7 @@
 #include <node/context.h>
 #include <test/util/mining.h>
 #include <test/util/setup_common.h>
-#include <test/util/wallet.h>
+#include <wallet/test/util.h>
 #include <util/translation.h>
 #include <validationinterface.h>
 #include <wallet/context.h>
@@ -52,30 +52,6 @@ static void AddTx(CWallet& wallet)
     wallet.AddToWallet(MakeTransactionRef(mtx), TxStateInactive{});
 }
 
-static std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options)
-{
-    auto new_database = CreateMockWalletDatabase(options);
-
-    // Get a cursor to the original database
-    auto batch = database.MakeBatch();
-    batch->StartCursor();
-
-    // Get a batch for the new database
-    auto new_batch = new_database->MakeBatch();
-
-    // Read all records from the original database and write them to the new one
-    while (true) {
-        CDataStream key(SER_DISK, CLIENT_VERSION);
-        CDataStream value(SER_DISK, CLIENT_VERSION);
-        bool complete;
-        batch->ReadAtCursor(key, value, complete);
-        if (complete) break;
-        new_batch->Write(key, value);
-    }
-
-    return new_database;
-}
-
 static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet)
 {
     const auto test_setup = MakeNoLogFileContext<TestingSetup>();

src/coinjoin/client.cpp

@@ -2150,7 +2150,7 @@ bool CCoinJoinClientSession::CreateCollateralTransaction(CMutableTransaction& tx
     AssertLockHeld(m_wallet->cs_wallet);
 
     CCoinControl coin_control(CoinType::ONLY_COINJOIN_COLLATERAL);
-    std::vector<COutput> vCoins{AvailableCoinsListUnspent(*m_wallet, &coin_control).all()};
+    std::vector<COutput> vCoins{AvailableCoinsListUnspent(*m_wallet, &coin_control).All()};
     if (vCoins.empty()) {
         strReason = strprintf("%s requires a collateral transaction and could not locate an acceptable input!", gCoinJoinName);
         return false;

src/outputtype.cpp

@@ -13,20 +13,22 @@
 #include <util/vector.h>
 
 #include <assert.h>
+#include <optional>
 #include <string>
 
 static const std::string OUTPUT_TYPE_STRING_LEGACY = "legacy";
 static const std::string OUTPUT_TYPE_STRING_UNKNOWN = "unknown";
 
 const std::array<OutputType, 1> OUTPUT_TYPES = {OutputType::LEGACY};
 
-bool ParseOutputType(const std::string& type, OutputType& output_type)
+std::optional<OutputType> ParseOutputType(const std::string& type)
 {
     if (type == OUTPUT_TYPE_STRING_LEGACY) {
-        output_type = OutputType::LEGACY;
-        return true;
+        return OutputType::LEGACY;
+    } else if (type == OUTPUT_TYPE_STRING_UNKNOWN) {
+        return OutputType::UNKNOWN;
     }
-    return false;
+    return std::nullopt;
 }
 
 const std::string& FormatOutputType(OutputType type)

src/outputtype.h

@@ -10,6 +10,7 @@
 #include <script/standard.h>
 
 #include <array>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -20,7 +21,7 @@ enum class OutputType {
 
 extern const std::array<OutputType, 1> OUTPUT_TYPES;
 
-[[nodiscard]] bool ParseOutputType(const std::string& str, OutputType& output_type);
+std::optional<OutputType> ParseOutputType(const std::string& str);
 const std::string& FormatOutputType(OutputType type);
 
 /**

src/qt/sendcoinsdialog.cpp

@@ -324,7 +324,9 @@ bool SendCoinsDialog::send(const QList<SendCoinsRecipient>& recipients, QString&
 
     updateCoinControlState();
 
-    prepareStatus = model->prepareTransaction(*m_current_transaction, *m_coin_control);
+    CCoinControl coin_control = *m_coin_control;
+    coin_control.m_allow_other_inputs = !coin_control.HasSelected(); // future, could introduce a checkbox to customize this value.
+    prepareStatus = model->prepareTransaction(*m_current_transaction, coin_control);
 
     // process prepareStatus and on error generate message shown to user
     processSendCoinsReturn(prepareStatus,
@@ -895,8 +897,13 @@ void SendCoinsDialog::useAvailableBalance(SendCoinsEntry* entry)
     // Include watch-only for wallets without private key
     m_coin_control->fAllowWatchOnly = model->wallet().privateKeysDisabled() && !model->wallet().hasExternalSigner();
 
+    // Same behavior as send: if we have selected coins, only obtain their available balance.
+    // Copy to avoid modifying the member's data.
+    CCoinControl coin_control = *m_coin_control;
+    coin_control.m_allow_other_inputs = !coin_control.HasSelected();
+
     // Calculate available amount to send.
-    CAmount amount = model->wallet().getAvailableBalance(*m_coin_control);
+    CAmount amount = model->wallet().getAvailableBalance(coin_control);
     for (int i = 0; i < ui->entries->count(); ++i) {
         SendCoinsEntry* e = qobject_cast<SendCoinsEntry*>(ui->entries->itemAt(i)->widget());
         if (e && !e->isHidden() && e != entry) {

src/rpc/masternode.cpp

@@ -160,7 +160,7 @@ static RPCHelpMan masternode_outputs()
     CCoinControl coin_control(CoinType::ONLY_MASTERNODE_COLLATERAL);
 
     UniValue outputsArr(UniValue::VARR);
-    for (const auto& out : WITH_LOCK(wallet->cs_wallet, return AvailableCoinsListUnspent(*wallet, &coin_control).all())) {
+    for (const auto& out : WITH_LOCK(wallet->cs_wallet, return AvailableCoinsListUnspent(*wallet, &coin_control).All())) {
         outputsArr.push_back(out.outpoint.ToStringShort());
     }