Refactor: separate wallet from node #10973

 105 | +        auto it = ::mapBlockIndex.find(hash);
 106 | +        if (it == ::mapBlockIndex.end()) {
 107 | +            return false;
 108 | +        }
 109 | +        if (block) {
 110 | +            if (!ReadBlockFromDisk(*block, it->second, Params().GetConsensus())) {

---

  25 | @@ -23,6 +26,68 @@ class Chain
  26 |      {
  27 |      public:
  28 |          virtual ~LockedState() {}
  29 | +

---

 887 | @@ -887,7 +888,7 @@ bool AppInitBasicSetup()
 888 |      return true;
 889 |  }
 890 |  
 891 | -bool AppInitParameterInteraction()
 892 | +bool AppInitParameterInteraction(ipc::Chain& ipc_chain, ipc::Chain::Clients& ipc_clients)

---

  13 | @@ -14,6 +14,26 @@ class Chain
  14 |  public:
  15 |      virtual ~Chain() {}
  16 |  
  17 | +    //! Interface for querying locked chain state, used by legacy code that
  18 | +    //! assumes state won't change between calls. New code should avoid using
  19 | +    //! the LockedState interface and instead call higher-level Chain methods
  20 | +    //! that return more information so the chain doesn't need to stay locked
  21 | +    //! between calls.
  22 | +    class LockedState

---

  36 | +#endif
  37 | +
  38 | +#include <algorithm>
  39 | +#include <future>
  40 | +#include <map>
  41 | +#include <memory>

---

 143 | @@ -144,7 +144,9 @@ void TestGUI()
 144 |      for (int i = 0; i < 5; ++i) {
 145 |          test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey()));
 146 |      }
 147 | -    CWallet wallet("mock", WalletDatabase::CreateMock());
 148 | +    auto node = interfaces::MakeNode();
 149 | +    node->parseParameters(0, nullptr);
 150 | +    CWallet wallet(&node->getChain(), "mock", WalletDatabase::CreateMock());

---

  35 | +        //! chain only contains genesis block, nothing if chain does not contain
  36 | +        //! any blocks).
  37 | +        virtual Optional<int> getHeight() = 0;
  38 | +
  39 | +        //! Get block height above genesis block. Returns 0 for genesis block,
  40 | +        //! for following block, and so on. Returns nothing for a block not

---

  71 | +        //! nothing if no block in the range is pruned. Range is inclusive.
  72 | +        virtual Optional<int> findPruned(int start_height = 0, Optional<int> stop_height = nullopt) = 0;
  73 | +
  74 | +        //! Return height of the highest block on the chain that is an ancestor
  75 | +        //! of the specified block. Also return the height of the specified
  76 | +        //! block as an optinal output parameter.

---

3794 |      }
3795 |      UniValue response(UniValue::VOBJ);
3796 | -    response.pushKV("start_height", pindexStart->nHeight);
3797 | -    response.pushKV("stop_height", stopBlock->nHeight);
3798 | +    response.pushKV("start_height", start_height);
3799 | +    response.pushKV("stop_height", stop_height ? UniValue(*stop_height) : tip_height ? UniValue(*tip_height): UniValue());

---

1806 | -                if (pindex && !chainActive.Contains(pindex)) {
1807 | +                if (!locked_chain->getBlockHeight(block_hash)) {
1808 |                      // Abort scan if current block is no longer active, to prevent
1809 | -                    // marking transactions as coming from the wrong block.
1810 | -                    ret = pindex;
1811 | +                    // marking transac<tions as coming from the wrong block.

---

 339 | +        // inside the RPC request for wallet name and sending it directly to the
 340 | +        // right handler), but right now all wallets are in-process so there is
 341 | +        // only ever a single handler, and in the future it isn't clear if we
 342 | +        // will want we want to keep forwarding RPCs at all (clients could just
 343 | +        // listen for their own RPCs).
 344 | +        for (auto it = m_commands.begin(); it != m_commands.end();) {

---

 298 | @@ -296,6 +299,13 @@ class ChainImpl : public Chain
 299 |      void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); }
 300 |      void initWarning(const std::string& message) override { InitWarning(message); }
 301 |      void initError(const std::string& message) override { InitError(message); }
 302 | +    UniValue generateBlocks(std::shared_ptr<CReserveScript> coinbase_script,
 303 | +        int num_blocks,
 304 | +        uint64_t max_tries,
 305 | +        bool keep_script) override
 306 | +    {
 307 | +        return ::generateBlocks(coinbase_script, num_blocks, max_tries, keep_script);

---

  50 | @@ -50,6 +51,7 @@ class NodeImpl : public Node
  51 |  {
  52 |      void parseParameters(int argc, const char* const argv[]) override
  53 |      {
  54 | +        m_interfaces.chain = interfaces::MakeChain();

---

 215 | +        virtual void registerRpcs() = 0;
 216 | +
 217 | +        //! Prepare for execution, loading any needed state.
 218 | +        virtual bool prepare() = 0;
 219 | +
 220 | +        //! Start client execution and provide a scheduler. (Scheduler is

---

  34 | +    //! Interface for querying locked chain state, used by legacy code that
  35 | +    //! assumes state won't change between calls. New code should avoid using
  36 | +    //! the Lock interface and instead call higher-level Chain methods
  37 | +    //! that return more information so the chain doesn't need to stay locked
  38 | +    //! between calls.
  39 | +    class Lock

---

3416 | @@ -3388,7 +3417,8 @@ UniValue fundrawtransaction(const JSONRPCRequest& request)
3417 |      CAmount nFeeOut;
3418 |      std::string strFailReason;
3419 |  
3420 | -    if (!pwallet->FundTransaction(tx, nFeeOut, changePosition, strFailReason, lockUnspents, setSubtractFeeFromOutputs, coinControl)) {
3421 | +    auto locked_chain = pwallet->chain().assumeLocked();  // Temporary. Removed in upcoming lock cleanup

---

  81 | +        //! nothing if no block in the range is pruned. Range is inclusive.
  82 | +        virtual Optional<int> findPruned(int start_height = 0, Optional<int> stop_height = nullopt) = 0;
  83 | +
  84 | +        //! Return height of the highest block on the chain that is an ancestor
  85 | +        //! of the specified block. Also return the height of the specified
  86 | +        //! block as an optinal output parameter.

---

 696 | @@ -698,7 +697,7 @@ class CWallet final : public CCryptoKeyStore, public CValidationInterface
 697 |  
 698 |      /* Used by TransactionAddedToMemorypool/BlockConnected/Disconnected.
 699 |       * Should be called with pindexBlock and posInBlock if this is for a transaction that is included in a block. */

---

 750 | @@ -744,7 +751,7 @@ class CWallet final : public CCryptoKeyStore, public CValidationInterface
 751 |       *
 752 |       * Protected by cs_main (see BlockUntilSyncedToCurrentChain)

---

 755 | @@ -747,8 +756,9 @@ UniValue dumpwallet(const JSONRPCRequest& request)
 756 |      // produce output
 757 |      file << strprintf("# Wallet dump created by Bitcoin %s\n", CLIENT_BUILD);
 758 |      file << strprintf("# * Created on %s\n", FormatISO8601DateTime(GetTime()));
 759 | -    file << strprintf("# * Best block at time of backup was %i (%s),\n", chainActive.Height(), chainActive.Tip()->GetBlockHash().ToString());
 760 | -    file << strprintf("#   mined on %s\n", FormatISO8601DateTime(chainActive.Tip()->GetBlockTime()));
 761 | +    const Optional<int> tip_height = locked_chain->getHeight();
 762 | +    file << strprintf("# * Best block at time of backup was %i (%s),\n", tip_height ? *tip_height : -1, tip_height ? locked_chain->getBlockHash(*tip_height).ToString() : "(missing block hash)");

---

2226 | @@ -2214,39 +2227,41 @@ UniValue listsinceblock(const JSONRPCRequest& request)
2227 |  
2228 |      bool include_removed = (request.params[3].isNull() || request.params[3].get_bool());
2229 |  
2230 | -    int depth = pindex ? (1 + chainActive.Height() - pindex->nHeight) : -1;
2231 | +    const Optional<int> tip_height = locked_chain->getHeight();
2232 | +    int depth = tip_height && height ? (1 + *tip_height - *height) : -1;

---

1778 | -            LOCK(cs_main);
1779 | -            tip = chainActive.Tip();
1780 | -            dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex);
1781 | -            dProgressTip = GuessVerificationProgress(chainParams.TxData(), tip);
1782 | +            auto locked_chain = m_chain->lock();
1783 | +            if (Optional<int> tip_height = locked_chain->getHeight()) {

---

3817 | @@ -3795,8 +3818,9 @@ void CWallet::GetKeyBirthTimes(std::map<CTxDestination, int64_t> &mapKeyBirth) c
3818 |      }
3819 |  
3820 |      // map in which we'll infer heights of other keys
3821 | -    CBlockIndex *pindexMax = chainActive[std::max(0, chainActive.Height() - 144)]; // the tip can be reorganized; use a 144-block safety margin
3822 | -    std::map<CKeyID, CBlockIndex*> mapKeyFirstBlock;
3823 | +    const Optional<int> tip_height = locked_chain.getHeight();
3824 | +    const int pindexMax = tip_height && *tip_height > 144 ? *tip_height - 144 : 0; // the tip can be reorganized; use a 144-block safety margin

---

4191 | +        if (chain.getPruneMode())
4192 |          {
4193 | -            CBlockIndex *block = chainActive.Tip();
4194 | -            while (block && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA) && block->pprev->nTx > 0 && pindexRescan != block)
4195 | -                block = block->pprev;
4196 | +            int block = *tip_height;

---

  62 | +        virtual int64_t getBlockTime(int height) = 0;
  63 | +
  64 | +        //! Get block median time past.
  65 | +        virtual int64_t getBlockMedianTimePast(int height) = 0;
  66 | +
  67 | +        //! Check if block is empty.

---

  71 | +        //! greater than the given time, or nothing if there is no block with a
  72 | +        //! high enough timestamp.
  73 | +        virtual Optional<int> findEarliestAtLeast(int64_t time) = 0;
  74 | +
  75 | +        //! Return height of last block in chain with timestamp less than the
  76 | +        //! given, and height less than or equal to the given, or nothing if

---

 339 | +        m_commands.erase(std::remove(m_commands.begin(), m_commands.end(), &command));
 340 | +    }
 341 | +
 342 | +    UniValue forwardRequest(const JSONRPCRequest& request) const
 343 | +    {
 344 | +        // Simple forwarding of RPC requests. This just sends the request to the

---

 181 | @@ -182,6 +182,9 @@ class Chain
 182 |          uint64_t max_tries,
 183 |          bool keep_script) = 0;
 184 |  
 185 | +    //! Parse confirm target.
 186 | +    virtual unsigned int parseConfirmTarget(const UniValue& value) = 0;

---

  65 | @@ -66,11 +66,6 @@ bool EnsureWalletIsAvailable(CWallet * const pwallet, bool avoidException)
  66 |      if (pwallet) return true;
  67 |      if (avoidException) return false;
  68 |      if (!HasWallets()) {
  69 | -        // Note: It isn't currently possible to trigger this error because
  70 | -        // wallet RPC methods aren't registered unless a wallet is loaded. But
  71 | -        // this error is being kept as a precaution, because it's possible in
  72 | -        // the future that wallet RPC methods might get or remain registered
  73 | -        // when no wallets are loaded.

---

   0 | @@ -0,0 +1,18 @@
   1 | +// Copyright (c) 2017 The Bitcoin Core developers
   2 | +// Distributed under the MIT software license, see the accompanying
   3 | +// file COPYING or http://www.opensource.org/licenses/mit-license.php.
   4 | +
   5 | +#ifndef BITCOIN_OPTIONAL_H
   6 | +#define BITCOIN_OPTIONAL_H
   7 | +
   8 | +#include <boost/none.hpp>
   9 | +#include <boost/optional/optional.hpp>

---

2740 | @@ -2726,7 +2741,8 @@ bool CWallet::CreateTransaction(interfaces::Chain::Lock& locked_chain, const std
2741 |      // enough, that fee sniping isn't a problem yet, but by implementing a fix
2742 |      // now we ensure code won't be written that makes assumptions about
2743 |      // nLockTime that preclude a fix later.
2744 | -    txNew.nLockTime = chainActive.Height();
2745 | +    const Optional<int> tip_height = locked_chain.getHeight();
2746 | +    txNew.nLockTime = tip_height ? *tip_height : std::numeric_limits<uint32_t>::max();

---

  40 | @@ -41,6 +41,11 @@ bench_bench_bitcoin_LDADD = \
  41 |    $(LIBBITCOIN_COMMON) \
  42 |    $(LIBBITCOIN_UTIL) \
  43 |    $(LIBBITCOIN_SERVER) \
  44 | +  $(LIBBITCOIN_COMMON) \
  45 | +  $(LIBBITCOIN_UTIL) \
  46 | +  $(LIBBITCOIN_WALLET) \
  47 | +  $(LIBBITCOIN_SERVER) \
  48 | +  $(LIBBITCOIN_UTIL) \

---

  18 |  
  19 | -    RegisterWalletRPCCommands(tableRPC);
  20 | +    m_chain_client->registerRpcs();
  21 |  }
  22 |  
  23 |  WalletTestingSetup::~WalletTestingSetup()

---

  76 | +    int64_t getBlockTime(int height) override { return ::chainActive[height]->GetBlockTime(); }
  77 | +    int64_t getBlockMedianTimePast(int height) override { return ::chainActive[height]->GetMedianTimePast(); }
  78 | +    bool haveBlockOnDisk(int height) override
  79 | +    {
  80 | +        CBlockIndex* block = ::chainActive[height];
  81 | +        return block && (block->nStatus & BLOCK_HAVE_DATA) && block->nTx > 0;

---

 100 | +    Optional<int> findPruned(int start_height, Optional<int> stop_height) override
 101 | +    {
 102 | +        if (::fPruneMode) {
 103 | +            CBlockIndex* block = stop_height ? ::chainActive[*stop_height] : ::chainActive.Tip();
 104 | +            while (block && block->nHeight >= start_height) {
 105 | +                if (!(block->nStatus & BLOCK_HAVE_DATA)) {

---

 116 | +        auto it = ::mapBlockIndex.find(hash);
 117 | +        if (it != ::mapBlockIndex.end()) {
 118 | +            block = it->second;
 119 | +            fork = ::chainActive.FindFork(block);
 120 | +        }
 121 | +        if (height) {

---

 163 | +};
 164 | +
 165 | +class HandlerImpl : public Handler
 166 | +{
 167 | +public:
 168 | +    HandlerImpl(Chain::Notifications& notifications) : m_forwarder(notifications) {}

---

 177 | +    }
 178 | +    void disconnect() override { m_forwarder.disconnect(); }
 179 | +
 180 | +    struct Forwarder : CValidationInterface
 181 | +    {
 182 | +        Forwarder(Chain::Notifications& notifications) : m_notifications(&notifications)

---

 207 | +        void BlockDisconnected(const std::shared_ptr<const CBlock>& block) override
 208 | +        {
 209 | +            m_notifications->BlockDisconnected(*block);
 210 | +        }
 211 | +        void ChainStateFlushed(const CBlockLocator& locator) override { m_notifications->ChainStateFlushed(locator); }
 212 | +        void ResendWalletTransactions(int64_t best_block_time, CConnman* connman) override

---

 310 | +        size_t nLimitDescendantSize = gArgs.GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT) * 1000;
 311 | +        std::string errString;
 312 | +        LOCK(::mempool.cs);
 313 | +        if (!::mempool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize,
 314 | +                nLimitDescendants, nLimitDescendantSize, errString)) {
 315 | +            return false;

---

 357 | +
 358 | +//! Forwarder for one RPC method.
 359 | +class ChainImpl::RpcForwarder
 360 | +{
 361 | +public:
 362 | +    RpcForwarder(const CRPCCommand& command) : m_command(command)

---

 368 | +
 369 | +    void addCommand(const CRPCCommand& command) { m_commands.emplace_back(&command); }
 370 | +
 371 | +    void removeCommand(const CRPCCommand& command)
 372 | +    {
 373 | +        m_commands.erase(std::remove(m_commands.begin(), m_commands.end(), &command));

---

 427 | +            m_forwarder->removeCommand(m_command);
 428 | +            m_forwarder = nullptr;
 429 | +        }
 430 | +    }
 431 | +
 432 | +    ~RpcHandler() { disconnect(); }

---

 946 | @@ -945,19 +947,19 @@ void CWallet::LoadToWallet(const CWalletTx& wtxIn)
 947 |      }
 948 |  }
 949 |  
 950 | -bool CWallet::AddToWalletIfInvolvingMe(const CTransactionRef& ptx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate)
 951 | +bool CWallet::AddToWalletIfInvolvingMe(interfaces::Chain::Lock& locked_chain, const CTransactionRef& ptx, const uint256& block_hash, int posInBlock, bool fUpdate)

---

  53 | @@ -54,4 +54,15 @@
  54 |  #define ASSERT_EXCLUSIVE_LOCK(...)
  55 |  #endif // __GNUC__
  56 |  
  57 | +// Utility class to indicate mutex is locked for thread analysis when it can't
  58 | +// be determined otherwise.
  59 | +struct SCOPED_LOCKABLE LockAnnotation
  60 | +{
  61 | +    template <typename Mutex>
  62 | +    LockAnnotation(Mutex& mutex) EXCLUSIVE_LOCK_FUNCTION(mutex)

---

 204 | +            const std::vector<CTransactionRef>& tx_conflicted)
 205 | +        {
 206 | +        }
 207 | +        virtual void BlockDisconnected(const CBlock& block) {}
 208 | +        virtual void ChainStateFlushed(const CBlockLocator& locator) {}
 209 | +        virtual void Inventory(const uint256& hash) {}

---

   0 | @@ -0,0 +1,453 @@
   1 | +#include <interfaces/chain.h>

---

   0 | @@ -0,0 +1,265 @@
   1 | +#ifndef BITCOIN_INTERFACES_CHAIN_H

---

 319 | +    }
 320 | +    CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override
 321 | +    {
 322 | +        return ::feeEstimator.estimateSmartFee(num_blocks, calc, conservative);
 323 | +    }
 324 | +    int estimateMaxBlocks() override { return ::feeEstimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); }

---

 405 | +        // This will only be reached if m_commands is empty. (Because the RPC
 406 | +        // server provides an appendCommand, but no removeCommand method, it
 407 | +        // will keep sending requests here even if there are no clients left to
 408 | +        // forward to.)
 409 | +        throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found");
 410 | +    };

---

 764 | +public:
 765 | +    CoinFill(CCoinsViewCache& cache, const COutPoint &output, Coin&& coin, CCoinsView &backend) : m_cache(cache), m_output(output), m_coin(std::move(coin)), m_backend(backend) {
 766 | +        m_cache.SetBackend(*this);
 767 | +        m_cache.AccessCoin(output);
 768 | +    }
 769 | +    ~CoinFill() {

---

 306 | +    bool checkChainLimits(CTransactionRef tx) override
 307 | +    {
 308 | +        LockPoints lp;
 309 | +        CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp);
 310 | +        CTxMemPool::setEntries setAncestors;
 311 | +        size_t nLimitAncestors = gArgs.GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT);

---

  88 | @@ -90,10 +89,10 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr
  89 |      return feerate_needed;
  90 |  }
  91 |  
  92 | -CFeeRate GetDiscardRate(const CWallet& wallet, const CBlockPolicyEstimator& estimator)
  93 | +CFeeRate GetDiscardRate(const CWallet& wallet)
  94 |  {
  95 | -    unsigned int highest_target = estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE);
  96 | -    CFeeRate discard_rate = estimator.estimateSmartFee(highest_target, nullptr /* FeeCalculation */, false /* conservative */);
  97 | +    unsigned int highest_target = wallet.chain().estimateMaxBlocks();

---

1257 | @@ -1243,7 +1258,11 @@ bool AppInitMain()
1258 |      }
1259 |  
1260 |      // ********************************************************* Step 5: verify wallet database integrity
1261 | -    if (!g_wallet_init_interface.Verify()) return false;
1262 | +    for (const auto& client : interfaces.chain_clients) {
1263 | +        if (!client->verify()) {
1264 | +            return false;

---

1580 | @@ -1562,7 +1581,11 @@ bool AppInitMain()
1581 |      }
1582 |  
1583 |      // ********************************************************* Step 9: load wallet
1584 | -    if (!g_wallet_init_interface.Open()) return false;
1585 | +    for (const auto& client : interfaces.chain_clients) {
1586 | +        if (!client->load()) {
1587 | +            return false;

---

 801 | +    }
 802 | +    ~CoinFill() override {
 803 | +        m_cache.SetBackend(m_backend);
 804 | +    }
 805 | +
 806 | +    private:

---

 802 | +    ~CoinFill() override {
 803 | +        m_cache.SetBackend(m_backend);
 804 | +    }
 805 | +
 806 | +    private:
 807 | +    bool GetCoin(const COutPoint &output, Coin &coin) const override {

---

 162 | @@ -160,6 +163,9 @@ class Chain
 163 |          int64_t* time = nullptr,
 164 |          int64_t* max_time = nullptr) = 0;
 165 |  
 166 | +    //! Get unspent outputs associated with a transaction.
 167 | +    virtual std::vector<Coin> findCoins(const std::vector<COutPoint>& outputs) = 0;

---

 809 | +        coin = std::move(m_coin);
 810 | +        return true;
 811 | +    }
 812 | +    CCoinsViewCache& m_cache;
 813 | +    const COutPoint& m_output;
 814 | +    Coin&& m_coin;

---

 192 | @@ -193,9 +193,8 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest&
 193 |          jreq.strMethod = strMethod;
 194 |          try
 195 |          {
 196 | -            rpcfn_type pfn = pcmd->actor;
 197 | -            if (setDone.insert(pfn).second)
 198 | -                (*pfn)(jreq);
 199 | +            if (setDone.insert(pcmd->unique_id).second)

---

 831 | -            view.AccessCoin(txin.prevout); // Load entries from viewChain into view; can fail.
 832 | +            outputs.emplace_back(txin.prevout);
 833 | +        }
 834 | +        std::vector<Coin> coins = chain.findCoins(outputs);
 835 | +        for (size_t i = 0; i < coins.size(); ++i) {
 836 | +            CoinFill(view, outputs[i], std::move(coins[i]), viewDummy);

---

 439 | +
 440 | +std::unique_ptr<Handler> ChainImpl::handleRpc(const CRPCCommand& command)
 441 | +{
 442 | +    auto inserted = m_rpc_forwarders.emplace(
 443 | +        std::piecewise_construct, std::forward_as_tuple(command.name), std::forward_as_tuple(command));
 444 | +    if (inserted.second && !inserted.first->second.registerForwarder()) {

---

4155 | @@ -4156,8 +4156,8 @@ static const CRPCCommand commands[] =
4156 |  };
4157 |  // clang-format on
4158 |  
4159 | -void RegisterWalletRPCCommands(CRPCTable &t)
4160 | +void RegisterWalletRPCCommands(interfaces::Chain& chain, std::vector<std::unique_ptr<interfaces::Handler>>& handlers)
4161 |  {
4162 |      for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++)
4163 | -        t.appendCommand(commands[vcidx].name, &commands[vcidx]);
4164 | +        handlers.emplace_back(chain.handleRpc(commands[vcidx]));

---