Merge pull request #5837 from knst/bp-v21-p16

backport: bitcoin#17002, #19267, #19738, #19765, #19914, #20006, #20119, #20279, bitcoin-core/gui#43, #97,

.travis.yml

@@ -2,24 +2,9 @@
 # - sudo/dist/group are set so as to get Blue Box VMs, necessary for [loopback]
 #   IPv6 support
 
-# The test build matrix (stage: test) is constructed to test a wide range of
-# configurations, rather than a single pass/fail. This helps to catch build
-# failures and logic errors that present on platforms other than the ones the
-# author has tested.
-#
-# Some builders use the dependency-generator in `./depends`, rather than using
-# apt-get to install build dependencies. This guarantees that the tester is
-# using the same versions as Gitian, so the build results are nearly identical
-# to what would be found in a final release.
-#
-# In order to avoid rebuilding all dependencies for each build, the binaries
-# are cached and re-used when possible. Changes in the dependency-generator
-# will trigger cache-invalidation and rebuilds as necessary.
-#
-
 version: ~> 1.0
 
-dist: xenial
+dist: bionic
 
 os: linux
 language: minimal

ci/README.md

@@ -1,12 +1,8 @@
-## ci scripts
+## CI Scripts
 
 This directory contains scripts for each build step in each build stage.
 
-Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the
-[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named
-and numbered according to which stage and lifecycle step it belongs to.
-
-### Running a stage locally
+### Running a Stage Locally
 
 Be aware that the tests will be built and run in-place, so please run at your own risk.
 If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
@@ -36,3 +32,34 @@ To run the test stage with a specific configuration,
 ```
 FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
 ```
+
+### Configurations
+
+The test files (`FILE_ENV`) are constructed to test a wide range of
+configurations, rather than a single pass/fail. This helps to catch build
+failures and logic errors that present on platforms other than the ones the
+author has tested.
+
+Some builders use the dependency-generator in `./depends`, rather than using
+the system package manager to install build dependencies. This guarantees that
+the tester is using the same versions as the release builds, which also use
+`./depends`.
+
+If no `FILE_ENV` has been specified or values are left out, `00_setup_env.sh`
+is used as the default configuration with fallback values.
+
+It is also possible to force a specific configuration without modifying the
+file. For example,
+
+```
+MAKEJOBS="-j1" FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+```
+
+The files starting with `0n` (`n` greater than 0) are the scripts that are run
+in order.
+
+### Cache
+
+In order to avoid rebuilding all dependencies for each build, the binaries are
+cached and re-used when possible. Changes in the dependency-generator will
+trigger cache-invalidation and rebuilds as necessary.

ci/test/00_setup_env_arm.sh

@@ -18,7 +18,6 @@ if [ -n "$QEMU_USER_CMD" ]; then
 fi
 export CONTAINER_NAME=ci_arm_linux
 # Use debian to avoid 404 apt errors when cross compiling
-export DOCKER_NAME_TAG="debian:focal"
 export CHECK_DOC=1
 export USE_BUSY_BOX=true
 export RUN_UNIT_TESTS=false

ci/test/00_setup_env_native_fuzz.sh

@@ -6,7 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-export DOCKER_NAME_TAG="ubuntu:20.04"
 export CONTAINER_NAME=ci_native_fuzz
 export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev"
 export DEP_OPTS="NO_UPNP=1 DEBUG=1"

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -6,7 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-export DOCKER_NAME_TAG="ubuntu:20.04"
 export CONTAINER_NAME=ci_native_fuzz_valgrind
 export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev valgrind"
 export NO_DEPENDS=1

ci/test/00_setup_env_s390x.sh

@@ -18,7 +18,6 @@ if [ -n "$QEMU_USER_CMD" ]; then
 fi
 # Use debian to avoid 404 apt errors
 export CONTAINER_NAME=ci_s390x
-export DOCKER_NAME_TAG="debian:focal"
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=true
 export GOAL="install"

doc/bips.md

@@ -1,5 +1,5 @@
 BIPs that are implemented by Bitcoin Core, some of them are relevant for Dash Core, some are just mentioned as a reference.
-Specified versions, PRs are relevant to Bitcoin's core.
+Versions and PRs are relevant to Bitcoin's core if not mentioned other.
 
 * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1**  ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575))
 * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)).
@@ -38,7 +38,7 @@ Specified versions, PRs are relevant to Bitcoin's core.
 * [`BIP 147`](https://github.com/bitcoin/bips/blob/master/bip-0147.mediawiki): NULLDUMMY softfork as of **v0.13.1** ([PR 8636](https://github.com/bitcoin/bitcoin/pull/8636) and [PR 8937](https://github.com/bitcoin/bitcoin/pull/8937)), *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)).
 * [`BIP 152`](https://github.com/bitcoin/bips/blob/master/bip-0152.mediawiki): Compact block transfer and related optimizations are used as of **v0.13.0** ([PR 8068](https://github.com/bitcoin/bitcoin/pull/8068)).
 * [`BIP 155`](https://github.com/bitcoin/bips/blob/master/bip-0155.mediawiki): The 'addrv2' and 'sendaddrv2' messages which enable relay of Tor V3 addresses (and other networks) are supported as of **v18.0** ([PR 19954](https://github.com/bitcoin/bitcoin/pull/19954)).
-- [`BIP 158`](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki): Compact Block Filters for Light Clients can be indexed as of **v18.0** ([PR #14121](https://github.com/bitcoin/bitcoin/pull/14121)).
+* [`BIP 158`](https://github.com/bitcoin/bips/blob/master/bip-0158.mediawiki): Compact Block Filters for Light Clients can be indexed as of **Dash Core v18.0** ([PR dash#4314](https://github.com/dashpay/dash/pull/4314), [PR #14121](https://github.com/bitcoin/bitcoin/pull/14121)).
 * [`BIP 159`](https://github.com/bitcoin/bips/blob/master/bip-0159.mediawiki): The `NODE_NETWORK_LIMITED` service bit is signalled as of **v0.16.0** ([PR 11740](https://github.com/bitcoin/bitcoin/pull/11740)), and such nodes are connected to as of **v0.17.0** ([PR 10387](https://github.com/bitcoin/bitcoin/pull/10387)).
 * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v18.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)).
 * [`BIP 339`](https://github.com/bitcoin/bips/blob/master/bip-0339.mediawiki): Relay of transactions by wtxid is supported as of **v0.21.0** ([PR 18044](https://github.com/bitcoin/bitcoin/pull/18044)).

doc/release-process.md

@@ -25,8 +25,8 @@ Before every minor and major release:
 Before every major release:
 
 * [ ] Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/dashpay/dash/pull/5692) for an example.
-* [ ] Update [`src/chainparams.cpp`](/src/chainparams.cpp) `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see [this](#how-to-calculate-m_assumed_blockchain_size-and-m_assumed_chain_state_size) for information on how to calculate them).
-* [ ] Update `src/chainparams.cpp` `chainTxData` with statistics about the transaction count and rate. Use the output of the RPC `getchaintxstats`, see
+* [ ] Update [`src/chainparams.cpp`](/src/chainparams.cpp) `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see [this](#how-to-calculate-assumed-blockchain-and-chain-state-size) for information on how to calculate them).
+* [ ] Update [`src/chainparams.cpp`](/src/chainparams.cpp) `chainTxData` with statistics about the transaction count and rate. Use the output of the `getchaintxstats` RPC, see
   [this pull request](https://github.com/dashpay/dash/pull/5692) for an example. Reviewers can verify the results by running `getchaintxstats <window_block_count> <window_last_block_hash>` with the `window_block_count` and `window_last_block_hash` from your output.
 
 ### First time / New builders
@@ -269,13 +269,13 @@ checks. If the app is successfully notarized, the command line will include a li
 
 ### Additional information
 
-#### How to calculate `m_assumed_blockchain_size` and `m_assumed_chain_state_size`
+#### <a name="how-to-calculate-assumed-blockchain-and-chain-state-size"></a>How to calculate `m_assumed_blockchain_size` and `m_assumed_chain_state_size`
 
 Both variables are used as a guideline for how much space the user needs on their drive in total, not just strictly for the blockchain.
 Note that all values should be taken from a **fully synced** node and have an overhead of 5-10% added on top of its base value.
 
 To calculate `m_assumed_blockchain_size`:
-- For `mainnet` -> Take the size of the Dash Core data directory, excluding `/regtest` and `/testnet3` directories.
+- For `mainnet` -> Take the size of the data directory, excluding `/regtest` and `/testnet3` directories.
 - For `testnet` -> Take the size of the `/testnet3` directory.
 
 

src/net_processing.cpp

@@ -240,7 +240,7 @@ public:
     bool SendMessages(CNode* pto) override EXCLUSIVE_LOCKS_REQUIRED(pto->cs_sendProcessing);
 
     /** Implement PeerManager */
-    void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams) override;
+    void CheckForStaleTipAndEvictPeers() override;
     bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) override;
     bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; }
     void RelayTransaction(const uint256& txid) override;
@@ -1582,13 +1582,12 @@ PeerManagerImpl::PeerManagerImpl(const CChainParams& chainparams, CConnman& conn
       m_ignore_incoming_txs(ignore_incoming_txs)
 {
     assert(std::addressof(g_chainman) == std::addressof(m_chainman));
-    const Consensus::Params& consensusParams = Params().GetConsensus();
     // Stale tip checking and peer eviction are on two different timers, but we
     // don't want them to get out of sync due to drift in the scheduler, so we
     // combine them in one function and schedule at the quicker (peer-eviction)
     // timer.
     static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer");
-    scheduler.scheduleEvery([this, consensusParams] { this->CheckForStaleTipAndEvictPeers(consensusParams); }, std::chrono::seconds{EXTRA_PEER_CHECK_INTERVAL});
+    scheduler.scheduleEvery([this] { this->CheckForStaleTipAndEvictPeers(); }, std::chrono::seconds{EXTRA_PEER_CHECK_INTERVAL});
 
     // schedule next run for 10-15 minutes in the future
     const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5});
@@ -1897,6 +1896,8 @@ void PeerManagerImpl::RelayTransaction(const uint256& txid)
 
 static void RelayAddress(const CAddress& addr, bool fReachable, const CConnman& connman)
 {
+    if (!fReachable && !addr.IsRelayable()) return;
+
     // Relay to a limited number of other nodes
     // Use deterministic randomness to send to the same nodes for 24 hours
     // at a time so the m_addr_knowns of the chosen nodes prevent repeats
@@ -4611,7 +4612,7 @@ void PeerManagerImpl::EvictExtraOutboundPeers(int64_t time_in_seconds)
     }
 }
 
-void PeerManagerImpl::CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams)
+void PeerManagerImpl::CheckForStaleTipAndEvictPeers()
 {
     LOCK(cs_main);
 
@@ -4655,7 +4656,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto)
 {
     assert(m_llmq_ctx);
 
-    const Consensus::Params& consensusParams = Params().GetConsensus();
+    const Consensus::Params& consensusParams = m_chainparams.GetConsensus();
 
     // We must call MaybeDiscourageAndDisconnect first, to ensure that we'll
     // disconnect misbehaving peers even before the version handshake is complete.

src/net_processing.h

@@ -68,7 +68,7 @@ public:
      * Evict extra outbound peers. If we think our tip may be stale, connect to an extra outbound.
      * Public for unit testing.
      */
-    virtual void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams) = 0;
+    virtual void CheckForStaleTipAndEvictPeers() = 0;
 
     /** Process a single message from a peer. Public for fuzz testing */
     virtual void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv,

src/netaddress.h

@@ -227,6 +227,14 @@ public:
     friend bool operator!=(const CNetAddr& a, const CNetAddr& b) { return !(a == b); }
     friend bool operator<(const CNetAddr& a, const CNetAddr& b);
 
+    /**
+     * Whether this address should be relayed to other peers even if we can't reach it ourselves.
+     */
+    bool IsRelayable() const
+    {
+        return IsIPv4() || IsIPv6() || IsTor();
+    }
+
     /**
      * Serialize to a stream.
      */

src/qt/bitcoingui.cpp

@@ -904,13 +904,14 @@ void BitcoinGUI::addWallet(WalletModel* walletModel)
 {
     if (!walletFrame) return;
     if (!walletFrame->addWallet(walletModel)) return;
-    const QString display_name = walletModel->getDisplayName();
-    setWalletActionsEnabled(true);
     rpcConsole->addWallet(walletModel);
-    m_wallet_selector->addItem(display_name, QVariant::fromValue(walletModel));
-    if (m_wallet_selector->count() == 2) {
+    if (m_wallet_selector->count() == 0) {
+        setWalletActionsEnabled(true);
+    } else if (m_wallet_selector->count() == 1) {
         m_wallet_selector_action->setVisible(true);
     }
+    const QString display_name = walletModel->getDisplayName();
+    m_wallet_selector->addItem(display_name, QVariant::fromValue(walletModel));
 }
 
 void BitcoinGUI::removeWallet(WalletModel* walletModel)

src/qt/sendcoinsdialog.cpp

@@ -28,6 +28,8 @@
 #include <wallet/fees.h>
 #include <wallet/wallet.h>
 
+#include <validation.h>
+
 #include <QFontMetrics>
 #include <QScrollBar>
 #include <QSettings>
@@ -158,7 +160,7 @@ void SendCoinsDialog::setClientModel(ClientModel *_clientModel)
     this->clientModel = _clientModel;
 
     if (_clientModel) {
-        connect(_clientModel, &ClientModel::numBlocksChanged, this, &SendCoinsDialog::updateSmartFeeLabel);
+        connect(_clientModel, &ClientModel::numBlocksChanged, this, &SendCoinsDialog::updateNumberOfBlocks);
     }
 }
 
@@ -838,6 +840,12 @@ void SendCoinsDialog::updateCoinControlState(CCoinControl& ctrl)
     ctrl.fAllowWatchOnly = model->wallet().privateKeysDisabled();
 }
 
+void SendCoinsDialog::updateNumberOfBlocks(int count, const QDateTime& blockDate, const QString& blockHash, double nVerificationProgress, bool header, SynchronizationState sync_state) {
+    if (sync_state == SynchronizationState::POST_INIT) {
+        updateSmartFeeLabel();
+    }
+}
+
 void SendCoinsDialog::updateSmartFeeLabel()
 {
     if(!model || !model->getOptionsModel())

src/qt/sendcoinsdialog.h

@@ -19,6 +19,7 @@ class CCoinControl;
 class ClientModel;
 class SendCoinsEntry;
 class SendCoinsRecipient;
+enum class SynchronizationState;
 
 namespace Ui {
     class SendCoinsDialog;
@@ -101,6 +102,7 @@ private Q_SLOTS:
     void coinControlClipboardLowOutput();
     void coinControlClipboardChange();
     void updateFeeSectionControls();
+    void updateNumberOfBlocks(int count, const QDateTime& blockDate, const QString& blockHash, double nVerificationProgress, bool header, SynchronizationState sync_state);
     void updateSmartFeeLabel();
     void keepChangeAddressChanged(bool);
 

src/rpc/blockchain.cpp

@@ -662,8 +662,10 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request)
                 RPCResult::Type::ARR, "", "",
                 {{RPCResult::Type::STR_HEX, "", "The transaction id of an in-mempool ancestor transaction"}}},
             RPCResult{"for verbose = true",
-                RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()
-            },
+                RPCResult::Type::OBJ_DYN, "", "",
+                {
+                    {RPCResult::Type::OBJ, "transactionid", "", MempoolEntryDescription()},
+                }},
         },
         RPCExamples{
             HelpExampleCli("getmempoolancestors", "\"mytxid\"")
@@ -697,7 +699,6 @@ static UniValue getmempoolancestors(const JSONRPCRequest& request)
         for (CTxMemPool::txiter ancestorIt : setAncestors) {
             o.push_back(ancestorIt->GetTx().GetHash().ToString());
         }
-
         return o;
     } else {
         UniValue o(UniValue::VOBJ);

src/script/script_error.cpp

@@ -87,7 +87,7 @@ std::string ScriptErrorString(const ScriptError serror)
         case SCRIPT_ERR_PUBKEYTYPE:
             return "Public key is neither compressed or uncompressed";
         case SCRIPT_ERR_CLEANSTACK:
-            return "Extra items left on stack after execution";
+            return "Stack size must be exactly one after execution";
         case SCRIPT_ERR_OP_CODESEPARATOR:
             return "Using OP_CODESEPARATOR";
         case SCRIPT_ERR_SIG_FINDANDDELETE:

src/test/denialofservice_tests.cpp

@@ -156,7 +156,6 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management)
                                        *m_node.chainman, *m_node.mempool, *governance, m_node.cj_ctx,
                                        m_node.llmq_ctx, false);
 
-    const Consensus::Params& consensusParams = Params().GetConsensus();
     constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS;
     CConnman::Options options;
     options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS;
@@ -171,18 +170,18 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management)
         AddRandomOutboundPeer(vNodes, *peerLogic, connman.get());
     }
 
-    peerLogic->CheckForStaleTipAndEvictPeers(consensusParams);
+    peerLogic->CheckForStaleTipAndEvictPeers();
 
     // No nodes should be marked for disconnection while we have no extra peers
     for (const CNode *node : vNodes) {
         BOOST_CHECK(node->fDisconnect == false);
     }
 
-    SetMockTime(GetTime() + 3*consensusParams.nPowTargetSpacing + 1);
+    SetMockTime(GetTime() + 3 * chainparams.GetConsensus().nPowTargetSpacing + 1);
 
     // Now tip should definitely be stale, and we should look for an extra
     // outbound peer
-    peerLogic->CheckForStaleTipAndEvictPeers(consensusParams);
+    peerLogic->CheckForStaleTipAndEvictPeers();
     BOOST_CHECK(connman->GetTryNewOutboundPeer());
 
     // Still no peers should be marked for disconnection
@@ -195,8 +194,8 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management)
     // required time connected check should be satisfied).
     AddRandomOutboundPeer(vNodes, *peerLogic, connman.get());
 
-    peerLogic->CheckForStaleTipAndEvictPeers(consensusParams);
-    for (int i=0; i<max_outbound_full_relay; ++i) {
+    peerLogic->CheckForStaleTipAndEvictPeers();
+    for (int i = 0; i < max_outbound_full_relay; ++i) {
         BOOST_CHECK(vNodes[i]->fDisconnect == false);
     }
     // Last added node should get marked for eviction
@@ -208,8 +207,8 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management)
     // peer, and check that the next newest node gets evicted.
     UpdateLastBlockAnnounceTime(vNodes.back()->GetId(), GetTime());
 
-    peerLogic->CheckForStaleTipAndEvictPeers(consensusParams);
-    for (int i=0; i<max_outbound_full_relay-1; ++i) {
+    peerLogic->CheckForStaleTipAndEvictPeers();
+    for (int i = 0; i < max_outbound_full_relay - 1; ++i) {
         BOOST_CHECK(vNodes[i]->fDisconnect == false);
     }
     BOOST_CHECK(vNodes[max_outbound_full_relay-1]->fDisconnect == true);

src/wallet/wallet.cpp

@@ -2713,6 +2713,7 @@ std::map<CTxDestination, std::vector<COutput>> CWallet::ListCoins() const
 
 const CTxOut& CWallet::FindNonChangeParentOutput(const CTransaction& tx, int output) const
 {
+    AssertLockHeld(cs_wallet);
     const CTransaction* ptx = &tx;
     int n = output;
     while (IsChange(ptx->vout[n]) && ptx->vin.size() > 0) {
@@ -3984,6 +3985,7 @@ bool CWallet::DelAddressBook(const CTxDestination& address)
         return false;
     }
 
+    WalletBatch batch(GetDatabase());
     {
         LOCK(cs_wallet);
 
@@ -3991,15 +3993,15 @@ bool CWallet::DelAddressBook(const CTxDestination& address)
         std::string strAddress = EncodeDestination(address);
         for (const std::pair<const std::string, std::string> &item : m_address_book[address].destdata)
         {
-            WalletBatch(GetDatabase()).EraseDestData(strAddress, item.first);
+            batch.EraseDestData(strAddress, item.first);
         }
         m_address_book.erase(address);
     }
 
     NotifyAddressBookChanged(this, address, "", IsMine(address) != ISMINE_NO, "", CT_DELETED);
 
-    WalletBatch(GetDatabase()).ErasePurpose(EncodeDestination(address));
-    return WalletBatch(GetDatabase()).EraseName(EncodeDestination(address));
+    batch.ErasePurpose(EncodeDestination(address));
+    return batch.EraseName(EncodeDestination(address));
 }
 
 size_t CWallet::KeypoolCountExternalKeys() const