From be6a045b8cbedbaa6285d16987da83e99c14d14c Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 30 May 2020 12:22:03 -0400 Subject: [PATCH] Merge #18807: [doc / test / mempool] unbroadcast follow-ups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 9e1cb1adf1800efe429e348650931f2669b0d2c0 [trivial/doc] Fix comment type (Amiti Uttarwar) 8f30260a67166a6ab7c0f33f7ec1990d3c31761e [doc] Update unbroadcast description in RPC results (Amiti Uttarwar) 750456d6f29c63d57af05bfbdd6035bb9c965de2 [trivial] Remove misleading 'const' (Amiti Uttarwar) fa32e676e5833a5c5fc735ef00c0a80f5fab7a2c [test] Manage node connections better in mempool persist test (Amiti Uttarwar) 1f94bb0c744a103b633c1051e8fbc01e612097dc [doc] Provide rationale for randomization in scheduling. (Amiti Uttarwar) 9c8a55d9cb0ec73f10b196e79b637aa601c0a6b7 [mempool] Don't throw expected error message when upgrading (Amiti Uttarwar) ba5498318233ab81decbc585e9619d8ffe2df1b0 [test] Test that wallet transactions aren't rebroadcast before 12 hours (Amiti Uttarwar) 00d44a534b4e5ae249b8011360c6b0f7dc731581 [test] P2P connection behavior should meet expectations (Amiti Uttarwar) bd093ca15de762fdaf0937a0877d17b0c2bce16e [test] updates to unbroadcast test (Amiti Uttarwar) dab298d9ab5a5a41685f437db9081fa7b395fa73 [docs] add release notes (Amiti Uttarwar) Pull request description: This PR is a follow up to #18038 which introduced the idea of an unbroadcast set & focuses mostly on documentation updates and test fixes. One small functionality update to not throw an expected error in `LoadMempool` when you upgrade software versions. #18895 is another follow up to that addresses other functionality updates. Background context: The unbroadcast set is a mechanism for the mempool to track locally submitted transactions (via wallet or RPC). The node does a best-effort of delivering the transactions to the network via retries every 10-15 minutes until either a `GETDATA` is received or the transaction is removed from the mempool. ACKs for top commit: MarcoFalke: ACK 9e1cb1adf1 👁 gzhao408: ACK [`9e1cb1a`](https://github.com/bitcoin/bitcoin/pull/18807/commits/9e1cb1adf1800efe429e348650931f2669b0d2c0) Tree-SHA512: 0cd51c4ca368b9dce92d50d73ec6e9df278a259e609eef2858f24cb8595ad07acc3db781d9eb0c351715f18fca5a2b4526838981fdb34a522427e9dc868bdaa6 --- doc/release-notes-18807.md | 33 +++++++++++++++++++ src/net_processing.cpp | 3 +- src/rpc/blockchain.cpp | 2 +- src/txmempool.h | 6 ++-- src/validation.cpp | 14 +++++--- test/functional/mempool_persist.py | 10 ++++-- test/functional/mempool_unbroadcast.py | 12 ++++--- test/functional/test_framework/mininode.py | 2 ++ .../wallet_resendwallettransactions.py | 12 +++++-- 9 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 doc/release-notes-18807.md diff --git a/doc/release-notes-18807.md b/doc/release-notes-18807.md new file mode 100644 index 0000000000..e0dcd1ecda --- /dev/null +++ b/doc/release-notes-18807.md @@ -0,0 +1,33 @@ +P2P and network changes +----------------------- + +- The mempool now tracks whether transactions submitted via the wallet or RPCs + have been successfully broadcast. Every 10-15 minutes, the node will try to + announce unbroadcast transactions until a peer requests it via a `getdata` + message or the transaction is removed from the mempool for other reasons. + The node will not track the broadcast status of transactions submitted to the + node using P2P relay. This version reduces the initial broadcast guarantees + for wallet transactions submitted via P2P to a node running the wallet. (#18038) + +Updated RPCs +------------ + +- `getmempoolinfo` now returns an additional `unbroadcastcount` field. The + mempool tracks locally submitted transactions until their initial broadcast + is acknowledged by a peer. This field returns the count of transactions + waiting for acknowledgement. + +- Mempool RPCs such as `getmempoolentry` and `getrawmempool` with + `verbose=true` now return an additional `unbroadcast` field. This indicates + whether initial broadcast of the transaction has been acknowledged by a + peer. `getmempoolancestors` and `getmempooldescendants` are also updated. + +Wallet +------ + +- To improve wallet privacy, the frequency of wallet rebroadcast attempts is + reduced from approximately once every 15 minutes to once every 12-36 hours. + To maintain a similar level of guarantee for initial broadcast of wallet + transactions, the mempool tracks these transactions as a part of the newly + introduced unbroadcast set. See the "P2P and network changes" section for + more information on the unbroadcast set. (#18038) diff --git a/src/net_processing.cpp b/src/net_processing.cpp index ded85c706d..681ecf74dc 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -965,7 +965,8 @@ void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const } } - // schedule next run for 10-15 minutes in the future + // Schedule next run for 10-15 minutes in the future. + // We add randomness on every cycle to avoid the possibility of P2P fingerprinting. const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 2689086ba4..67a3ba564c 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -494,7 +494,7 @@ static std::vector MempoolEntryDescription() { return { {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "parent transaction id"}}}, RPCResult{RPCResult::Type::ARR, "spentby", "unconfirmed transactions spending outputs from this transaction", {RPCResult{RPCResult::Type::STR_HEX, "transactionid", "child transaction id"}}}, - RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet confirmed)"}, + RPCResult{RPCResult::Type::BOOL, "unbroadcast", "Whether this transaction is currently unbroadcast (initial broadcast not yet acknowledged by any peers)"}, RPCResult{RPCResult::Type::BOOL, "time", "True if this transaction was locked via InstantSend"} };} diff --git a/src/txmempool.h b/src/txmempool.h index fc646324d2..ce5b4b4f62 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -750,7 +750,7 @@ public: /** Adds a transaction to the unbroadcast set */ void AddUnbroadcastTx(const uint256& txid) { LOCK(cs); - /** Sanity Check: the transaction should also be in the mempool */ + // Sanity Check: the transaction should also be in the mempool if (exists(txid)) { m_unbroadcast_txids.insert(txid); } @@ -760,12 +760,12 @@ public: void RemoveUnbroadcastTx(const uint256& txid, const bool unchecked = false); /** Returns transactions in unbroadcast set */ - const std::set GetUnbroadcastTxs() const { + std::set GetUnbroadcastTxs() const { LOCK(cs); return m_unbroadcast_txids; } - // Returns if a txid is in the unbroadcast set + /** Returns whether a txid is in the unbroadcast set */ bool IsUnbroadcastTx(const uint256& txid) const { LOCK(cs); return (m_unbroadcast_txids.count(txid) != 0); diff --git a/src/validation.cpp b/src/validation.cpp index 46600a9aa2..6508f71a7a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5403,12 +5403,18 @@ bool LoadMempool(CTxMemPool& pool) pool.PrioritiseTransaction(i.first, i.second); } - std::set unbroadcast_txids; - file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); + // TODO: remove this try except in v0.22 + try { + std::set unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { + for (const auto& txid : unbroadcast_txids) { pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception&) { + // mempool.dat files created prior to v0.21 will not have an + // unbroadcast set. No need to log a failure if parsing fails here. } } catch (const std::exception& e) { diff --git a/test/functional/mempool_persist.py b/test/functional/mempool_persist.py index aef51c55cb..4e68ddcc22 100755 --- a/test/functional/mempool_persist.py +++ b/test/functional/mempool_persist.py @@ -79,7 +79,9 @@ class MempoolPersistTest(BitcoinTestFramework): assert_greater_than_or_equal(tx_creation_time_higher, tx_creation_time) # disconnect nodes & make a txn that remains in the unbroadcast set. - self.disconnect_nodes(0, 2) + self.disconnect_nodes(0, 1) + assert(len(self.nodes[0].getpeerinfo()) == 0) + assert(len(self.nodes[0].p2ps) == 0) self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), Decimal("12")) self.connect_nodes(0, 2) @@ -152,8 +154,10 @@ class MempoolPersistTest(BitcoinTestFramework): # clear out mempool node0.generate(1) - # disconnect nodes to make a txn that remains in the unbroadcast set. - self.disconnect_nodes(0, 1) + # ensure node0 doesn't have any connections + # make a transaction that will remain in the unbroadcast set + assert(len(node0.getpeerinfo()) == 0) + assert(len(node0.p2ps) == 0) node0.sendtoaddress(self.nodes[1].getnewaddress(), Decimal("12")) # shutdown, then startup with wallet disabled diff --git a/test/functional/mempool_unbroadcast.py b/test/functional/mempool_unbroadcast.py index 3e00e9470a..0d514f7257 100755 --- a/test/functional/mempool_unbroadcast.py +++ b/test/functional/mempool_unbroadcast.py @@ -14,6 +14,7 @@ from test_framework.util import ( create_confirmed_utxos, ) +MAX_INITIAL_BROADCAST_DELAY = 15 * 60 # 15 minutes in seconds class MempoolUnbroadcastTest(BitcoinTestFramework): def set_test_params(self): @@ -70,7 +71,7 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.connect_nodes(0, 1) # fast forward into the future & ensure that the second node has the txns - node.mockscheduler(15 * 60) # 15 min in seconds + node.mockscheduler(MAX_INITIAL_BROADCAST_DELAY) self.sync_mempools(timeout=30) mempool = self.nodes[1].getrawmempool() assert rpc_tx_hsh in mempool @@ -84,15 +85,16 @@ class MempoolUnbroadcastTest(BitcoinTestFramework): self.log.info("Add another connection & ensure transactions aren't broadcast again") conn = node.add_p2p_connection(P2PTxInvStore()) - node.mockscheduler(15 * 60) - time.sleep(5) + node.mockscheduler(MAX_INITIAL_BROADCAST_DELAY) + time.sleep(2) # allow sufficient time for possibility of broadcast assert_equal(len(conn.get_invs()), 0) + self.disconnect_nodes(0, 1) + node.disconnect_p2ps() + def test_txn_removal(self): self.log.info("Test that transactions removed from mempool are removed from unbroadcast set") node = self.nodes[0] - self.disconnect_nodes(0, 1) - node.disconnect_p2ps() # since the node doesn't have any connections, it will not receive # any GETDATAs & thus the transaction will remain in the unbroadcast set. diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index e18101b9da..bbb7463d17 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -740,6 +740,8 @@ class P2PTxInvStore(P2PInterface): # save txid self.tx_invs_received[i.hash] += 1 + super().on_inv(message) + def get_invs(self): with mininode_lock: return list(self.tx_invs_received.keys()) diff --git a/test/functional/wallet_resendwallettransactions.py b/test/functional/wallet_resendwallettransactions.py index fad5f11137..6893b2308a 100755 --- a/test/functional/wallet_resendwallettransactions.py +++ b/test/functional/wallet_resendwallettransactions.py @@ -52,10 +52,16 @@ class ResendWalletTransactionsTest(BitcoinTestFramework): block.solve() node.submitblock(ToHex(block)) - # Transaction should not be rebroadcast node.syncwithvalidationinterfacequeue() - node.p2ps[1].sync_with_ping() - assert_equal(node.p2ps[1].tx_invs_received[txid], 0) + + # Transaction should not be rebroadcast within first 12 hours + # Leave 2 mins for buffer + twelve_hrs = 12 * 60 * 60 + two_min = 2 * 60 + node.setmocktime(self.mocktime + twelve_hrs - two_min) + self.mocktime = self.mocktime + twelve_hrs - two_min + time.sleep(2) # ensure enough time has passed for rebroadcast attempt to occur + assert_equal(txid in node.p2ps[1].get_invs(), False) self.log.info("Bump time & check that transaction is rebroadcast") # Transaction should be rebroadcast approximately 2 hours in the future,