From bda74458b577c71a2f1b3279be2110ed67413052 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 20 Aug 2021 17:38:38 +0200 Subject: [PATCH] (partial) Merge bitcoin/bitcoin#22707: test: refactor use of getrawmempool in functional tests for efficiency 47c48b5f35b4910fcf87caa6e37407e67d879c80 test: only use verbose for getrawmempool when necessary in functional tests (Michael Dietz) 77349713b189e80f2c140db4df50177353a1cb83 test: use getmempoolentry instead of getrawmempool in functional tests when appropriate (Michael Dietz) 86dbd54ae8a8f9c693c0ea67114bbff24a0754df test: improve mempool_updatefrom efficiency by using getmempoolentry for specific txns (Michael Dietz) Pull request description: I don't think this changes the intention of the test. But it does shave ~30 seconds off the time it takes to run. From what I've seen our CI `macOS 11 native [gui] [no depends]` runs `mempool_updatefrom.py` in ~135 seconds. After this PR it should run in ~105 seconds I noticed this improvement should probably be made when testing performance/runtimes of https://github.com/bitcoin/bitcoin/pull/22698. But I wanted to separate this out from that PR so the affects of each is decoupled Edit: The major change in this PR is improving mempool_updatefrom.py's runtime as this is a very long running test. Then made the same efficiency improvements across all the functional tests as it made since to do that here ACKs for top commit: theStack: Tested ACK 47c48b5f35b4910fcf87caa6e37407e67d879c80 Tree-SHA512: 40f553715f3d4649dc18c2738554eafaca9ea800c4b028c099217896cc1c466ff457ae814d59cf8564c782a8964d8fac3eda60c1b6ffb08bbee1439b2d34434b Signed-off-by: Vijay --- test/functional/mempool_compatibility.py | 3 +- test/functional/mempool_package_limits.py | 2 +- test/functional/mempool_package_onemore.py | 4 +- test/functional/mempool_packages.py | 63 +++++++++++----------- test/functional/mempool_updatefromblock.py | 11 ++-- test/functional/rpc_fundrawtransaction.py | 10 ++-- 6 files changed, 46 insertions(+), 47 deletions(-) diff --git a/test/functional/mempool_compatibility.py b/test/functional/mempool_compatibility.py index b2b51d9288..ef66f188d9 100755 --- a/test/functional/mempool_compatibility.py +++ b/test/functional/mempool_compatibility.py @@ -68,8 +68,7 @@ class MempoolCompatibilityTest(BitcoinTestFramework): self.log.info("Add unbroadcasted tx to mempool on new node and shutdown") unbroadcasted_tx_hash = new_wallet.send_self_transfer(from_node=new_node)['txid'] assert unbroadcasted_tx_hash in new_node.getrawmempool() - mempool = new_node.getrawmempool(True) - assert mempool[unbroadcasted_tx_hash]['unbroadcast'] + assert new_node.getmempoolentry(unbroadcasted_tx_hash)['unbroadcast'] self.stop_node(1) self.log.info("Move mempool.dat from new to old node") diff --git a/test/functional/mempool_package_limits.py b/test/functional/mempool_package_limits.py index 07829a619f..f4868587d1 100755 --- a/test/functional/mempool_package_limits.py +++ b/test/functional/mempool_package_limits.py @@ -72,7 +72,7 @@ class MempoolPackageLimitsTest(BitcoinTestFramework): txid = tx.rehash() if i < mempool_count: node.sendrawtransaction(txhex) - assert_equal(node.getrawmempool(verbose=True)[txid]["ancestorcount"], i + 1) + assert_equal(node.getmempoolentry(txid)["ancestorcount"], i + 1) else: chain_hex.append(txhex) chain_txns.append(tx) diff --git a/test/functional/mempool_package_onemore.py b/test/functional/mempool_package_onemore.py index 999f8c5d52..940d6dbd1d 100755 --- a/test/functional/mempool_package_onemore.py +++ b/test/functional/mempool_package_onemore.py @@ -64,7 +64,7 @@ class MempoolPackagesTest(BitcoinTestFramework): (second_chain, second_chain_value) = self.chain_transaction(self.nodes[0], [utxo[1]['txid']], [utxo[1]['vout']], utxo[1]['amount'], fee, 1) # Check mempool has MAX_ANCESTORS + 1 transactions in it - assert_equal(len(self.nodes[0].getrawmempool(True)), MAX_ANCESTORS + 1) + assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 1) # Adding one more transaction on to the chain should fail. assert_raises_rpc_error(-26, "too-long-mempool-chain, too many unconfirmed ancestors [limit: 25]", self.chain_transaction, self.nodes[0], [txid], [0], value, fee, 1) @@ -81,7 +81,7 @@ class MempoolPackagesTest(BitcoinTestFramework): self.chain_transaction(self.nodes[0], [second_chain], [0], second_chain_value, fee, 1) # Finally, check that we added two transactions - assert_equal(len(self.nodes[0].getrawmempool(True)), MAX_ANCESTORS + 3) + assert_equal(len(self.nodes[0].getrawmempool()), MAX_ANCESTORS + 3) if __name__ == '__main__': MempoolPackagesTest().main() diff --git a/test/functional/mempool_packages.py b/test/functional/mempool_packages.py index 5b3fc4bea9..16e4e9606e 100755 --- a/test/functional/mempool_packages.py +++ b/test/functional/mempool_packages.py @@ -98,28 +98,28 @@ class MempoolPackagesTest(BitcoinTestFramework): assert_equal(entry, mempool[x]) # Check that the descendant calculations are correct - assert_equal(mempool[x]['descendantcount'], descendant_count) - descendant_fees += mempool[x]['fee'] - assert_equal(mempool[x]['modifiedfee'], mempool[x]['fee']) - assert_equal(mempool[x]['fees']['base'], mempool[x]['fee']) - assert_equal(mempool[x]['fees']['modified'], mempool[x]['modifiedfee']) - assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN) - assert_equal(mempool[x]['fees']['descendant'], descendant_fees) - descendant_vsize += mempool[x]['vsize'] - assert_equal(mempool[x]['descendantsize'], descendant_vsize) + assert_equal(entry['descendantcount'], descendant_count) + descendant_fees += entry['fee'] + assert_equal(entry['modifiedfee'], entry['fee']) + assert_equal(entry['fees']['base'], entry['fee']) + assert_equal(entry['fees']['modified'], entry['modifiedfee']) + assert_equal(entry['descendantfees'], descendant_fees * COIN) + assert_equal(entry['fees']['descendant'], descendant_fees) + descendant_vsize += entry['vsize'] + assert_equal(entry['descendantsize'], descendant_vsize) descendant_count += 1 # Check that ancestor calculations are correct - assert_equal(mempool[x]['ancestorcount'], ancestor_count) - assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN) - assert_equal(mempool[x]['ancestorsize'], ancestor_vsize) - ancestor_vsize -= mempool[x]['vsize'] - ancestor_fees -= mempool[x]['fee'] + assert_equal(entry['ancestorcount'], ancestor_count) + assert_equal(entry['ancestorfees'], ancestor_fees * COIN) + assert_equal(entry['ancestorsize'], ancestor_vsize) + ancestor_vsize -= entry['vsize'] + ancestor_fees -= entry['fee'] ancestor_count -= 1 # Check that parent/child list is correct - assert_equal(mempool[x]['spentby'], descendants[-1:]) - assert_equal(mempool[x]['depends'], ancestors[-2:-1]) + assert_equal(entry['spentby'], descendants[-1:]) + assert_equal(entry['depends'], ancestors[-2:-1]) # Check that getmempooldescendants is correct assert_equal(sorted(descendants), sorted(self.nodes[0].getmempooldescendants(x))) @@ -162,12 +162,12 @@ class MempoolPackagesTest(BitcoinTestFramework): # Check that ancestor modified fees includes fee deltas from # prioritisetransaction self.nodes[0].prioritisetransaction(chain[0], 1000) - mempool = self.nodes[0].getrawmempool(True) ancestor_fees = 0 for x in chain: - ancestor_fees += mempool[x]['fee'] - assert_equal(mempool[x]['fees']['ancestor'], ancestor_fees + Decimal('0.00001')) - assert_equal(mempool[x]['ancestorfees'], ancestor_fees * COIN + 1000) + entry = self.nodes[0].getmempoolentry(x) + ancestor_fees += entry['fee'] + assert_equal(entry['fees']['ancestor'], ancestor_fees + Decimal('0.00001')) + assert_equal(entry['ancestorfees'], ancestor_fees * COIN + 1000) # Undo the prioritisetransaction for later tests self.nodes[0].prioritisetransaction(chain[0], -1000) @@ -175,13 +175,13 @@ class MempoolPackagesTest(BitcoinTestFramework): # Check that descendant modified fees includes fee deltas from # prioritisetransaction self.nodes[0].prioritisetransaction(chain[-1], 1000) - mempool = self.nodes[0].getrawmempool(True) descendant_fees = 0 for x in reversed(chain): - descendant_fees += mempool[x]['fee'] - assert_equal(mempool[x]['fees']['descendant'], descendant_fees + Decimal('0.00001')) - assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 1000) + entry = self.nodes[0].getmempoolentry(x) + descendant_fees += entry['fee'] + assert_equal(entry['fees']['descendant'], descendant_fees + Decimal('0.00001')) + assert_equal(entry['descendantfees'], descendant_fees * COIN + 1000) # Adding one more transaction on to the chain should fail. assert_raises_rpc_error(-26, "too-long-mempool-chain", self.chain_transaction, self.nodes[0], txid, vout, value, fee, 1) @@ -199,16 +199,15 @@ class MempoolPackagesTest(BitcoinTestFramework): self.nodes[1].invalidateblock(self.nodes[1].getbestblockhash()) # Now check that the transaction is in the mempool, with the right modified fee - mempool = self.nodes[0].getrawmempool(True) - descendant_fees = 0 for x in reversed(chain): - descendant_fees += mempool[x]['fee'] + entry = self.nodes[0].getmempoolentry(x) + descendant_fees += entry['fee'] if (x == chain[-1]): - assert_equal(mempool[x]['modifiedfee'], mempool[x]['fee']+satoshi_round(0.00002)) - assert_equal(mempool[x]['fees']['modified'], mempool[x]['fee']+satoshi_round(0.00002)) - assert_equal(mempool[x]['descendantfees'], descendant_fees * COIN + 2000) - assert_equal(mempool[x]['fees']['descendant'], descendant_fees+satoshi_round(0.00002)) + assert_equal(entry['modifiedfee'], entry['fee']+satoshi_round(0.00002)) + assert_equal(entry['fees']['modified'], entry['fee']+satoshi_round(0.00002)) + assert_equal(entry['descendantfees'], descendant_fees * COIN + 2000) + assert_equal(entry['fees']['descendant'], descendant_fees+satoshi_round(0.00002)) # Check that node1's mempool is as expected (-> custom ancestor limit) mempool0 = self.nodes[0].getrawmempool(False) @@ -264,7 +263,7 @@ class MempoolPackagesTest(BitcoinTestFramework): # - txs from previous ancestor test (-> custom ancestor limit) # - parent tx for descendant test # - txs chained off parent tx (-> custom descendant limit) - self.wait_until(lambda: len(self.nodes[1].getrawmempool(False)) == + self.wait_until(lambda: len(self.nodes[1].getrawmempool()) == MAX_ANCESTORS_CUSTOM + 1 + MAX_DESCENDANTS_CUSTOM, timeout=10) mempool0 = self.nodes[0].getrawmempool(False) mempool1 = self.nodes[1].getrawmempool(False) diff --git a/test/functional/mempool_updatefromblock.py b/test/functional/mempool_updatefromblock.py index 8baf974a0a..4cd11e9d11 100755 --- a/test/functional/mempool_updatefromblock.py +++ b/test/functional/mempool_updatefromblock.py @@ -86,7 +86,7 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): unsigned_raw_tx = self.nodes[0].createrawtransaction(inputs, outputs) signed_raw_tx = self.nodes[0].signrawtransactionwithwallet(unsigned_raw_tx) tx_id.append(self.nodes[0].sendrawtransaction(signed_raw_tx['hex'])) - tx_size.append(self.nodes[0].getrawmempool(True)[tx_id[-1]]['vsize']) + tx_size.append(self.nodes[0].getmempoolentry(tx_id[-1])['vsize']) if tx_count in n_tx_to_mine: # The created transactions are mined into blocks by batches. @@ -109,10 +109,11 @@ class MempoolUpdateFromBlockTest(BitcoinTestFramework): self.log.info('Checking descendants/ancestors properties of all of the in-mempool transactions...') for k, tx in enumerate(tx_id): self.log.debug('Check transaction #{}.'.format(k)) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantcount'], size - k) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['descendantsize'], sum(tx_size[k:size])) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorcount'], k + 1) - assert_equal(self.nodes[0].getrawmempool(True)[tx]['ancestorsize'], sum(tx_size[0:(k + 1)])) + entry = self.nodes[0].getmempoolentry(tx) + assert_equal(entry['descendantcount'], size - k) + assert_equal(entry['descendantsize'], sum(tx_size[k:size])) + assert_equal(entry['ancestorcount'], k + 1) + assert_equal(entry['ancestorsize'], sum(tx_size[0:(k + 1)])) def run_test(self): # Use batch size limited by DEFAULT_ANCESTOR_LIMIT = 25 to not fire "too many unconfirmed parents" error. diff --git a/test/functional/rpc_fundrawtransaction.py b/test/functional/rpc_fundrawtransaction.py index 300f3d7fcf..bf33d1092d 100755 --- a/test/functional/rpc_fundrawtransaction.py +++ b/test/functional/rpc_fundrawtransaction.py @@ -368,7 +368,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 11) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) @@ -391,7 +391,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendmany("", outputs) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) @@ -415,7 +415,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 11) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) @@ -456,7 +456,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[0].sendtoaddress(mSigObj, 11) - signedFee = self.nodes[0].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[0].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee) @@ -589,7 +589,7 @@ class RawTransactionsTest(BitcoinTestFramework): # Create same transaction over sendtoaddress. txId = self.nodes[1].sendmany("", outputs) - signedFee = self.nodes[1].getrawmempool(True)[txId]['fee'] + signedFee = self.nodes[1].getmempoolentry(txId)['fee'] # Compare fee. feeDelta = Decimal(fundedTx['fee']) - Decimal(signedFee)