From fa2ecc48fb84f5d74807acb162b46ccece6f4caa Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 4 Dec 2016 16:16:11 +0100 Subject: [PATCH 1/2] [qa] pruning: Use cached utxo set to run faster --- qa/rpc-tests/mempool_limit.py | 6 +++--- qa/rpc-tests/prioritise_transaction.py | 2 +- qa/rpc-tests/pruning.py | 15 +++++++++++---- qa/rpc-tests/test_framework/util.py | 14 +++++++++----- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/qa/rpc-tests/mempool_limit.py b/qa/rpc-tests/mempool_limit.py index 4438c152df..154ae59c26 100755 --- a/qa/rpc-tests/mempool_limit.py +++ b/qa/rpc-tests/mempool_limit.py @@ -26,7 +26,7 @@ class MempoolLimitTest(BitcoinTestFramework): def run_test(self): txids = [] - utxos = create_confirmed_utxos(self.relayfee, self.nodes[0], 90) + utxos = create_confirmed_utxos(self.relayfee, self.nodes[0], 91) #create a mempool tx that will be evicted us0 = utxos.pop() @@ -41,9 +41,9 @@ class MempoolLimitTest(BitcoinTestFramework): relayfee = self.nodes[0].getnetworkinfo()['relayfee'] base_fee = relayfee*100 - for i in range (4): + for i in range (3): txids.append([]) - txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[30*i:30*i+30], (i+1)*base_fee) + txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[30*i:30*i+30], 30, (i+1)*base_fee) # by now, the tx should be evicted, check confirmation state assert(txid not in self.nodes[0].getrawmempool()) diff --git a/qa/rpc-tests/prioritise_transaction.py b/qa/rpc-tests/prioritise_transaction.py index 85afeab2e3..b7459c80cb 100755 --- a/qa/rpc-tests/prioritise_transaction.py +++ b/qa/rpc-tests/prioritise_transaction.py @@ -39,7 +39,7 @@ class PrioritiseTransactionTest(BitcoinTestFramework): txids.append([]) start_range = i * range_size end_range = start_range + range_size - txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[start_range:end_range], (i+1)*base_fee) + txids[i] = create_lots_of_big_transactions(self.nodes[0], self.txouts, utxos[start_range:end_range], end_range - start_range, (i+1)*base_fee) # Make sure that the size of each group of transactions exceeds # MAX_BLOCK_BASE_SIZE -- otherwise the test needs to be revised to create diff --git a/qa/rpc-tests/pruning.py b/qa/rpc-tests/pruning.py index 6635b0dff2..78b8938e4a 100755 --- a/qa/rpc-tests/pruning.py +++ b/qa/rpc-tests/pruning.py @@ -13,6 +13,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +import time +import os + def calc_usage(blockdir): return sum(os.path.getsize(blockdir+f) for f in os.listdir(blockdir) if os.path.isfile(blockdir+f)) / (1024. * 1024.) @@ -24,6 +27,10 @@ class PruneTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 3 + # Cache for utxos, as the listunspent may take a long time later in the test + self.utxo_cache_0 = [] + self.utxo_cache_1 = [] + def setup_network(self): self.nodes = [] self.is_network_split = False @@ -48,7 +55,7 @@ class PruneTest(BitcoinTestFramework): self.nodes[0].generate(150) # Then mine enough full blocks to create more than 550MiB of data for i in range(645): - mine_large_block(self.nodes[0]) + mine_large_block(self.nodes[0], self.utxo_cache_0) sync_blocks(self.nodes[0:3]) @@ -60,7 +67,7 @@ class PruneTest(BitcoinTestFramework): print("Mining 25 more blocks should cause the first block file to be pruned") # Pruning doesn't run until we're allocating another chunk, 20 full blocks past the height cutoff will ensure this for i in range(25): - mine_large_block(self.nodes[0]) + mine_large_block(self.nodes[0], self.utxo_cache_0) waitstart = time.time() while os.path.isfile(self.prunedir+"blk00000.dat"): @@ -87,13 +94,13 @@ class PruneTest(BitcoinTestFramework): # Mine 24 blocks in node 1 for i in range(24): if j == 0: - mine_large_block(self.nodes[1]) + mine_large_block(self.nodes[1], self.utxo_cache_1) else: self.nodes[1].generate(1) #tx's already in mempool from previous disconnects # Reorg back with 25 block chain from node 0 for i in range(25): - mine_large_block(self.nodes[0]) + mine_large_block(self.nodes[0], self.utxo_cache_0) # Create connections in the order so both nodes can see the reorg at the same time connect_nodes(self.nodes[1], 0) diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 1d6c9aa230..85898d9f32 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -654,10 +654,10 @@ def create_tx(node, coinbase, to_address, amount): # Create a spend of each passed-in utxo, splicing in "txouts" to each raw # transaction to make it large. See gen_return_txouts() above. -def create_lots_of_big_transactions(node, txouts, utxos, fee): +def create_lots_of_big_transactions(node, txouts, utxos, num, fee): addr = node.getnewaddress() txids = [] - for _ in range(len(utxos)): + for _ in range(num): t = utxos.pop() inputs=[{ "txid" : t["txid"], "vout" : t["vout"]}] outputs = {} @@ -672,13 +672,17 @@ def create_lots_of_big_transactions(node, txouts, utxos, fee): txids.append(txid) return txids -def mine_large_block(node): +def mine_large_block(node, utxos=None): # generate a 66k transaction, # and 14 of them is close to the 1MB block limit + num = 14 txouts = gen_return_txouts() - utxos = node.listunspent()[:14] + utxos = utxos if utxos is not None else [] + if len(utxos) < num: + utxos.clear() + utxos.extend(node.listunspent()) fee = 100 * node.getnetworkinfo()["relayfee"] - create_lots_of_big_transactions(node, txouts, utxos, fee=fee) + create_lots_of_big_transactions(node, txouts, utxos, num, fee=fee) node.generate(1) def get_bip9_status(node, key): From fab1af31d43f0db58f590992d1cc5e302f1133f8 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 5 Dec 2016 00:27:47 +0100 Subject: [PATCH 2/2] [qa] maxuploadtarget: Use cached utxo set --- qa/rpc-tests/maxuploadtarget.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/maxuploadtarget.py b/qa/rpc-tests/maxuploadtarget.py index 83168a7ce7..9340e899eb 100755 --- a/qa/rpc-tests/maxuploadtarget.py +++ b/qa/rpc-tests/maxuploadtarget.py @@ -86,6 +86,9 @@ class MaxUploadTest(BitcoinTestFramework): self.setup_clean_chain = True self.num_nodes = 1 + # Cache for utxos, as the listunspent may take a long time later in the test + self.utxo_cache = [] + def setup_network(self): # Start a node with maxuploadtarget of 200 MB (/24h) self.nodes = [] @@ -118,7 +121,7 @@ class MaxUploadTest(BitcoinTestFramework): # Test logic begins here # Now mine a big block - mine_large_block(self.nodes[0]) + mine_large_block(self.nodes[0], self.utxo_cache) # Store the hash; we'll request this later big_old_block = self.nodes[0].getbestblockhash() @@ -129,7 +132,7 @@ class MaxUploadTest(BitcoinTestFramework): self.nodes[0].setmocktime(int(time.time()) - 2*60*60*24) # Mine one more block, so that the prior block looks old - mine_large_block(self.nodes[0]) + mine_large_block(self.nodes[0], self.utxo_cache) # We'll be requesting this new block too big_new_block = self.nodes[0].getbestblockhash()