Merge #8185: [0.12.2] Various qa and test backports

dc38a53 [qa] Move create_tx() to util.py (MarcoFalke)
c0fe8b5 [qa] smartfees: Properly use ordered dict (MarcoFalke)
493b89e [qa] test_framework: Properly print exceptions and assert empty dict (MarcoFalke)
7a83489 Tests: Fix deserialization of reject messages (Suhas Daftuar)
e0b1bbe [qa] pull-tester: Don't mute zmq ImportError (MarcoFalke)
697ed8c Check if zmq is installed in tests, update docs (Elliot Olds)
d5a9de3 tests: Check Content-Type header returned from RPC server (Wladimir J. van der Laan)
ed2f0e3 [qa] maxblocksinflight: Actually enable test (MarcoFalke)
3036282 [qa] httpbasics: Actually test second connection (MarcoFalke)
afbc000 test_framework: python3.4 authproxy compat (Wladimir J. van der Laan)
80b6bfa test_framework: detect failure of bitcoind startup (Wladimir J. van der Laan)
4ffd309 Create SingleNodeConnCB class for RPC tests (Alex Morcos)
4fd6008 travis: 'make check' in parallel and verbose (Cory Fields)
2d2b045 Reenable multithread scheduler test (Pavel Janík)
658307e test: Add more thorough test for dbwrapper iterators (Wladimir J. van der Laan)

.travis.yml

@@ -75,7 +75,7 @@ script:
     - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
     - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
     - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
-    - if [ "$RUN_TESTS" = "true" ]; then make check; fi
+    - if [ "$RUN_TESTS" = "true" ]; then make $MAKEJOBS check VERBOSE=1; fi
     - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage; fi
 after_script:
     - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then (echo "Upload goes here. Something like: scp -r $BASE_OUTDIR server" || echo "upload failed"); fi

README.md

@@ -55,10 +55,10 @@ submit new unit tests for old code. Unit tests can be compiled and run
 
 There are also [regression and integration tests](/qa) of the RPC interface, written
 in Python, that are run automatically on the build server.
-These tests can be run with: `qa/pull-tester/rpc-tests.py`
+These tests can be run (if the [test dependencies](/qa) are installed) with: `qa/pull-tester/rpc-tests.py`
 
 The Travis CI system makes sure that every pull request is built for Windows
-and Linux, OSX, and that unit and sanity tests are automatically run.
+and Linux, OS X, and that unit and sanity tests are automatically run.
 
 ### Manual Quality Assurance (QA) Testing
 

qa/README.md

@@ -5,6 +5,17 @@ Every pull request to the bitcoin repository is built and run through
 the regression test suite. You can also run all or only individual
 tests locally.
 
+Test dependencies
+=================
+Before running the tests, the following must be installed.
+
+Unix
+----
+The python-zmq library is required. On Ubuntu or Debian it can be installed via: 
+```
+sudo apt-get install python-zmq
+```
+
 Running tests
 =============
 

qa/pull-tester/rpc-tests.py

@@ -67,11 +67,25 @@ if "BITCOIND" not in os.environ:
 if "BITCOINCLI" not in os.environ:
     os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT
 
-#Disable Windows tests by default
 if EXEEXT == ".exe" and "-win" not in opts:
-    print "Win tests currently disabled.  Use -win option to enable"
+    # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
+    # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
+    print "Win tests currently disabled by default.  Use -win option to enable"
     sys.exit(0)
 
+if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
+    print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled"
+    sys.exit(0)
+
+# python-zmq may not be installed. Handle this gracefully and with some helpful info
+if ENABLE_ZMQ:
+    try:
+        import zmq
+    except ImportError as e:
+        print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or " \
+            "to run zmq tests, see dependency info in /qa/README.md.")
+        raise e
+
 #Tests
 testScripts = [
     'bip68-112-113-p2p.py',
@@ -109,6 +123,9 @@ testScripts = [
     'abandonconflict.py',
     'p2p-versionbits-warning.py',
 ]
+if ENABLE_ZMQ:
+    testScripts.append('zmq_test.py')
+
 testScriptsExt = [
     'bip9-softforks.py',
     'bip65-cltv.py',
@@ -132,11 +149,6 @@ testScriptsExt = [
     'replace-by-fee.py',
 ]
 
-#Enable ZMQ tests
-if ENABLE_ZMQ == 1:
-    testScripts.append('zmq_test.py')
-
-
 def runtests():
     coverage = None
 
@@ -144,53 +156,49 @@ def runtests():
         coverage = RPCCoverage()
         print("Initializing coverage directory at %s\n" % coverage.dir)
 
-    if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
+    rpcTestDir = buildDir + '/qa/rpc-tests/'
-        rpcTestDir = buildDir + '/qa/rpc-tests/'
+    run_extended = '-extended' in opts
-        run_extended = '-extended' in opts
+    cov_flag = coverage.flag if coverage else ''
-        cov_flag = coverage.flag if coverage else ''
+    flags = " --srcdir %s/src %s %s" % (buildDir, cov_flag, passOn)
-        flags = " --srcdir %s/src %s %s" % (buildDir, cov_flag, passOn)
 
-        #Run Tests
+    #Run Tests
-        for i in range(len(testScripts)):
+    for i in range(len(testScripts)):
-            if (len(opts) == 0
+        if (len(opts) == 0
-                    or (len(opts) == 1 and "-win" in opts )
+                or (len(opts) == 1 and "-win" in opts )
-                    or run_extended
+                or run_extended
-                    or testScripts[i] in opts
+                or testScripts[i] in opts
-                    or re.sub(".py$", "", testScripts[i]) in opts ):
+                or re.sub(".py$", "", testScripts[i]) in opts ):
 
-                print("Running testscript %s%s%s ..." % (bold[1], testScripts[i], bold[0]))
+            print("Running testscript %s%s%s ..." % (bold[1], testScripts[i], bold[0]))
-                time0 = time.time()
+            time0 = time.time()
-                subprocess.check_call(
+            subprocess.check_call(
-                    rpcTestDir + testScripts[i] + flags, shell=True)
+                rpcTestDir + testScripts[i] + flags, shell=True)
-                print("Duration: %s s\n" % (int(time.time() - time0)))
+            print("Duration: %s s\n" % (int(time.time() - time0)))
 
-                # exit if help is called so we print just one set of
+            # exit if help is called so we print just one set of
-                # instructions
+            # instructions
-                p = re.compile(" -h| --help")
+            p = re.compile(" -h| --help")
-                if p.match(passOn):
+            if p.match(passOn):
-                    sys.exit(0)
+                sys.exit(0)
 
-        # Run Extended Tests
+    # Run Extended Tests
-        for i in range(len(testScriptsExt)):
+    for i in range(len(testScriptsExt)):
-            if (run_extended or testScriptsExt[i] in opts
+        if (run_extended or testScriptsExt[i] in opts
-                    or re.sub(".py$", "", testScriptsExt[i]) in opts):
+                or re.sub(".py$", "", testScriptsExt[i]) in opts):
 
-                print(
+            print(
-                    "Running 2nd level testscript "
+                "Running 2nd level testscript "
-                    + "%s%s%s ..." % (bold[1], testScriptsExt[i], bold[0]))
+                + "%s%s%s ..." % (bold[1], testScriptsExt[i], bold[0]))
-                time0 = time.time()
+            time0 = time.time()
-                subprocess.check_call(
+            subprocess.check_call(
-                    rpcTestDir + testScriptsExt[i] + flags, shell=True)
+                rpcTestDir + testScriptsExt[i] + flags, shell=True)
-                print("Duration: %s s\n" % (int(time.time() - time0)))
+            print("Duration: %s s\n" % (int(time.time() - time0)))
 
-        if coverage:
+    if coverage:
-            coverage.report_rpc_coverage()
+        coverage.report_rpc_coverage()
 
-            print("Cleaning up coverage data")
+        print("Cleaning up coverage data")
-            coverage.cleanup()
+        coverage.cleanup()
-
-    else:
-        print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled"
 
 
 class RPCCoverage(object):

qa/rpc-tests/httpbasics.py

@@ -36,13 +36,13 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.connect()
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
-        assert('"error":null' in out1)
+        assert(b'"error":null' in out1)
         assert(conn.sock!=None) #according to http/1.1 connection must still be open!
 
         #send 2nd request without closing connection
         conn.request('POST', '/', '{"method": "getchaintips"}', headers)
-        out2 = conn.getresponse().read()
+        out1 = conn.getresponse().read()
-        assert('"error":null' in out1) #must also response with a correct json-rpc message
+        assert(b'"error":null' in out1) #must also response with a correct json-rpc message
         assert(conn.sock!=None) #according to http/1.1 connection must still be open!
         conn.close()
 
@@ -53,13 +53,13 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.connect()
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
-        assert('"error":null' in out1)
+        assert(b'"error":null' in out1)
         assert(conn.sock!=None) #according to http/1.1 connection must still be open!
 
         #send 2nd request without closing connection
         conn.request('POST', '/', '{"method": "getchaintips"}', headers)
-        out2 = conn.getresponse().read()
+        out1 = conn.getresponse().read()
-        assert('"error":null' in out1) #must also response with a correct json-rpc message
+        assert(b'"error":null' in out1) #must also response with a correct json-rpc message
         assert(conn.sock!=None) #according to http/1.1 connection must still be open!
         conn.close()
 
@@ -70,7 +70,7 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.connect()
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
-        assert('"error":null' in out1)
+        assert(b'"error":null' in out1)
         assert(conn.sock==None) #now the connection must be closed after the response
 
         #node1 (2nd node) is running with disabled keep-alive option
@@ -82,7 +82,7 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.connect()
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
-        assert('"error":null' in out1)
+        assert(b'"error":null' in out1)
 
         #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on
         urlNode2 = urlparse.urlparse(self.nodes[2].url)
@@ -93,7 +93,7 @@ class HTTPBasicsTest (BitcoinTestFramework):
         conn.connect()
         conn.request('POST', '/', '{"method": "getbestblockhash"}', headers)
         out1 = conn.getresponse().read()
-        assert('"error":null' in out1)
+        assert(b'"error":null' in out1)
         assert(conn.sock!=None) #connection must be closed because bitcoind should use keep-alive by default
 
         # Check excessive request size

qa/rpc-tests/maxblocksinflight.py

@@ -41,40 +41,36 @@ class TestManager(NodeConnCB):
         self.disconnectOkay = False
 
     def run(self):
-        try:
+        self.connection.rpc.generate(1)  # Leave IBD
-            fail = False
-            self.connection.rpc.generate(1) # Leave IBD
 
-            numBlocksToGenerate = [ 8, 16, 128, 1024 ]
+        numBlocksToGenerate = [8, 16, 128, 1024]
-            for count in range(len(numBlocksToGenerate)):
+        for count in range(len(numBlocksToGenerate)):
-                current_invs = []
+            current_invs = []
-                for i in range(numBlocksToGenerate[count]):
+            for i in range(numBlocksToGenerate[count]):
-                    current_invs.append(CInv(2, random.randrange(0, 1<<256)))
+                current_invs.append(CInv(2, random.randrange(0, 1 << 256)))
-                    if len(current_invs) >= 50000:
+                if len(current_invs) >= 50000:
-                        self.connection.send_message(msg_inv(current_invs))
-                        current_invs = []
-                if len(current_invs) > 0:
                     self.connection.send_message(msg_inv(current_invs))
-                
+                    current_invs = []
-                # Wait and see how many blocks were requested
+            if len(current_invs) > 0:
-                time.sleep(2)
+                self.connection.send_message(msg_inv(current_invs))
 
-                total_requests = 0
+            # Wait and see how many blocks were requested
-                with mininode_lock:
+            time.sleep(2)
-                    for key in self.blockReqCounts:
+
-                        total_requests += self.blockReqCounts[key]
+            total_requests = 0
-                        if self.blockReqCounts[key] > 1:
+            with mininode_lock:
-                            raise AssertionError("Error, test failed: block %064x requested more than once" % key)
+                for key in self.blockReqCounts:
-                if total_requests > MAX_REQUESTS:
+                    total_requests += self.blockReqCounts[key]
-                    raise AssertionError("Error, too many blocks (%d) requested" % total_requests)
+                    if self.blockReqCounts[key] > 1:
-                print "Round %d: success (total requests: %d)" % (count, total_requests)
+                        raise AssertionError("Error, test failed: block %064x requested more than once" % key)
-        except AssertionError as e:
+            if total_requests > MAX_REQUESTS:
-            print "TEST FAILED: ", e.args
+                raise AssertionError("Error, too many blocks (%d) requested" % total_requests)
+            print "Round %d: success (total requests: %d)" % (count, total_requests)
 
         self.disconnectOkay = True
         self.connection.disconnect_node()
 
-        
+
 class MaxBlocksInFlightTest(BitcoinTestFramework):
     def add_options(self, parser):
         parser.add_option("--testbinary", dest="testbinary",
@@ -86,7 +82,7 @@ class MaxBlocksInFlightTest(BitcoinTestFramework):
         initialize_chain_clean(self.options.tmpdir, 1)
 
     def setup_network(self):
-        self.nodes = start_nodes(1, self.options.tmpdir, 
+        self.nodes = start_nodes(1, self.options.tmpdir,
                                  extra_args=[['-debug', '-whitelist=127.0.0.1']],
                                  binary=[self.options.testbinary])
 

qa/rpc-tests/maxuploadtarget.py

@@ -7,7 +7,6 @@
 from test_framework.mininode import *
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import *
-from test_framework.comptool import wait_until
 import time
 
 '''

qa/rpc-tests/mempool_reorg.py

@@ -25,14 +25,6 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         self.is_network_split = False
         self.sync_all()
 
-    def create_tx(self, from_txid, to_address, amount):
-        inputs = [{ "txid" : from_txid, "vout" : 0}]
-        outputs = { to_address : amount }
-        rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
-        signresult = self.nodes[0].signrawtransaction(rawtx)
-        assert_equal(signresult["complete"], True)
-        return signresult["hex"]
-
     def run_test(self):
         start_count = self.nodes[0].getblockcount()
 
@@ -52,9 +44,9 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         # and make sure the mempool code behaves correctly.
         b = [ self.nodes[0].getblockhash(n) for n in range(101, 105) ]
         coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ]
-        spend_101_raw = self.create_tx(coinbase_txids[1], node1_address, 50)
+        spend_101_raw = create_tx(self.nodes[0], coinbase_txids[1], node1_address, 50)
-        spend_102_raw = self.create_tx(coinbase_txids[2], node0_address, 50)
+        spend_102_raw = create_tx(self.nodes[0], coinbase_txids[2], node0_address, 50)
-        spend_103_raw = self.create_tx(coinbase_txids[3], node0_address, 50)
+        spend_103_raw = create_tx(self.nodes[0], coinbase_txids[3], node0_address, 50)
 
         # Create a block-height-locked transaction which will be invalid after reorg
         timelock_tx = self.nodes[0].createrawtransaction([{"txid": coinbase_txids[0], "vout": 0}], {node0_address: 50})
@@ -71,8 +63,8 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         assert_raises(JSONRPCException, self.nodes[0].sendrawtransaction, timelock_tx)
 
         # Create 102_1 and 103_1:
-        spend_102_1_raw = self.create_tx(spend_102_id, node1_address, 50)
+        spend_102_1_raw = create_tx(self.nodes[0], spend_102_id, node1_address, 50)
-        spend_103_1_raw = self.create_tx(spend_103_id, node1_address, 50)
+        spend_103_1_raw = create_tx(self.nodes[0], spend_103_id, node1_address, 50)
 
         # Broadcast and mine 103_1:
         spend_103_1_id = self.nodes[0].sendrawtransaction(spend_103_1_raw)

qa/rpc-tests/mempool_resurrect_test.py

@@ -21,14 +21,6 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
         self.nodes.append(start_node(0, self.options.tmpdir, args))
         self.is_network_split = False
 
-    def create_tx(self, from_txid, to_address, amount):
-        inputs = [{ "txid" : from_txid, "vout" : 0}]
-        outputs = { to_address : amount }
-        rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
-        signresult = self.nodes[0].signrawtransaction(rawtx)
-        assert_equal(signresult["complete"], True)
-        return signresult["hex"]
-
     def run_test(self):
         node0_address = self.nodes[0].getnewaddress()
         # Spend block 1/2/3's coinbase transactions
@@ -43,13 +35,13 @@ class MempoolCoinbaseTest(BitcoinTestFramework):
 
         b = [ self.nodes[0].getblockhash(n) for n in range(1, 4) ]
         coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ]
-        spends1_raw = [ self.create_tx(txid, node0_address, 50) for txid in coinbase_txids ]
+        spends1_raw = [ create_tx(self.nodes[0], txid, node0_address, 50) for txid in coinbase_txids ]
         spends1_id = [ self.nodes[0].sendrawtransaction(tx) for tx in spends1_raw ]
 
         blocks = []
         blocks.extend(self.nodes[0].generate(1))
 
-        spends2_raw = [ self.create_tx(txid, node0_address, 49.99) for txid in spends1_id ]
+        spends2_raw = [ create_tx(self.nodes[0], txid, node0_address, 49.99) for txid in spends1_id ]
         spends2_id = [ self.nodes[0].sendrawtransaction(tx) for tx in spends2_raw ]
 
         blocks.extend(self.nodes[0].generate(1))

qa/rpc-tests/mempool_spendcoinbase.py

@@ -26,14 +26,6 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework):
         self.nodes.append(start_node(0, self.options.tmpdir, args))
         self.is_network_split = False
 
-    def create_tx(self, from_txid, to_address, amount):
-        inputs = [{ "txid" : from_txid, "vout" : 0}]
-        outputs = { to_address : amount }
-        rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
-        signresult = self.nodes[0].signrawtransaction(rawtx)
-        assert_equal(signresult["complete"], True)
-        return signresult["hex"]
-
     def run_test(self):
         chain_height = self.nodes[0].getblockcount()
         assert_equal(chain_height, 200)
@@ -44,7 +36,7 @@ class MempoolSpendCoinbaseTest(BitcoinTestFramework):
         # is too immature to spend.
         b = [ self.nodes[0].getblockhash(n) for n in range(101, 103) ]
         coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ]
-        spends_raw = [ self.create_tx(txid, node0_address, 50) for txid in coinbase_txids ]
+        spends_raw = [ create_tx(self.nodes[0], txid, node0_address, 50) for txid in coinbase_txids ]
 
         spend_101_id = self.nodes[0].sendrawtransaction(spends_raw[0])
 

qa/rpc-tests/smartfees.py

@@ -7,6 +7,7 @@
 # Test fee estimation code
 #
 
+from collections import OrderedDict
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import *
 
@@ -22,7 +23,7 @@ SCRIPT_SIG = ["0451025175", "0451025275"]
 def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee_increment):
     '''
     Create and send a transaction with a random fee.
-    The transaction pays to a trival P2SH script, and assumes that its inputs
+    The transaction pays to a trivial P2SH script, and assumes that its inputs
     are of the same form.
     The function takes a list of confirmed outputs and unconfirmed outputs
     and attempts to use the confirmed list first for its inputs.
@@ -49,10 +50,10 @@ def small_txpuzzle_randfee(from_node, conflist, unconflist, amount, min_fee, fee
         if total_in <= amount + fee:
             raise RuntimeError("Insufficient funds: need %d, have %d"%(amount+fee, total_in))
     outputs = {}
-    outputs[P2SH_1] = total_in - amount - fee
+    outputs = OrderedDict([(P2SH_1, total_in - amount - fee),
-    outputs[P2SH_2] = amount
+                           (P2SH_2, amount)])
     rawtx = from_node.createrawtransaction(inputs, outputs)
-    # Createrawtransaction constructions a transaction that is ready to be signed
+    # createrawtransaction constructs a transaction that is ready to be signed.
     # These transactions don't need to be signed, but we still have to insert the ScriptSig
     # that will satisfy the ScriptPubKey.
     completetx = rawtx[0:10]
@@ -78,12 +79,10 @@ def split_inputs(from_node, txins, txouts, initial_split = False):
     '''
     prevtxout = txins.pop()
     inputs = []
-    outputs = {}
     inputs.append({ "txid" : prevtxout["txid"], "vout" : prevtxout["vout"] })
     half_change = satoshi_round(prevtxout["amount"]/2)
     rem_change = prevtxout["amount"] - half_change  - Decimal("0.00001000")
-    outputs[P2SH_1] = half_change
+    outputs = OrderedDict([(P2SH_1, half_change), (P2SH_2, rem_change)])
-    outputs[P2SH_2] = rem_change
     rawtx = from_node.createrawtransaction(inputs, outputs)
     # If this is the initial split we actually need to sign the transaction
     # Otherwise we just need to insert the property ScriptSig
@@ -224,7 +223,7 @@ class EstimateFeeTest(BitcoinTestFramework):
             sync_mempools(self.nodes[0:3],.1)
             mined = mining_node.getblock(mining_node.generate(1)[0],True)["tx"]
             sync_blocks(self.nodes[0:3],.1)
-            #update which txouts are confirmed
+            # update which txouts are confirmed
             newmem = []
             for utx in self.memutxo:
                 if utx["txid"] in mined:

qa/rpc-tests/test_framework/authproxy.py

@@ -61,7 +61,7 @@ class JSONRPCException(Exception):
 
 def EncodeDecimal(o):
     if isinstance(o, decimal.Decimal):
-        return round(o, 8)
+        return str(o)
     raise TypeError(repr(o) + " is not JSON serializable")
 
 class AuthServiceProxy(object):
@@ -92,11 +92,10 @@ class AuthServiceProxy(object):
             self.__conn = connection
         elif self.__url.scheme == 'https':
             self.__conn = httplib.HTTPSConnection(self.__url.hostname, port,
-                                                  None, None, False,
+                                                  timeout=timeout)
-                                                  timeout)
         else:
             self.__conn = httplib.HTTPConnection(self.__url.hostname, port,
-                                                 False, timeout)
+                                                 timeout=timeout)
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
@@ -155,6 +154,11 @@ class AuthServiceProxy(object):
             raise JSONRPCException({
                 'code': -342, 'message': 'missing HTTP response from server'})
 
+        content_type = http_response.getheader('Content-Type')
+        if content_type != 'application/json':
+            raise JSONRPCException({
+                'code': -342, 'message': 'non-JSON HTTP response with \'%i %s\' from server' % (http_response.status, http_response.reason)})
+
         responsedata = http_response.read().decode('utf8')
         response = json.loads(responsedata, parse_float=decimal.Decimal)
         if "error" in response and response["error"] is None:

qa/rpc-tests/test_framework/comptool.py

@@ -27,20 +27,6 @@ generator that returns TestInstance objects.  See below for definition.
 
 global mininode_lock
 
-def wait_until(predicate, attempts=float('inf'), timeout=float('inf')):
-    attempt = 0
-    elapsed = 0
-
-    while attempt < attempts and elapsed < timeout:
-        with mininode_lock:
-            if predicate():
-                return True
-        attempt += 1
-        elapsed += 0.05
-        time.sleep(0.05)
-
-    return False
-
 class RejectResult(object):
     '''
     Outcome that expects rejection of a transaction or block.

qa/rpc-tests/test_framework/mininode.py

@@ -93,7 +93,7 @@ def deser_uint256(f):
 
 
 def ser_uint256(u):
-    rs = ""
+    rs = b""
     for i in xrange(8):
         rs += struct.pack("<I", u & 0xFFFFFFFFL)
         u >>= 32
@@ -191,7 +191,7 @@ def deser_string_vector(f):
 
 
 def ser_string_vector(l):
-    r = ""
+    r = b""
     if len(l) < 253:
         r = struct.pack("B", len(l))
     elif len(l) < 0x10000:
@@ -624,7 +624,7 @@ class CAlert(object):
         self.vchSig = deser_string(f)
 
     def serialize(self):
-        r = ""
+        r = b""
         r += ser_string(self.vchMsg)
         r += ser_string(self.vchSig)
         return r
@@ -983,25 +983,28 @@ class msg_headers(object):
 
 class msg_reject(object):
     command = b"reject"
+    REJECT_MALFORMED = 1
 
     def __init__(self):
         self.message = b""
         self.code = 0
-        self.reason = ""
+        self.reason = b""
         self.data = 0L
 
     def deserialize(self, f):
         self.message = deser_string(f)
         self.code = struct.unpack("<B", f.read(1))[0]
         self.reason = deser_string(f)
-        if (self.message == "block" or self.message == "tx"):
+        if (self.code != self.REJECT_MALFORMED and
+                (self.message == b"block" or self.message == b"tx")):
             self.data = deser_uint256(f)
 
     def serialize(self):
         r = ser_string(self.message)
         r += struct.pack("<B", self.code)
         r += ser_string(self.reason)
-        if (self.message == "block" or self.message == "tx"):
+        if (self.code != self.REJECT_MALFORMED and
+                (self.message == b"block" or self.message == b"tx")):
             r += ser_uint256(self.data)
         return r
 
@@ -1009,6 +1012,21 @@ class msg_reject(object):
         return "msg_reject: %s %d %s [%064x]" \
             % (self.message, self.code, self.reason, self.data)
 
+# Helper function
+def wait_until(predicate, attempts=float('inf'), timeout=float('inf')):
+    attempt = 0
+    elapsed = 0
+
+    while attempt < attempts and elapsed < timeout:
+        with mininode_lock:
+            if predicate():
+                return True
+        attempt += 1
+        elapsed += 0.05
+        time.sleep(0.05)
+
+    return False
+
 # This is what a callback should look like for NodeConn
 # Reimplement the on_* functions to provide handling for events
 class NodeConnCB(object):
@@ -1085,6 +1103,32 @@ class NodeConnCB(object):
     def on_mempool(self, conn): pass
     def on_pong(self, conn, message): pass
 
+# More useful callbacks and functions for NodeConnCB's which have a single NodeConn
+class SingleNodeConnCB(NodeConnCB):
+    def __init__(self):
+        NodeConnCB.__init__(self)
+        self.connection = None
+        self.ping_counter = 1
+        self.last_pong = msg_pong()
+
+    def add_connection(self, conn):
+        self.connection = conn
+
+    # Wrapper for the NodeConn's send_message function
+    def send_message(self, message):
+        self.connection.send_message(message)
+
+    def on_pong(self, conn, message):
+        self.last_pong = message
+
+    # Sync up with the node
+    def sync_with_ping(self, timeout=30):
+        def received_pong():
+            return (self.last_pong.nonce == self.ping_counter)
+        self.send_message(msg_ping(nonce=self.ping_counter))
+        success = wait_until(received_pong, timeout)
+        self.ping_counter += 1
+        return success
 
 # The actual NodeConn class
 # This class provides an interface for a p2p connection to a specified node

qa/rpc-tests/test_framework/test_framework.py

@@ -143,7 +143,7 @@ class BitcoinTestFramework(object):
             print("Assertion failed: "+ str(e))
             traceback.print_tb(sys.exc_info()[2])
         except Exception as e:
-            print("Unexpected exception caught during testing: "+str(e))
+            print("Unexpected exception caught during testing: " + repr(e))
             traceback.print_tb(sys.exc_info()[2])
 
         if not self.options.noshutdown:

qa/rpc-tests/test_framework/util.py

@@ -20,6 +20,7 @@ import shutil
 import subprocess
 import time
 import re
+import errno
 
 from . import coverage
 from .authproxy import AuthServiceProxy, JSONRPCException
@@ -143,11 +144,33 @@ def initialize_datadir(dirname, n):
         f.write("listenonion=0\n")
     return datadir
 
+def rpc_url(i, rpchost=None):
+    return "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i))
+
+def wait_for_bitcoind_start(process, url, i):
+    '''
+    Wait for bitcoind to start. This means that RPC is accessible and fully initialized.
+    Raise an exception if bitcoind exits during initialization.
+    '''
+    while True:
+        if process.poll() is not None:
+            raise Exception('bitcoind exited with status %i during initialization' % process.returncode)
+        try:
+            rpc = get_rpc_proxy(url, i)
+            blocks = rpc.getblockcount()
+            break # break out of loop on success
+        except IOError as e:
+            if e.errno != errno.ECONNREFUSED: # Port not yet open?
+                raise # unknown IO error
+        except JSONRPCException as e: # Initialization phase
+            if e.error['code'] != -28: # RPC in warmup?
+                raise # unkown JSON RPC exception
+        time.sleep(0.25)
+
 def initialize_chain(test_dir):
     """
     Create (or copy from cache) a 200-block-long chain and
     4 wallets.
-    bitcoind and bitcoin-cli must be in search path.
     """
 
     if (not os.path.isdir(os.path.join("cache","node0"))
@@ -160,7 +183,6 @@ def initialize_chain(test_dir):
             if os.path.isdir(os.path.join("cache","node"+str(i))):
                 shutil.rmtree(os.path.join("cache","node"+str(i)))
 
-        devnull = open(os.devnull, "w")
         # Create cache directories, run bitcoinds:
         for i in range(4):
             datadir=initialize_datadir("cache", i)
@@ -169,19 +191,15 @@ def initialize_chain(test_dir):
                 args.append("-connect=127.0.0.1:"+str(p2p_port(0)))
             bitcoind_processes[i] = subprocess.Popen(args)
             if os.getenv("PYTHON_DEBUG", ""):
-                print "initialize_chain: bitcoind started, calling bitcoin-cli -rpcwait getblockcount"
+                print "initialize_chain: bitcoind started, waiting for RPC to come up"
-            subprocess.check_call([ os.getenv("BITCOINCLI", "bitcoin-cli"), "-datadir="+datadir,
+            wait_for_bitcoind_start(bitcoind_processes[i], rpc_url(i), i)
-                                    "-rpcwait", "getblockcount"], stdout=devnull)
             if os.getenv("PYTHON_DEBUG", ""):
-                print "initialize_chain: bitcoin-cli -rpcwait getblockcount completed"
+                print "initialize_chain: RPC succesfully started"
-        devnull.close()
 
         rpcs = []
-
         for i in range(4):
             try:
-                url = "http://rt:rt@127.0.0.1:%d" % (rpc_port(i),)
+                rpcs.append(get_rpc_proxy(rpc_url(i), i))
-                rpcs.append(get_rpc_proxy(url, i))
             except:
                 sys.stderr.write("Error connecting to "+url+"\n")
                 sys.exit(1)
@@ -257,17 +275,12 @@ def start_node(i, dirname, extra_args=None, rpchost=None, timewait=None, binary=
     args = [ binary, "-datadir="+datadir, "-server", "-keypool=1", "-discover=0", "-rest", "-blockprioritysize=50000", "-mocktime="+str(get_mocktime()) ]
     if extra_args is not None: args.extend(extra_args)
     bitcoind_processes[i] = subprocess.Popen(args)
-    devnull = open(os.devnull, "w")
     if os.getenv("PYTHON_DEBUG", ""):
-        print "start_node: bitcoind started, calling bitcoin-cli -rpcwait getblockcount"
+        print "start_node: bitcoind started, waiting for RPC to come up"
-    subprocess.check_call([ os.getenv("BITCOINCLI", "bitcoin-cli"), "-datadir="+datadir] +
+    url = rpc_url(i, rpchost)
-                          _rpchost_to_args(rpchost)  +
+    wait_for_bitcoind_start(bitcoind_processes[i], url, i)
-                          ["-rpcwait", "getblockcount"], stdout=devnull)
     if os.getenv("PYTHON_DEBUG", ""):
-        print "start_node: calling bitcoin-cli -rpcwait getblockcount returned"
+        print "start_node: RPC succesfully started"
-    devnull.close()
-    url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', rpc_port(i))
-
     proxy = get_rpc_proxy(url, i, timeout=timewait)
 
     if COVERAGE_DIR:
@@ -281,7 +294,14 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None):
     """
     if extra_args is None: extra_args = [ None for i in range(num_nodes) ]
     if binary is None: binary = [ None for i in range(num_nodes) ]
-    return [ start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]) for i in range(num_nodes) ]
+    rpcs = []
+    try:
+        for i in range(num_nodes):
+            rpcs.append(start_node(i, dirname, extra_args[i], rpchost, binary=binary[i]))
+    except: # If one node failed to start, stop the others
+        stop_nodes(rpcs)
+        raise
+    return rpcs
 
 def log_filename(dirname, n_node, logname):
     return os.path.join(dirname, "node"+str(n_node), "regtest", logname)
@@ -468,7 +488,7 @@ def assert_array_result(object_array, to_match, expected, should_not_find = Fals
         in object_array
         """
     if should_not_find == True:
-        expected = { }
+        assert_equal(expected, { })
     num_matched = 0
     for item in object_array:
         all_match = True

src/chain.cpp

@@ -93,6 +93,7 @@ CBlockIndex* CBlockIndex::GetAncestor(int height)
             pindexWalk = pindexWalk->pskip;
             heightWalk = heightSkip;
         } else {
+            assert(pindexWalk->pprev);
             pindexWalk = pindexWalk->pprev;
             heightWalk--;
         }

src/scheduler.cpp

@@ -79,6 +79,7 @@ void CScheduler::serviceQueue()
         }
     }
     --nThreadsServicingQueue;
+    newTaskScheduled.notify_one();
 }
 
 void CScheduler::stop(bool drain)

src/test/dbwrapper_tests.cpp

@@ -203,5 +203,125 @@ BOOST_AUTO_TEST_CASE(existing_data_reindex)
     BOOST_CHECK(odbw.Read(key, res3));
     BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString());
 }
- 
+
+BOOST_AUTO_TEST_CASE(iterator_ordering)
+{
+    path ph = temp_directory_path() / unique_path();
+    CDBWrapper dbw(ph, (1 << 20), true, false, false);
+    for (int x=0x00; x<256; ++x) {
+        uint8_t key = x;
+        uint32_t value = x*x;
+        BOOST_CHECK(dbw.Write(key, value));
+    }
+
+    boost::scoped_ptr<CDBIterator> it(const_cast<CDBWrapper*>(&dbw)->NewIterator());
+    for (int c=0; c<2; ++c) {
+        int seek_start;
+        if (c == 0)
+            seek_start = 0x00;
+        else
+            seek_start = 0x80;
+        it->Seek((uint8_t)seek_start);
+        for (int x=seek_start; x<256; ++x) {
+            uint8_t key;
+            uint32_t value;
+            BOOST_CHECK(it->Valid());
+            if (!it->Valid()) // Avoid spurious errors about invalid iterator's key and value in case of failure
+                break;
+            BOOST_CHECK(it->GetKey(key));
+            BOOST_CHECK(it->GetValue(value));
+            BOOST_CHECK_EQUAL(key, x);
+            BOOST_CHECK_EQUAL(value, x*x);
+            it->Next();
+        }
+        BOOST_CHECK(!it->Valid());
+    }
+}
+
+struct StringContentsSerializer {
+    // Used to make two serialized objects the same while letting them have a different lengths
+    // This is a terrible idea
+    string str;
+    StringContentsSerializer() {}
+    StringContentsSerializer(const string& inp) : str(inp) {}
+
+    StringContentsSerializer& operator+=(const string& s) {
+        str += s;
+        return *this;
+    }
+    StringContentsSerializer& operator+=(const StringContentsSerializer& s) { return *this += s.str; }
+
+    ADD_SERIALIZE_METHODS;
+
+    template <typename Stream, typename Operation>
+    inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) {
+        if (ser_action.ForRead()) {
+            str.clear();
+            char c = 0;
+            while (true) {
+                try {
+                    READWRITE(c);
+                    str.push_back(c);
+                } catch (const std::ios_base::failure& e) {
+                    break;
+                }
+            }
+        } else {
+            for (size_t i = 0; i < str.size(); i++)
+                READWRITE(str[i]);
+        }
+    }
+};
+
+BOOST_AUTO_TEST_CASE(iterator_string_ordering)
+{
+    char buf[10];
+
+    path ph = temp_directory_path() / unique_path();
+    CDBWrapper dbw(ph, (1 << 20), true, false, false);
+    for (int x=0x00; x<10; ++x) {
+        for (int y = 0; y < 10; y++) {
+            sprintf(buf, "%d", x);
+            StringContentsSerializer key(buf);
+            for (int z = 0; z < y; z++)
+                key += key;
+            uint32_t value = x*x;
+            BOOST_CHECK(dbw.Write(key, value));
+        }
+    }
+
+    boost::scoped_ptr<CDBIterator> it(const_cast<CDBWrapper*>(&dbw)->NewIterator());
+    for (int c=0; c<2; ++c) {
+        int seek_start;
+        if (c == 0)
+            seek_start = 0;
+        else
+            seek_start = 5;
+        sprintf(buf, "%d", seek_start);
+        StringContentsSerializer seek_key(buf);
+        it->Seek(seek_key);
+        for (int x=seek_start; x<10; ++x) {
+            for (int y = 0; y < 10; y++) {
+                sprintf(buf, "%d", x);
+                string exp_key(buf);
+                for (int z = 0; z < y; z++)
+                    exp_key += exp_key;
+                StringContentsSerializer key;
+                uint32_t value;
+                BOOST_CHECK(it->Valid());
+                if (!it->Valid()) // Avoid spurious errors about invalid iterator's key and value in case of failure
+                    break;
+                BOOST_CHECK(it->GetKey(key));
+                BOOST_CHECK(it->GetValue(value));
+                BOOST_CHECK_EQUAL(key.str, exp_key);
+                BOOST_CHECK_EQUAL(value, x*x);
+                it->Next();
+            }
+        }
+        BOOST_CHECK(!it->Valid());
+    }
+}
+
+
+
 BOOST_AUTO_TEST_SUITE_END()

src/test/scheduler_tests.cpp

@@ -40,7 +40,6 @@ static void MicroSleep(uint64_t n)
 #endif
 }
 
-#if 0 /* Disabled for now because there is a race condition issue in this test - see #6540 */
 BOOST_AUTO_TEST_CASE(manythreads)
 {
     seed_insecure_rand(false);
@@ -116,6 +115,5 @@ BOOST_AUTO_TEST_CASE(manythreads)
     }
     BOOST_CHECK_EQUAL(counterSum, 200);
 }
-#endif
 
 BOOST_AUTO_TEST_SUITE_END()