2016-05-06 11:23:48 +02:00
#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
2015-08-26 12:05:36 +02:00
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
2015-10-11 07:41:19 +02:00
"""
Run Regression Test Suite
This module calls down into individual test cases via subprocess . It will
forward all unrecognized arguments onto the individual test scripts , other
than :
- ` - extended ` : run the " extended " test suite in addition to the basic one .
- ` - win ` : signal that this is running in a Windows environment , and we
should run the tests .
- ` - - coverage ` : this generates a basic coverage report for the RPC
interface .
For a description of arguments recognized by test scripts , see
` qa / pull - tester / test_framework / test_framework . py : BitcoinTestFramework . main ` .
"""
2015-08-26 12:05:36 +02:00
import os
2015-11-30 14:53:07 +01:00
import time
2015-10-11 07:41:19 +02:00
import shutil
2015-08-26 12:05:36 +02:00
import sys
import subprocess
2015-10-11 07:41:19 +02:00
import tempfile
2015-08-26 12:05:36 +02:00
import re
2015-10-11 07:41:19 +02:00
2015-08-26 12:05:36 +02:00
from tests_config import *
2016-05-09 17:01:55 +02:00
BOLD = ( " " , " " )
if os . name == ' posix ' :
# primitive formatting on supported
# terminal via ANSI escape sequences:
BOLD = ( ' \033 [0m ' , ' \033 [1m ' )
RPC_TESTS_DIR = BUILDDIR + ' /qa/rpc-tests/ '
2015-08-26 12:05:36 +02:00
#If imported values are not defined then set to zero (or disabled)
2016-03-19 21:36:32 +01:00
if ' ENABLE_WALLET ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_WALLET = 0
2016-03-19 21:36:32 +01:00
if ' ENABLE_BITCOIND ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_BITCOIND = 0
2016-03-19 21:36:32 +01:00
if ' ENABLE_UTILS ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_UTILS = 0
2016-03-19 21:36:32 +01:00
if ' ENABLE_ZMQ ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_ZMQ = 0
2016-03-11 08:42:45 +01:00
# python-zmq may not be installed. Handle this gracefully and with some helpful info
if ENABLE_ZMQ :
try :
import zmq
except ImportError :
print ( " WARNING: \" import zmq \" failed. Setting ENABLE_ZMQ=0. " \
" To run zmq tests, see dependency info in /qa/README.md. " )
ENABLE_ZMQ = 0
2015-08-26 12:05:36 +02:00
2015-10-11 07:41:19 +02:00
ENABLE_COVERAGE = 0
2016-05-09 17:01:55 +02:00
#Create a set to store arguments and create the passon string
2015-10-11 07:41:19 +02:00
opts = set ( )
2016-05-09 17:01:55 +02:00
passon_args = " "
PASSON_REGEX = re . compile ( " ^-- " )
2015-10-11 07:41:19 +02:00
2016-05-09 17:01:55 +02:00
print_help = False
2015-11-30 14:53:07 +01:00
2015-10-11 07:41:19 +02:00
for arg in sys . argv [ 1 : ] :
2016-05-09 17:01:55 +02:00
if arg == " --help " or arg == " -h " or arg == " -? " :
print_help = True
break
2015-10-11 07:41:19 +02:00
if arg == ' --coverage ' :
ENABLE_COVERAGE = 1
2016-05-09 17:01:55 +02:00
elif PASSON_REGEX . match ( arg ) :
passon_args + = " " + arg
2015-08-26 12:05:36 +02:00
else :
2015-10-11 07:41:19 +02:00
opts . add ( arg )
2015-08-26 12:05:36 +02:00
#Set env vars
2016-03-04 08:25:16 +01:00
if " DASHD " not in os . environ :
2016-05-09 17:01:55 +02:00
os . environ [ " DASHD " ] = BUILDDIR + ' /src/dashd ' + EXEEXT
2016-03-04 08:25:16 +01:00
if " DASHCLI " not in os . environ :
2016-05-09 17:01:55 +02:00
os . environ [ " DASHCLI " ] = BUILDDIR + ' /src/dash-cli ' + EXEEXT
2015-10-08 10:22:50 +02:00
2015-08-26 12:05:36 +02:00
if EXEEXT == " .exe " and " -win " not in opts :
2016-04-09 21:14:18 +02:00
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
2016-05-06 11:23:48 +02:00
print ( " Win tests currently disabled by default. Use -win option to enable " )
2015-08-26 12:05:36 +02:00
sys . exit ( 0 )
2016-04-09 21:14:18 +02:00
if not ( ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1 ) :
2016-05-06 11:23:48 +02:00
print ( " No rpc tests to run. Wallet, utils, and bitcoind must all be enabled " )
2015-08-26 12:05:36 +02:00
sys . exit ( 0 )
2016-05-06 11:23:48 +02:00
# python3-zmq may not be installed. Handle this gracefully and with some helpful info
2016-04-09 21:14:18 +02:00
if ENABLE_ZMQ :
try :
import zmq
except ImportError as e :
2016-05-09 15:11:41 +02:00
print ( " WARNING: \" import zmq \" failed. Set ENABLE_ZMQ=0 or " \
2016-04-09 21:14:18 +02:00
" to run zmq tests, see dependency info in /qa/README.md. " )
2016-05-09 15:11:41 +02:00
ENABLE_ZMQ = 0
2016-04-09 21:14:18 +02:00
2015-08-26 12:05:36 +02:00
#Tests
testScripts = [
2016-03-11 00:36:55 +01:00
' bip68-112-113-p2p.py ' ,
2015-08-26 12:05:36 +02:00
' wallet.py ' ,
2017-05-29 13:51:40 +02:00
' wallet-hd.py ' ,
2015-08-26 12:05:36 +02:00
' listtransactions.py ' ,
2015-11-14 21:44:35 +01:00
' receivedby.py ' ,
2015-08-26 12:05:36 +02:00
' mempool_resurrect_test.py ' ,
' txn_doublespend.py --mineblock ' ,
' txn_clone.py ' ,
' getchaintips.py ' ,
' rawtransactions.py ' ,
' rest.py ' ,
' mempool_spendcoinbase.py ' ,
2015-08-27 03:15:04 +02:00
' mempool_reorg.py ' ,
2015-11-30 15:42:27 +01:00
' mempool_limit.py ' ,
2015-08-26 12:05:36 +02:00
' httpbasics.py ' ,
2015-11-11 16:49:32 +01:00
' multi_rpc.py ' ,
2015-08-26 12:05:36 +02:00
' zapwallettxes.py ' ,
' proxy_test.py ' ,
' merkle_blocks.py ' ,
' fundrawtransaction.py ' ,
2017-09-20 22:31:12 +02:00
' fundrawtransaction-hd.py ' ,
2015-08-26 12:05:36 +02:00
' signrawtransactions.py ' ,
' walletbackup.py ' ,
' nodehandling.py ' ,
' reindex.py ' ,
2016-03-08 22:15:49 +01:00
' addressindex.py ' ,
2016-03-22 23:11:04 +01:00
' timestampindex.py ' ,
2016-04-05 21:53:38 +02:00
' spentindex.py ' ,
2015-08-26 12:05:36 +02:00
' decodescript.py ' ,
2016-10-22 18:52:14 +02:00
' p2p-fullblocktest.py ' , # NOTE: needs dash_hash to pass
2015-10-08 10:22:50 +02:00
' blockchain.py ' ,
2015-11-09 08:40:46 +01:00
' disablewallet.py ' ,
2016-10-22 18:52:14 +02:00
' sendheaders.py ' , # NOTE: needs dash_hash to pass
2015-11-30 14:53:07 +01:00
' keypool.py ' ,
2017-09-20 22:31:12 +02:00
' keypool-hd.py ' ,
2015-11-19 02:55:52 +01:00
' prioritise_transaction.py ' ,
2016-10-22 18:52:14 +02:00
' invalidblockrequest.py ' , # NOTE: needs dash_hash to pass
' invalidtxrequest.py ' , # NOTE: needs dash_hash to pass
2016-01-07 22:31:12 +01:00
' abandonconflict.py ' ,
2016-03-15 17:09:16 +01:00
' p2p-versionbits-warning.py ' ,
2016-03-29 11:14:47 +02:00
' importprunedfunds.py ' ,
2016-05-05 12:54:05 +02:00
' signmessages.py '
2015-08-26 12:05:36 +02:00
]
2016-04-09 21:14:18 +02:00
if ENABLE_ZMQ :
testScripts . append ( ' zmq_test.py ' )
2015-08-26 12:05:36 +02:00
testScriptsExt = [
2016-03-16 06:30:04 +01:00
' bip9-softforks.py ' ,
2015-06-28 20:42:17 +02:00
' bip65-cltv.py ' ,
2016-10-22 18:52:14 +02:00
' bip65-cltv-p2p.py ' , # NOTE: needs dash_hash to pass
2016-02-13 16:42:24 +01:00
' bip68-sequence.py ' ,
2016-10-22 18:52:14 +02:00
' bipdersig-p2p.py ' , # NOTE: needs dash_hash to pass
2015-08-26 12:05:36 +02:00
' bipdersig.py ' ,
2016-03-06 16:14:39 +01:00
' getblocktemplate_longpoll.py ' , # FIXME: "socket.error: [Errno 54] Connection reset by peer" on my Mac, same as https://github.com/bitcoin/bitcoin/issues/6651
2015-08-26 12:05:36 +02:00
' getblocktemplate_proposals.py ' ,
' txn_doublespend.py ' ,
' txn_clone.py --mineblock ' ,
' forknotify.py ' ,
' invalidateblock.py ' ,
2015-10-15 20:37:59 +02:00
# 'rpcbind_test.py', #temporary, bug in libevent, see #6655
2015-08-26 12:05:36 +02:00
' smartfees.py ' ,
' maxblocksinflight.py ' ,
2016-10-22 18:52:14 +02:00
' p2p-acceptblock.py ' , # NOTE: needs dash_hash to pass
2015-08-26 12:05:36 +02:00
' mempool_packages.py ' ,
2015-09-18 21:59:55 +02:00
' maxuploadtarget.py ' ,
2016-07-29 07:30:19 +02:00
# 'replace-by-fee.py', # RBF is disabled in Dash Core
2016-03-21 18:02:47 +01:00
' p2p-feefilter.py ' ,
# 'pruning.py', # leave pruning last as it takes a REALLY long time #### Prune mode is incompatible with -txindex.
2015-08-26 12:05:36 +02:00
]
2015-10-11 07:41:19 +02:00
def runtests ( ) :
2016-05-09 17:01:55 +02:00
test_list = [ ]
if ' -extended ' in opts :
test_list = testScripts + testScriptsExt
elif len ( opts ) == 0 or ( len ( opts ) == 1 and " -win " in opts ) :
test_list = testScripts
else :
for t in testScripts + testScriptsExt :
if t in opts or re . sub ( " .py$ " , " " , t ) in opts :
test_list . append ( t )
if print_help :
# Only print help of the first script and exit
subprocess . check_call ( RPC_TESTS_DIR + test_list [ 0 ] + ' -h ' , shell = True )
sys . exit ( 0 )
2015-10-11 07:41:19 +02:00
coverage = None
if ENABLE_COVERAGE :
coverage = RPCCoverage ( )
2015-11-30 14:53:07 +01:00
print ( " Initializing coverage directory at %s \n " % coverage . dir )
2016-05-09 17:01:55 +02:00
flags = " --srcdir %s /src %s %s " % ( BUILDDIR , coverage . flag if coverage else ' ' , passon_args )
2016-04-09 21:14:18 +02:00
#Run Tests
2016-05-09 17:01:55 +02:00
for t in test_list :
print ( " Running testscript %s %s %s ... " % ( BOLD [ 1 ] , t , BOLD [ 0 ] ) )
2016-04-09 21:14:18 +02:00
time0 = time . time ( )
subprocess . check_call (
2016-05-09 17:01:55 +02:00
RPC_TESTS_DIR + t + flags , shell = True )
2016-04-09 21:14:18 +02:00
print ( " Duration: %s s \n " % ( int ( time . time ( ) - time0 ) ) )
if coverage :
coverage . report_rpc_coverage ( )
print ( " Cleaning up coverage data " )
coverage . cleanup ( )
2015-10-11 07:41:19 +02:00
class RPCCoverage ( object ) :
"""
Coverage reporting utilities for pull - tester .
Coverage calculation works by having each test script subprocess write
coverage files into a particular directory . These files contain the RPC
commands invoked during testing , as well as a complete listing of RPC
commands per ` bitcoin - cli help ` ( ` rpc_interface . txt ` ) .
After all tests complete , the commands run are combined and diff ' d against
the complete list to calculate uncovered RPC commands .
See also : qa / rpc - tests / test_framework / coverage . py
"""
def __init__ ( self ) :
self . dir = tempfile . mkdtemp ( prefix = " coverage " )
self . flag = ' --coveragedir %s ' % self . dir
def report_rpc_coverage ( self ) :
"""
Print out RPC commands that were unexercised by tests .
"""
uncovered = self . _get_uncovered_rpc_commands ( )
if uncovered :
print ( " Uncovered RPC commands: " )
print ( " " . join ( ( " - %s \n " % i ) for i in sorted ( uncovered ) ) )
else :
print ( " All RPC commands covered. " )
def cleanup ( self ) :
return shutil . rmtree ( self . dir )
def _get_uncovered_rpc_commands ( self ) :
"""
Return a set of currently untested RPC commands .
"""
# This is shared from `qa/rpc-tests/test-framework/coverage.py`
REFERENCE_FILENAME = ' rpc_interface.txt '
COVERAGE_FILE_PREFIX = ' coverage. '
coverage_ref_filename = os . path . join ( self . dir , REFERENCE_FILENAME )
coverage_filenames = set ( )
all_cmds = set ( )
covered_cmds = set ( )
if not os . path . isfile ( coverage_ref_filename ) :
raise RuntimeError ( " No coverage reference found " )
with open ( coverage_ref_filename , ' r ' ) as f :
all_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
for root , dirs , files in os . walk ( self . dir ) :
for filename in files :
if filename . startswith ( COVERAGE_FILE_PREFIX ) :
coverage_filenames . add ( os . path . join ( root , filename ) )
for filename in coverage_filenames :
with open ( filename , ' r ' ) as f :
covered_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
return all_cmds - covered_cmds
if __name__ == ' __main__ ' :
runtests ( )