2015-08-26 12:05:36 +02:00
#!/usr/bin/env python2
2015-12-13 14:51:43 +01:00
# Copyright (c) 2014-2015 The Bitcoin Core developers
2015-08-26 12:05:36 +02:00
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
2015-10-11 07:41:19 +02:00
"""
Run Regression Test Suite
This module calls down into individual test cases via subprocess . It will
forward all unrecognized arguments onto the individual test scripts , other
than :
- ` - extended ` : run the " extended " test suite in addition to the basic one .
- ` - win ` : signal that this is running in a Windows environment , and we
should run the tests .
- ` - - coverage ` : this generates a basic coverage report for the RPC
interface .
For a description of arguments recognized by test scripts , see
` qa / pull - tester / test_framework / test_framework . py : BitcoinTestFramework . main ` .
"""
2015-08-26 12:05:36 +02:00
import os
2015-11-30 14:53:07 +01:00
import time
2015-10-11 07:41:19 +02:00
import shutil
2015-08-26 12:05:36 +02:00
import sys
import subprocess
2015-10-11 07:41:19 +02:00
import tempfile
2015-08-26 12:05:36 +02:00
import re
2015-10-11 07:41:19 +02:00
2015-08-26 12:05:36 +02:00
from tests_config import *
#If imported values are not defined then set to zero (or disabled)
2016-03-19 21:36:32 +01:00
if ' ENABLE_WALLET ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_WALLET = 0
2016-03-19 21:36:32 +01:00
if ' ENABLE_BITCOIND ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_BITCOIND = 0
2016-03-19 21:36:32 +01:00
if ' ENABLE_UTILS ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_UTILS = 0
2016-03-19 21:36:32 +01:00
if ' ENABLE_ZMQ ' not in vars ( ) :
2015-08-26 12:05:36 +02:00
ENABLE_ZMQ = 0
2015-10-11 07:41:19 +02:00
ENABLE_COVERAGE = 0
2015-08-26 12:05:36 +02:00
#Create a set to store arguments and create the passOn string
2015-10-11 07:41:19 +02:00
opts = set ( )
2015-08-26 12:05:36 +02:00
passOn = " "
p = re . compile ( " ^-- " )
2015-10-11 07:41:19 +02:00
2015-11-30 14:53:07 +01:00
bold = ( " " , " " )
if ( os . name == ' posix ' ) :
bold = ( ' \033 [0m ' , ' \033 [1m ' )
2015-10-11 07:41:19 +02:00
for arg in sys . argv [ 1 : ] :
if arg == ' --coverage ' :
ENABLE_COVERAGE = 1
elif ( p . match ( arg ) or arg == " -h " ) :
passOn + = " " + arg
2015-08-26 12:05:36 +02:00
else :
2015-10-11 07:41:19 +02:00
opts . add ( arg )
2015-08-26 12:05:36 +02:00
#Set env vars
buildDir = BUILDDIR
2024-02-05 23:00:22 +01:00
if " NEOBYTESD " not in os . environ :
os . environ [ " NEOBYTESD " ] = buildDir + ' /src/neobytesd ' + EXEEXT
if " NEOBYTESCLI " not in os . environ :
os . environ [ " NEOBYTESCLI " ] = buildDir + ' /src/neobytes-cli ' + EXEEXT
2015-10-08 10:22:50 +02:00
2015-08-26 12:05:36 +02:00
if EXEEXT == " .exe " and " -win " not in opts :
2016-04-09 21:14:18 +02:00
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
print " Win tests currently disabled by default. Use -win option to enable "
2015-08-26 12:05:36 +02:00
sys . exit ( 0 )
2016-04-09 21:14:18 +02:00
if not ( ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1 ) :
print " No rpc tests to run. Wallet, utils, and bitcoind must all be enabled "
2015-08-26 12:05:36 +02:00
sys . exit ( 0 )
2016-04-09 21:14:18 +02:00
# python-zmq may not be installed. Handle this gracefully and with some helpful info
if ENABLE_ZMQ :
try :
import zmq
except ImportError as e :
print ( " ERROR: \" import zmq \" failed. Set ENABLE_ZMQ=0 or " \
" to run zmq tests, see dependency info in /qa/README.md. " )
raise e
2015-08-26 12:05:36 +02:00
#Tests
testScripts = [
2016-03-11 00:36:55 +01:00
' bip68-112-113-p2p.py ' ,
2015-08-26 12:05:36 +02:00
' wallet.py ' ,
' listtransactions.py ' ,
2015-11-14 21:44:35 +01:00
' receivedby.py ' ,
2015-08-26 12:05:36 +02:00
' mempool_resurrect_test.py ' ,
' txn_doublespend.py --mineblock ' ,
' txn_clone.py ' ,
' getchaintips.py ' ,
' rawtransactions.py ' ,
' rest.py ' ,
' mempool_spendcoinbase.py ' ,
2015-08-27 03:15:04 +02:00
' mempool_reorg.py ' ,
2015-11-30 15:42:27 +01:00
' mempool_limit.py ' ,
2015-08-26 12:05:36 +02:00
' httpbasics.py ' ,
2015-11-11 16:49:32 +01:00
' multi_rpc.py ' ,
2015-08-26 12:05:36 +02:00
' zapwallettxes.py ' ,
' proxy_test.py ' ,
' merkle_blocks.py ' ,
' fundrawtransaction.py ' ,
' signrawtransactions.py ' ,
' walletbackup.py ' ,
' nodehandling.py ' ,
' reindex.py ' ,
2016-03-08 22:15:49 +01:00
' addressindex.py ' ,
2016-03-22 23:11:04 +01:00
' timestampindex.py ' ,
2016-04-05 21:53:38 +02:00
' spentindex.py ' ,
2015-08-26 12:05:36 +02:00
' decodescript.py ' ,
2024-02-05 23:00:22 +01:00
' p2p-fullblocktest.py ' , # NOTE: needs neobytes_hash to pass
2015-10-08 10:22:50 +02:00
' blockchain.py ' ,
2015-11-09 08:40:46 +01:00
' disablewallet.py ' ,
2024-12-16 19:40:22 +01:00
' sendheaders.py ' , # NOTE: needs neobytes_hash to pass
2015-11-30 14:53:07 +01:00
' keypool.py ' ,
2015-11-19 02:55:52 +01:00
' prioritise_transaction.py ' ,
2024-02-05 23:00:22 +01:00
' invalidblockrequest.py ' , # NOTE: needs neobytes_hash to pass
' invalidtxrequest.py ' , # NOTE: needs neobytes_hash to pass
2016-01-07 22:31:12 +01:00
' abandonconflict.py ' ,
2016-03-15 17:09:16 +01:00
' p2p-versionbits-warning.py ' ,
2015-08-26 12:05:36 +02:00
]
2016-04-09 21:14:18 +02:00
if ENABLE_ZMQ :
testScripts . append ( ' zmq_test.py ' )
2015-08-26 12:05:36 +02:00
testScriptsExt = [
2016-03-16 06:30:04 +01:00
' bip9-softforks.py ' ,
2015-06-28 20:42:17 +02:00
' bip65-cltv.py ' ,
2024-02-05 23:00:22 +01:00
' bip65-cltv-p2p.py ' , # NOTE: needs neobytes_hash to pass
2016-02-13 16:42:24 +01:00
' bip68-sequence.py ' ,
2024-02-05 23:00:22 +01:00
' bipdersig-p2p.py ' , # NOTE: needs neobytes_hash to pass
2015-08-26 12:05:36 +02:00
' bipdersig.py ' ,
2016-03-06 16:14:39 +01:00
' getblocktemplate_longpoll.py ' , # FIXME: "socket.error: [Errno 54] Connection reset by peer" on my Mac, same as https://github.com/bitcoin/bitcoin/issues/6651
2015-08-26 12:05:36 +02:00
' getblocktemplate_proposals.py ' ,
' txn_doublespend.py ' ,
' txn_clone.py --mineblock ' ,
2016-03-06 16:14:39 +01:00
# 'pruning.py', # Prune mode is incompatible with -txindex.
2015-08-26 12:05:36 +02:00
' forknotify.py ' ,
' invalidateblock.py ' ,
2015-10-15 20:37:59 +02:00
# 'rpcbind_test.py', #temporary, bug in libevent, see #6655
2015-08-26 12:05:36 +02:00
' smartfees.py ' ,
' maxblocksinflight.py ' ,
2024-02-05 23:00:22 +01:00
' p2p-acceptblock.py ' , # NOTE: needs neobytes_hash to pass
2015-08-26 12:05:36 +02:00
' mempool_packages.py ' ,
2015-09-18 21:59:55 +02:00
' maxuploadtarget.py ' ,
2024-12-16 19:40:22 +01:00
# 'replace-by-fee.py', # RBF is disabled in Neobytes Core
2015-08-26 12:05:36 +02:00
]
2015-10-11 07:41:19 +02:00
def runtests ( ) :
coverage = None
if ENABLE_COVERAGE :
coverage = RPCCoverage ( )
2015-11-30 14:53:07 +01:00
print ( " Initializing coverage directory at %s \n " % coverage . dir )
2015-10-11 07:41:19 +02:00
2016-04-09 21:14:18 +02:00
rpcTestDir = buildDir + ' /qa/rpc-tests/ '
run_extended = ' -extended ' in opts
cov_flag = coverage . flag if coverage else ' '
flags = " --srcdir %s /src %s %s " % ( buildDir , cov_flag , passOn )
#Run Tests
for i in range ( len ( testScripts ) ) :
if ( len ( opts ) == 0
or ( len ( opts ) == 1 and " -win " in opts )
or run_extended
or testScripts [ i ] in opts
or re . sub ( " .py$ " , " " , testScripts [ i ] ) in opts ) :
print ( " Running testscript %s %s %s ... " % ( bold [ 1 ] , testScripts [ i ] , bold [ 0 ] ) )
time0 = time . time ( )
subprocess . check_call (
rpcTestDir + testScripts [ i ] + flags , shell = True )
print ( " Duration: %s s \n " % ( int ( time . time ( ) - time0 ) ) )
# exit if help is called so we print just one set of
# instructions
p = re . compile ( " -h| --help " )
if p . match ( passOn ) :
sys . exit ( 0 )
# Run Extended Tests
for i in range ( len ( testScriptsExt ) ) :
if ( run_extended or testScriptsExt [ i ] in opts
or re . sub ( " .py$ " , " " , testScriptsExt [ i ] ) in opts ) :
print (
" Running 2nd level testscript "
+ " %s %s %s ... " % ( bold [ 1 ] , testScriptsExt [ i ] , bold [ 0 ] ) )
time0 = time . time ( )
subprocess . check_call (
rpcTestDir + testScriptsExt [ i ] + flags , shell = True )
print ( " Duration: %s s \n " % ( int ( time . time ( ) - time0 ) ) )
if coverage :
coverage . report_rpc_coverage ( )
print ( " Cleaning up coverage data " )
coverage . cleanup ( )
2015-10-11 07:41:19 +02:00
class RPCCoverage ( object ) :
"""
Coverage reporting utilities for pull - tester .
Coverage calculation works by having each test script subprocess write
coverage files into a particular directory . These files contain the RPC
commands invoked during testing , as well as a complete listing of RPC
commands per ` bitcoin - cli help ` ( ` rpc_interface . txt ` ) .
After all tests complete , the commands run are combined and diff ' d against
the complete list to calculate uncovered RPC commands .
See also : qa / rpc - tests / test_framework / coverage . py
"""
def __init__ ( self ) :
self . dir = tempfile . mkdtemp ( prefix = " coverage " )
self . flag = ' --coveragedir %s ' % self . dir
def report_rpc_coverage ( self ) :
"""
Print out RPC commands that were unexercised by tests .
"""
uncovered = self . _get_uncovered_rpc_commands ( )
if uncovered :
print ( " Uncovered RPC commands: " )
print ( " " . join ( ( " - %s \n " % i ) for i in sorted ( uncovered ) ) )
else :
print ( " All RPC commands covered. " )
def cleanup ( self ) :
return shutil . rmtree ( self . dir )
def _get_uncovered_rpc_commands ( self ) :
"""
Return a set of currently untested RPC commands .
"""
# This is shared from `qa/rpc-tests/test-framework/coverage.py`
REFERENCE_FILENAME = ' rpc_interface.txt '
COVERAGE_FILE_PREFIX = ' coverage. '
coverage_ref_filename = os . path . join ( self . dir , REFERENCE_FILENAME )
coverage_filenames = set ( )
all_cmds = set ( )
covered_cmds = set ( )
if not os . path . isfile ( coverage_ref_filename ) :
raise RuntimeError ( " No coverage reference found " )
with open ( coverage_ref_filename , ' r ' ) as f :
all_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
for root , dirs , files in os . walk ( self . dir ) :
for filename in files :
if filename . startswith ( COVERAGE_FILE_PREFIX ) :
coverage_filenames . add ( os . path . join ( root , filename ) )
for filename in coverage_filenames :
with open ( filename , ' r ' ) as f :
covered_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
return all_cmds - covered_cmds
if __name__ == ' __main__ ' :
runtests ( )