Merge #10331: Share config between util and functional tests

8ad5bde Merge bctest.py into bitcoin-util-test.py (John Newbery)
95836c5 Use shared config file for functional and util tests (John Newbery)
89fcd35 Use an .ini config file for environment vars in bitcoin-util-test.py (John Newbery)
e9265df Change help_text in bitcoin-util-test.py to a docstring. (John Newbery)
ce58e93 Change bitcoin-util-test.py to use Python3 (John Newbery)

Tree-SHA512: 66dab0b4a8546aee0dfaef134a165f1447aff4c0ec335754bbc7d9e55909721c62f09cdbf4b22d02ac1fcd5a9b66780f91e1cc4d8687fae7288cc9072a23a78f

.gitignore

@@ -88,7 +88,6 @@ Dash-Qt.app
 # Unit-tests
 Makefile.test
 dash-qt_test
-src/test/buildenv.py
 
 # Resources cpp
 qrc_*.cpp
@@ -109,8 +108,7 @@ coverage_percent.txt
 linux-coverage-build
 linux-build
 win32-build
-test/functional/config.ini
+test/config.ini
-test/util/buildenv.py
 test/cache/*
 
 !src/leveldb*/Makefile

Makefile.am

@@ -223,7 +223,6 @@ dist_noinst_SCRIPTS = autogen.sh
 EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
 
 EXTRA_DIST += \
-    test/util/bctest.py \
     test/util/bitcoin-util-test.py \
     test/util/data/bitcoin-util-test.json \
     test/util/data/blanktxv1.hex \
@@ -265,9 +264,6 @@ EXTRA_DIST += \
 
 CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER)
 
-# This file is problematic for out-of-tree builds if it exists.
-DISTCLEANFILES = test/util/buildenv.pyc
-
 .INTERMEDIATE: $(COVERAGE_INFO)
 
 DISTCHECK_CONFIGURE_FLAGS = --enable-man

configure.ac

@@ -1218,13 +1218,11 @@ AC_SUBST(ZMQ_LIBS)
 AC_SUBST(PROTOBUF_LIBS)
 AC_SUBST(QR_LIBS)
 AC_SUBST(DSYMUTIL_FLAT)
-AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini])
+AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini])
-AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py])
 AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh])
 AC_CONFIG_FILES([doc/Doxyfile])
 AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py])
 AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py])
-AC_CONFIG_LINKS([test/util/bctest.py:test/util/bctest.py])
 
 dnl boost's m4 checks do something really nasty: they export these vars. As a
 dnl result, they leak into secp256k1's configure and crazy things happen.
@@ -1272,8 +1270,8 @@ esac
 dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows
 case ${OS} in
    *Windows*)
-     sed  's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/'  test/functional/config.ini > test/functional/config-2.ini
+     sed  's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/'  test/config.ini > test/config-2.ini
-     mv test/functional/config-2.ini test/functional/config.ini
+     mv test/config-2.ini test/config.ini
    ;;
 esac
 

src/Makefile.test.include

@@ -142,7 +142,7 @@ dash_test_clean : FORCE
 
 check-local:
 	@echo "Running test/util/bitcoin-util-test.py..."
-	$(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py
+	$(top_builddir)/test/util/bitcoin-util-test.py
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check
 if EMBEDDED_UNIVALUE
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check

test/config.ini.in

@@ -3,7 +3,7 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 # These environment variables are set by the build process and read by
-# test/functional/test_runner.py
+# test/functional/test_runner.py and test/util/bitcoin-util-test.py
 
 [environment]
 SRCDIR=@abs_top_srcdir@

test/functional/test_runner.py

@@ -194,7 +194,7 @@ def main():
 
     # Read config generated by configure.
     config = configparser.ConfigParser()
-    configfile = os.path.abspath(os.path.dirname(__file__)) + "/config.ini"
+    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
     config.read_file(open(configfile))
 
     passon_args.append("--configfile=%s" % configfile)

test/util/bctest.py

@@ -1,139 +0,0 @@
-# Copyright 2014 BitPay Inc.
-# Copyright 2016 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-from __future__ import division,print_function,unicode_literals
-import subprocess
-import os
-import json
-import sys
-import binascii
-import difflib
-import logging
-import pprint
-
-def parse_output(a, fmt):
-    """Parse the output according to specified format.
-
-    Raise an error if the output can't be parsed."""
-    if fmt == 'json': # json: compare parsed data
-        return json.loads(a)
-    elif fmt == 'hex': # hex: parse and compare binary data
-        return binascii.a2b_hex(a.strip())
-    else:
-        raise NotImplementedError("Don't know how to compare %s" % fmt)
-
-def bctest(testDir, testObj, buildenv):
-    """Runs a single test, comparing output and RC to expected output and RC.
-
-    Raises an error if input can't be read, executable fails, or output/RC
-    are not as expected. Error is caught by bctester() and reported.
-    """
-    # Get the exec names and arguments
-    execprog = buildenv.BUILDDIR + "/src/" + testObj['exec'] + buildenv.exeext
-    execargs = testObj['args']
-    execrun = [execprog] + execargs
-
-    # Read the input data (if there is any)
-    stdinCfg = None
-    inputData = None
-    if "input" in testObj:
-        filename = testDir + "/" + testObj['input']
-        inputData = open(filename).read()
-        stdinCfg = subprocess.PIPE
-
-    # Read the expected output data (if there is any)
-    outputFn = None
-    outputData = None
-    if "output_cmp" in testObj:
-        outputFn = testObj['output_cmp']
-        outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
-        try:
-            outputData = open(testDir + "/" + outputFn).read()
-        except:
-            logging.error("Output file " + outputFn + " can not be opened")
-            raise
-        if not outputData:
-            logging.error("Output data missing for " + outputFn)
-            raise Exception
-
-    # Run the test
-    proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
-    try:
-        outs = proc.communicate(input=inputData)
-    except OSError:
-        logging.error("OSError, Failed to execute " + execprog)
-        raise
-
-    if outputData:
-        data_mismatch, formatting_mismatch = False, False
-        # Parse command output and expected output
-        try:
-            a_parsed = parse_output(outs[0], outputType)
-        except Exception as e:
-            logging.error('Error parsing command output as %s: %s' % (outputType,e))
-            raise
-        try:
-            b_parsed = parse_output(outputData, outputType)
-        except Exception as e:
-            logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
-            raise
-        # Compare data
-        if a_parsed != b_parsed:
-            logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
-            data_mismatch = True
-        # Compare formatting
-        if outs[0] != outputData:
-            error_message = "Output formatting mismatch for " + outputFn + ":\n"
-            error_message += "".join(difflib.context_diff(outputData.splitlines(True),
-                                                          outs[0].splitlines(True),
-                                                          fromfile=outputFn,
-                                                          tofile="returned"))
-            logging.error(error_message)
-            formatting_mismatch = True
-
-        assert not data_mismatch and not formatting_mismatch
-
-    # Compare the return code to the expected return code
-    wantRC = 0
-    if "return_code" in testObj:
-        wantRC = testObj['return_code']
-    if proc.returncode != wantRC:
-        logging.error("Return code mismatch for " + outputFn)
-        raise Exception
-
-    if "error_txt" in testObj:
-        want_error = testObj["error_txt"]
-        # Compare error text
-        # TODO: ideally, we'd compare the strings exactly and also assert
-        # That stderr is empty if no errors are expected. However, bitcoin-tx
-        # emits DISPLAY errors when running as a windows application on
-        # linux through wine. Just assert that the expected error text appears
-        # somewhere in stderr.
-        if want_error not in outs[1]:
-            logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
-            raise Exception
-
-def bctester(testDir, input_basename, buildenv):
-    """ Loads and parses the input file, runs all tests and reports results"""
-    input_filename = testDir + "/" + input_basename
-    raw_data = open(input_filename).read()
-    input_data = json.loads(raw_data)
-
-    failed_testcases = []
-
-    for testObj in input_data:
-        try:
-            bctest(testDir, testObj, buildenv)
-            logging.info("PASSED: " + testObj["description"])
-        except:
-            logging.info("FAILED: " + testObj["description"])
-            failed_testcases.append(testObj["description"])
-
-    if failed_testcases:
-        error_message = "FAILED_TESTCASES:\n"
-        error_message += pprint.pformat(failed_testcases, width=400)
-        logging.error(error_message)
-        sys.exit(1)
-    else:
-        sys.exit(0)

test/util/bitcoin-util-test.py

@@ -1,26 +1,30 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # Copyright 2014 BitPay Inc.
-# Copyright 2016 The Bitcoin Core developers
+# Copyright 2016-2017 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
-from __future__ import division,print_function,unicode_literals
+"""Test framework for bitcoin utils.
-import os
-import sys
-import argparse
-import logging
 
-help_text="""Test framework for bitcoin utils.
+Runs automatically during `make check`.
-
-Runs automatically during `make check`. 
 
 Can also be run manually."""
 
-if __name__ == '__main__':
+import argparse
-    sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+import binascii
-    import buildenv
+import configparser
-    import bctest
+import difflib
+import json
+import logging
+import os
+import pprint
+import subprocess
+import sys
 
-    parser = argparse.ArgumentParser(description=help_text)
+def main():
+    config = configparser.ConfigParser()
+    config.read_file(open(os.path.dirname(__file__) + "/../config.ini"))
+
+    parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('-v', '--verbose', action='store_true')
     args = parser.parse_args()
     verbose = args.verbose
@@ -31,6 +35,135 @@ if __name__ == '__main__':
         level = logging.ERROR
     formatter = '%(asctime)s - %(levelname)s - %(message)s'
     # Add the format/level to the logger
-    logging.basicConfig(format = formatter, level=level)
+    logging.basicConfig(format=formatter, level=level)
 
-    bctest.bctester(buildenv.SRCDIR + "/test/util/data", "bitcoin-util-test.json", buildenv)
+    bctester(config["environment"]["SRCDIR"] + "/test/util/data", "bitcoin-util-test.json", config["environment"])
+
+def bctester(testDir, input_basename, buildenv):
+    """ Loads and parses the input file, runs all tests and reports results"""
+    input_filename = testDir + "/" + input_basename
+    raw_data = open(input_filename).read()
+    input_data = json.loads(raw_data)
+
+    failed_testcases = []
+
+    for testObj in input_data:
+        try:
+            bctest(testDir, testObj, buildenv)
+            logging.info("PASSED: " + testObj["description"])
+        except:
+            logging.info("FAILED: " + testObj["description"])
+            failed_testcases.append(testObj["description"])
+
+    if failed_testcases:
+        error_message = "FAILED_TESTCASES:\n"
+        error_message += pprint.pformat(failed_testcases, width=400)
+        logging.error(error_message)
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+def bctest(testDir, testObj, buildenv):
+    """Runs a single test, comparing output and RC to expected output and RC.
+
+    Raises an error if input can't be read, executable fails, or output/RC
+    are not as expected. Error is caught by bctester() and reported.
+    """
+    # Get the exec names and arguments
+    execprog = buildenv["BUILDDIR"] + "/src/" + testObj['exec'] + buildenv["EXEEXT"]
+    execargs = testObj['args']
+    execrun = [execprog] + execargs
+
+    # Read the input data (if there is any)
+    stdinCfg = None
+    inputData = None
+    if "input" in testObj:
+        filename = testDir + "/" + testObj['input']
+        inputData = open(filename).read()
+        stdinCfg = subprocess.PIPE
+
+    # Read the expected output data (if there is any)
+    outputFn = None
+    outputData = None
+    if "output_cmp" in testObj:
+        outputFn = testObj['output_cmp']
+        outputType = os.path.splitext(outputFn)[1][1:]  # output type from file extension (determines how to compare)
+        try:
+            outputData = open(testDir + "/" + outputFn).read()
+        except:
+            logging.error("Output file " + outputFn + " can not be opened")
+            raise
+        if not outputData:
+            logging.error("Output data missing for " + outputFn)
+            raise Exception
+
+    # Run the test
+    proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+    try:
+        outs = proc.communicate(input=inputData)
+    except OSError:
+        logging.error("OSError, Failed to execute " + execprog)
+        raise
+
+    if outputData:
+        data_mismatch, formatting_mismatch = False, False
+        # Parse command output and expected output
+        try:
+            a_parsed = parse_output(outs[0], outputType)
+        except Exception as e:
+            logging.error('Error parsing command output as %s: %s' % (outputType, e))
+            raise
+        try:
+            b_parsed = parse_output(outputData, outputType)
+        except Exception as e:
+            logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
+            raise
+        # Compare data
+        if a_parsed != b_parsed:
+            logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
+            data_mismatch = True
+        # Compare formatting
+        if outs[0] != outputData:
+            error_message = "Output formatting mismatch for " + outputFn + ":\n"
+            error_message += "".join(difflib.context_diff(outputData.splitlines(True),
+                                                          outs[0].splitlines(True),
+                                                          fromfile=outputFn,
+                                                          tofile="returned"))
+            logging.error(error_message)
+            formatting_mismatch = True
+
+        assert not data_mismatch and not formatting_mismatch
+
+    # Compare the return code to the expected return code
+    wantRC = 0
+    if "return_code" in testObj:
+        wantRC = testObj['return_code']
+    if proc.returncode != wantRC:
+        logging.error("Return code mismatch for " + outputFn)
+        raise Exception
+
+    if "error_txt" in testObj:
+        want_error = testObj["error_txt"]
+        # Compare error text
+        # TODO: ideally, we'd compare the strings exactly and also assert
+        # That stderr is empty if no errors are expected. However, bitcoin-tx
+        # emits DISPLAY errors when running as a windows application on
+        # linux through wine. Just assert that the expected error text appears
+        # somewhere in stderr.
+        if want_error not in outs[1]:
+            logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
+            raise Exception
+
+def parse_output(a, fmt):
+    """Parse the output according to specified format.
+
+    Raise an error if the output can't be parsed."""
+    if fmt == 'json':  # json: compare parsed data
+        return json.loads(a)
+    elif fmt == 'hex':  # hex: parse and compare binary data
+        return binascii.a2b_hex(a.strip())
+    else:
+        raise NotImplementedError("Don't know how to compare %s" % fmt)
+
+if __name__ == '__main__':
+    main()

test/util/buildenv.py.in

@@ -1,4 +0,0 @@
-#!/usr/bin/env python
-exeext="@EXEEXT@"
-SRCDIR="@abs_top_srcdir@"
-BUILDDIR="@abs_top_builddir@"