Add logging to bitcoin-util-test.py

- Use the python standard logging library
- Run all tests and report all failing test-cases (rather than stop after one test case fails)
- If output is different from expected output, log a contextual diff.

src/test/bctest.py

@@ -7,6 +7,8 @@ import os
 import json
 import sys
 import binascii
+import difflib
+import logging
 
 def parse_output(a, fmt):
 	if fmt == 'json': # json: compare parsed data
@@ -33,53 +35,70 @@ def bctest(testDir, testObj, exeext):
 	if "output_cmp" in testObj:
 		outputFn = testObj['output_cmp']
 		outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
+		try:
 			outputData = open(testDir + "/" + outputFn).read()
+		except:
+			logging.error("Output file " + outputFn + " can not be opened")
+			raise
 		if not outputData:
-			print("Output data missing for " + outputFn)
+			logging.error("Output data missing for " + outputFn)
-			sys.exit(1)
+			raise Exception
+
 	proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
 	try:
 		outs = proc.communicate(input=inputData)
 	except OSError:
-		print("OSError, Failed to execute " + execprog)
+		logging.error("OSError, Failed to execute " + execprog)
-		sys.exit(1)
+		raise
 
 	if outputData:
 		try:
 			a_parsed = parse_output(outs[0], outputType)
 		except Exception as e:
-			print('Error parsing command output as %s: %s' % (outputType,e))
+			logging.error('Error parsing command output as %s: %s' % (outputType,e))
-			sys.exit(1)
+			raise
 		try:
 			b_parsed = parse_output(outputData, outputType)
 		except Exception as e:
-			print('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
+			logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
-			sys.exit(1)
+			raise
 		if a_parsed != b_parsed:
-			print("Output data mismatch for " + outputFn + " (format " + outputType + ")")
+			logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
-			sys.exit(1)
+			raise Exception
 		if outs[0] != outputData:
-			print("Output formatting mismatch for " + outputFn + " (format " + outputType + ")")
+			error_message = "Output formatting mismatch for " + outputFn + ":\n"
-			sys.exit(1)
+			error_message += "".join(difflib.context_diff(outputData.splitlines(True),
+				                                          outs[0].splitlines(True),
+				                                          fromfile=outputFn,
+				                                          tofile="returned"))
+			logging.error(error_message)
+			raise Exception
 
 	wantRC = 0
 	if "return_code" in testObj:
 		wantRC = testObj['return_code']
 	if proc.returncode != wantRC:
-		print("Return code mismatch for " + outputFn)
+		logging.error("Return code mismatch for " + outputFn)
-		sys.exit(1)
+		raise Exception
 
-def bctester(testDir, input_basename, buildenv, verbose = False):
+def bctester(testDir, input_basename, buildenv):
 	input_filename = testDir + "/" + input_basename
 	raw_data = open(input_filename).read()
 	input_data = json.loads(raw_data)
 
-	for testObj in input_data:
+	failed_testcases = []
-		if verbose and "description" in testObj:
-			print ("Testing: " + testObj["description"])
-		bctest(testDir, testObj, buildenv.exeext)
-		if verbose and "description" in testObj:
-			print ("PASS")
 
+	for testObj in input_data:
+		try:
+			bctest(testDir, testObj, buildenv.exeext)
+			logging.info("PASSED: " + testObj["description"])
+		except:
+			logging.info("FAILED: " + testObj["description"])
+			failed_testcases.append(testObj["description"])
+
+	if failed_testcases:
+		logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]")
+		sys.exit(1)
+	else:
 		sys.exit(0)
 

src/test/bitcoin-util-test.py

@@ -4,9 +4,11 @@
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 from __future__ import division,print_function,unicode_literals
 import os
+import sys
 import bctest
 import buildenv
 import argparse
+import logging
 
 help_text="""Test framework for bitcoin utils.
 
@@ -19,9 +21,9 @@ test/bitcoin-util-test.py --src=[srcdir]
 
 
 if __name__ == '__main__':
-    verbose = False
     try:
         srcdir = os.environ["srcdir"]
+        verbose = False
     except:
         parser = argparse.ArgumentParser(description=help_text)
         parser.add_argument('-s', '--srcdir')
@@ -29,4 +31,13 @@ if __name__ == '__main__':
         args = parser.parse_args()
         srcdir = args.srcdir
         verbose = args.verbose
-    bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv, verbose = verbose)
+
+    if verbose:
+        level = logging.DEBUG
+    else:
+        level = logging.ERROR
+    formatter = '%(asctime)s - %(levelname)s - %(message)s'
+    # Add the format/level to the logger
+    logging.basicConfig(format = formatter, level=level)
+
+    bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv)