bitcoin#15295: Add test/fuzz/test_runner.py and run it in travis

.travis.yml

@@ -59,6 +59,7 @@ builddocker: &builddocker
     - RUN_UNIT_TESTS=true
     - RUN_FUNCTIONAL_TESTS=true
     - RUN_BENCH=false  # Set to true for any one job that has debug enabled, to quickly check bench is not crashing or hitting assertions
+    - RUN_FUZZ_TESTS=false
     - DOCKER_NAME_TAG=ubuntu:18.04
     - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
     - CCACHE_SIZE=100M
@@ -252,7 +253,7 @@ after_success:
         PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev"
         DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
         GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-fuzz --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
+        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\""
 # x86_64 Linux (xenial, no depends, only system libs)
     - stage: test
       env: >-
@@ -281,6 +282,17 @@ after_success:
         GOAL="install"
         BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=undefined CC=clang CXX=clang++"
 # x86_64 Linux, No wallet
+    - stage: test
+      env: >-
+        HOST=x86_64-unknown-linux-gnu
+        PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
+        NO_DEPENDS=1
+        RUN_UNIT_TESTS=false
+        RUN_FUNCTIONAL_TESTS=false
+        RUN_FUZZ_TESTS=true
+        GOAL="install"
+        BITCOIN_CONFIG="--disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++"
+
     - stage: test
       env: >-
         HOST=x86_64-unknown-linux-gnu

Makefile.am

@@ -225,7 +225,11 @@ endif
 
 dist_noinst_SCRIPTS = autogen.sh
 
-EXTRA_DIST = $(DIST_SHARE) test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+EXTRA_DIST = $(DIST_SHARE) $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+
+EXTRA_DIST += \
+    test/functional \
+    test/fuzz
 
 EXTRA_DIST += \
     test/util/bitcoin-util-test.py \

doc/fuzzing.md

@@ -5,6 +5,29 @@ A special test harness in `src/test/fuzz/` is provided for each fuzz target to
 provide an easy entry point for fuzzers and the like. In this document we'll
 describe how to use it with AFL and libFuzzer.
 
+## Preparing fuzzing
+
+AFL needs an input directory with examples, and an output directory where it
+will place examples that it found. These can be anywhere in the file system,
+we'll define environment variables to make it easy to reference them.
+
+libFuzzer will use the input directory as output directory.
+
+Extract the example seeds (or other starting inputs) into the inputs
+directory before starting fuzzing.
+
+```
+git clone https://github.com/bitcoin-core/qa-assets
+export DIR_FUZZ_IN=$PWD/qa-assets/fuzz_seed_corpus
+```
+
+Only for AFL:
+
+```
+mkdir outputs
+export AFLOUT=$PWD/outputs
+```
+
 ## AFL
 
 ### Building AFL
@@ -23,7 +46,7 @@ export AFLPATH=$PWD
 To build Dash Core using AFL instrumentation (this assumes that the
 `AFLPATH` was set as above):
 ```
-./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
+./configure --disable-ccache --disable-shared --enable-tests --enable-fuzz --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no CC=${AFLPATH}/afl-gcc CXX=${AFLPATH}/afl-g++
 export AFL_HARDEN=1
 cd src/
 make
@@ -39,31 +62,14 @@ binary will be instrumented in such a way that the AFL
 features "persistent mode" and "deferred forkserver" can be used. See
 https://github.com/mcarpenter/afl/tree/master/llvm_mode for details.
 
-### Preparing fuzzing
-
-AFL needs an input directory with examples, and an output directory where it
-will place examples that it found. These can be anywhere in the file system,
-we'll define environment variables to make it easy to reference them.
-
-```
-mkdir inputs
-AFLIN=$PWD/inputs
-mkdir outputs
-AFLOUT=$PWD/outputs
-```
-
-Example inputs are available from:
-
-- https://download.visucore.com/bitcoin/bitcoin_fuzzy_in.tar.xz
-- http://strateman.ninja/fuzzing.tar.xz
-
-Extract these (or other starting inputs) into the `inputs` directory before starting fuzzing.
-
 ### Fuzzing
 
 To start the actual fuzzing use:
+
 ```
-$AFLPATH/afl-fuzz -i ${AFLIN} -o ${AFLOUT} -m52 -- test/fuzz/fuzz_target_foo
+export FUZZ_TARGET=fuzz_target_foo  # Pick a fuzz_target
+mkdir ${AFLOUT}/${FUZZ_TARGET}
+$AFLPATH/afl-fuzz -i ${DIR_FUZZ_IN}/${FUZZ_TARGET} -o ${AFLOUT}/${FUZZ_TARGET} -m52 -- test/fuzz/${FUZZ_TARGET}
 ```
 
 You may have to change a few kernel parameters to test optimally - `afl-fuzz`
@@ -74,10 +80,10 @@ will print an error and suggestion if so.
 A recent version of `clang`, the address sanitizer and libFuzzer is needed (all
 found in the `compiler-rt` runtime libraries package).
 
-To build the `test/test_dash_fuzzy` executable run
+To build all fuzz targets with libFuzzer, run
 
 ```
-./configure --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
+./configure --disable-ccache --disable-wallet --disable-bench --with-utils=no --with-daemon=no --with-libs=no --with-gui=no --enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++
 make
 ```
 
@@ -86,3 +92,6 @@ interchangeably between libFuzzer and AFL.
 
 See https://llvm.org/docs/LibFuzzer.html#running on how to run the libFuzzer
 instrumented executable.
+
+Alternatively run the script in `./test/fuzz/test_runner.py` and provide it
+with the `${DIR_FUZZ_IN}` created earlier.

src/Makefile.test.include

@@ -3,7 +3,6 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-bin_PROGRAMS += test/test_dash
 
 FUZZ_TARGETS = \
   test/fuzz/address_deserialize \
@@ -29,6 +28,8 @@ FUZZ_TARGETS = \
 
 if ENABLE_FUZZ
 noinst_PROGRAMS += $(FUZZ_TARGETS:=)
+else
+bin_PROGRAMS += test/test_dash
 endif
 
 TEST_SRCDIR = test

test/config.ini.in

@@ -16,4 +16,5 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
 @ENABLE_WALLET_TRUE@ENABLE_WALLET=true
 @BUILD_BITCOIN_UTILS_TRUE@ENABLE_UTILS=true
 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true
+@ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true
 @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true

test/fuzz/test_runner.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Run fuzz test targets.
+"""
+
+import argparse
+import configparser
+import os
+import sys
+import subprocess
+import logging
+
+
+def main():
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        "-l",
+        "--loglevel",
+        dest="loglevel",
+        default="INFO",
+        help="log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console.",
+    )
+    parser.add_argument(
+        '--export_coverage',
+        action='store_true',
+        help='If true, export coverage information to files in the seed corpus',
+    )
+    parser.add_argument(
+        'seed_dir',
+        help='The seed corpus to run on (must contain subfolders for each fuzz target).',
+    )
+    parser.add_argument(
+        'target',
+        nargs='*',
+        help='The target(s) to run. Default is to run all targets.',
+    )
+
+    args = parser.parse_args()
+
+    # Set up logging
+    logging.basicConfig(
+        format='%(message)s',
+        level=int(args.loglevel) if args.loglevel.isdigit() else args.loglevel.upper(),
+    )
+
+    # Read config generated by configure.
+    config = configparser.ConfigParser()
+    configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
+    config.read_file(open(configfile, encoding="utf8"))
+
+    if not config["components"].getboolean("ENABLE_FUZZ"):
+        logging.error("Must have fuzz targets built")
+        sys.exit(1)
+
+    # Build list of tests
+    test_list_all = parse_test_list(makefile=os.path.join(config["environment"]["SRCDIR"], 'src', 'Makefile.test.include'))
+
+    if not test_list_all:
+        logging.error("No fuzz targets found")
+        sys.exit(1)
+
+    logging.info("Fuzz targets found: {}".format(test_list_all))
+
+    args.target = args.target or test_list_all  # By default run all
+    test_list_error = list(set(args.target).difference(set(test_list_all)))
+    if test_list_error:
+        logging.error("Unknown fuzz targets selected: {}".format(test_list_error))
+    test_list_selection = list(set(test_list_all).intersection(set(args.target)))
+    if not test_list_selection:
+        logging.error("No fuzz targets selected")
+    logging.info("Fuzz targets selected: {}".format(test_list_selection))
+
+    try:
+        help_output = subprocess.run(
+            args=[
+                os.path.join(config["environment"]["BUILDDIR"], 'src', 'test', 'fuzz', test_list_selection[0]),
+                '-help=1',
+            ],
+            timeout=1,
+            check=True,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+        ).stderr
+        if "libFuzzer" not in help_output:
+            logging.error("Must be built with libFuzzer")
+            sys.exit(1)
+    except subprocess.TimeoutExpired:
+        logging.error("subprocess timed out: Currently only libFuzzer is supported")
+        sys.exit(1)
+
+    run_once(
+        corpus=args.seed_dir,
+        test_list=test_list_selection,
+        build_dir=config["environment"]["BUILDDIR"],
+        export_coverage=args.export_coverage,
+    )
+
+
+def run_once(*, corpus, test_list, build_dir, export_coverage):
+    for t in test_list:
+        args = [
+            os.path.join(build_dir, 'src', 'test', 'fuzz', t),
+            '-runs=1',
+            os.path.join(corpus, t),
+        ]
+        logging.debug('Run {} with args {}'.format(t, args))
+        output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr
+        logging.debug('Output: {}'.format(output))
+        if not export_coverage:
+            continue
+        for l in output.splitlines():
+            if 'INITED' in l:
+                with open(os.path.join(corpus, t + '_coverage'), 'w', encoding='utf-8') as cov_file:
+                    cov_file.write(l)
+                    break
+
+
+def parse_test_list(makefile):
+    with open(makefile, encoding='utf-8') as makefile_test:
+        test_list_all = []
+        read_targets = False
+        for line in makefile_test.readlines():
+            line = line.strip().replace('test/fuzz/', '').replace(' \\', '')
+            if read_targets:
+                if not line:
+                    break
+                test_list_all.append(line)
+                continue
+
+            if line == 'FUZZ_TARGETS =':
+                read_targets = True
+    return test_list_all
+
+
+if __name__ == '__main__':
+    main()