2019-01-16 17:49:01 +01:00
#!/usr/bin/env python3
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
""" Run fuzz test targets.
"""
2020-04-17 21:44:29 +02:00
from concurrent . futures import ThreadPoolExecutor , as_completed
2019-01-16 17:49:01 +01:00
import argparse
import configparser
2020-04-17 21:44:29 +02:00
import logging
2019-01-16 17:49:01 +01:00
import os
import subprocess
2020-04-17 21:44:29 +02:00
import sys
2019-01-16 17:49:01 +01:00
def main ( ) :
2020-03-09 16:21:27 +01:00
parser = argparse . ArgumentParser (
formatter_class = argparse . ArgumentDefaultsHelpFormatter ,
description = ''' Run the fuzz targets with all inputs from the seed_dir once. ''' ,
)
2019-01-16 17:49:01 +01:00
parser . add_argument (
" -l " ,
" --loglevel " ,
dest = " loglevel " ,
default = " INFO " ,
help = " log events at this level and higher to the console. Can be set to DEBUG, INFO, WARNING, ERROR or CRITICAL. Passing --loglevel DEBUG will output all logs to console. " ,
)
2020-02-16 18:47:50 +01:00
parser . add_argument (
' --valgrind ' ,
action = ' store_true ' ,
2020-02-17 14:34:51 +01:00
help = ' If true, run fuzzing binaries under the valgrind memory error detector ' ,
)
parser . add_argument (
' -x ' ,
' --exclude ' ,
help = " A comma-separated list of targets to exclude " ,
2020-02-16 18:47:50 +01:00
)
2020-04-17 21:44:29 +02:00
parser . add_argument (
' --par ' ,
2020-05-09 21:55:29 +02:00
' -j ' ,
2020-04-17 21:44:29 +02:00
type = int ,
default = 4 ,
help = ' How many targets to merge or execute in parallel. ' ,
)
2019-01-16 17:49:01 +01:00
parser . add_argument (
' seed_dir ' ,
help = ' The seed corpus to run on (must contain subfolders for each fuzz target). ' ,
)
parser . add_argument (
' target ' ,
nargs = ' * ' ,
help = ' The target(s) to run. Default is to run all targets. ' ,
)
2020-03-09 16:21:27 +01:00
parser . add_argument (
' --m_dir ' ,
help = ' Merge inputs from this directory into the seed_dir. Needs /target subdirectory. ' ,
)
2020-08-04 18:42:29 +02:00
parser . add_argument (
' -g ' ,
' --generate ' ,
action = ' store_true ' ,
help = ' Create new corpus seeds (or extend the existing ones) by running '
' the given targets for a finite number of times. Outputs them to '
' the passed seed_dir. '
)
2019-01-16 17:49:01 +01:00
args = parser . parse_args ( )
# Set up logging
logging . basicConfig (
format = ' %(message)s ' ,
level = int ( args . loglevel ) if args . loglevel . isdigit ( ) else args . loglevel . upper ( ) ,
)
# Read config generated by configure.
config = configparser . ConfigParser ( )
configfile = os . path . abspath ( os . path . dirname ( __file__ ) ) + " /../config.ini "
config . read_file ( open ( configfile , encoding = " utf8 " ) )
if not config [ " components " ] . getboolean ( " ENABLE_FUZZ " ) :
logging . error ( " Must have fuzz targets built " )
sys . exit ( 1 )
# Build list of tests
2022-09-19 09:03:12 +02:00
test_list_all = parse_test_list ( fuzz_bin = os . path . join ( config [ " environment " ] [ " BUILDDIR " ] , ' src ' , ' test ' , ' fuzz ' , ' fuzz ' ) )
2019-01-16 17:49:01 +01:00
if not test_list_all :
logging . error ( " No fuzz targets found " )
sys . exit ( 1 )
2020-02-17 14:34:51 +01:00
logging . debug ( " {} fuzz target(s) found: {} " . format ( len ( test_list_all ) , " " . join ( sorted ( test_list_all ) ) ) )
2019-01-16 17:49:01 +01:00
args . target = args . target or test_list_all # By default run all
test_list_error = list ( set ( args . target ) . difference ( set ( test_list_all ) ) )
if test_list_error :
logging . error ( " Unknown fuzz targets selected: {} " . format ( test_list_error ) )
test_list_selection = list ( set ( test_list_all ) . intersection ( set ( args . target ) ) )
if not test_list_selection :
logging . error ( " No fuzz targets selected " )
2020-02-17 14:34:51 +01:00
if args . exclude :
for excluded_target in args . exclude . split ( " , " ) :
if excluded_target not in test_list_selection :
logging . error ( " Target \" {} \" not found in current target list. " . format ( excluded_target ) )
continue
test_list_selection . remove ( excluded_target )
test_list_selection . sort ( )
logging . info ( " {} of {} detected fuzz target(s) selected: {} " . format ( len ( test_list_selection ) , len ( test_list_all ) , " " . join ( test_list_selection ) ) )
2019-01-16 17:49:01 +01:00
2020-08-04 18:42:29 +02:00
if not args . generate :
test_list_seedless = [ ]
for t in test_list_selection :
corpus_path = os . path . join ( args . seed_dir , t )
if not os . path . exists ( corpus_path ) or len ( os . listdir ( corpus_path ) ) == 0 :
test_list_seedless . append ( t )
test_list_seedless . sort ( )
if test_list_seedless :
logging . info (
" Fuzzing harnesses lacking a seed corpus: {} " . format (
" " . join ( test_list_seedless )
)
2020-03-09 18:20:52 +01:00
)
2020-08-04 18:42:29 +02:00
logging . info ( " Please consider adding a fuzz seed corpus at https://github.com/bitcoin-core/qa-assets " )
2020-03-09 18:20:52 +01:00
2019-01-16 17:49:01 +01:00
try :
help_output = subprocess . run (
args = [
2022-09-19 09:03:12 +02:00
os . path . join ( config [ " environment " ] [ " BUILDDIR " ] , ' src ' , ' test ' , ' fuzz ' , ' fuzz ' ) ,
2019-01-16 17:49:01 +01:00
' -help=1 ' ,
] ,
2022-09-19 09:03:12 +02:00
env = {
' FUZZ ' : test_list_selection [ 0 ]
} ,
2020-03-02 22:26:49 +01:00
timeout = 20 ,
2019-01-16 17:49:01 +01:00
check = True ,
stderr = subprocess . PIPE ,
universal_newlines = True ,
) . stderr
if " libFuzzer " not in help_output :
logging . error ( " Must be built with libFuzzer " )
sys . exit ( 1 )
except subprocess . TimeoutExpired :
logging . error ( " subprocess timed out: Currently only libFuzzer is supported " )
sys . exit ( 1 )
2020-04-17 21:44:29 +02:00
with ThreadPoolExecutor ( max_workers = args . par ) as fuzz_pool :
2020-08-04 18:42:29 +02:00
if args . generate :
return generate_corpus_seeds (
fuzz_pool = fuzz_pool ,
build_dir = config [ " environment " ] [ " BUILDDIR " ] ,
seed_dir = args . seed_dir ,
targets = test_list_selection ,
)
2020-04-17 21:44:29 +02:00
if args . m_dir :
merge_inputs (
fuzz_pool = fuzz_pool ,
corpus = args . seed_dir ,
test_list = test_list_selection ,
build_dir = config [ " environment " ] [ " BUILDDIR " ] ,
merge_dir = args . m_dir ,
)
return
run_once (
fuzz_pool = fuzz_pool ,
2020-03-09 16:21:27 +01:00
corpus = args . seed_dir ,
test_list = test_list_selection ,
build_dir = config [ " environment " ] [ " BUILDDIR " ] ,
2020-04-17 21:44:29 +02:00
use_valgrind = args . valgrind ,
2020-03-09 16:21:27 +01:00
)
2019-01-16 17:49:01 +01:00
2020-08-04 18:42:29 +02:00
def generate_corpus_seeds ( * , fuzz_pool , build_dir , seed_dir , targets ) :
""" Generates new corpus seeds.
Run { targets } without input , and outputs the generated corpus seeds to
{ seed_dir } .
"""
logging . info ( " Generating corpus seeds to {} " . format ( seed_dir ) )
2022-09-19 09:03:12 +02:00
def job ( command , t ) :
2020-08-04 18:42:29 +02:00
logging . debug ( " Running ' {} ' \n " . format ( " " . join ( command ) ) )
logging . debug ( " Command ' {} ' output: \n ' {} ' \n " . format (
' ' . join ( command ) ,
2022-09-19 09:03:12 +02:00
subprocess . run (
command ,
env = {
' FUZZ ' : t
} ,
check = True ,
stderr = subprocess . PIPE ,
universal_newlines = True ,
) . stderr ) )
2020-08-04 18:42:29 +02:00
futures = [ ]
for target in targets :
target_seed_dir = os . path . join ( seed_dir , target )
os . makedirs ( target_seed_dir , exist_ok = True )
command = [
2022-09-19 09:03:12 +02:00
os . path . join ( build_dir , ' src ' , ' test ' , ' fuzz ' , ' fuzz ' ) ,
2020-08-04 18:42:29 +02:00
" -runs=100000 " ,
target_seed_dir ,
]
2022-09-19 09:03:12 +02:00
futures . append ( fuzz_pool . submit ( job , command , target ) )
2020-08-04 18:42:29 +02:00
for future in as_completed ( futures ) :
future . result ( )
2020-04-17 21:44:29 +02:00
def merge_inputs ( * , fuzz_pool , corpus , test_list , build_dir , merge_dir ) :
2020-03-09 16:21:27 +01:00
logging . info ( " Merge the inputs in the passed dir into the seed_dir. Passed dir {} " . format ( merge_dir ) )
2020-04-17 21:44:29 +02:00
jobs = [ ]
2020-03-09 16:21:27 +01:00
for t in test_list :
args = [
2022-09-19 09:03:12 +02:00
os . path . join ( build_dir , ' src ' , ' test ' , ' fuzz ' , ' fuzz ' ) ,
2020-03-09 16:21:27 +01:00
' -merge=1 ' ,
2020-04-08 18:52:43 +02:00
' -use_value_profile=1 ' , # Also done by oss-fuzz https://github.com/google/oss-fuzz/issues/1406#issuecomment-387790487
2020-03-09 16:21:27 +01:00
os . path . join ( corpus , t ) ,
os . path . join ( merge_dir , t ) ,
]
os . makedirs ( os . path . join ( corpus , t ) , exist_ok = True )
os . makedirs ( os . path . join ( merge_dir , t ) , exist_ok = True )
2020-04-17 21:44:29 +02:00
def job ( t , args ) :
output = ' Run {} with args {} \n ' . format ( t , " " . join ( args ) )
2022-09-19 09:03:12 +02:00
output + = subprocess . run (
args ,
env = {
' FUZZ ' : t
} ,
check = True ,
stderr = subprocess . PIPE ,
universal_newlines = True ,
) . stderr
2020-04-17 21:44:29 +02:00
logging . debug ( output )
jobs . append ( fuzz_pool . submit ( job , t , args ) )
for future in as_completed ( jobs ) :
future . result ( )
2020-03-09 16:21:27 +01:00
2020-04-17 21:44:29 +02:00
def run_once ( * , fuzz_pool , corpus , test_list , build_dir , use_valgrind ) :
jobs = [ ]
2019-01-16 17:49:01 +01:00
for t in test_list :
2021-08-11 05:52:06 +02:00
corpus_path = os . path . join ( corpus , t )
2020-03-09 18:20:52 +01:00
os . makedirs ( corpus_path , exist_ok = True )
2019-01-16 17:49:01 +01:00
args = [
2022-09-19 09:03:12 +02:00
os . path . join ( build_dir , ' src ' , ' test ' , ' fuzz ' , ' fuzz ' ) ,
2019-01-16 17:49:01 +01:00
' -runs=1 ' ,
2021-08-11 05:52:06 +02:00
corpus_path ,
2019-01-16 17:49:01 +01:00
]
2020-02-16 18:47:50 +01:00
if use_valgrind :
2020-02-17 14:34:51 +01:00
args = [ ' valgrind ' , ' --quiet ' , ' --error-exitcode=1 ' ] + args
2020-04-17 21:44:29 +02:00
def job ( t , args ) :
output = ' Run {} with args {} ' . format ( t , args )
2022-09-19 09:03:12 +02:00
result = subprocess . run ( args , env = { ' FUZZ ' : t } , stderr = subprocess . PIPE , universal_newlines = True )
2020-04-17 21:44:29 +02:00
output + = result . stderr
return output , result
jobs . append ( fuzz_pool . submit ( job , t , args ) )
for future in as_completed ( jobs ) :
output , result = future . result ( )
logging . debug ( output )
2020-02-17 14:34:51 +01:00
try :
result . check_returncode ( )
2020-02-19 17:20:27 +01:00
except subprocess . CalledProcessError as e :
if e . stdout :
logging . info ( e . stdout )
if e . stderr :
logging . info ( e . stderr )
logging . info ( " Target \" {} \" failed with exit code {} : {} " . format ( t , e . returncode , " " . join ( args ) ) )
sys . exit ( 1 )
2020-02-17 14:34:51 +01:00
except subprocess . CalledProcessError as e :
if e . stdout :
logging . info ( e . stdout )
if e . stderr :
logging . info ( e . stderr )
2020-04-17 21:44:29 +02:00
logging . info ( " Target \" {} \" failed with exit code {} " . format ( " " . join ( result . args ) , e . returncode ) )
2020-02-17 14:34:51 +01:00
sys . exit ( 1 )
2019-01-16 17:49:01 +01:00
2022-09-19 09:03:12 +02:00
def parse_test_list ( * , fuzz_bin ) :
test_list_all = subprocess . run (
fuzz_bin ,
env = {
' PRINT_ALL_FUZZ_TARGETS_AND_ABORT ' : ' '
} ,
stdout = subprocess . PIPE ,
stderr = subprocess . DEVNULL ,
universal_newlines = True ,
) . stdout . splitlines ( )
2019-01-16 17:49:01 +01:00
return test_list_all
if __name__ == ' __main__ ' :
main ( )