diff --git a/.gitignore b/.gitignore index ebcbe3927c..682b74bb19 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,9 @@ dist/ *.background.tiff /guix-build-* + +# cppcheck cache-directory +.cppcheck/* + +# flake8 cache location +.cache/* diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index 313fb215c3..291b94e0cd 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -39,7 +39,8 @@ RUN pip3 install \ jinja2 \ pyzmq \ vulture==2.3 \ - yq + yq \ + multiprocess # dash_hash RUN git clone --depth 1 --no-tags https://github.com/dashpay/dash_hash diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index 98a1a24b28..50a2705d89 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -5,6 +5,7 @@ import sys import re +from multiprocess import Pool MAPPING = { 'core_read.cpp': 'core_io.cpp', @@ -37,7 +38,7 @@ if __name__=="__main__": RE = re.compile("^#include <(.*)>") - def handle_module(module): + def handle_module(arg): module = module_name(arg) if module is None: print("Ignoring file %s (does not constitute module)\n" % arg) @@ -45,6 +46,26 @@ if __name__=="__main__": files[arg] = module deps[module] = set() + def handle_module2(module): + # Build the transitive closure of dependencies of module + closure = dict() + for dep in deps[module]: + closure[dep] = [] + while True: + old_size = len(closure) + old_closure_keys = sorted(closure.keys()) + for src in old_closure_keys: + for dep in deps[src]: + if dep not in closure: + closure[dep] = closure[src] + [src] + if len(closure) == old_size: + break + # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest + if module in closure: + return [module] + closure[module] + + return None + # Iterate over files, and create list of modules for arg in sys.argv[1:]: @@ -71,32 +92,20 @@ if __name__=="__main__": def shortest_c_dep(): have_cycle = False - def handle_module(module, shortest_cycle): - - # Build the transitive closure of dependencies of module - closure = dict() - for dep in deps[module]: - closure[dep] = [] - while True: - old_size = len(closure) - old_closure_keys = sorted(closure.keys()) - for src in old_closure_keys: - for dep in deps[src]: - if dep not in closure: - closure[dep] = closure[src] + [src] - if len(closure) == old_size: - break - # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest - if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)): - shortest_cycle = [module] + closure[module] - - return shortest_cycle + sorted_keys = None while True: shortest_cycles = None - for module in sorted(deps.keys()): - shortest_cycles = handle_module(module, shortest_cycles) + if sorted_keys is None: + sorted_keys = sorted(deps.keys()) + + with Pool(8) as pool: + cycles = pool.map(handle_module2, sorted_keys) + + for cycle in cycles: + if cycle is not None and (shortest_cycles is None or len(cycle) < len(shortest_cycles)): + shortest_cycles = cycle if shortest_cycles is None: break @@ -104,7 +113,8 @@ if __name__=="__main__": module = shortest_cycles[0] print("Circular dependency: %s" % (" -> ".join(shortest_cycles + [module]))) # And then break the dependency to avoid repeating in other cycles - deps[shortest_cycles[-1]] = deps[shortest_cycles[-1]] - set([module]) + deps[shortest_cycles[-1]] -= {module} + sorted_keys = None have_cycle = True if have_cycle: diff --git a/test/lint/lint-cppcheck-dash.sh b/test/lint/lint-cppcheck-dash.sh index ec9f145648..72a9364772 100755 --- a/test/lint/lint-cppcheck-dash.sh +++ b/test/lint/lint-cppcheck-dash.sh @@ -48,6 +48,8 @@ IGNORED_WARNINGS=( # "Consider using std::count_if algorithm instead of a raw loop." # "Consider using std::find_if algorithm instead of a raw loop." # "Member variable '.*' is not initialized in the constructor." + + "unusedFunction" ) # We should attempt to update this with all dash specific code @@ -108,8 +110,14 @@ function join_array { ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") FILES_REGEXP=$(join_array "|" "${FILES[@]}") +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +CPPCHECK_DIR=$SCRIPT_DIR/.cppcheck/ +if [ ! -d $CPPCHECK_DIR ] +then + mkdir $CPPCHECK_DIR +fi WARNINGS=$(echo "${FILES}" | \ - xargs cppcheck --enable=all --inline-suppr -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \ + xargs cppcheck --enable=all --inline-suppr --cppcheck-build-dir=$CPPCHECK_DIR -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \ grep -E "${ENABLED_CHECKS_REGEXP}" | \ grep -vE "${IGNORED_WARNINGS_REGEXP}" | \ grep -E "${FILES_REGEXP}") diff --git a/test/lint/lint-format-strings.py b/test/lint/lint-format-strings.py index b6752e9bb8..66de402515 100755 --- a/test/lint/lint-format-strings.py +++ b/test/lint/lint-format-strings.py @@ -11,6 +11,8 @@ import argparse import re import sys +from functools import partial +from multiprocessing import Pool FALSE_POSITIVES = [ ("src/batchedlogger.h", "strprintf(fmt, args...)"), @@ -261,6 +263,28 @@ def count_format_specifiers(format_string): return n +def handle_filename(filename, args): + exit_code = 0 + with open(filename, "r", encoding="utf-8") as f: + for function_call_str in parse_function_calls(args.function_name, f.read()): + parts = parse_function_call_and_arguments(args.function_name, function_call_str) + relevant_function_call_str = unescape("".join(parts))[:512] + if (f.name, relevant_function_call_str) in FALSE_POSITIVES: + continue + if len(parts) < 3 + args.skip_arguments: + exit_code = 1 + print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) + continue + argument_count = len(parts) - 3 - args.skip_arguments + format_str = parse_string_content(parts[1 + args.skip_arguments]) + format_specifier_count = count_format_specifiers(format_str) + if format_specifier_count != argument_count: + exit_code = 1 + print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) + continue + return exit_code + + def main(): parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed " "to a variadic format string function matches the number of format " @@ -270,26 +294,12 @@ def main(): parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None) parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)") args = parser.parse_args() - exit_code = 0 - for filename in args.file: - with open(filename, "r", encoding="utf-8") as f: - for function_call_str in parse_function_calls(args.function_name, f.read()): - parts = parse_function_call_and_arguments(args.function_name, function_call_str) - relevant_function_call_str = unescape("".join(parts))[:512] - if (f.name, relevant_function_call_str) in FALSE_POSITIVES: - continue - if len(parts) < 3 + args.skip_arguments: - exit_code = 1 - print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str)) - continue - argument_count = len(parts) - 3 - args.skip_arguments - format_str = parse_string_content(parts[1 + args.skip_arguments]) - format_specifier_count = count_format_specifiers(format_str) - if format_specifier_count != argument_count: - exit_code = 1 - print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str)) - continue - sys.exit(exit_code) + exit_codes = [] + + with Pool(8) as pool: + exit_codes = pool.map(partial(handle_filename, args=args), args.file) + + sys.exit(max(exit_codes)) if __name__ == "__main__": diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh index 0e3730e46d..83b83e5e15 100755 --- a/test/lint/lint-python.sh +++ b/test/lint/lint-python.sh @@ -88,7 +88,15 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then exit 0 fi -PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$( +FLAKECMD=flake8 + +if command -v flake8-cached > /dev/null; then + FLAKECMD=flake8-cached +else + echo "Consider install flake8-cached for cached flake8 results." +fi + +PYTHONWARNINGS="ignore" $FLAKECMD --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$( if [[ $# == 0 ]]; then git ls-files "*.py" else