perf: enable more multi-threading and caching in linters (#4807)

* perf: enable more multi-threading and caching in linters

20s -> 6s

* ci: add multiprocess to ci dockerfile

* Update test/lint/lint-cppcheck-dash.sh

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

.gitignore

@@ -160,3 +160,9 @@ dist/
 *.background.tiff
 
 /guix-build-*
+
+# cppcheck cache-directory
+.cppcheck/*
+
+# flake8 cache location
+.cache/*

contrib/containers/ci/Dockerfile

@@ -39,7 +39,8 @@ RUN pip3 install \
     jinja2 \
     pyzmq \
     vulture==2.3 \
-    yq
+    yq \
+    multiprocess
 
 # dash_hash
 RUN git clone --depth 1 --no-tags https://github.com/dashpay/dash_hash

contrib/devtools/circular-dependencies.py

@@ -5,6 +5,7 @@
 
 import sys
 import re
+from multiprocess import Pool
 
 MAPPING = {
     'core_read.cpp': 'core_io.cpp',
@@ -37,7 +38,7 @@ if __name__=="__main__":
 
     RE = re.compile("^#include <(.*)>")
 
-    def handle_module(module):
+    def handle_module(arg):
         module = module_name(arg)
         if module is None:
             print("Ignoring file %s (does not constitute module)\n" % arg)
@@ -45,6 +46,26 @@ if __name__=="__main__":
             files[arg] = module
             deps[module] = set()
 
+    def handle_module2(module):
+        # Build the transitive closure of dependencies of module
+        closure = dict()
+        for dep in deps[module]:
+            closure[dep] = []
+        while True:
+            old_size = len(closure)
+            old_closure_keys = sorted(closure.keys())
+            for src in old_closure_keys:
+                for dep in deps[src]:
+                    if dep not in closure:
+                        closure[dep] = closure[src] + [src]
+            if len(closure) == old_size:
+                break
+        # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
+        if module in closure:
+            return [module] + closure[module]
+
+        return None
+
 
     # Iterate over files, and create list of modules
     for arg in sys.argv[1:]:
@@ -71,32 +92,20 @@ if __name__=="__main__":
     def shortest_c_dep():
         have_cycle = False
 
-        def handle_module(module, shortest_cycle):
+        sorted_keys = None
-
-            # Build the transitive closure of dependencies of module
-            closure = dict()
-            for dep in deps[module]:
-                closure[dep] = []
-            while True:
-                old_size = len(closure)
-                old_closure_keys = sorted(closure.keys())
-                for src in old_closure_keys:
-                    for dep in deps[src]:
-                        if dep not in closure:
-                            closure[dep] = closure[src] + [src]
-                if len(closure) == old_size:
-                    break
-            # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
-            if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)):
-                shortest_cycle = [module] + closure[module]
-
-            return shortest_cycle
 
         while True:
 
             shortest_cycles = None
-            for module in sorted(deps.keys()):
+            if sorted_keys is None:
-                shortest_cycles = handle_module(module, shortest_cycles)
+                sorted_keys = sorted(deps.keys())
+
+            with Pool(8) as pool:
+                cycles = pool.map(handle_module2, sorted_keys)
+
+            for cycle in cycles:
+                if cycle is not None and (shortest_cycles is None or len(cycle) < len(shortest_cycles)):
+                    shortest_cycles = cycle
 
             if shortest_cycles is None:
                 break
@@ -104,7 +113,8 @@ if __name__=="__main__":
             module = shortest_cycles[0]
             print("Circular dependency: %s" % (" -> ".join(shortest_cycles + [module])))
             # And then break the dependency to avoid repeating in other cycles
-            deps[shortest_cycles[-1]] = deps[shortest_cycles[-1]] - set([module])
+            deps[shortest_cycles[-1]] -= {module}
+            sorted_keys = None
             have_cycle = True
 
         if have_cycle:

test/lint/lint-cppcheck-dash.sh

@@ -48,6 +48,8 @@ IGNORED_WARNINGS=(
 #    "Consider using std::count_if algorithm instead of a raw loop."
 #    "Consider using std::find_if algorithm instead of a raw loop."
 #    "Member variable '.*' is not initialized in the constructor."
+
+    "unusedFunction"
 )
 
 # We should attempt to update this with all dash specific code
@@ -108,8 +110,14 @@ function join_array {
 ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}")
 IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}")
 FILES_REGEXP=$(join_array "|" "${FILES[@]}")
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CPPCHECK_DIR=$SCRIPT_DIR/.cppcheck/
+if [ ! -d $CPPCHECK_DIR ]
+then
+    mkdir $CPPCHECK_DIR
+fi
 WARNINGS=$(echo "${FILES}" | \
-    xargs cppcheck --enable=all --inline-suppr -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
+    xargs cppcheck --enable=all --inline-suppr --cppcheck-build-dir=$CPPCHECK_DIR -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
     grep -E "${ENABLED_CHECKS_REGEXP}" | \
     grep -vE "${IGNORED_WARNINGS_REGEXP}" | \
     grep -E "${FILES_REGEXP}")

test/lint/lint-format-strings.py

@@ -11,6 +11,8 @@
 import argparse
 import re
 import sys
+from functools import partial
+from multiprocessing import Pool
 
 FALSE_POSITIVES = [
     ("src/batchedlogger.h", "strprintf(fmt, args...)"),
@@ -261,6 +263,28 @@ def count_format_specifiers(format_string):
     return n
 
 
+def handle_filename(filename, args):
+    exit_code = 0
+    with open(filename, "r", encoding="utf-8") as f:
+        for function_call_str in parse_function_calls(args.function_name, f.read()):
+            parts = parse_function_call_and_arguments(args.function_name, function_call_str)
+            relevant_function_call_str = unescape("".join(parts))[:512]
+            if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
+                continue
+            if len(parts) < 3 + args.skip_arguments:
+                exit_code = 1
+                print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
+                continue
+            argument_count = len(parts) - 3 - args.skip_arguments
+            format_str = parse_string_content(parts[1 + args.skip_arguments])
+            format_specifier_count = count_format_specifiers(format_str)
+            if format_specifier_count != argument_count:
+                exit_code = 1
+                print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
+                continue
+    return exit_code
+
+
 def main():
     parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
                                      "to a variadic format string function matches the number of format "
@@ -270,26 +294,12 @@ def main():
     parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None)
     parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)")
     args = parser.parse_args()
-    exit_code = 0
+    exit_codes = []
-    for filename in args.file:
+
-        with open(filename, "r", encoding="utf-8") as f:
+    with Pool(8) as pool:
-            for function_call_str in parse_function_calls(args.function_name, f.read()):
+        exit_codes = pool.map(partial(handle_filename, args=args), args.file)
-                parts = parse_function_call_and_arguments(args.function_name, function_call_str)
+
-                relevant_function_call_str = unescape("".join(parts))[:512]
+    sys.exit(max(exit_codes))
-                if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
-                    continue
-                if len(parts) < 3 + args.skip_arguments:
-                    exit_code = 1
-                    print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
-                    continue
-                argument_count = len(parts) - 3 - args.skip_arguments
-                format_str = parse_string_content(parts[1 + args.skip_arguments])
-                format_specifier_count = count_format_specifiers(format_str)
-                if format_specifier_count != argument_count:
-                    exit_code = 1
-                    print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
-                    continue
-    sys.exit(exit_code)
 
 
 if __name__ == "__main__":

test/lint/lint-python.sh

@@ -88,7 +88,15 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then
     exit 0
 fi
 
-PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
+FLAKECMD=flake8
+
+if command -v flake8-cached > /dev/null; then
+    FLAKECMD=flake8-cached
+else
+    echo "Consider install flake8-cached for cached flake8 results."
+fi
+
+PYTHONWARNINGS="ignore" $FLAKECMD --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
     if [[ $# == 0 ]]; then
         git ls-files "*.py"
     else