perf: enable more multi-threading and caching in linters (#4807)

* perf: enable more multi-threading and caching in linters 20s -> 6s * ci: add multiprocess to ci dockerfile * Update test/lint/lint-cppcheck-dash.sh Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com> Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
2024-12-24 19:42:46 +01:00 · 2022-04-27 13:14:40 -05:00 · 2022-04-27 13:14:40 -05:00 · b421cfacae
commit b421cfacae
parent ba4a7b1d07
6 changed files with 90 additions and 47 deletions
--- a/.gitignore
+++ b/.gitignore
@ -160,3 +160,9 @@ dist/
 *.background.tiff

 /guix-build-*
+
+# cppcheck cache-directory
+.cppcheck/*
+
+# flake8 cache location
+.cache/*
--- a/contrib/containers/ci/Dockerfile
+++ b/contrib/containers/ci/Dockerfile
@ -39,7 +39,8 @@ RUN pip3 install \
    jinja2 \
    pyzmq \
    vulture==2.3 \
-    yq
+    yq \
+    multiprocess

 # dash_hash
 RUN git clone --depth 1 --no-tags https://github.com/dashpay/dash_hash
--- a/contrib/devtools/circular-dependencies.py
+++ b/contrib/devtools/circular-dependencies.py
@ -5,6 +5,7 @@

 import sys
 import re
+from multiprocess import Pool

 MAPPING = {
    'core_read.cpp': 'core_io.cpp',
@ -37,7 +38,7 @@ if __name__=="__main__":

    RE = re.compile("^#include <(.*)>")

-    def handle_module(module):
+    def handle_module(arg):
        module = module_name(arg)
        if module is None:
            print("Ignoring file %s (does not constitute module)\n" % arg)
@ -45,6 +46,26 @@ if __name__=="__main__":
            files[arg] = module
            deps[module] = set()

+    def handle_module2(module):
+        # Build the transitive closure of dependencies of module
+        closure = dict()
+        for dep in deps[module]:
+            closure[dep] = []
+        while True:
+            old_size = len(closure)
+            old_closure_keys = sorted(closure.keys())
+            for src in old_closure_keys:
+                for dep in deps[src]:
+                    if dep not in closure:
+                        closure[dep] = closure[src] + [src]
+            if len(closure) == old_size:
+                break
+        # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
+        if module in closure:
+            return [module] + closure[module]
+
+        return None
+

    # Iterate over files, and create list of modules
    for arg in sys.argv[1:]:
@ -71,32 +92,20 @@ if __name__=="__main__":
    def shortest_c_dep():
        have_cycle = False

-        def handle_module(module, shortest_cycle):
-
-            # Build the transitive closure of dependencies of module
-            closure = dict()
-            for dep in deps[module]:
-                closure[dep] = []
-            while True:
-                old_size = len(closure)
-                old_closure_keys = sorted(closure.keys())
-                for src in old_closure_keys:
-                    for dep in deps[src]:
-                        if dep not in closure:
-                            closure[dep] = closure[src] + [src]
-                if len(closure) == old_size:
-                    break
-            # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
-            if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)):
-                shortest_cycle = [module] + closure[module]
-
-            return shortest_cycle
+        sorted_keys = None

        while True:

            shortest_cycles = None
-            for module in sorted(deps.keys()):
-                shortest_cycles = handle_module(module, shortest_cycles)
+            if sorted_keys is None:
+                sorted_keys = sorted(deps.keys())
+
+            with Pool(8) as pool:
+                cycles = pool.map(handle_module2, sorted_keys)
+
+            for cycle in cycles:
+                if cycle is not None and (shortest_cycles is None or len(cycle) < len(shortest_cycles)):
+                    shortest_cycles = cycle

            if shortest_cycles is None:
                break
@ -104,7 +113,8 @@ if __name__=="__main__":
            module = shortest_cycles[0]
            print("Circular dependency: %s" % (" -> ".join(shortest_cycles + [module])))
            # And then break the dependency to avoid repeating in other cycles
-            deps[shortest_cycles[-1]] = deps[shortest_cycles[-1]] - set([module])
+            deps[shortest_cycles[-1]] -= {module}
+            sorted_keys = None
            have_cycle = True

        if have_cycle:
--- a/test/lint/lint-cppcheck-dash.sh
+++ b/test/lint/lint-cppcheck-dash.sh
@ -48,6 +48,8 @@ IGNORED_WARNINGS=(
 #    "Consider using std::count_if algorithm instead of a raw loop."
 #    "Consider using std::find_if algorithm instead of a raw loop."
 #    "Member variable '.*' is not initialized in the constructor."
+
+    "unusedFunction"
 )

 # We should attempt to update this with all dash specific code
@ -108,8 +110,14 @@ function join_array {
 ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}")
 IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}")
 FILES_REGEXP=$(join_array "|" "${FILES[@]}")
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CPPCHECK_DIR=$SCRIPT_DIR/.cppcheck/
+if [ ! -d $CPPCHECK_DIR ]
+then
+    mkdir $CPPCHECK_DIR
+fi
 WARNINGS=$(echo "${FILES}" | \
-    xargs cppcheck --enable=all --inline-suppr -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
+    xargs cppcheck --enable=all --inline-suppr --cppcheck-build-dir=$CPPCHECK_DIR -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
    grep -E "${ENABLED_CHECKS_REGEXP}" | \
    grep -vE "${IGNORED_WARNINGS_REGEXP}" | \
    grep -E "${FILES_REGEXP}")
--- a/test/lint/lint-format-strings.py
+++ b/test/lint/lint-format-strings.py
@ -11,6 +11,8 @@
 import argparse
 import re
 import sys
+from functools import partial
+from multiprocessing import Pool

 FALSE_POSITIVES = [
    ("src/batchedlogger.h", "strprintf(fmt, args...)"),
@ -261,17 +263,8 @@ def count_format_specifiers(format_string):
    return n


-def main():
-    parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
-                                     "to a variadic format string function matches the number of format "
-                                     "specifiers in the format string.")
-    parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string "
-                        "argument (e.g. 1 in the case of fprintf)", default=0)
-    parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None)
-    parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)")
-    args = parser.parse_args()
+def handle_filename(filename, args):
    exit_code = 0
-    for filename in args.file:
    with open(filename, "r", encoding="utf-8") as f:
        for function_call_str in parse_function_calls(args.function_name, f.read()):
            parts = parse_function_call_and_arguments(args.function_name, function_call_str)
@ -289,7 +282,24 @@ def main():
                exit_code = 1
                print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
                continue
-    sys.exit(exit_code)
+    return exit_code
+
+
+def main():
+    parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
+                                     "to a variadic format string function matches the number of format "
+                                     "specifiers in the format string.")
+    parser.add_argument("--skip-arguments", type=int, help="number of arguments before the format string "
+                        "argument (e.g. 1 in the case of fprintf)", default=0)
+    parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None)
+    parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)")
+    args = parser.parse_args()
+    exit_codes = []
+
+    with Pool(8) as pool:
+        exit_codes = pool.map(partial(handle_filename, args=args), args.file)
+
+    sys.exit(max(exit_codes))


 if __name__ == "__main__":
--- a/test/lint/lint-python.sh
+++ b/test/lint/lint-python.sh
@ -88,7 +88,15 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then
    exit 0
 fi

-PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
+FLAKECMD=flake8
+
+if command -v flake8-cached > /dev/null; then
+    FLAKECMD=flake8-cached
+else
+    echo "Consider install flake8-cached for cached flake8 results."
+fi
+
+PYTHONWARNINGS="ignore" $FLAKECMD --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
    if [[ $# == 0 ]]; then
        git ls-files "*.py"
    else