From 06998263bbcfad940bf55e83583fbc717eff62ae Mon Sep 17 00:00:00 2001 From: PastaPastaPasta <6443210+PastaPastaPasta@users.noreply.github.com> Date: Thu, 30 Dec 2021 11:34:36 -0500 Subject: [PATCH] Optimize linters (#4637) * optimize: somehow optimize circular-dependencies.py Signed-off-by: pasta * optimize: use parallel if available to lint in parallel Signed-off-by: pasta * suggestions * more suggestions Co-authored-by: UdjinM6 --- contrib/containers/ci/Dockerfile | 2 +- contrib/devtools/circular-dependencies.py | 126 +++++++++++++--------- test/lint/lint-all.sh | 28 +++-- 3 files changed, 97 insertions(+), 59 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index a2aac50ad3..a4210c1d35 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && apt-get install $APT_ARGS g++-arm-linux-gnueabihf && rm -r RUN apt-get update && apt-get install $APT_ARGS g++-mingw-w64-x86-64 && rm -rf /var/lib/apt/lists/* RUN apt-get update && apt-get install $APT_ARGS wine-stable wine32 wine64 bc nsis xorriso libncurses5 && rm -rf /var/lib/apt/lists/* RUN apt-get update && apt-get install $APT_ARGS python3-zmq && rm -rf /var/lib/apt/lists/* -RUN apt-get update && apt-get install $APT_ARGS gawk jq && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install $APT_ARGS gawk jq parallel && rm -rf /var/lib/apt/lists/* RUN apt-get update && apt-get install $APT_ARGS imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools && rm -rf /var/lib/apt/lists/* ARG CPPCHECK_VERSION=2.4 diff --git a/contrib/devtools/circular-dependencies.py b/contrib/devtools/circular-dependencies.py index 2e4657f1dd..5b73a1cd74 100755 --- a/contrib/devtools/circular-dependencies.py +++ b/contrib/devtools/circular-dependencies.py @@ -28,61 +28,83 @@ def module_name(path): return path[:-4] return None -files = dict() -deps = dict() +if __name__=="__main__": + files = dict() + deps = dict() -RE = re.compile("^#include <(.*)>") + RE = re.compile("^#include <(.*)>") -# Iterate over files, and create list of modules -for arg in sys.argv[1:]: - module = module_name(arg) - if module is None: - print("Ignoring file %s (does not constitute module)\n" % arg) - else: - files[arg] = module - deps[module] = set() + def handle_module(module): + module = module_name(arg) + if module is None: + print("Ignoring file %s (does not constitute module)\n" % arg) + else: + files[arg] = module + deps[module] = set() -# Iterate again, and build list of direct dependencies for each module -# TODO: implement support for multiple include directories -for arg in sorted(files.keys()): - module = files[arg] - with open(arg, 'r', encoding="utf8") as f: - for line in f: - match = RE.match(line) - if match: - include = match.group(1) - included_module = module_name(include) - if included_module is not None and included_module in deps and included_module != module: - deps[module].add(included_module) -# Loop to find the shortest (remaining) circular dependency -have_cycle = False -while True: - shortest_cycle = None - for module in sorted(deps.keys()): - # Build the transitive closure of dependencies of module - closure = dict() - for dep in deps[module]: - closure[dep] = [] + # Iterate over files, and create list of modules + for arg in sys.argv[1:]: + handle_module(arg) + + def build_list_direct(arg): + module = files[arg] + with open(arg, 'r', encoding="utf8") as f: + for line in f: + match = RE.match(line) + if match: + include = match.group(1) + included_module = module_name(include) + if included_module is not None and included_module in deps and included_module != module: + deps[module].add(included_module) + + + # Iterate again, and build list of direct dependencies for each module + # TODO: implement support for multiple include directories + for arg in sorted(files.keys()): + build_list_direct(arg) + # Loop to find the shortest (remaining) circular dependency + + def shortest_c_dep(): + have_cycle = False + + def handle_module(module, shortest_cycle): + + # Build the transitive closure of dependencies of module + closure = dict() + for dep in deps[module]: + closure[dep] = [] + while True: + old_size = len(closure) + old_closure_keys = sorted(closure.keys()) + for src in old_closure_keys: + for dep in deps[src]: + if dep not in closure: + closure[dep] = closure[src] + [src] + if len(closure) == old_size: + break + # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest + if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)): + shortest_cycle = [module] + closure[module] + + return shortest_cycle + while True: - old_size = len(closure) - old_closure_keys = sorted(closure.keys()) - for src in old_closure_keys: - for dep in deps[src]: - if dep not in closure: - closure[dep] = closure[src] + [src] - if len(closure) == old_size: - break - # If module is in its own transitive closure, it's a circular dependency; check if it is the shortest - if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)): - shortest_cycle = [module] + closure[module] - if shortest_cycle is None: - break - # We have the shortest circular dependency; report it - module = shortest_cycle[0] - print("Circular dependency: %s" % (" -> ".join(shortest_cycle + [module]))) - # And then break the dependency to avoid repeating in other cycles - deps[shortest_cycle[-1]] = deps[shortest_cycle[-1]] - set([module]) - have_cycle = True -sys.exit(1 if have_cycle else 0) + shortest_cycles = None + for module in sorted(deps.keys()): + shortest_cycles = handle_module(module, shortest_cycles) + + if shortest_cycles is None: + break + # We have the shortest circular dependency; report it + module = shortest_cycles[0] + print("Circular dependency: %s" % (" -> ".join(shortest_cycles + [module]))) + # And then break the dependency to avoid repeating in other cycles + deps[shortest_cycles[-1]] = deps[shortest_cycles[-1]] - set([module]) + have_cycle = True + + if have_cycle: + return True + + sys.exit(1 if shortest_c_dep() else 0) diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh index fabc24c91b..43327baf2f 100755 --- a/test/lint/lint-all.sh +++ b/test/lint/lint-all.sh @@ -18,13 +18,29 @@ LINTALL=$(basename "${BASH_SOURCE[0]}") EXIT_CODE=0 -for f in "${SCRIPTDIR}"/lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - EXIT_CODE=1 +if ! command -v parallel > /dev/null; then + for f in "${SCRIPTDIR}"/lint-*.sh; do + if [ "$(basename "$f")" != "$LINTALL" ]; then + if ! "$f"; then + echo "^---- failure generated from $f" + EXIT_CODE=1 + fi fi + done +else + SCRIPTS=() + + for f in "${SCRIPTDIR}"/lint-*.sh; do + if [ "$(basename "$f")" != "$LINTALL" ]; then + SCRIPTS+=("$f") + fi + done + + if ! parallel --jobs 100% --will-cite --joblog parallel_out.log bash ::: "${SCRIPTS[@]}"; then + echo "^---- failure generated" + EXIT_CODE=1 fi -done + column -t parallel_out.log && rm parallel_out.log +fi exit ${EXIT_CODE}