mirror of
https://github.com/dashpay/dash.git
synced 2024-12-24 11:32:46 +01:00
perf: enable more multi-threading and caching in linters (#4807)
* perf: enable more multi-threading and caching in linters 20s -> 6s * ci: add multiprocess to ci dockerfile * Update test/lint/lint-cppcheck-dash.sh Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com> Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
This commit is contained in:
PastaPastaPasta
GitHub
parent
ba4a7b1d07
commit
b421cfacae
6
.gitignore
vendored
6
.gitignore
vendored
@ -160,3 +160,9 @@ dist/
|
||||
*.background.tiff
|
||||
|
||||
/guix-build-*
|
||||
|
||||
# cppcheck cache-directory
|
||||
.cppcheck/*
|
||||
|
||||
# flake8 cache location
|
||||
.cache/*
|
||||
|
||||
@ -39,7 +39,8 @@ RUN pip3 install \
|
||||
jinja2 \
|
||||
pyzmq \
|
||||
vulture==2.3 \
|
||||
yq
|
||||
yq \
|
||||
multiprocess
|
||||
|
||||
# dash_hash
|
||||
RUN git clone --depth 1 --no-tags https://github.com/dashpay/dash_hash
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
|
||||
import sys
|
||||
import re
|
||||
from multiprocess import Pool
|
||||
|
||||
MAPPING = {
|
||||
'core_read.cpp': 'core_io.cpp',
|
||||
@ -37,7 +38,7 @@ if __name__=="__main__":
|
||||
|
||||
RE = re.compile("^#include <(.*)>")
|
||||
|
||||
def handle_module(module):
|
||||
def handle_module(arg):
|
||||
module = module_name(arg)
|
||||
if module is None:
|
||||
print("Ignoring file %s (does not constitute module)\n" % arg)
|
||||
@ -45,6 +46,26 @@ if __name__=="__main__":
|
||||
files[arg] = module
|
||||
deps[module] = set()
|
||||
|
||||
def handle_module2(module):
|
||||
# Build the transitive closure of dependencies of module
|
||||
closure = dict()
|
||||
for dep in deps[module]:
|
||||
closure[dep] = []
|
||||
while True:
|
||||
old_size = len(closure)
|
||||
old_closure_keys = sorted(closure.keys())
|
||||
for src in old_closure_keys:
|
||||
for dep in deps[src]:
|
||||
if dep not in closure:
|
||||
closure[dep] = closure[src] + [src]
|
||||
if len(closure) == old_size:
|
||||
break
|
||||
# If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
|
||||
if module in closure:
|
||||
return [module] + closure[module]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Iterate over files, and create list of modules
|
||||
for arg in sys.argv[1:]:
|
||||
@ -71,32 +92,20 @@ if __name__=="__main__":
|
||||
def shortest_c_dep():
|
||||
have_cycle = False
|
||||
|
||||
def handle_module(module, shortest_cycle):
|
||||
|
||||
# Build the transitive closure of dependencies of module
|
||||
closure = dict()
|
||||
for dep in deps[module]:
|
||||
closure[dep] = []
|
||||
while True:
|
||||
old_size = len(closure)
|
||||
old_closure_keys = sorted(closure.keys())
|
||||
for src in old_closure_keys:
|
||||
for dep in deps[src]:
|
||||
if dep not in closure:
|
||||
closure[dep] = closure[src] + [src]
|
||||
if len(closure) == old_size:
|
||||
break
|
||||
# If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
|
||||
if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)):
|
||||
shortest_cycle = [module] + closure[module]
|
||||
|
||||
return shortest_cycle
|
||||
sorted_keys = None
|
||||
|
||||
while True:
|
||||
|
||||
shortest_cycles = None
|
||||
for module in sorted(deps.keys()):
|
||||
shortest_cycles = handle_module(module, shortest_cycles)
|
||||
if sorted_keys is None:
|
||||
sorted_keys = sorted(deps.keys())
|
||||
|
||||
with Pool(8) as pool:
|
||||
cycles = pool.map(handle_module2, sorted_keys)
|
||||
|
||||
for cycle in cycles:
|
||||
if cycle is not None and (shortest_cycles is None or len(cycle) < len(shortest_cycles)):
|
||||
shortest_cycles = cycle
|
||||
|
||||
if shortest_cycles is None:
|
||||
break
|
||||
@ -104,7 +113,8 @@ if __name__=="__main__":
|
||||
module = shortest_cycles[0]
|
||||
print("Circular dependency: %s" % (" -> ".join(shortest_cycles + [module])))
|
||||
# And then break the dependency to avoid repeating in other cycles
|
||||
deps[shortest_cycles[-1]] = deps[shortest_cycles[-1]] - set([module])
|
||||
deps[shortest_cycles[-1]] -= {module}
|
||||
sorted_keys = None
|
||||
have_cycle = True
|
||||
|
||||
if have_cycle:
|
||||
|
||||
@ -48,6 +48,8 @@ IGNORED_WARNINGS=(
|
||||
# "Consider using std::count_if algorithm instead of a raw loop."
|
||||
# "Consider using std::find_if algorithm instead of a raw loop."
|
||||
# "Member variable '.*' is not initialized in the constructor."
|
||||
|
||||
"unusedFunction"
|
||||
)
|
||||
|
||||
# We should attempt to update this with all dash specific code
|
||||
@ -108,8 +110,14 @@ function join_array {
|
||||
ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}")
|
||||
IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}")
|
||||
FILES_REGEXP=$(join_array "|" "${FILES[@]}")
|
||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
||||
CPPCHECK_DIR=$SCRIPT_DIR/.cppcheck/
|
||||
if [ ! -d $CPPCHECK_DIR ]
|
||||
then
|
||||
mkdir $CPPCHECK_DIR
|
||||
fi
|
||||
WARNINGS=$(echo "${FILES}" | \
|
||||
xargs cppcheck --enable=all --inline-suppr -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
|
||||
xargs cppcheck --enable=all --inline-suppr --cppcheck-build-dir=$CPPCHECK_DIR -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
|
||||
grep -E "${ENABLED_CHECKS_REGEXP}" | \
|
||||
grep -vE "${IGNORED_WARNINGS_REGEXP}" | \
|
||||
grep -E "${FILES_REGEXP}")
|
||||
|
||||
@ -11,6 +11,8 @@
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from functools import partial
|
||||
from multiprocessing import Pool
|
||||
|
||||
FALSE_POSITIVES = [
|
||||
("src/batchedlogger.h", "strprintf(fmt, args...)"),
|
||||
@ -261,6 +263,28 @@ def count_format_specifiers(format_string):
|
||||
return n
|
||||
|
||||
|
||||
def handle_filename(filename, args):
|
||||
exit_code = 0
|
||||
with open(filename, "r", encoding="utf-8") as f:
|
||||
for function_call_str in parse_function_calls(args.function_name, f.read()):
|
||||
parts = parse_function_call_and_arguments(args.function_name, function_call_str)
|
||||
relevant_function_call_str = unescape("".join(parts))[:512]
|
||||
if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
|
||||
continue
|
||||
if len(parts) < 3 + args.skip_arguments:
|
||||
exit_code = 1
|
||||
print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
|
||||
continue
|
||||
argument_count = len(parts) - 3 - args.skip_arguments
|
||||
format_str = parse_string_content(parts[1 + args.skip_arguments])
|
||||
format_specifier_count = count_format_specifiers(format_str)
|
||||
if format_specifier_count != argument_count:
|
||||
exit_code = 1
|
||||
print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
|
||||
continue
|
||||
return exit_code
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
|
||||
"to a variadic format string function matches the number of format "
|
||||
@ -270,26 +294,12 @@ def main():
|
||||
parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None)
|
||||
parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)")
|
||||
args = parser.parse_args()
|
||||
exit_code = 0
|
||||
for filename in args.file:
|
||||
with open(filename, "r", encoding="utf-8") as f:
|
||||
for function_call_str in parse_function_calls(args.function_name, f.read()):
|
||||
parts = parse_function_call_and_arguments(args.function_name, function_call_str)
|
||||
relevant_function_call_str = unescape("".join(parts))[:512]
|
||||
if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
|
||||
continue
|
||||
if len(parts) < 3 + args.skip_arguments:
|
||||
exit_code = 1
|
||||
print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
|
||||
continue
|
||||
argument_count = len(parts) - 3 - args.skip_arguments
|
||||
format_str = parse_string_content(parts[1 + args.skip_arguments])
|
||||
format_specifier_count = count_format_specifiers(format_str)
|
||||
if format_specifier_count != argument_count:
|
||||
exit_code = 1
|
||||
print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
|
||||
continue
|
||||
sys.exit(exit_code)
|
||||
exit_codes = []
|
||||
|
||||
with Pool(8) as pool:
|
||||
exit_codes = pool.map(partial(handle_filename, args=args), args.file)
|
||||
|
||||
sys.exit(max(exit_codes))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -88,7 +88,15 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
|
||||
FLAKECMD=flake8
|
||||
|
||||
if command -v flake8-cached > /dev/null; then
|
||||
FLAKECMD=flake8-cached
|
||||
else
|
||||
echo "Consider install flake8-cached for cached flake8 results."
|
||||
fi
|
||||
|
||||
PYTHONWARNINGS="ignore" $FLAKECMD --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
|
||||
if [[ $# == 0 ]]; then
|
||||
git ls-files "*.py"
|
||||
else
|
||||
|
||||
Loading…
Reference in New Issue
Block a user