sikkienl/dash
1
0
Fork 0
mirror of https://github.com/dashpay/dash.git synced 2024-12-24 19:42:46 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

perf: enable more multi-threading and caching in linters (#4807)

Browse Source 
* perf: enable more multi-threading and caching in linters

20s -> 6s

* ci: add multiprocess to ci dockerfile

* Update test/lint/lint-cppcheck-dash.sh

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
This commit is contained in:
PastaPastaPasta 2022-04-27 13:14:40 -05:00 committed by GitHub
parent ba4a7b1d07
commit b421cfacae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 90 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -160,3 +160,9 @@ dist/
*.background.tiff
/guix-build-*
# cppcheck cache-directory
.cppcheck/*
# flake8 cache location
.cache/*

3
contrib/containers/ci/Dockerfile
View File

@ -39,7 +39,8 @@ RUN pip3 install \
jinja2 \
pyzmq \
vulture==2.3 \
yq
yq \
multiprocess
# dash_hash
RUN git clone --depth 1 --no-tags https://github.com/dashpay/dash_hash

58
contrib/devtools/circular-dependencies.py
View File

@ -5,6 +5,7 @@
import sys
import re
from multiprocess import Pool
MAPPING = {
'core_read.cpp': 'core_io.cpp',
@ -37,7 +38,7 @@ if __name__=="__main__":
RE = re.compile("^#include <(.*)>")
def handle_module(module):
def handle_module(arg):
module = module_name(arg)
if module is None:
print("Ignoring file %s (does not constitute module)\n" % arg)
@ -45,6 +46,26 @@ if __name__=="__main__":
files[arg] = module
deps[module] = set()
def handle_module2(module):
# Build the transitive closure of dependencies of module
closure = dict()
for dep in deps[module]:
closure[dep] = []
while True:
old_size = len(closure)
old_closure_keys = sorted(closure.keys())
for src in old_closure_keys:
for dep in deps[src]:
if dep not in closure:
closure[dep] = closure[src] + [src]
if len(closure) == old_size:
break
# If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
if module in closure:
return [module] + closure[module]
return None
# Iterate over files, and create list of modules
for arg in sys.argv[1:]:
@ -71,32 +92,20 @@ if __name__=="__main__":
def shortest_c_dep():
have_cycle = False
def handle_module(module, shortest_cycle):
# Build the transitive closure of dependencies of module
closure = dict()
for dep in deps[module]:
closure[dep] = []
while True:
old_size = len(closure)
old_closure_keys = sorted(closure.keys())
for src in old_closure_keys:
for dep in deps[src]:
if dep not in closure:
closure[dep] = closure[src] + [src]
if len(closure) == old_size:
break
# If module is in its own transitive closure, it's a circular dependency; check if it is the shortest
if module in closure and (shortest_cycle is None or len(closure[module]) + 1 < len(shortest_cycle)):
shortest_cycle = [module] + closure[module]
return shortest_cycle
sorted_keys = None
while True:
shortest_cycles = None
for module in sorted(deps.keys()):
shortest_cycles = handle_module(module, shortest_cycles)
if sorted_keys is None:
sorted_keys = sorted(deps.keys())
with Pool(8) as pool:
cycles = pool.map(handle_module2, sorted_keys)
for cycle in cycles:
if cycle is not None and (shortest_cycles is None or len(cycle) < len(shortest_cycles)):
shortest_cycles = cycle
if shortest_cycles is None:
break
@ -104,7 +113,8 @@ if __name__=="__main__":
module = shortest_cycles[0]
print("Circular dependency: %s" % (" -> ".join(shortest_cycles + [module])))
# And then break the dependency to avoid repeating in other cycles
deps[shortest_cycles[-1]] = deps[shortest_cycles[-1]] - set([module])
deps[shortest_cycles[-1]] -= {module}
sorted_keys = None
have_cycle = True
if have_cycle:

10
test/lint/lint-cppcheck-dash.sh
View File

@ -48,6 +48,8 @@ IGNORED_WARNINGS=(
# "Consider using std::count_if algorithm instead of a raw loop."
# "Consider using std::find_if algorithm instead of a raw loop."
# "Member variable '.*' is not initialized in the constructor."
"unusedFunction"
)
# We should attempt to update this with all dash specific code
@ -108,8 +110,14 @@ function join_array {
ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}")
IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}")
FILES_REGEXP=$(join_array "|" "${FILES[@]}")
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
CPPCHECK_DIR=$SCRIPT_DIR/.cppcheck/
if [ ! -d $CPPCHECK_DIR ]
then
mkdir $CPPCHECK_DIR
fi
WARNINGS=$(echo "${FILES}" | \
xargs cppcheck --enable=all --inline-suppr -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
xargs cppcheck --enable=all --inline-suppr --cppcheck-build-dir=$CPPCHECK_DIR -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCLIENT_VERSION_REVISION -DCOPYRIGHT_YEAR -DDEBUG -DHAVE_WORKING_BOOST_SLEEP_FOR -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \
grep -E "${ENABLED_CHECKS_REGEXP}" | \
grep -vE "${IGNORED_WARNINGS_REGEXP}" | \
grep -E "${FILES_REGEXP}")

50
test/lint/lint-format-strings.py
View File

@ -11,6 +11,8 @@
import argparse
import re
import sys
from functools import partial
from multiprocessing import Pool
FALSE_POSITIVES = [
("src/batchedlogger.h", "strprintf(fmt, args...)"),
@ -261,6 +263,28 @@ def count_format_specifiers(format_string):
return n
def handle_filename(filename, args):
exit_code = 0
with open(filename, "r", encoding="utf-8") as f:
for function_call_str in parse_function_calls(args.function_name, f.read()):
parts = parse_function_call_and_arguments(args.function_name, function_call_str)
relevant_function_call_str = unescape("".join(parts))[:512]
if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
continue
if len(parts) < 3 + args.skip_arguments:
exit_code = 1
print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
continue
argument_count = len(parts) - 3 - args.skip_arguments
format_str = parse_string_content(parts[1 + args.skip_arguments])
format_specifier_count = count_format_specifiers(format_str)
if format_specifier_count != argument_count:
exit_code = 1
print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
continue
return exit_code
def main():
parser = argparse.ArgumentParser(description="This program checks that the number of arguments passed "
"to a variadic format string function matches the number of format "
@ -270,26 +294,12 @@ def main():
parser.add_argument("function_name", help="function name (e.g. fprintf)", default=None)
parser.add_argument("file", nargs="*", help="C++ source code file (e.g. foo.cpp)")
args = parser.parse_args()
exit_code = 0
for filename in args.file:
with open(filename, "r", encoding="utf-8") as f:
for function_call_str in parse_function_calls(args.function_name, f.read()):
parts = parse_function_call_and_arguments(args.function_name, function_call_str)
relevant_function_call_str = unescape("".join(parts))[:512]
if (f.name, relevant_function_call_str) in FALSE_POSITIVES:
continue
if len(parts) < 3 + args.skip_arguments:
exit_code = 1
print("{}: Could not parse function call string \"{}(...)\": {}".format(f.name, args.function_name, relevant_function_call_str))
continue
argument_count = len(parts) - 3 - args.skip_arguments
format_str = parse_string_content(parts[1 + args.skip_arguments])
format_specifier_count = count_format_specifiers(format_str)
if format_specifier_count != argument_count:
exit_code = 1
print("{}: Expected {} argument(s) after format string but found {} argument(s): {}".format(f.name, format_specifier_count, argument_count, relevant_function_call_str))
continue
sys.exit(exit_code)
exit_codes = []
with Pool(8) as pool:
exit_codes = pool.map(partial(handle_filename, args=args), args.file)
sys.exit(max(exit_codes))
if __name__ == "__main__":

10
test/lint/lint-python.sh
View File

@ -88,7 +88,15 @@ elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then
exit 0
fi
PYTHONWARNINGS="ignore" flake8 --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
FLAKECMD=flake8
if command -v flake8-cached > /dev/null; then
FLAKECMD=flake8-cached
else
echo "Consider install flake8-cached for cached flake8 results."
fi
PYTHONWARNINGS="ignore" $FLAKECMD --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}")$(
if [[ $# == 0 ]]; then
git ls-files "*.py"
else