From 2d4db48606b8dc365eaae55f8076c437e4e84771 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 13 Apr 2020 11:44:59 -0400 Subject: [PATCH] Merge #18581: ci: Print ccache stats, add pip cache, and cleanups 73f83036298e4932772704b2d5359c78d81bd7cc ci: Add pip cache (Hennadii Stepanov) 6d480667b24dc630e7c3fe7ff032cbe1a958ab77 ci: Print ccache statistics summary (Hennadii Stepanov) a6eea715ff549febe6d4bd52f17acb50abd99120 ci: Drop unused WINEDEBUG variable (Hennadii Stepanov) Pull request description: The Travis [pip cache](https://docs.travis-ci.com/user/caching/#pip-cache) is free and saves a dozen of seconds :) Here are some excerpts from the Travis logs with `ccache` statistics (I found useful): 2) [Arm64](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507749) ``` ccache version 3.6 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf stats updated Fri Apr 10 18:21:06 2020 stats zeroed Fri Apr 10 18:03:04 2020 cache hit (direct) 156 cache hit (preprocessed) 198 cache miss 143 cache hit rate 71.23 % called for link 8 cleanups performed 9 files in cache 1255 cache size 80.6 MB max cache size 100.0 MB ``` 3) [s390x](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507750) ``` ccache version 3.4.1 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf stats zero time Fri Apr 10 17:15:35 2020 cache hit (direct) 115 cache hit (preprocessed) 163 cache miss 94 cache hit rate 74.73 % called for link 6 cleanups performed 0 files in cache 804 cache size 304.8 MB max cache size 5.0 GB ``` 4) [Win64](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507751) ``` ccache version 3.4.1 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf stats zero time Fri Apr 10 17:38:07 2020 cache hit (direct) 147 cache hit (preprocessed) 199 cache miss 138 cache hit rate 71.49 % called for link 7 cleanups performed 7 files in cache 1242 cache size 87.0 MB max cache size 100.0 MB ``` 5) [CentOS 7](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507752) ``` ccache version 3.7.7 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf stats updated Fri Apr 10 17:45:59 2020 stats zeroed Fri Apr 10 17:34:27 2020 cache hit (direct) 152 cache hit (preprocessed) 200 cache miss 144 cache hit rate 70.97 % called for link 8 cleanups performed 4 files in cache 1352 cache size 86.1 MB max cache size 100.0 MB ``` 6) [bionic](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507753) ``` ccache version 3.4.1 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf stats zero time Fri Apr 10 17:21:10 2020 cache hit (direct) 136 cache hit (preprocessed) 35 cache miss 330 cache hit rate 34.13 % called for link 8 cleanups performed 18 files in cache 1302 cache size 83.8 MB max cache size 100.0 MB ``` 7) [xenial](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507754) ``` ccache version 3.2.4 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf cache hit (direct) 161 cache hit (preprocessed) 174 cache miss 95 called for link 7 files in cache 3411 cache size 1.1 GB max cache size 5.0 GB ``` 10) [focal](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507757) ``` ccache version 3.7.7 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf stats updated Fri Apr 10 17:35:57 2020 stats zeroed Fri Apr 10 17:21:32 2020 cache hit (direct) 78 cache hit (preprocessed) 125 cache miss 107 cache hit rate 65.48 % called for link 120 cleanups performed 0 files in cache 6218 cache size 1.8 GB max cache size 5.0 GB ``` 14) [macOS 10.12](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507761) ``` ccache version 3.4.1 cache directory /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /home/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /etc/ccache.conf stats zero time Fri Apr 10 17:47:19 2020 cache hit (direct) 28 cache hit (preprocessed) 1 cache miss 469 cache hit rate 5.82 % called for link 8 cleanups performed 17 files in cache 1946 cache size 82.6 MB max cache size 100.0 MB ``` 15) [macOS 10.14 native](https://travis-ci.org/github/bitcoin/bitcoin/jobs/673507762) ``` ccache version 3.7.8 cache directory /Users/travis/build/bitcoin/bitcoin/ci/scratch/.ccache primary config /Users/travis/build/bitcoin/bitcoin/ci/scratch/.ccache/ccache.conf secondary config (readonly) /usr/local/Cellar/ccache/3.7.8/etc/ccache.conf stats updated Fri Apr 10 17:38:44 2020 stats zeroed Fri Apr 10 17:22:36 2020 cache hit (direct) 213 cache hit (preprocessed) 293 cache miss 144 cache hit rate 77.85 % called for link 11 called for preprocessing 56 compile failed 30 preprocessor error 56 bad compiler arguments 14 autoconf compile/link 77 no input file 66 cleanups performed 0 files in cache 1564 cache size 284.6 MB max cache size 5.0 GB ``` Top commit has no ACKs. Tree-SHA512: f9f2c6d72680bafc7a982efc04dc37ab9bba77ffd2a4cbbc56bb0e86c5c3501d57aa2f0ebeac0113e9d8daac1ee4eac1eddf74c1172c267b37ee2d9324366c8a --- .travis.yml | 2 +- ci/dash/build_src.sh | 3 +++ ci/test/00_setup_env.sh | 3 +-- ci/test/04_install.sh | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index e135852c39..36b0c8e24e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -192,7 +192,7 @@ after_success: - stage: lint name: 'lint' env: - cache: false + cache: pip language: python python: '3.8' # Oldest supported version according to doc/dependencies.md install: diff --git a/ci/dash/build_src.sh b/ci/dash/build_src.sh index c12bb895e1..328acc07d8 100755 --- a/ci/dash/build_src.sh +++ b/ci/dash/build_src.sh @@ -31,6 +31,7 @@ if [ "$CHECK_DOC" = 1 ]; then test/lint/extended-lint-all.sh fi +command -v ccache > /dev/null && ccache --zero-stats ccache --max-size=$CCACHE_SIZE if [ -n "$CONFIG_SHELL" ]; then @@ -53,6 +54,8 @@ cd dashcore-$BUILD_TARGET make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) +command -v ccache > /dev/null && ccache --version | head -n 1 && ccache --show-stats + if [ -n "$USE_VALGRIND" ]; then echo "valgrind in USE!" ${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 3311a78524..5b4da6b81b 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -29,7 +29,7 @@ MAKEJOBS="-j$(nproc)" export MAKEJOBS # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) # This folder only exists on the ci host. -export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} +export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch} # What host to compile for. See also ./depends/README.md # Tests that need cross-compilation export the appropriate HOST. # Tests that run natively guess the host @@ -60,7 +60,6 @@ export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} # Folder where the build is done (bin and lib). export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export WINEDEBUG=${WINEDEBUG:-fixme-all} export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 68c48dfae4..99fec18e16 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -25,7 +25,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order= export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan" export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" -env | grep -E '^(BASE_|QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env +env | grep -E '^(BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env if [[ $HOST = *-mingw32 ]]; then DOCKER_ADMIN="--cap-add SYS_ADMIN" elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)