From b8850608022023f0fe8ed0ef48d3bac5a9575974 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Tue, 3 Dec 2019 11:00:58 -0500 Subject: [PATCH] Merge #17549: ci: misc cleanups fad88e6f86d1dd32cf01db2287df9c63e66c5116 ci: Remove use of cd (MarcoFalke) fa2941bbf47a8a6b79b8db4a87e1aedcf6a29a5e ci: Remove unmaintained extended_lint (MarcoFalke) fa041875284d01602647519cc452185ba1ad5a8f scripted-diff: Use ci DEPENDS_DIR, remove BASE_BUILD_DIR (MarcoFalke) fa0656d1ebc2a01d900fead4ce055d59eb818a1f ci: Add DEPENDS_DIR variable, Add documentation for folders (MarcoFalke) faeeca87b65dd98e0efbc54443b3f8854cae9c00 scripted-diff: Move various folders to ci scratch dir (MarcoFalke) Pull request description: Some minor cleanups: * Remove unused and unmaintained extended_lint to avoid ci bottlenecks and waste of CPU * Move all folders that hold temporary ci files to the ci scratch dir (except for the build dirs) * Add some documentation to folders and remove the `BASE_BUILD_DIR` alias for the root directory * Fixes #17178 by removing `cd` Top commit has no ACKs. Tree-SHA512: f6eb9d47ab98c08af59a79c2a4bf62fc331f0f6f3174909a28f2c2f48b0234423da5aee876225d2e188619b71b008d882a20d29a7aca68248039ea5080be5af5 --- .travis.yml | 13 ------------- ci/dash/build-docker.sh | 9 +++------ ci/dash/build_depends.sh | 10 +++++----- ci/dash/build_src.sh | 2 +- ci/dash/test_integrationtests.sh | 2 +- ci/dash/test_unittests.sh | 2 +- ci/extended_lint/04_install.sh | 12 ------------ ci/extended_lint/06_script.sh | 9 --------- ci/test/00_setup_env.sh | 14 +++++++++----- ci/test/04_install.sh | 18 ++++++++++-------- ci/test/05_before_script.sh | 4 ++-- 11 files changed, 32 insertions(+), 63 deletions(-) delete mode 100755 ci/extended_lint/04_install.sh delete mode 100755 ci/extended_lint/06_script.sh diff --git a/.travis.yml b/.travis.yml index 355753bf70..dcf8dd2027 100644 --- a/.travis.yml +++ b/.travis.yml @@ -212,19 +212,6 @@ after_success: script: - set -o errexit; source ./ci/lint/06_script.sh - - stage: extended-lint - name: 'extended lint [runtime >= 60 seconds]' - env: - cache: false - language: python - python: '3.5' - install: - - set -o errexit; source ./ci/extended_lint/04_install.sh - before_script: - - set -o errexit; source ./ci/lint/05_before_script.sh - script: - - set -o errexit; source ./ci/extended_lint/06_script.sh - - stage: test name: 'ARM [GOAL: install] [bionic] [unit tests, functional tests]' arch: arm64 diff --git a/ci/dash/build-docker.sh b/ci/dash/build-docker.sh index fd541928e8..053f28e6eb 100755 --- a/ci/dash/build-docker.sh +++ b/ci/dash/build-docker.sh @@ -12,17 +12,14 @@ DOCKER_IMAGE=${DOCKER_IMAGE:-dashpay/dashd-develop} DOCKER_TAG=${DOCKER_TAG:-latest} DOCKER_RELATIVE_PATH=contrib/containers/deploy -BASE_BUILD_DIR=${BASE_BUILD_DIR:-.} - - if [ -d $DOCKER_RELATIVE_PATH/bin ]; then rm $DOCKER_RELATIVE_PATH/bin/* fi mkdir $DOCKER_RELATIVE_PATH/bin -cp "$BASE_BUILD_DIR"/src/dashd $DOCKER_RELATIVE_PATH/bin/ -cp "$BASE_BUILD_DIR"/src/dash-cli $DOCKER_RELATIVE_PATH/bin/ -cp "$BASE_BUILD_DIR"/src/dash-tx $DOCKER_RELATIVE_PATH/bin/ +cp "$BASE_ROOT_DIR"/src/dashd $DOCKER_RELATIVE_PATH/bin/ +cp "$BASE_ROOT_DIR"/src/dash-cli $DOCKER_RELATIVE_PATH/bin/ +cp "$BASE_ROOT_DIR"/src/dash-tx $DOCKER_RELATIVE_PATH/bin/ strip $DOCKER_RELATIVE_PATH/bin/dashd strip $DOCKER_RELATIVE_PATH/bin/dash-cli strip $DOCKER_RELATIVE_PATH/bin/dash-tx diff --git a/ci/dash/build_depends.sh b/ci/dash/build_depends.sh index cefd55047c..e25f55df41 100755 --- a/ci/dash/build_depends.sh +++ b/ci/dash/build_depends.sh @@ -17,19 +17,19 @@ unset DISPLAY mkdir -p $CACHE_DIR/depends mkdir -p $CACHE_DIR/sdk-sources -ln -s $CACHE_DIR/depends depends/built -ln -s $CACHE_DIR/sdk-sources depends/sdk-sources +ln -s $CACHE_DIR/depends ${DEPENDS_DIR}/built +ln -s $CACHE_DIR/sdk-sources ${DEPENDS_DIR}/sdk-sources -mkdir -p depends/SDKs +mkdir -p ${DEPENDS_DIR}/SDKs if [ -n "$XCODE_VERSION" ]; then OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" - OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" if [ ! -f "$OSX_SDK_PATH" ]; then curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi if [ -f "$OSX_SDK_PATH" ]; then - tar -C depends/SDKs -xf "$OSX_SDK_PATH" + tar -C ${DEPENDS_DIR}/SDKs -xf "$OSX_SDK_PATH" fi fi diff --git a/ci/dash/build_src.sh b/ci/dash/build_src.sh index d2af35179f..9ee0860eef 100755 --- a/ci/dash/build_src.sh +++ b/ci/dash/build_src.sh @@ -40,7 +40,7 @@ if [ -n "$CONFIG_SHELL" ]; then export CONFIG_SHELL="$CONFIG_SHELL" fi -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" ( test -n "$CONFIG_SHELL" && eval '"$CONFIG_SHELL" -c "./autogen.sh"' ) || ./autogen.sh diff --git a/ci/dash/test_integrationtests.sh b/ci/dash/test_integrationtests.sh index f236c8d9d0..de5d8a19c2 100755 --- a/ci/dash/test_integrationtests.sh +++ b/ci/dash/test_integrationtests.sh @@ -18,7 +18,7 @@ if [ "$RUN_INTEGRATION_TESTS" != "true" ]; then exit 0 fi -export LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib +export LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib cd build-ci/dashcore-$BUILD_TARGET diff --git a/ci/dash/test_unittests.sh b/ci/dash/test_unittests.sh index 595effffaf..e97d234348 100755 --- a/ci/dash/test_unittests.sh +++ b/ci/dash/test_unittests.sh @@ -17,7 +17,7 @@ if [ "$RUN_UNIT_TESTS" != "true" ]; then fi export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} -export LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib +export LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib export WINEDEBUG=fixme-all export BOOST_TEST_LOG_LEVEL=test_suite diff --git a/ci/extended_lint/04_install.sh b/ci/extended_lint/04_install.sh deleted file mode 100755 index 123d874a84..0000000000 --- a/ci/extended_lint/04_install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -CPPCHECK_VERSION=1.86 -curl -s https://codeload.github.com/danmar/cppcheck/tar.gz/${CPPCHECK_VERSION} | tar -zxf - --directory /tmp/ -(cd /tmp/cppcheck-${CPPCHECK_VERSION}/ && make CFGDIR=/tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ > /dev/null) -export PATH="$PATH:/tmp/cppcheck-${CPPCHECK_VERSION}/" diff --git a/ci/extended_lint/06_script.sh b/ci/extended_lint/06_script.sh deleted file mode 100755 index e8228c9c4d..0000000000 --- a/ci/extended_lint/06_script.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -test/lint/extended-lint-all.sh diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 2fb73c291f..2ff88d07f1 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -28,6 +28,7 @@ echo "Fallback to default values in env (if not yet set)" MAKEJOBS="-j$(nproc)" export MAKEJOBS # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) +# This folder only exists on the ci host. export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} # What host to compile for. See also ./depends/README.md # Tests that need cross-compilation export the appropriate HOST. @@ -47,16 +48,19 @@ export CACHE_DIR=${CACHE_DIR:-$HOST_CACHE_DIR} export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} +# The cache dir. +# This folder exists on the ci host and ci guest. Changes are propagated back and forth. export CCACHE_DIR=${CCACHE_DIR:-$CACHE_DIR/ccache} -# Folder where the build is done (depends and dist). Can not be changed and is equal to the root of the git repo -export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR} -# Folder where the build is done (bin and lib). Can not be changed. -export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out} +# The depends dir. +# This folder exists on the ci host and ci guest. Changes are propagated back and forth. +export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} +# Folder where the build is done (bin and lib). +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git} export GOAL=${GOAL:-install} -export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} +export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"} # Dash's Docker-specifics diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 015364c276..061a7276a0 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -51,25 +51,27 @@ elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (A DOCKER_ADMIN="--cap-add SYS_PTRACE" fi +export P_CI_DIR="$PWD" + if [ -z "$RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ - --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ - --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \ - -w $BASE_BUILD_DIR \ + --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ + -w $BASE_ROOT_DIR \ --env-file /tmp/env \ $DOCKER_NAME_TAG) DOCKER_EXEC () { - docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" + docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" } else echo "Running on host system without docker wrapper" DOCKER_EXEC () { - bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" + bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*" } fi export -f DOCKER_EXEC @@ -101,11 +103,11 @@ if [ ! -d ${DIR_QA_ASSETS} ]; then fi export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ -DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" +DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" if [ -z "$RUN_CI_ON_HOST" ]; then - echo "Create $BASE_BUILD_DIR" - DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR + echo "Create $BASE_ROOT_DIR" + DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR fi if [ "$USE_BUSY_BOX" = "true" ]; then diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 8a14be4197..131be54c4a 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -13,13 +13,13 @@ else DOCKER_EXEC echo \> \$HOME/.dashcore fi -DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources +DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then - DOCKER_EXEC tar -C "depends/SDKs" -xf "$OSX_SDK_PATH" + DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" fi if [[ $HOST = *-mingw32 ]]; then DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)