Merge #17549: ci: misc cleanups

fad88e6f86d1dd32cf01db2287df9c63e66c5116 ci: Remove use of cd (MarcoFalke)
fa2941bbf47a8a6b79b8db4a87e1aedcf6a29a5e ci: Remove unmaintained extended_lint (MarcoFalke)
fa041875284d01602647519cc452185ba1ad5a8f scripted-diff: Use ci DEPENDS_DIR, remove BASE_BUILD_DIR (MarcoFalke)
fa0656d1ebc2a01d900fead4ce055d59eb818a1f ci: Add DEPENDS_DIR variable, Add documentation for folders (MarcoFalke)
faeeca87b65dd98e0efbc54443b3f8854cae9c00 scripted-diff: Move various folders to ci scratch dir (MarcoFalke)

Pull request description:

  Some minor cleanups:

  * Remove unused and unmaintained extended_lint to avoid ci bottlenecks and waste of CPU
  * Move all folders that hold temporary ci files to the ci scratch dir (except for the build dirs)
  * Add some documentation to folders and remove the `BASE_BUILD_DIR` alias for the root directory
  * Fixes #17178 by removing `cd`

Top commit has no ACKs.

Tree-SHA512: f6eb9d47ab98c08af59a79c2a4bf62fc331f0f6f3174909a28f2c2f48b0234423da5aee876225d2e188619b71b008d882a20d29a7aca68248039ea5080be5af5
This commit is contained in:
MarcoFalke 2019-12-03 11:00:58 -05:00 committed by UdjinM6
parent 5066e83d59
commit b885060802
11 changed files with 32 additions and 63 deletions

View File

@ -212,19 +212,6 @@ after_success:
script:
- set -o errexit; source ./ci/lint/06_script.sh
- stage: extended-lint
name: 'extended lint [runtime >= 60 seconds]'
env:
cache: false
language: python
python: '3.5'
install:
- set -o errexit; source ./ci/extended_lint/04_install.sh
before_script:
- set -o errexit; source ./ci/lint/05_before_script.sh
script:
- set -o errexit; source ./ci/extended_lint/06_script.sh
- stage: test
name: 'ARM [GOAL: install] [bionic] [unit tests, functional tests]'
arch: arm64

View File

@ -12,17 +12,14 @@ DOCKER_IMAGE=${DOCKER_IMAGE:-dashpay/dashd-develop}
DOCKER_TAG=${DOCKER_TAG:-latest}
DOCKER_RELATIVE_PATH=contrib/containers/deploy
BASE_BUILD_DIR=${BASE_BUILD_DIR:-.}
if [ -d $DOCKER_RELATIVE_PATH/bin ]; then
rm $DOCKER_RELATIVE_PATH/bin/*
fi
mkdir $DOCKER_RELATIVE_PATH/bin
cp "$BASE_BUILD_DIR"/src/dashd $DOCKER_RELATIVE_PATH/bin/
cp "$BASE_BUILD_DIR"/src/dash-cli $DOCKER_RELATIVE_PATH/bin/
cp "$BASE_BUILD_DIR"/src/dash-tx $DOCKER_RELATIVE_PATH/bin/
cp "$BASE_ROOT_DIR"/src/dashd $DOCKER_RELATIVE_PATH/bin/
cp "$BASE_ROOT_DIR"/src/dash-cli $DOCKER_RELATIVE_PATH/bin/
cp "$BASE_ROOT_DIR"/src/dash-tx $DOCKER_RELATIVE_PATH/bin/
strip $DOCKER_RELATIVE_PATH/bin/dashd
strip $DOCKER_RELATIVE_PATH/bin/dash-cli
strip $DOCKER_RELATIVE_PATH/bin/dash-tx

View File

@ -17,19 +17,19 @@ unset DISPLAY
mkdir -p $CACHE_DIR/depends
mkdir -p $CACHE_DIR/sdk-sources
ln -s $CACHE_DIR/depends depends/built
ln -s $CACHE_DIR/sdk-sources depends/sdk-sources
ln -s $CACHE_DIR/depends ${DEPENDS_DIR}/built
ln -s $CACHE_DIR/sdk-sources ${DEPENDS_DIR}/sdk-sources
mkdir -p depends/SDKs
mkdir -p ${DEPENDS_DIR}/SDKs
if [ -n "$XCODE_VERSION" ]; then
OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}"
if [ ! -f "$OSX_SDK_PATH" ]; then
curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
fi
if [ -f "$OSX_SDK_PATH" ]; then
tar -C depends/SDKs -xf "$OSX_SDK_PATH"
tar -C ${DEPENDS_DIR}/SDKs -xf "$OSX_SDK_PATH"
fi
fi

View File

@ -40,7 +40,7 @@ if [ -n "$CONFIG_SHELL" ]; then
export CONFIG_SHELL="$CONFIG_SHELL"
fi
BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
( test -n "$CONFIG_SHELL" && eval '"$CONFIG_SHELL" -c "./autogen.sh"' ) || ./autogen.sh

View File

@ -18,7 +18,7 @@ if [ "$RUN_INTEGRATION_TESTS" != "true" ]; then
exit 0
fi
export LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib
export LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib
cd build-ci/dashcore-$BUILD_TARGET

View File

@ -17,7 +17,7 @@ if [ "$RUN_UNIT_TESTS" != "true" ]; then
fi
export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1}
export LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib
export LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib
export WINEDEBUG=fixme-all
export BOOST_TEST_LOG_LEVEL=test_suite

View File

@ -1,12 +0,0 @@
#!/usr/bin/env bash
#
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
export LC_ALL=C
CPPCHECK_VERSION=1.86
curl -s https://codeload.github.com/danmar/cppcheck/tar.gz/${CPPCHECK_VERSION} | tar -zxf - --directory /tmp/
(cd /tmp/cppcheck-${CPPCHECK_VERSION}/ && make CFGDIR=/tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ > /dev/null)
export PATH="$PATH:/tmp/cppcheck-${CPPCHECK_VERSION}/"

View File

@ -1,9 +0,0 @@
#!/usr/bin/env bash
#
# Copyright (c) 2019 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
export LC_ALL=C
test/lint/extended-lint-all.sh

View File

@ -28,6 +28,7 @@ echo "Fallback to default values in env (if not yet set)"
MAKEJOBS="-j$(nproc)"
export MAKEJOBS
# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
# This folder only exists on the ci host.
export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/}
# What host to compile for. See also ./depends/README.md
# Tests that need cross-compilation export the appropriate HOST.
@ -47,16 +48,19 @@ export CACHE_DIR=${CACHE_DIR:-$HOST_CACHE_DIR}
export CCACHE_SIZE=${CCACHE_SIZE:-100M}
export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
# The cache dir.
# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
export CCACHE_DIR=${CCACHE_DIR:-$CACHE_DIR/ccache}
# Folder where the build is done (depends and dist). Can not be changed and is equal to the root of the git repo
export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR}
# Folder where the build is done (bin and lib). Can not be changed.
export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out}
# The depends dir.
# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
# Folder where the build is done (bin and lib).
export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
export WINEDEBUG=${WINEDEBUG:-fixme-all}
export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git}
export GOAL=${GOAL:-install}
export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"}
# Dash's Docker-specifics

View File

@ -51,25 +51,27 @@ elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (A
DOCKER_ADMIN="--cap-add SYS_PTRACE"
fi
export P_CI_DIR="$PWD"
if [ -z "$RUN_CI_ON_HOST" ]; then
echo "Creating $DOCKER_NAME_TAG container to run in"
${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
--mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \
--mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
--mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
--mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \
-w $BASE_BUILD_DIR \
--mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \
-w $BASE_ROOT_DIR \
--env-file /tmp/env \
$DOCKER_NAME_TAG)
DOCKER_EXEC () {
docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
}
else
echo "Running on host system without docker wrapper"
DOCKER_EXEC () {
bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
}
fi
export -f DOCKER_EXEC
@ -101,11 +103,11 @@ if [ ! -d ${DIR_QA_ASSETS} ]; then
fi
export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
if [ -z "$RUN_CI_ON_HOST" ]; then
echo "Create $BASE_BUILD_DIR"
DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR
echo "Create $BASE_ROOT_DIR"
DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR
fi
if [ "$USE_BUSY_BOX" = "true" ]; then

View File

@ -13,13 +13,13 @@ else
DOCKER_EXEC echo \> \$HOME/.dashcore
fi
DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources
DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources
if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then
DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
fi
if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then
DOCKER_EXEC tar -C "depends/SDKs" -xf "$OSX_SDK_PATH"
DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
fi
if [[ $HOST = *-mingw32 ]]; then
DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)