From 1c20456a52371abd880c8513bd9a840a17dcf926 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 20 Nov 2019 16:41:45 -0500 Subject: [PATCH] Merge #17423: ci: Make ci system read-only on the git work tree fa7523d3aa75b0266015af59901c2397b52265b5 ci: Extend docs (MarcoFalke) fa493ef08830efe493150d07411af85518959804 ci: Make ci system read-only on the git work tree (MarcoFalke) fab133329281cdaa3804585a2cdadd0478fefa4f ci: Remove git from required packages on host (MarcoFalke) fa00393bce0c6128c6188afc7a1d50cc01b0277f ci: Make all filesystem operations inside docker (MarcoFalke) Pull request description: Running the ci completely in a docker, without leaving any traces on the host system is not possible right now because the ccache and depends dir needs to be propagated back and picked up by the host for caching. Fixes #17372 ACKs for top commit: JeremyRubin: tested ACK fa7523d3aa75b0266015af59901c2397b52265b5 Tree-SHA512: 4bce1a0f883bcbdb34abf409bdbc80d420c5da2045d2f9c5536ac433f9e5b490f23df084546c8c049f688b487572bbfc4f9c4029e9e672f4d9279739d066ed2e --- ci/README.md | 15 +++++++++++---- ci/test/00_setup_env.sh | 2 +- ci/test/04_install.sh | 26 +++++++++++++++++++------- ci/test/05_before_script.sh | 2 +- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/ci/README.md b/ci/README.md index fb1cd7460b..880e49b459 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,11 +8,21 @@ and numbered according to which stage and lifecycle step it belongs to. ### Running a stage locally +Be aware that the tests will be built and run in-place, so please run at your own risk. +If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. + +The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory. +While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories, +such as the depends cache or ccache, are mounted as read-write into the docker container. While it should be fine to run +the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and +testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci +system in a virtual machine with a Linux operating system of your choice. + To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage requires `docker` to be installed. To install all requirements on Ubuntu, run ``` -sudo apt install docker.io bash git +sudo apt install docker.io bash ``` To run the default test stage, @@ -26,6 +36,3 @@ To run the test stage with a specific configuration, ``` FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh ``` - -Be aware that the tests will be build and run in-place, so please run at your own risk. -If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 1c7428f863..378dfa4fce 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -41,7 +41,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export WINEDEBUG=${WINEDEBUG:-fixme-all} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index d8febcaab8..40d68908e7 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -9,12 +9,6 @@ export LC_ALL=C.UTF-8 mkdir -p "${BASE_SCRATCH_DIR}" mkdir -p "${CCACHE_DIR}" -if [ ! -d ${DIR_QA_ASSETS} ]; then - git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} -fi -export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - -mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan" export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(TSAN|UBSAN)_OPTIONS)' | tee /tmp/env @@ -28,7 +22,13 @@ if [ -z "$RUN_CI_ON_HOST" ]; then echo "Creating $DOCKER_NAME_TAG container to run in" ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" - DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) + DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \ + --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \ + --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ + --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \ + -w $BASE_BUILD_DIR \ + --env-file /tmp/env \ + $DOCKER_NAME_TAG) DOCKER_EXEC () { docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*" @@ -51,6 +51,18 @@ fi ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES +if [ ! -d ${DIR_QA_ASSETS} ]; then + DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} +fi +export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + +DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" + +if [ -z "$RUN_CI_ON_HOST" ]; then + echo "Create $BASE_BUILD_DIR" + DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR +fi + if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/ diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index eac3dfcc69..1ec02cc32a 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -10,7 +10,7 @@ DOCKER_EXEC echo \> \$HOME/.dashcore # Make sure default datadir does not exist OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}" -mkdir -p depends/SDKs depends/sdk-sources +DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"