From 4c2211dd41b5bd94284947bfc05712e1f6f48a85 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 27 Aug 2018 09:33:59 -0400 Subject: [PATCH] Partial merge #13863: travis: move script sections to files in `.travis/` subject to shellcheck 414326952c use export LC_ALL=C.UTF-8 (Julian Fleischer) 728c82d029 make script exit if a command fails (Julian Fleischer) 506890b24d move remaining travis build steps into individual files (Julian Fleischer) 272306ea57 number .travis/ script according to build lifecycle and add README to explain (Julian Fleischer) 519e2739cf move lint stage up to resemble travis build ui (Julian Fleischer) 86d34f0e65 abort script in END_FOLD on non-zero exit code (Julian Fleischer) 4f2f88c7b0 move script sections info individual files and comply with shellcheck (Julian Fleischer) Pull request description: This PR is extracted from https://github.com/bitcoin/bitcoin/pull/13816 to make that one easier to review. It follows on https://github.com/bitcoin/bitcoin/pull/13849 and https://github.com/bitcoin/bitcoin/pull/13851 In here the shell script parts from `travis.yml` are extracted into `.travis/before_install.sh`, `.travis/install.sh`, `.travis/before_script.sh`, `.travis/script.sh`, and `.travis/lint.sh`. This has the benefit that `test/lint/lint-shell.sh` will also shellcheck these parts. Also it makes the individual script parts more readable. Tree-SHA512: c497e1687ceb1c1d795de177d3fc35af908bc8e3f781a871afabdecf031e581d4db229290627249e35ef7c09952bc34884e4734ea91d40f57b4a9efb85bba2e3 --- .travis.yml | 119 +++++++++++++++++++++++++++++- .travis/README.md | 8 ++ .travis/lint_04_install.sh | 9 +++ .travis/lint_05_before_script.sh | 9 +++ .travis/lint_06_script.sh | 24 ++++++ .travis/test_03_before_install.sh | 25 +++++++ .travis/test_04_install.sh | 26 +++++++ .travis/test_05_before_script.sh | 25 +++++++ ci/build_depends.sh | 2 +- ci/build_src.sh | 2 +- ci/matrix.sh | 2 +- ci/test_integrationtests.sh | 5 +- ci/test_unittests.sh | 2 +- test/lint/lint-shell-locale.sh | 5 +- 14 files changed, 252 insertions(+), 11 deletions(-) create mode 100644 .travis/README.md create mode 100755 .travis/lint_04_install.sh create mode 100755 .travis/lint_05_before_script.sh create mode 100755 .travis/lint_06_script.sh create mode 100755 .travis/test_03_before_install.sh create mode 100755 .travis/test_04_install.sh create mode 100755 .travis/test_05_before_script.sh diff --git a/.travis.yml b/.travis.yml index 87688015d0..4b121ef222 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ cache: ccache: true directories: - $HOME/cache - env: global: # DOCKER_HUB_USER @@ -56,6 +55,31 @@ builddocker: &builddocker # no need to run tests again here - if [ "$DOCKER_BUILD" = "true" ]; then $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh && $DOCKER_RUN_IN_BUILDER ./ci/build_src.sh && BUILD_DIR=build-ci/dashcore-$BUILD_TARGET ./docker/build-docker.sh; fi + - MAKEJOBS=-j3 + - RUN_UNIT_TESTS=true + - RUN_FUNCTIONAL_TESTS=true + - RUN_BENCH=false # Set to true for any one job that has debug enabled, to quickly check bench is not crashing or hitting assertions + - DOCKER_NAME_TAG=ubuntu:18.04 + - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID + - CCACHE_SIZE=100M + - CCACHE_TEMPDIR=/tmp/.ccache-temp + - CCACHE_COMPRESS=1 + - CCACHE_DIR=$HOME/.ccache + - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out + - SDK_URL=https://bitcoincore.org/depends-sources/sdks + - WINEDEBUG=fixme-all + - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" +before_install: + - set -o errexit; source .travis/test_03_before_install.sh +install: + - set -o errexit; source .travis/test_04_install.sh +before_script: + - set -o errexit; source .travis/test_05_before_script.sh +script: + - set -o errexit; source .travis/test_06_script.sh +after_script: + - echo $TRAVIS_COMMIT_RANGE + - echo $TRAVIS_COMMIT_LOG jobs: include: # build depends @@ -176,3 +200,96 @@ after_script: - echo $TRAVIS_COMMIT_LOG after_success: - if [ "$DOCKER_BUILD" = "true" ]; then docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_PASSWORD && ./docker/push-docker.sh; fi +# lint stage + - stage: lint + env: + sudo: false + cache: false + language: python + python: '3.6' + install: + - set -o errexit; source .travis/lint_04_install.sh + before_script: + - set -o errexit; source .travis/lint_05_before_script.sh + script: + - set -o errexit; source .travis/lint_06_script.sh +# ARM + - stage: test + env: >- + HOST=arm-linux-gnueabihf + PACKAGES="g++-arm-linux-gnueabihf" + DEP_OPTS="NO_QT=1" + RUN_UNIT_TESTS=false + RUN_FUNCTIONAL_TESTS=false + GOAL="install" + BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +# Win32 + - stage: test + env: >- + HOST=i686-w64-mingw32 + DPKG_ADD_ARCH="i386" + DEP_OPTS="NO_QT=1" + PACKAGES="python3 nsis g++-mingw-w64-i686 wine-binfmt wine32" + GOAL="install" + BITCOIN_CONFIG="--enable-reduce-exports" +# Win64 + - stage: test + env: >- + HOST=x86_64-w64-mingw32 + DEP_OPTS="NO_QT=1" + PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" + GOAL="install" + BITCOIN_CONFIG="--enable-reduce-exports" +# 32-bit + dash + - stage: test + env: >- + HOST=i686-pc-linux-gnu + PACKAGES="g++-multilib python3-zmq" + DEP_OPTS="NO_QT=1" + GOAL="install" + BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" + CONFIG_SHELL="/bin/dash" +# x86_64 Linux (uses qt5 dev package instead of depends Qt to speed up build and avoid timeout) + - stage: test + env: >- + HOST=x86_64-unknown-linux-gnu + PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" + DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" + GOAL="install" + BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CXXFLAGS=\"-g0 -O2\"" +# x86_64 Linux (no depends, only system libs) + - stage: test + env: >- + HOST=x86_64-unknown-linux-gnu + PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" + NO_DEPENDS=1 + GOAL="install" + BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER" +# x86_64 Linux (sanitizers) + - stage: test + env: >- + HOST=x86_64-unknown-linux-gnu + PACKAGES="clang python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev" + NO_DEPENDS=1 + RUN_BENCH=true + RUN_FUNCTIONAL_TESTS=false # Disabled for now, can be combined with the other x86_64 linux NO_DEPENDS job when functional tests pass the sanitizers + GOAL="install" + BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --enable-glibc-back-compat --enable-reduce-exports --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=undefined CC=clang CXX=clang++" +# x86_64 Linux, No wallet + - stage: test + env: >- + HOST=x86_64-unknown-linux-gnu + PACKAGES="python3" + DEP_OPTS="NO_WALLET=1" + GOAL="install" + BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +# Cross-Mac + - stage: test + env: >- + HOST=x86_64-apple-darwin14 + PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" + OSX_SDK=10.11 + RUN_UNIT_TESTS=false + RUN_FUNCTIONAL_TESTS=false + GOAL="all deploy" + BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror" diff --git a/.travis/README.md b/.travis/README.md new file mode 100644 index 0000000000..21d1b9cc03 --- /dev/null +++ b/.travis/README.md @@ -0,0 +1,8 @@ +## travis build scripts + +The `.travis` directory contains scripts for each build step in each build stage. +Currently the travis build defines two stages `lint` and `test`. Each stage has +it's own [lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle). +Every script in here is named and numbered according to which stage and lifecycle +step it belongs to. + diff --git a/.travis/lint_04_install.sh b/.travis/lint_04_install.sh new file mode 100755 index 0000000000..34118a57c3 --- /dev/null +++ b/.travis/lint_04_install.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C + +travis_retry pip install flake8 diff --git a/.travis/lint_05_before_script.sh b/.travis/lint_05_before_script.sh new file mode 100755 index 0000000000..28bcbb47f7 --- /dev/null +++ b/.travis/lint_05_before_script.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C + +git fetch --unshallow diff --git a/.travis/lint_06_script.sh b/.travis/lint_06_script.sh new file mode 100755 index 0000000000..6191d82571 --- /dev/null +++ b/.travis/lint_06_script.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C + +if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then + test/lint/commit-script-check.sh $TRAVIS_COMMIT_RANGE +fi + +test/lint/git-subtree-check.sh src/crypto/ctaes +test/lint/git-subtree-check.sh src/secp256k1 +test/lint/git-subtree-check.sh src/univalue +test/lint/git-subtree-check.sh src/leveldb +test/lint/check-doc.py +test/lint/check-rpc-mappings.py . +test/lint/lint-all.sh + +if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then + while read -r LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys && + travis_wait 50 contrib/verify-commits/verify-commits.py; +fi diff --git a/.travis/test_03_before_install.sh b/.travis/test_03_before_install.sh new file mode 100755 index 0000000000..d091a67ca9 --- /dev/null +++ b/.travis/test_03_before_install.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") +export PATH + +BEGIN_FOLD () { + echo "" + CURRENT_FOLD_NAME=$1 + echo "travis_fold:start:${CURRENT_FOLD_NAME}" +} + +END_FOLD () { + RET=$? + echo "travis_fold:end:${CURRENT_FOLD_NAME}" + if [ $RET != 0 ]; then + echo "${CURRENT_FOLD_NAME} failed with status code ${RET}" + fi +} + diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh new file mode 100755 index 0000000000..ef595287b7 --- /dev/null +++ b/.travis/test_04_install.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +travis_retry docker pull "$DOCKER_NAME_TAG" +env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL)' | tee /tmp/env +if [[ $HOST = *-mingw32 ]]; then + DOCKER_ADMIN="--cap-add SYS_ADMIN" +fi +DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) + +DOCKER_EXEC () { + docker exec $DOCKER_ID bash -c "cd $PWD && $*" +} + +if [ -n "$DPKG_ADD_ARCH" ]; then + DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" +fi + +travis_retry DOCKER_EXEC apt-get update +travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES + diff --git a/.travis/test_05_before_script.sh b/.travis/test_05_before_script.sh new file mode 100755 index 0000000000..70caee979c --- /dev/null +++ b/.travis/test_05_before_script.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +DOCKER_EXEC echo \> \$HOME/.bitcoin # Make sure default datadir does not exist and is never read by creating a dummy file + +mkdir -p depends/SDKs depends/sdk-sources + +if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +fi +if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz +fi +if [[ $HOST = *-mingw32 ]]; then + DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\) +fi +if [ -z "$NO_DEPENDS" ]; then + DOCKER_EXEC CONFIG_SHELL= make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS +fi + diff --git a/ci/build_depends.sh b/ci/build_depends.sh index fec2cad4de..ecacf6807d 100755 --- a/ci/build_depends.sh +++ b/ci/build_depends.sh @@ -2,7 +2,7 @@ # # This script is executed inside the builder image -export LC_ALL=C +export LC_ALL=C.UTF-8 set -e diff --git a/ci/build_src.sh b/ci/build_src.sh index 4fa3e9de37..d727d11538 100755 --- a/ci/build_src.sh +++ b/ci/build_src.sh @@ -2,7 +2,7 @@ # # This script is executed inside the builder image -export LC_ALL=C +export LC_ALL=C.UTF-8 set -e diff --git a/ci/matrix.sh b/ci/matrix.sh index e6c47f5bc0..be6b92516f 100755 --- a/ci/matrix.sh +++ b/ci/matrix.sh @@ -3,7 +3,7 @@ # This script is meant to be sourced into the actual build script. It contains the build matrix and will set all # necessary environment variables for the request build target -export LC_ALL=C +export LC_ALL=C.UTF-8 export BUILD_TARGET=${BUILD_TARGET:-linux64} export PULL_REQUEST=${PULL_REQUEST:-false} diff --git a/ci/test_integrationtests.sh b/ci/test_integrationtests.sh index bcaa22e39f..ac68ca36dd 100755 --- a/ci/test_integrationtests.sh +++ b/ci/test_integrationtests.sh @@ -2,7 +2,7 @@ # # This script is executed inside the builder image -export LC_ALL=C +export LC_ALL=C.UTF-8 set -e @@ -15,9 +15,6 @@ if [ "$RUN_INTEGRATIONTESTS" != "true" ]; then exit 0 fi -# override LC_ALL to allow special characters and emojis in filenames -export LC_ALL=C.UTF-8 - export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib cd build-ci/dashcore-$BUILD_TARGET diff --git a/ci/test_unittests.sh b/ci/test_unittests.sh index 0848fc5b50..ef711eff64 100755 --- a/ci/test_unittests.sh +++ b/ci/test_unittests.sh @@ -2,7 +2,7 @@ # # This script is executed inside the builder image -export LC_ALL=C +export LC_ALL=C.UTF-8 set -e diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh index edcd254559..084dc93f76 100755 --- a/test/lint/lint-shell-locale.sh +++ b/test/lint/lint-shell-locale.sh @@ -5,7 +5,8 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using "export LC_ALL=C", or +# a.) explicitly opt out of locale dependence using +# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or # b.) explicitly opt in to locale dependence using the annotation below. export LC_ALL=C @@ -16,7 +17,7 @@ for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(secp256k1|univalue continue fi FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" ]]; then + if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" EXIT_CODE=1 fi