dash/ci/test/04_install.sh

114 lines
4.4 KiB
Bash
Raw Normal View History

#!/usr/bin/env bash
#
# Copyright (c) 2018-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
export LC_ALL=C.UTF-8
Merge #17635: ci: Add CentOS 7 build 711e0449cf4a0f15cabe0d64094e3add24ad44b0 ci: Remove trusty build (Hennadii Stepanov) 7f3ae224685efaeb6fe714de90e8871d12e55f34 ci: Add CentOS 7 build (Hennadii Stepanov) Pull request description: Arguably, CentOS is the most conservative distro of all the popular ones. Thus, it could be a good way to check the Bitcoin Core compatibility with aged dependencies. Currently, CentOS 7 has: - Berkeley DB == 4.8.30 - Boost == 1.53.0 - GCC == 4.8.5 - libevent == 2.0.21 < minimum required [2.0.22](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md), but tests passed - MiniUPnPc == 2.0 - Python == 3.6.8 - qrencode == 3.4.1 - Qt == 5.9.7 - ZeroMQ == 4.1.4 ~Please note that this PR is based on the bugfix #17634.~ Also trusty build has been removed for the following reasons: - https://github.com/bitcoin/bitcoin/issues/17628#issuecomment-559448201: > Maybe it'd make sense to replace Ubuntu Trusty with Centos 7 as the "check ancient backward compatibililty" Travis run. It's supported until 2024, apparently. - https://github.com/bitcoin/bitcoin/pull/17635#discussion_r354811792: > Our travis is currently running at its limit and this doesn't seem like it is adding a lot new coverage compared to the other builds. Close #17628 ACKs for top commit: MarcoFalke: ACK 711e0449cf4a0f15cabe0d64094e3add24ad44b0 🚠 Tree-SHA512: 614ec8394943f482a5867067f7119bffd052924a51e32ffda9a08e10c392c4a955a3539e2f8907cb65bfd9347dadf0ba62f6d1530bbc49927c347360a5a7f73c
2019-12-07 05:32:16 +01:00
if [[ $DOCKER_NAME_TAG == centos* ]]; then
export LC_ALL=en_US.utf8
fi
if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
export LC_ALL=C
fi
Merge #17635: ci: Add CentOS 7 build 711e0449cf4a0f15cabe0d64094e3add24ad44b0 ci: Remove trusty build (Hennadii Stepanov) 7f3ae224685efaeb6fe714de90e8871d12e55f34 ci: Add CentOS 7 build (Hennadii Stepanov) Pull request description: Arguably, CentOS is the most conservative distro of all the popular ones. Thus, it could be a good way to check the Bitcoin Core compatibility with aged dependencies. Currently, CentOS 7 has: - Berkeley DB == 4.8.30 - Boost == 1.53.0 - GCC == 4.8.5 - libevent == 2.0.21 < minimum required [2.0.22](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md), but tests passed - MiniUPnPc == 2.0 - Python == 3.6.8 - qrencode == 3.4.1 - Qt == 5.9.7 - ZeroMQ == 4.1.4 ~Please note that this PR is based on the bugfix #17634.~ Also trusty build has been removed for the following reasons: - https://github.com/bitcoin/bitcoin/issues/17628#issuecomment-559448201: > Maybe it'd make sense to replace Ubuntu Trusty with Centos 7 as the "check ancient backward compatibililty" Travis run. It's supported until 2024, apparently. - https://github.com/bitcoin/bitcoin/pull/17635#discussion_r354811792: > Our travis is currently running at its limit and this doesn't seem like it is adding a lot new coverage compared to the other builds. Close #17628 ACKs for top commit: MarcoFalke: ACK 711e0449cf4a0f15cabe0d64094e3add24ad44b0 🚠 Tree-SHA512: 614ec8394943f482a5867067f7119bffd052924a51e32ffda9a08e10c392c4a955a3539e2f8907cb65bfd9347dadf0ba62f6d1530bbc49927c347360a5a7f73c
2019-12-07 05:32:16 +01:00
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
export PATH="/usr/local/opt/ccache/libexec:$PATH"
${CI_RETRY_EXE} pip3 install $PIP_PACKAGES
fi
mkdir -p "${BASE_SCRATCH_DIR}"
mkdir -p "${CCACHE_DIR}"
Merge #12134: Build previous releases and run functional tests c456145b2c65f580683df03bf10cd39000cf24d5 [test] add 0.19 backwards compatibility tests (Sjors Provoost) b769cd142deda74fe46e231cc7b687a86514f2f1 [test] add v0.17.1 wallet upgrade test (Sjors Provoost) 9d9390dab716f07057c94e8e21f3c7dd06192f35 [tests] add wallet backwards compatility tests (Sjors Provoost) c7ca6308968b29a0e0edc485cd06e68e5edb7c7d [scripts] support release candidates of earlier releases (Sjors Provoost) 8b1460dbd1b732f06d4cebe1fa6844286c7a0056 [tests] check v0.17.1 and v0.18.1 backwards compatibility (Sjors Provoost) ae379cf7d12943fc192d58176673bcfe7d53da53 [scripts] build earlier releases (Sjors Provoost) Pull request description: This PR adds binaries for 0.17, 0.18 and 0.19 to Travis and runs a basic block propagation test. Includes test for upgrading v0.17.1 wallets and opening master wallets with older versions. Usage: ```sh contrib/devtools/previous_release.sh -f -b v0.19.0.1 v0.18.1 v0.17.1 test/functional/backwards_compatibility.py ``` Travis caches these earlier releases, so it should be able to run these tests with little performance impact. Additional scenarios where it might be useful to run tests against earlier releases: * creating a wallet with #11403's segwit implementation, copying it to an older node and making sure the user didn't lose any funds (although this PR doesn't support `v0.15.1`) * future consensus changes * P2P changes (e.g. to make sure we don't accidentally ban old nodes) ACKs for top commit: MarcoFalke: ACK c456145b2c65f580683df03bf10cd39000cf24d5 🔨 Tree-SHA512: 360bd870603f95b14dc0cd629532cc147344f632b808617c18e1b585dfb1f082b401e5d493a48196b719e0aeaee533ae0a773dfc9f217f704aae898576c19232
2020-02-12 15:19:50 +01:00
mkdir -p "${PREVIOUS_RELEASES_DIR}"
Merge #17674: tests: Add initialization order fiasco detection in Travis 1f9d5af4f197e7cc0469a0bb25dcbc51dfa537f4 tests: Add initialization order fiasco detection in Travis (practicalswift) Pull request description: Add initialization order fiasco detection in Travis :) Context: https://github.com/bitcoin/bitcoin/pull/17670#issuecomment-562035813 This would have caught the `events_hasher` initialization order issue introduced in #17573 and fixed in #17670. Output in case of an initialization order fiasco: ``` ==7934==ERROR: AddressSanitizer: initialization-order-fiasco on address 0x557098d79200 at pc 0x55709796b9a3 bp 0x7ffde524dc30 sp 0x7ffde524dc28 READ of size 8 at 0x557098d79200 thread T0 #0 0x55709796b9a2 in CSHA256::Finalize(unsigned char*) src/crypto/sha256.cpp:667:25 #1 0x5570978150e9 in SeedEvents(CSHA512&) src/random.cpp:462:19 #2 0x5570978145e1 in SeedSlow(CSHA512&) src/random.cpp:482:5 #3 0x5570978149a3 in SeedStartup(CSHA512&, (anonymous namespace)::RNGState&) src/random.cpp:527:5 #4 0x55709781102d in ProcRand(unsigned char*, int, RNGLevel) src/random.cpp:571:9 #5 0x557097810d19 in GetRandBytes(unsigned char*, int) src/random.cpp:576:59 #6 0x557096c2f9d5 in (anonymous namespace)::CSignatureCache::CSignatureCache() src/script/sigcache.cpp:34:9 #7 0x557096511977 in __cxx_global_var_init.7 src/script/sigcache.cpp:67:24 #8 0x5570965119f8 in _GLOBAL__sub_I_sigcache.cpp src/script/sigcache.cpp #9 0x557097bba4ac in __libc_csu_init (src/bitcoind+0x18554ac) #10 0x7f214b1c2b27 in __libc_start_main /build/glibc-OTsEL5/glibc-2.27/csu/../csu/libc-start.c:266 #11 0x5570965347d9 in _start (src/bitcoind+0x1cf7d9) 0x557098d79200 is located 96 bytes inside of global variable 'events_hasher' defined in 'random.cpp:456:16' (0x557098d791a0) of size 104 registered at: #0 0x557096545dfd in __asan_register_globals compiler-rt/lib/asan/asan_globals.cpp:360:3 #1 0x557097817f8b in asan.module_ctor (src/bitcoind+0x14b2f8b) SUMMARY: AddressSanitizer: initialization-order-fiasco src/crypto/sha256.cpp:667:25 in CSHA256::Finalize(unsigned char*) ``` ACKs for top commit: promag: Tested ACK 1f9d5af4f197e7cc0469a0bb25dcbc51dfa537f4, got MarcoFalke: ACK 1f9d5af4f197e7cc0469a0bb25dcbc51dfa537f4 👔 Tree-SHA512: f24ac0a313df7549193bd7f4fcfdf9b72bdfc6a6ee31d0b08e6d0752e5108fbd532106b6c86377ae0641258c9adb4921872e5d9a0154c0284e03315e0777102c
2019-12-05 21:56:12 +01:00
export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan"
export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
env | grep -E '^(BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|TEST_PREVIOUS_RELEASES|PREVIOUS_RELEASES_DIR))' | tee /tmp/env
if [[ $HOST = *-mingw32 ]]; then
DOCKER_ADMIN="--cap-add SYS_ADMIN"
elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
DOCKER_ADMIN="--cap-add SYS_PTRACE"
fi
export P_CI_DIR="$PWD"
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
echo "Creating $DOCKER_NAME_TAG container to run in"
${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
--mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
--mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
--mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \
Merge #12134: Build previous releases and run functional tests c456145b2c65f580683df03bf10cd39000cf24d5 [test] add 0.19 backwards compatibility tests (Sjors Provoost) b769cd142deda74fe46e231cc7b687a86514f2f1 [test] add v0.17.1 wallet upgrade test (Sjors Provoost) 9d9390dab716f07057c94e8e21f3c7dd06192f35 [tests] add wallet backwards compatility tests (Sjors Provoost) c7ca6308968b29a0e0edc485cd06e68e5edb7c7d [scripts] support release candidates of earlier releases (Sjors Provoost) 8b1460dbd1b732f06d4cebe1fa6844286c7a0056 [tests] check v0.17.1 and v0.18.1 backwards compatibility (Sjors Provoost) ae379cf7d12943fc192d58176673bcfe7d53da53 [scripts] build earlier releases (Sjors Provoost) Pull request description: This PR adds binaries for 0.17, 0.18 and 0.19 to Travis and runs a basic block propagation test. Includes test for upgrading v0.17.1 wallets and opening master wallets with older versions. Usage: ```sh contrib/devtools/previous_release.sh -f -b v0.19.0.1 v0.18.1 v0.17.1 test/functional/backwards_compatibility.py ``` Travis caches these earlier releases, so it should be able to run these tests with little performance impact. Additional scenarios where it might be useful to run tests against earlier releases: * creating a wallet with #11403's segwit implementation, copying it to an older node and making sure the user didn't lose any funds (although this PR doesn't support `v0.15.1`) * future consensus changes * P2P changes (e.g. to make sure we don't accidentally ban old nodes) ACKs for top commit: MarcoFalke: ACK c456145b2c65f580683df03bf10cd39000cf24d5 🔨 Tree-SHA512: 360bd870603f95b14dc0cd629532cc147344f632b808617c18e1b585dfb1f082b401e5d493a48196b719e0aeaee533ae0a773dfc9f217f704aae898576c19232
2020-02-12 15:19:50 +01:00
--mount type=bind,src=$PREVIOUS_RELEASES_DIR,dst=$PREVIOUS_RELEASES_DIR \
-w $BASE_ROOT_DIR \
--env-file /tmp/env \
--name $CONTAINER_NAME \
$DOCKER_NAME_TAG)
DOCKER_EXEC () {
docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
}
else
echo "Running on host system without docker wrapper"
DOCKER_EXEC () {
bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
}
fi
export -f DOCKER_EXEC
if [ -n "$DPKG_ADD_ARCH" ]; then
DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
fi
Merge #17635: ci: Add CentOS 7 build 711e0449cf4a0f15cabe0d64094e3add24ad44b0 ci: Remove trusty build (Hennadii Stepanov) 7f3ae224685efaeb6fe714de90e8871d12e55f34 ci: Add CentOS 7 build (Hennadii Stepanov) Pull request description: Arguably, CentOS is the most conservative distro of all the popular ones. Thus, it could be a good way to check the Bitcoin Core compatibility with aged dependencies. Currently, CentOS 7 has: - Berkeley DB == 4.8.30 - Boost == 1.53.0 - GCC == 4.8.5 - libevent == 2.0.21 < minimum required [2.0.22](https://github.com/bitcoin/bitcoin/blob/master/doc/dependencies.md), but tests passed - MiniUPnPc == 2.0 - Python == 3.6.8 - qrencode == 3.4.1 - Qt == 5.9.7 - ZeroMQ == 4.1.4 ~Please note that this PR is based on the bugfix #17634.~ Also trusty build has been removed for the following reasons: - https://github.com/bitcoin/bitcoin/issues/17628#issuecomment-559448201: > Maybe it'd make sense to replace Ubuntu Trusty with Centos 7 as the "check ancient backward compatibililty" Travis run. It's supported until 2024, apparently. - https://github.com/bitcoin/bitcoin/pull/17635#discussion_r354811792: > Our travis is currently running at its limit and this doesn't seem like it is adding a lot new coverage compared to the other builds. Close #17628 ACKs for top commit: MarcoFalke: ACK 711e0449cf4a0f15cabe0d64094e3add24ad44b0 🚠 Tree-SHA512: 614ec8394943f482a5867067f7119bffd052924a51e32ffda9a08e10c392c4a955a3539e2f8907cb65bfd9347dadf0ba62f6d1530bbc49927c347360a5a7f73c
2019-12-07 05:32:16 +01:00
if [[ $DOCKER_NAME_TAG == centos* ]]; then
${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release
${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES
elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
${CI_RETRY_EXE} DOCKER_EXEC apt-get update
${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
if [ -n "$PIP_PACKAGES" ]; then
${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
fi
fi
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
top -l 1 -s 0 | awk ' /PhysMem/ {print}'
echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
else
DOCKER_EXEC free -m -h
DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
DOCKER_EXEC echo $(lscpu | grep Endian)
DOCKER_EXEC echo "Free disk space:"
DOCKER_EXEC df -h
fi
if [ ! -d ${DIR_QA_ASSETS} ]; then
if [ "$RUN_FUZZ_TESTS" = "true" ]; then
DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
fi
fi
export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
echo "Create $BASE_ROOT_DIR"
DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR
fi
if [ "$USE_BUSY_BOX" = "true" ]; then
echo "Setup to use BusyBox utils"
DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/
# tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
# find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version)
# ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
# shellcheck disable=SC1010
DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) $BASE_SCRATCH_DIR/bins/\$util\; done
# Print BusyBox version
DOCKER_EXEC patch --help
fi