sikkienl/dash
1
0
Fork 0
mirror of https://github.com/dashpay/dash.git synced 2024-12-25 12:02:48 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #17767: ci: Fix qemu issues

Browse Source 
fa569e1a9c5ad1bf8bdf866235b21aff56112224 ci: Set LC_ALL=C when running in qemu-s390x (MarcoFalke)
fa3d77623ecb25dde6b5f4be11626ed16966eb29 ci: Use debian to avoid apt install 404 errors (MarcoFalke)
fabb946090be2f604da3d7d4b1bbe93b79baf23e ci: Install needed gcc and qemu-user iff cross-compiling (MarcoFalke)
faba4672b64fb5ba89e5cb6299479887494b571a ci: Fix QEMU_USER_CMD parse issues (MarcoFalke)
fa5d709fb266c97d4db15006bf855e887a6c123b ci: Move wrap-qemu into separate script (MarcoFalke)

Pull request description:

  Now that the ci system no longer propagates files from the docker container back to the host, the wrap-qemu script is broken.

  To fix it, every statement in the script needs to be executed in the docker (with `DOCKER_EXEC`). Instead of juggling with triple escape sequences like `\\\"`, just move the script to a separate file and call it with `DOCKER_EXEC`.

  Also, fix a bunch of other bugs that prevent running the ci system in qemu

  See the `ci/README.md` on how to test. TLDR: Can be tested with (replace `arm` with `s390x` to run the s390x build):

  ```
  FILE_ENV="./ci/test/00_setup_env_arm.sh" MAKEJOBS="-j9" ./ci/test_run_all.sh
  ```

ACKs for top commit:
  laanwj:
    Code review ACK fa569e1a9c5ad1bf8bdf866235b21aff56112224

Tree-SHA512: 84ebc44a4f0261ee6c29605a6896a1833ff6c81d729e6d08dd111941f570ce73221422bd3303e1108a266ec5eab2148bd5ee1cf6bc01477d8cc9a6c5bf2b34c2
This commit is contained in:
Wladimir J. van der Laan 2020-01-22 21:09:07 +01:00 committed by PastaPastaPasta
parent 8d340ed4e9
commit c31cdcd3d7
4 changed files with 32 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ci/test/00_setup_env.sh
View File

@ -58,7 +58,7 @@ export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
export WINEDEBUG=${WINEDEBUG:-fixme-all}
export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git}
export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps}
export GOAL=${GOAL:-install}
export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH

12
ci/test/00_setup_env_arm.sh
View File

@ -9,11 +9,15 @@ export LC_ALL=C.UTF-8
export HOST=arm-linux-gnueabihf
# The host arch is unknown, so we run the tests through qemu.
# If the host is arm and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-arm -L /usr/arm-linux-gnueabihf/"}"
# We don't know whether the host can run the cross compiled binaries. To run them, either qemu-user or libc6:armhf for
# the target is required, so install both.
if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-arm -L /usr/arm-linux-gnueabihf/"}"; fi
export DPKG_ADD_ARCH="armhf"
export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
export PACKAGES="python3-zmq g++-arm-linux-gnueabihf busybox libc6:armhf libstdc++6:armhf libfontconfig1:armhf libxcb1:armhf"
if [ -n "$QEMU_USER_CMD" ]; then
# Likely cross-compiling, so install the needed gcc and qemu-user
export PACKAGES="$PACKAGES qemu-user"
fi
# Use debian to avoid 404 apt errors when cross compiling
export DOCKER_NAME_TAG="debian:focal"
export CHECK_DOC=1
export USE_BUSY_BOX=true
export RUN_UNIT_TESTS=false

11
ci/test/00_setup_env_s390x.sh
View File

@ -9,8 +9,15 @@ export LC_ALL=C.UTF-8
export HOST=s390x-linux-gnu
# The host arch is unknown, so we run the tests through qemu.
# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-s390x"}"
export PACKAGES="python3-zmq bsdmainutils qemu-user"
if [ -z ${QEMU_USER_CMD+x} ]; then export QEMU_USER_CMD="${QEMU_USER_CMD:-"qemu-s390x"}"; fi
export PACKAGES="python3-zmq"
if [ -n "$QEMU_USER_CMD" ]; then
# Likely cross-compiling, so install the needed gcc and qemu-user
export DPKG_ADD_ARCH="s390x"
export PACKAGES="$PACKAGES g++-s390x-linux-gnu qemu-user libc6:s390x libstdc++6:s390x libfontconfig1:s390x libxcb1:s390x"
fi
# Use debian to avoid 404 apt errors
export DOCKER_NAME_TAG="debian:focal"
export RUN_UNIT_TESTS=true
export RUN_FUNCTIONAL_TESTS=true
export GOAL="install"

25
ci/test/04_install.sh
View File

@ -9,6 +9,9 @@ export LC_ALL=C.UTF-8
if [[ $DOCKER_NAME_TAG == centos* ]]; then
export LC_ALL=en_US.utf8
fi
if [[ $QEMU_USER_CMD == qemu-s390* ]]; then
export LC_ALL=C
fi
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
set +o errexit
@ -44,7 +47,7 @@ export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=
export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan"
export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
env | grep -E '^(QEMU_|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
if [[ $HOST = *-mingw32 ]]; then
DOCKER_ADMIN="--cap-add SYS_ADMIN"
elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
@ -76,16 +79,6 @@ else
fi
export -f DOCKER_EXEC
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
top -l 1 -s 0 | awk ' /PhysMem/ {print}'
echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
else
DOCKER_EXEC free -m -h
DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
DOCKER_EXEC echo "Free disk space:"
DOCKER_EXEC df -h
fi
if [ -n "$DPKG_ADD_ARCH" ]; then
DOCKER_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH"
fi
@ -98,6 +91,16 @@ elif [ "$TRAVIS_OS_NAME" != "osx" ]; then
${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
fi
if [ "$TRAVIS_OS_NAME" == "osx" ]; then
top -l 1 -s 0 | awk ' /PhysMem/ {print}'
echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
else
DOCKER_EXEC free -m -h
DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
DOCKER_EXEC echo "Free disk space:"
DOCKER_EXEC df -h
fi
if [ ! -d ${DIR_QA_ASSETS} ]; then
DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
fi