From 4d8d40b5999f37117b3d1cfa5d50c72581959319 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Mon, 13 Dec 2021 12:24:31 +0530
Subject: [PATCH 1/8] merge bitcoin#16582: Rework ci (Use travis only as
 fallback env)

---
 .cirrus.yml                                   |  24 +++
 .travis.yml                                   |  37 ++--
 .travis/README.md                             |   8 -
 .travis/test_04_install.sh                    |  29 ----
 ci/README.md                                  |  25 +++
 .../extended_lint/04_install.sh               |   0
 .../extended_lint/06_script.sh                |   0
 .../lint/04_install.sh                        |   0
 .../lint/05_before_script.sh                  |   0
 .../lint_06_script.sh => ci/lint/06_script.sh |   0
 ci/retry/README.md                            | 123 +++++++++++++
 ci/retry/retry                                | 163 ++++++++++++++++++
 ci/test/00_setup_env.sh                       |  33 ++++
 .../test/03_before_install.sh                 |   1 -
 ci/test/04_install.sh                         |  47 +++++
 .../test/05_before_script.sh                  |   2 +-
 ci/test_run_all.sh                            |  14 ++
 17 files changed, 441 insertions(+), 65 deletions(-)
 delete mode 100644 .travis/README.md
 delete mode 100755 .travis/test_04_install.sh
 create mode 100644 ci/README.md
 rename .travis/extended_lint_04_install.sh => ci/extended_lint/04_install.sh (100%)
 rename .travis/extended_lint_06_script.sh => ci/extended_lint/06_script.sh (100%)
 rename .travis/lint_04_install.sh => ci/lint/04_install.sh (100%)
 rename .travis/lint_05_before_script.sh => ci/lint/05_before_script.sh (100%)
 rename .travis/lint_06_script.sh => ci/lint/06_script.sh (100%)
 create mode 100644 ci/retry/README.md
 create mode 100755 ci/retry/retry
 create mode 100755 ci/test/00_setup_env.sh
 rename .travis/test_03_before_install.sh => ci/test/03_before_install.sh (87%)
 create mode 100755 ci/test/04_install.sh
 rename .travis/test_05_before_script.sh => ci/test/05_before_script.sh (88%)
 create mode 100755 ci/test_run_all.sh

diff --git a/.cirrus.yml b/.cirrus.yml
index 5f8d969c6f..e1b100cb88 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -28,3 +28,27 @@ task:
     - gmake check ${MAKEJOBS} VERBOSE=1
   functional_test_script:
     - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast
+task:
+  name: "x86_64 Linux  [GOAL: install]  [bionic]  [Using ./ci/ system]"
+  container:
+    image: ubuntu:18.04
+    cpu: 8
+    memory: 8G
+  timeout_in: 60m
+  env:
+    MAKEJOBS: "-j9"
+    RUN_CI_ON_HOST: "1"
+    CCACHE_SIZE: "200M"
+    CCACHE_DIR: "/tmp/ccache_dir"
+  ccache_cache:
+    folder: "/tmp/ccache_dir"
+  depends_built_cache:
+    folder: "/tmp/cirrus-ci-build/depends/built"
+  depends_sdk_cache:
+    folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
+  install_script:
+    - apt-get update
+    - apt-get -y install git bash ccache
+    - ccache --max-size=${CCACHE_SIZE}
+  ci_script:
+    - ./ci/test_run_all.sh
diff --git a/.travis.yml b/.travis.yml
index e846125c68..b1755c1ae4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -53,33 +53,18 @@ builddocker: &builddocker
   script:
     # no need to run tests again here
     - if [ "$DOCKER_BUILD" = "true" ]; then $DOCKER_RUN_IN_BUILDER ./ci/dash/build_depends.sh && $DOCKER_RUN_IN_BUILDER ./ci/dash/build_src.sh && BUILD_DIR=build-ci/dashcore-$BUILD_TARGET ./ci/dash/build-docker.sh; fi
-
-    - MAKEJOBS=-j3
-    - RUN_UNIT_TESTS=true
-    - RUN_FUNCTIONAL_TESTS=true
-    - RUN_BENCH=false  # Set to true for any one job that has debug enabled, to quickly check bench is not crashing or hitting assertions
-    - RUN_FUZZ_TESTS=false
-    - DOCKER_NAME_TAG=ubuntu:18.04
-    - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
-    - CCACHE_SIZE=100M
-    - CCACHE_TEMPDIR=/tmp/.ccache-temp
-    - CCACHE_COMPRESS=1
-    - CCACHE_DIR=$HOME/.ccache
-    - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
-    - SDK_URL=https://bitcoincore.org/depends-sources/sdks
-    - WINEDEBUG=fixme-all
-    - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
+    - CI_RETRY_EXE="travis_retry"
 before_install:
-  - set -o errexit; source .travis/test_03_before_install.sh
+  - set -o errexit; source ./ci/test/00_setup_env.sh
+  - set -o errexit; source ./ci/test/03_before_install.sh
 install:
-  - set -o errexit; source .travis/test_04_install.sh
+  - set -o errexit; source ./ci/test/04_install.sh
 before_script:
-  - set -o errexit; source .travis/test_05_before_script.sh
+  - set -o errexit; source ./ci/test/05_before_script.sh
 script:
   - if [ $SECONDS -gt 1200 ]; then set +o errexit; echo "Travis early exit to cache current state"; false; else set -o errexit; source .travis/test_06_script.sh; fi
 after_script:
   - echo $TRAVIS_COMMIT_RANGE
-  - echo $TRAVIS_COMMIT_LOG
 jobs:
   include:
     # build depends
@@ -198,11 +183,11 @@ after_success:
       language: python
       python: '3.5' # Oldest supported version according to doc/dependencies.md
       install:
-        - set -o errexit; source .travis/lint_04_install.sh
+        - set -o errexit; source ./ci/lint/04_install.sh
       before_script:
-        - set -o errexit; source .travis/lint_05_before_script.sh
+        - set -o errexit; source ./ci/lint/05_before_script.sh
       script:
-        - set -o errexit; source .travis/lint_06_script.sh
+        - set -o errexit; source ./ci/lint/06_script.sh
 
     - stage: extended-lint
       name: 'extended lint [runtime >= 60 seconds]'
@@ -211,11 +196,11 @@ after_success:
       language: python
       python: '3.5'
       install:
-        - set -o errexit; source .travis/extended_lint_04_install.sh
+        - set -o errexit; source ./ci/extended_lint/04_install.sh
       before_script:
-        - set -o errexit; source .travis/lint_05_before_script.sh
+        - set -o errexit; source ./ci/lint/05_before_script.sh
       script:
-        - set -o errexit; source .travis/extended_lint_06_script.sh
+        - set -o errexit; source ./ci/extended_lint/06_script.sh
 
     - stage: test
       name: 'ARM  [GOAL: install]  [no unit or functional tests]'
diff --git a/.travis/README.md b/.travis/README.md
deleted file mode 100644
index 21d1b9cc03..0000000000
--- a/.travis/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-## travis build scripts
-
-The `.travis` directory contains scripts for each build step in each build stage.
-Currently the travis build defines two stages `lint` and `test`. Each stage has
-it's own [lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle).
-Every script in here is named and numbered according to which stage and lifecycle
-step it belongs to.
-
diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh
deleted file mode 100755
index ee724ddbbf..0000000000
--- a/.travis/test_04_install.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-free -m -h
-echo "Number of CPUs (nproc): $(nproc)"
-
-travis_retry docker pull "$DOCKER_NAME_TAG"
-export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan"
-export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
-env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
-if [[ $HOST = *-mingw32 ]]; then
-  DOCKER_ADMIN="--cap-add SYS_ADMIN"
-elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
-  DOCKER_ADMIN="--cap-add SYS_PTRACE"
-fi
-DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
-
-DOCKER_EXEC () {
-  docker exec $DOCKER_ID bash -c "cd $PWD && $*"
-}
-
-travis_retry DOCKER_EXEC apt-get update
-travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
-
diff --git a/ci/README.md b/ci/README.md
new file mode 100644
index 0000000000..754cbc7c95
--- /dev/null
+++ b/ci/README.md
@@ -0,0 +1,25 @@
+## ci scripts
+
+This directory contains scripts for each build step in each build stage.
+
+Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the
+[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named
+and numbered according to which stage and lifecycle step it belongs to.
+
+### Running a stage locally
+
+To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
+requires `docker` to be installed. To install all requirements on Ubuntu, run
+
+```
+sudo apt install docker.io ccache bash git
+```
+
+To run the test stage,
+
+```
+./ci/test_run_all.sh
+```
+
+Be aware that the tests will be build and run in-place, so please run at your own risk.
+If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
diff --git a/.travis/extended_lint_04_install.sh b/ci/extended_lint/04_install.sh
similarity index 100%
rename from .travis/extended_lint_04_install.sh
rename to ci/extended_lint/04_install.sh
diff --git a/.travis/extended_lint_06_script.sh b/ci/extended_lint/06_script.sh
similarity index 100%
rename from .travis/extended_lint_06_script.sh
rename to ci/extended_lint/06_script.sh
diff --git a/.travis/lint_04_install.sh b/ci/lint/04_install.sh
similarity index 100%
rename from .travis/lint_04_install.sh
rename to ci/lint/04_install.sh
diff --git a/.travis/lint_05_before_script.sh b/ci/lint/05_before_script.sh
similarity index 100%
rename from .travis/lint_05_before_script.sh
rename to ci/lint/05_before_script.sh
diff --git a/.travis/lint_06_script.sh b/ci/lint/06_script.sh
similarity index 100%
rename from .travis/lint_06_script.sh
rename to ci/lint/06_script.sh
diff --git a/ci/retry/README.md b/ci/retry/README.md
new file mode 100644
index 0000000000..983a498070
--- /dev/null
+++ b/ci/retry/README.md
@@ -0,0 +1,123 @@
+retry - The command line retry tool
+------------------------------------------
+
+Retry any shell command with exponential backoff or constant delay.
+
+### Instructions
+
+Install:
+
+retry is a shell script, so drop it somewhere and make sure it's added to your $PATH. Or you can use the following one-liner:
+
+```sh
+sudo sh -c "curl https://raw.githubusercontent.com/kadwanev/retry/master/retry -o /usr/local/bin/retry && chmod +x /usr/local/bin/retry"
+```
+
+If you're on OS X, retry is also on Homebrew:
+
+```
+brew pull 27283
+brew install retry
+```
+Not popular enough for homebrew-core. Please star this project to help.
+
+### Usage
+
+Help:
+
+`retry -?`
+
+    Usage: retry [options] -- execute command
+        -h, -?, --help
+        -v, --verbose                    Verbose output
+        -t, --tries=#                    Set max retries: Default 10
+        -s, --sleep=secs                 Constant sleep amount (seconds)
+        -m, --min=secs                   Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+        -x, --max=secs                   Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+        -f, --fail="script +cmds"        Fail Script: run in case of final failure
+
+### Examples
+
+No problem:
+
+`retry echo u work good`
+
+    u work good
+
+Test functionality:
+
+`retry 'echo "y u no work"; false'`
+
+    y u no work
+    Before retry #1: sleeping 0.3 seconds
+    y u no work
+    Before retry #2: sleeping 0.6 seconds
+    y u no work
+    Before retry #3: sleeping 1.2 seconds
+    y u no work
+    Before retry #4: sleeping 2.4 seconds
+    y u no work
+    Before retry #5: sleeping 4.8 seconds
+    y u no work
+    Before retry #6: sleeping 9.6 seconds
+    y u no work
+    Before retry #7: sleeping 19.2 seconds
+    y u no work
+    Before retry #8: sleeping 38.4 seconds
+    y u no work
+    Before retry #9: sleeping 60.0 seconds
+    y u no work
+    Before retry #10: sleeping 60.0 seconds
+    y u no work
+    etc..
+
+Limit retries:
+
+`retry -t 4 'echo "y u no work"; false'`
+
+    y u no work
+    Before retry #1: sleeping 0.3 seconds
+    y u no work
+    Before retry #2: sleeping 0.6 seconds
+    y u no work
+    Before retry #3: sleeping 1.2 seconds
+    y u no work
+    Before retry #4: sleeping 2.4 seconds
+    y u no work
+    Retries exhausted
+
+Bad command:
+
+`retry poop`
+
+    bash: poop: command not found
+
+Fail command:
+
+`retry -t 3 -f 'echo "oh poopsickles"' 'echo "y u no work"; false'`
+
+    y u no work
+    Before retry #1: sleeping 0.3 seconds
+    y u no work
+    Before retry #2: sleeping 0.6 seconds
+    y u no work
+    Before retry #3: sleeping 1.2 seconds
+    y u no work
+    Retries exhausted, running fail script
+    oh poopsickles
+
+Last attempt passed:
+
+`retry -t 3 -- 'if [ $RETRY_ATTEMPT -eq 3 ]; then echo Passed at attempt $RETRY_ATTEMPT; true; else echo Failed at attempt $RETRY_ATTEMPT; false; fi;'`
+
+    Failed at attempt 0
+    Before retry #1: sleeping 0.3 seconds
+    Failed at attempt 1
+    Before retry #2: sleeping 0.6 seconds
+    Failed at attempt 2
+    Before retry #3: sleeping 1.2 seconds
+    Passed at attempt 3
+
+### License
+
+Apache 2.0 - go nuts
diff --git a/ci/retry/retry b/ci/retry/retry
new file mode 100755
index 0000000000..0e5f6e9701
--- /dev/null
+++ b/ci/retry/retry
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+
+GETOPT_BIN=$IN_GETOPT_BIN
+GETOPT_BIN=${GETOPT_BIN:-getopt}
+
+__sleep_amount() {
+  if [ -n "$constant_sleep" ]; then
+    sleep_time=$constant_sleep
+  else
+    #TODO: check for awk
+    #TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc
+    sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"`
+  fi
+}
+
+__log_out() {
+  echo "$1" 1>&2
+}
+
+# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
+retry()
+{
+  local max_tries="$1"; shift
+  local min_sleep="$1"; shift
+  local max_sleep="$1"; shift
+  local constant_sleep="$1"; shift
+  local fail_script="$1"; shift
+  if [ -n "$VERBOSE" ]; then
+    __log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep"
+    if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi
+    __log_out ""
+    __log_out "Execution Command: $*"
+    __log_out ""
+  fi
+
+  local attempts=0
+  local return_code=1
+
+
+  while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do
+    if [ $attempts -gt 0 ]; then
+      __sleep_amount
+      __log_out "Before retry #$attempts: sleeping $sleep_time seconds"
+      sleep $sleep_time
+    fi
+
+    P="$1"
+    for param in "${@:2}"; do P="$P '$param'"; done
+    #TODO: replace single quotes in each arg with '"'"' ?
+    export RETRY_ATTEMPT=$attempts
+    bash -c "$P"
+    return_code=$?
+    #__log_out "Process returned $return_code on attempt $attempts"
+    if [ $return_code -eq 127 ]; then
+      # command not found
+      exit $return_code
+    elif [ $return_code -ne 0 ]; then
+      attempts=$[$attempts +1]
+    fi
+  done
+
+  if [ $attempts -gt $max_tries ]; then
+    if [ -n "$fail_script" ]; then
+      __log_out "Retries exhausted, running fail script"
+      eval $fail_script
+    else
+      __log_out "Retries exhausted"
+    fi
+  fi
+
+  exit $return_code
+}
+
+# If we're being sourced, don't worry about such things
+if [ "$BASH_SOURCE" == "$0" ]; then
+  # Prints the help text
+  help()
+  {
+    local retry=$(basename $0)
+    cat <<EOF
+Usage: $retry [options] -- execute command
+    -h, -?, --help
+    -v, --verbose                    Verbose output
+    -t, --tries=#                    Set max retries: Default 10
+    -s, --sleep=secs                 Constant sleep amount (seconds)
+    -m, --min=secs                   Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+    -x, --max=secs                   Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+    -f, --fail="script +cmds"        Fail Script: run in case of final failure
+EOF
+  }
+
+  # show help for no arguments if stdin is a terminal
+  if { [ -z "$1" ] && [ -t 0 ] ; } || [ "$1" == '-h' ] || [ "$1" == '-?' ] || [ "$1" == '--help' ]
+  then
+    help
+    exit 0
+  fi
+
+  $GETOPT_BIN --test > /dev/null
+  if [[ $? -ne 4 ]]; then
+      echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt."
+      exit 1
+  fi
+
+  OPTIONS=vt:s:m:x:f:
+  LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail:
+
+  PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@")
+  if [[ $? -ne 0 ]]; then
+    # e.g. $? == 1
+    #  then getopt has complained about wrong arguments to stdout
+    exit 2
+  fi
+  # read getopt’s output this way to handle the quoting right:
+  eval set -- "$PARSED"
+
+  max_tries=10
+  min_sleep=0.3
+  max_sleep=60.0
+  constant_sleep=
+  fail_script=
+
+  # now enjoy the options in order and nicely split until we see --
+  while true; do
+      case "$1" in
+          -v|--verbose)
+              VERBOSE=true
+              shift
+              ;;
+          -t|--tries)
+              max_tries="$2"
+              shift 2
+              ;;
+          -s|--sleep)
+              constant_sleep="$2"
+              shift 2
+              ;;
+          -m|--min)
+              min_sleep="$2"
+              shift 2
+              ;;
+          -x|--max)
+              max_sleep="$2"
+              shift 2
+              ;;
+          -f|--fail)
+              fail_script="$2"
+              shift 2
+              ;;
+          --)
+              shift
+              break
+              ;;
+          *)
+              echo "Programming error"
+              exit 3
+              ;;
+      esac
+  done
+
+  retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@"
+
+fi
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
new file mode 100755
index 0000000000..fae48ba176
--- /dev/null
+++ b/ci/test/00_setup_env.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+echo "Setting default values in env"
+
+BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_ROOT_DIR
+
+export MAKEJOBS=${MAKEJOBS:--j3}
+export HOST=${HOST:-x86_64-unknown-linux-gnu}
+export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
+export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
+export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
+export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04}
+export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID}
+export CCACHE_SIZE=${CCACHE_SIZE:-100M}
+export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
+export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
+export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache}
+export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}}
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
+export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
+export WINEDEBUG=${WINEDEBUG:-fixme-all}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
+export GOAL=${GOAL:-install}
+export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
+export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
+export CI_RETRY_EXE=${CI_RETRY_EXE:retry}
diff --git a/.travis/test_03_before_install.sh b/ci/test/03_before_install.sh
similarity index 87%
rename from .travis/test_03_before_install.sh
rename to ci/test/03_before_install.sh
index 3c9fcf3f98..5086114ba1 100755
--- a/.travis/test_03_before_install.sh
+++ b/ci/test/03_before_install.sh
@@ -6,7 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
 # Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers.
 PATH=$PATH:/usr/lib/llvm-6.0/bin/
 export PATH
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
new file mode 100755
index 0000000000..e8c17d92cf
--- /dev/null
+++ b/ci/test/04_install.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+ccache echo "Creating ccache dir if it didn't already exist"
+
+if [ ! -d ${DIR_QA_ASSETS} ]; then
+  git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
+fi
+export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
+
+mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
+export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan"
+export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
+env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
+if [[ $HOST = *-mingw32 ]]; then
+  DOCKER_ADMIN="--cap-add SYS_ADMIN"
+elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
+  DOCKER_ADMIN="--cap-add SYS_PTRACE"
+fi
+
+if [ -z "$RUN_CI_ON_HOST" ]; then
+  echo "Creating $DOCKER_NAME_TAG container to run in"
+  ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
+
+  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
+
+  DOCKER_EXEC () {
+    docker exec $DOCKER_ID bash -c "cd $PWD && $*"
+  }
+else
+  echo "Running on host system without docker wrapper"
+  DOCKER_EXEC () {
+    bash -c "cd $PWD && $*"
+  }
+fi
+
+DOCKER_EXEC free -m -h
+DOCKER_EXEC echo "Number of CPUs \(nproc\): $(nproc)"
+
+${CI_RETRY_EXE} DOCKER_EXEC apt-get update
+${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
+
diff --git a/.travis/test_05_before_script.sh b/ci/test/05_before_script.sh
similarity index 88%
rename from .travis/test_05_before_script.sh
rename to ci/test/05_before_script.sh
index 1061b385eb..30a22a063f 100755
--- a/.travis/test_05_before_script.sh
+++ b/ci/test/05_before_script.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-DOCKER_EXEC echo \> \$HOME/.bitcoin  # Make sure default datadir does not exist and is never read by creating a dummy file
+DOCKER_EXEC echo \> \$HOME/.dashcore  # Make sure default datadir does not exist and is never read by creating a dummy file
 OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
 OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
 
diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh
new file mode 100755
index 0000000000..0e9cf1d6ba
--- /dev/null
+++ b/ci/test_run_all.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+echo "Setting default values in env"
+
+set -o errexit; source ./ci/test/00_setup_env.sh
+set -o errexit; source ./ci/test/03_before_install.sh
+set -o errexit; source ./ci/test/04_install.sh
+set -o errexit; source ./ci/test/05_before_script.sh

From 9b1bd91f52786c1a99b68d70f00225995bcd7ad1 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Thu, 15 Aug 2019 16:51:31 -0400
Subject: [PATCH 2/8] merge bitcoin#16623: Add environment files for all
 settings

---
 .travis.yml                            | 75 ++++----------------------
 ci/README.md                           |  8 ++-
 ci/test/00_setup_env.sh                |  7 +++
 ci/test/00_setup_env_amd64_asan.sh     | 14 +++++
 ci/test/00_setup_env_amd64_fuzz.sh     | 16 ++++++
 ci/test/00_setup_env_amd64_nowallet.sh | 13 +++++
 ci/test/00_setup_env_amd64_qt5.sh      | 14 +++++
 ci/test/00_setup_env_amd64_trusty.sh   | 14 +++++
 ci/test/00_setup_env_amd64_tsan.sh     | 15 ++++++
 ci/test/00_setup_env_arm.sh            | 16 ++++++
 ci/test/00_setup_env_i686.sh           | 13 +++++
 ci/test/00_setup_env_mac.sh            | 16 ++++++
 ci/test/00_setup_env_win64.sh          | 13 +++++
 ci/test_run_all.sh                     |  2 -
 14 files changed, 169 insertions(+), 67 deletions(-)
 create mode 100644 ci/test/00_setup_env_amd64_asan.sh
 create mode 100644 ci/test/00_setup_env_amd64_fuzz.sh
 create mode 100644 ci/test/00_setup_env_amd64_nowallet.sh
 create mode 100644 ci/test/00_setup_env_amd64_qt5.sh
 create mode 100644 ci/test/00_setup_env_amd64_trusty.sh
 create mode 100644 ci/test/00_setup_env_amd64_tsan.sh
 create mode 100644 ci/test/00_setup_env_arm.sh
 create mode 100644 ci/test/00_setup_env_i686.sh
 create mode 100644 ci/test/00_setup_env_mac.sh
 create mode 100644 ci/test/00_setup_env_win64.sh

diff --git a/.travis.yml b/.travis.yml
index b1755c1ae4..7e98789fb4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -205,101 +205,48 @@ after_success:
     - stage: test
       name: 'ARM  [GOAL: install]  [no unit or functional tests]'
       env: >-
-        HOST=arm-linux-gnueabihf
-        PACKAGES="python3 g++-arm-linux-gnueabihf"
-        RUN_UNIT_TESTS=false
-        RUN_FUNCTIONAL_TESTS=false
-        GOAL="install"
-        # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
-        # This could be removed once the ABI change warning does not show up by default
-        BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi"
+        FILE_ENV="./ci/test/00_setup_env_arm.sh"
+
     - stage: test
       name: 'Win64  [GOAL: deploy]  [no gui or functional tests]'
       env: >-
-        HOST=x86_64-w64-mingw32
-        PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
-        RUN_FUNCTIONAL_TESTS=false
-        GOAL="deploy"
-        BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"
+        FILE_ENV="./ci/test/00_setup_env_win64.sh"
 
     - stage: test
       name: '32-bit + dash  [GOAL: install] [GUI: no BIP70]'
       env: >-
-        HOST=i686-pc-linux-gnu
-        PACKAGES="g++-multilib python3-zmq"
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --disable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++"
-        CONFIG_SHELL="/bin/dash"
+        FILE_ENV="./ci/test/00_setup_env_i686.sh"
 
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [bionic]  [uses qt5 dev package instead of depends Qt to speed up build and avoid timeout] [unsigned char]'
       env: >-
-        HOST=x86_64-unknown-linux-gnu
-        PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev"
-        DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
-        TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\""
+        FILE_ENV="./ci/test/00_setup_env_amd64_qt5.sh"
 
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [trusty]  [depends for now]'
       env: >-
-        HOST=x86_64-unknown-linux-gnu
-        DOCKER_NAME_TAG=ubuntu:14.04
-        PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
-        RUN_FUNCTIONAL_TESTS=false
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no"
+        FILE_ENV="./ci/test/00_setup_env_amd64_trusty.sh"
 
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [xenial]  [no depends, only system libs, sanitizers: thread (TSan), no wallet]'
       env: >-
-        HOST=x86_64-unknown-linux-gnu
-        DOCKER_NAME_TAG=ubuntu:16.04
-        PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
-        NO_DEPENDS=1
-        RUN_FUNCTIONAL_TESTS=false # Disabled for now. TODO identify suppressions or exclude specific tests
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++"
+        FILE_ENV="./ci/test/00_setup_env_amd64_tsan.sh"
 
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no depends, only system libs, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer]'
       env: >-
-        HOST=x86_64-unknown-linux-gnu
-        PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
-        NO_DEPENDS=1
-        RUN_BENCH=true
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=integer,undefined CC=clang CXX=clang++"
+        FILE_ENV="./ci/test/00_setup_env_amd64_asan.sh"
 
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [bionic]  [no wallet]'
       env: >-
-        HOST=x86_64-unknown-linux-gnu
-        PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev"
-        NO_DEPENDS=1
-        RUN_UNIT_TESTS=false
-        RUN_FUNCTIONAL_TESTS=false
-        RUN_FUZZ_TESTS=true
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++"
+        FILE_ENV="./ci/test/00_setup_env_amd64_fuzz.sh"
 
     - stage: test
       env: >-
-        HOST=x86_64-unknown-linux-gnu
-        PACKAGES="python3-zmq"
-        DEP_OPTS="NO_WALLET=1"
-        GOAL="install"
-        BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
+        FILE_ENV="./ci/test/00_setup_env_amd64_nowallet.sh"
 
     - stage: test
       name: 'macOS 10.12  [GOAL: deploy]'
       env: >-
-        HOST=x86_64-apple-darwin19
-        PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
-        export XCODE_VERSION=11.3.1
-        export XCODE_BUILD_ID=11C505
-        RUN_UNIT_TESTS=false
-        RUN_FUNCTIONAL_TESTS=false
-        GOAL="deploy"
-        BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"
+        FILE_ENV="./ci/test/00_setup_env_mac.sh"
diff --git a/ci/README.md b/ci/README.md
index 754cbc7c95..16c481158f 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -15,11 +15,17 @@ requires `docker` to be installed. To install all requirements on Ubuntu, run
 sudo apt install docker.io ccache bash git
 ```
 
-To run the test stage,
+To run the default test stage,
 
 ```
 ./ci/test_run_all.sh
 ```
 
+To run the test stage with a specific configuration,
+
+```
+FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
+```
+
 Be aware that the tests will be build and run in-place, so please run at your own risk.
 If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
index fae48ba176..09b37f8240 100755
--- a/ci/test/00_setup_env.sh
+++ b/ci/test/00_setup_env.sh
@@ -31,3 +31,10 @@ export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
 export CI_RETRY_EXE=${CI_RETRY_EXE:retry}
+
+echo "Setting specific values in env"
+if [ -n "${FILE_ENV}" ]; then
+  set -o errexit;
+  # shellcheck disable=SC1090
+  source "${FILE_ENV}"
+fi
diff --git a/ci/test/00_setup_env_amd64_asan.sh b/ci/test/00_setup_env_amd64_asan.sh
new file mode 100644
index 0000000000..0e5cb1ce9a
--- /dev/null
+++ b/ci/test/00_setup_env_amd64_asan.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-unknown-linux-gnu
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
+export NO_DEPENDS=1
+export RUN_BENCH=true
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=integer,undefined CC=clang CXX=clang++"
diff --git a/ci/test/00_setup_env_amd64_fuzz.sh b/ci/test/00_setup_env_amd64_fuzz.sh
new file mode 100644
index 0000000000..2b032ce1d5
--- /dev/null
+++ b/ci/test/00_setup_env_amd64_fuzz.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-unknown-linux-gnu
+export PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev"
+export NO_DEPENDS=1
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export RUN_FUZZ_TESTS=true
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address CC=clang CXX=clang++"
diff --git a/ci/test/00_setup_env_amd64_nowallet.sh b/ci/test/00_setup_env_amd64_nowallet.sh
new file mode 100644
index 0000000000..d5a2ba3111
--- /dev/null
+++ b/ci/test/00_setup_env_amd64_nowallet.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-unknown-linux-gnu
+export PACKAGES="python3-zmq"
+export DEP_OPTS="NO_WALLET=1"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
diff --git a/ci/test/00_setup_env_amd64_qt5.sh b/ci/test/00_setup_env_amd64_qt5.sh
new file mode 100644
index 0000000000..77b1531be4
--- /dev/null
+++ b/ci/test/00_setup_env_amd64_qt5.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-unknown-linux-gnu
+export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev"
+export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
+export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\""
diff --git a/ci/test/00_setup_env_amd64_trusty.sh b/ci/test/00_setup_env_amd64_trusty.sh
new file mode 100644
index 0000000000..c997d50f9e
--- /dev/null
+++ b/ci/test/00_setup_env_amd64_trusty.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-unknown-linux-gnu
+export DOCKER_NAME_TAG=ubuntu:14.04
+export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
+export RUN_FUNCTIONAL_TESTS=false
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no"
diff --git a/ci/test/00_setup_env_amd64_tsan.sh b/ci/test/00_setup_env_amd64_tsan.sh
new file mode 100644
index 0000000000..253222ada2
--- /dev/null
+++ b/ci/test/00_setup_env_amd64_tsan.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-unknown-linux-gnu
+export DOCKER_NAME_TAG=ubuntu:16.04
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libprotobuf-dev protobuf-compiler libqrencode-dev"
+export NO_DEPENDS=1
+export RUN_FUNCTIONAL_TESTS=false # Disabled for now. TODO identify suppressions or exclude specific tests
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++"
diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh
new file mode 100644
index 0000000000..ac7ace8c3b
--- /dev/null
+++ b/ci/test/00_setup_env_arm.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=arm-linux-gnueabihf
+export PACKAGES="python3 g++-arm-linux-gnueabihf"
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export GOAL="install"
+# -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
+# This could be removed once the ABI change warning does not show up by default
+export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi"
diff --git a/ci/test/00_setup_env_i686.sh b/ci/test/00_setup_env_i686.sh
new file mode 100644
index 0000000000..768e2ac558
--- /dev/null
+++ b/ci/test/00_setup_env_i686.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=i686-pc-linux-gnu
+export PACKAGES="g++-multilib python3-zmq"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --disable-bip70 --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++"
+export CONFIG_SHELL="/bin/dash"
diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh
new file mode 100644
index 0000000000..71eaedf455
--- /dev/null
+++ b/ci/test/00_setup_env_mac.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-apple-darwin19
+export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
+export XCODE_VERSION=11.3.1
+export XCODE_BUILD_ID=11C505
+export RUN_UNIT_TESTS=false
+export RUN_FUNCTIONAL_TESTS=false
+export GOAL="deploy"
+export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --enable-werror"
diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh
new file mode 100644
index 0000000000..1e04c4287a
--- /dev/null
+++ b/ci/test/00_setup_env_win64.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export HOST=x86_64-w64-mingw32
+export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64"
+export RUN_FUNCTIONAL_TESTS=false
+export GOAL="deploy"
+export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests"
diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh
index 0e9cf1d6ba..e082342d0a 100755
--- a/ci/test_run_all.sh
+++ b/ci/test_run_all.sh
@@ -6,8 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-echo "Setting default values in env"
-
 set -o errexit; source ./ci/test/00_setup_env.sh
 set -o errexit; source ./ci/test/03_before_install.sh
 set -o errexit; source ./ci/test/04_install.sh

From 0f1a215abb212ba82429cbe2c559541dd5e3d7a6 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Tue, 27 Aug 2019 15:44:57 -0400
Subject: [PATCH 3/8] merge bitcoin#16739: Pass down $MAKEJOBS to
 test_runner.py, other improvements

---
 .travis.yml             | 2 +-
 ci/test/00_setup_env.sh | 7 +++++--
 ci/test/04_install.sh   | 1 +
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7e98789fb4..f20a8a5bb3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,7 +13,7 @@ services:
 cache:
   ccache: true
   directories:
-    - $HOME/cache
+    - $TRAVIS_BUILD_DIR/ci/scratch/.ccache
 env:
   global:
     # DOCKER_HUB_USER
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
index 09b37f8240..51b5cfdd3f 100755
--- a/ci/test/00_setup_env.sh
+++ b/ci/test/00_setup_env.sh
@@ -11,7 +11,10 @@ echo "Setting default values in env"
 BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
 export BASE_ROOT_DIR
 
-export MAKEJOBS=${MAKEJOBS:--j3}
+# The number of parallel jobs to pass down to make and test_runner.py
+export MAKEJOBS=${MAKEJOBS:--j4}
+# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
+export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/}
 export HOST=${HOST:-x86_64-unknown-linux-gnu}
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
@@ -21,7 +24,7 @@ export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID}
 export CCACHE_SIZE=${CCACHE_SIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
-export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache}
+export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
 export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}}
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
index e8c17d92cf..12aec34428 100755
--- a/ci/test/04_install.sh
+++ b/ci/test/04_install.sh
@@ -6,6 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
+mkdir -p "${BASE_SCRATCH_DIR}"
 ccache echo "Creating ccache dir if it didn't already exist"
 
 if [ ! -d ${DIR_QA_ASSETS} ]; then

From 97725127906357b29e41c95a2c9f08c0e69143e2 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Tue, 24 Sep 2019 16:22:24 -0400
Subject: [PATCH 4/8] merge bitcoin#16959: Set $HOST before setting fallback
 values

---
 ci/test/00_setup_env.sh | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
index 51b5cfdd3f..94598835ac 100755
--- a/ci/test/00_setup_env.sh
+++ b/ci/test/00_setup_env.sh
@@ -6,11 +6,17 @@
 
 export LC_ALL=C.UTF-8
 
-echo "Setting default values in env"
+echo "Setting specific values in env"
+if [ -n "${FILE_ENV}" ]; then
+  set -o errexit;
+  # shellcheck disable=SC1090
+  source "${FILE_ENV}"
+fi
 
 BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
 export BASE_ROOT_DIR
 
+echo "Fallback to default values in env (if not yet set)"
 # The number of parallel jobs to pass down to make and test_runner.py
 export MAKEJOBS=${MAKEJOBS:--j4}
 # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
@@ -20,12 +26,16 @@ export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
 export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
 export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04}
-export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID}
+# Randomize test order.
+# See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html
+export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1}
 export CCACHE_SIZE=${CCACHE_SIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
 export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
-export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}}
+# Folder where the build is done (depends and dist). Can not be changed and is equal to the root of the git repo
+export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR}
+# Folder where the build is done (bin and lib). Can not be changed.
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export WINEDEBUG=${WINEDEBUG:-fixme-all}
@@ -34,10 +44,3 @@ export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
 export CI_RETRY_EXE=${CI_RETRY_EXE:retry}
-
-echo "Setting specific values in env"
-if [ -n "${FILE_ENV}" ]; then
-  set -o errexit;
-  # shellcheck disable=SC1090
-  source "${FILE_ENV}"
-fi

From f2b4607938137ad81bd12be486c6f52a34b6a50a Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Mon, 30 Sep 2019 17:26:22 -0400
Subject: [PATCH 5/8] merge bitcoin#17011: Use busybox utils for one build

---
 ci/test/00_setup_env.sh     |  2 ++
 ci/test/00_setup_env_arm.sh |  3 ++-
 ci/test/04_install.sh       | 17 ++++++++++++++---
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
index 94598835ac..8bc50da2c3 100755
--- a/ci/test/00_setup_env.sh
+++ b/ci/test/00_setup_env.sh
@@ -22,6 +22,8 @@ export MAKEJOBS=${MAKEJOBS:--j4}
 # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
 export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/}
 export HOST=${HOST:-x86_64-unknown-linux-gnu}
+# Whether to prefer BusyBox over GNU utilities
+export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
 export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh
index ac7ace8c3b..db640015b1 100644
--- a/ci/test/00_setup_env_arm.sh
+++ b/ci/test/00_setup_env_arm.sh
@@ -7,7 +7,8 @@
 export LC_ALL=C.UTF-8
 
 export HOST=arm-linux-gnueabihf
-export PACKAGES="python3 g++-arm-linux-gnueabihf"
+export PACKAGES="python3 g++-arm-linux-gnueabihf busybox"
+export USE_BUSY_BOX=true
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
index 12aec34428..017e2ba16a 100755
--- a/ci/test/04_install.sh
+++ b/ci/test/04_install.sh
@@ -31,18 +31,29 @@ if [ -z "$RUN_CI_ON_HOST" ]; then
   DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
 
   DOCKER_EXEC () {
-    docker exec $DOCKER_ID bash -c "cd $PWD && $*"
+    docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
   }
 else
   echo "Running on host system without docker wrapper"
   DOCKER_EXEC () {
-    bash -c "cd $PWD && $*"
+    bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
   }
 fi
 
 DOCKER_EXEC free -m -h
-DOCKER_EXEC echo "Number of CPUs \(nproc\): $(nproc)"
+DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
 
 ${CI_RETRY_EXE} DOCKER_EXEC apt-get update
 ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
 
+if [ "$USE_BUSY_BOX" = "true" ]; then
+  echo "Setup to use BusyBox utils"
+  DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/
+  # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)
+  # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version)
+  # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed)
+  # shellcheck disable=SC1010
+  DOCKER_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) $BASE_SCRATCH_DIR/bins/\$util\; done
+  # Print BusyBox version
+  DOCKER_EXEC patch --help
+fi

From 0f309b15eb078e0db3da08625f251a281f5930ef Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Thu, 3 Oct 2019 10:54:07 -0400
Subject: [PATCH 6/8] merge bitcoin#17040: Make apt-get more verbose, to debug
 travis timeouts

---
 ci/test/04_install.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
index 017e2ba16a..4e200ba1d8 100755
--- a/ci/test/04_install.sh
+++ b/ci/test/04_install.sh
@@ -44,7 +44,7 @@ DOCKER_EXEC free -m -h
 DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\)
 
 ${CI_RETRY_EXE} DOCKER_EXEC apt-get update
-${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
+${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
 
 if [ "$USE_BUSY_BOX" = "true" ]; then
   echo "Setup to use BusyBox utils"

From 275716c68f5bf2c8896078ac9fc4b3aa23ddf93d Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Wed, 2 Oct 2019 17:10:57 -0400
Subject: [PATCH 7/8] merge bitcoin#17041: Run tests on arm

---
 .travis.yml                 | 4 ++--
 ci/test/00_setup_env_arm.sh | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f20a8a5bb3..6145370996 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -203,12 +203,12 @@ after_success:
         - set -o errexit; source ./ci/extended_lint/06_script.sh
 
     - stage: test
-      name: 'ARM  [GOAL: install]  [no unit or functional tests]'
+      name: 'ARM  [GOAL: install]  [unit tests, no functional tests]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_arm.sh"
 
     - stage: test
-      name: 'Win64  [GOAL: deploy]  [no gui or functional tests]'
+      name: 'Win64  [GOAL: deploy]  [unit tests, no gui, no functional tests]'
       env: >-
         FILE_ENV="./ci/test/00_setup_env_win64.sh"
 
diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh
index db640015b1..9335f0b337 100644
--- a/ci/test/00_setup_env_arm.sh
+++ b/ci/test/00_setup_env_arm.sh
@@ -7,9 +7,10 @@
 export LC_ALL=C.UTF-8
 
 export HOST=arm-linux-gnueabihf
-export PACKAGES="python3 g++-arm-linux-gnueabihf busybox"
+export QEMU_USER_CMD="qemu-arm -L /usr/arm-linux-gnueabihf/"
+export PACKAGES="python3 g++-arm-linux-gnueabihf busybox qemu-user"
 export USE_BUSY_BOX=true
-export RUN_UNIT_TESTS=false
+export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"

From 4af1f1025ae6513fde57fba67fbf7d6927d1df0f Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com>
Date: Fri, 14 Jun 2019 15:20:42 -0400
Subject: [PATCH 8/8] merge bitcoin#13728: Run the CI lint stage on mac

---
 .travis.yml           | 22 ++++++++++++++++++++++
 ci/lint/04_install.sh | 21 +++++++++++++++++----
 2 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 6145370996..93e3678ae5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -202,6 +202,28 @@ after_success:
       script:
         - set -o errexit; source ./ci/extended_lint/06_script.sh
 
+    - stage: extended-lint
+      name: 'lint macOS 10.12 (compat)'
+      os: osx
+      # Use the earliest macOS that can build our lint dependencies:
+      # Xcode 8.3.3, macOS 10.12, JDK 1.8.0_112-b16
+      # https://docs.travis-ci.com/user/reference/osx/#OS-X-Version
+      osx_image: xcode8.3
+      # TODO: if you're updating osx_image, try using "rvm:" to supply the
+      # version of ruby required by homebrew. Despite this "rvm:" declaration,
+      # brew update installs ruby 2.3.7 as its first action.
+      language: ruby
+      rvm:
+        - 2.3.7
+      env:
+      cache: false
+      install:
+        - set -o errexit; source ./ci/lint/04_install.sh
+      before_script:
+        - set -o errexit; source ./ci/lint/05_before_script.sh
+      script:
+        - set -o errexit; source ./ci/lint/06_script.sh
+
     - stage: test
       name: 'ARM  [GOAL: install]  [unit tests, no functional tests]'
       env: >-
diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh
index 18af460b83..5fb7007fb0 100755
--- a/ci/lint/04_install.sh
+++ b/ci/lint/04_install.sh
@@ -6,11 +6,24 @@
 
 export LC_ALL=C
 
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  # update first to install required ruby dependency
+  travis_retry brew update
+  travis_retry brew reinstall git -- --with-pcre2 # for  --perl-regexp
+  travis_retry brew install grep # gnu grep for --perl-regexp support
+  PATH="$(brew --prefix grep)/libexec/gnubin:$PATH"
+  travis_retry brew install shellcheck
+  travis_retry brew upgrade python
+  PATH="$(brew --prefix python)/bin:$PATH"
+  export PATH
+else
+  SHELLCHECK_VERSION=v0.7.1
+  curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/
+  PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}"
+  export PATH
+fi
+
 travis_retry pip3 install codespell==1.17.1
 travis_retry pip3 install flake8==3.8.3
 travis_retry pip3 install vulture==2.3
 travis_retry pip3 install yq
-
-SHELLCHECK_VERSION=v0.7.1
-curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/
-export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}"