Partial Merge #17517: ci: Bump to clang-8 for asan build to avoid segfaults on ppc64le

fa40e48c50d8ccf42ce5e66c12390e2ed4b60e75 ci: Remove unparseable lines from supp file for old xenial clang tsan (MarcoFalke)
fa1bfc476c9208a4c412c8ca74d05f52bb47766f ci: ubsan report_error_type=1 and add suppressions (MarcoFalke)
fa69cef13e5aab8264339eb3d50a9e89d59efd87 test: Print stderr when subprocess fails (MarcoFalke)
2222c305866a77065ab5be24c1c252bae252bb59 test: Use char instead of unsigned char (MarcoFalke)
faa8023ce9a47b282e1fac3ca8b3a7bb0042935a ci: Bump to clang-8 for asan build to avoid segfaults on ppc64le (MarcoFalke)

Pull request description:

  Use clang-8 instead of default clang (which is clang-6 on Bionic) to avoid spurious segfaults when running the ci system on ppc64le

ACKs for top commit:
  practicalswift:
    ACK fa40e48c50d8ccf42ce5e66c12390e2ed4b60e75 assuming Travis is happy -- diff looks correct :)

Tree-SHA512: f4f26232d3a0ef38da245869340f723d279a3db9823befbc735fb5a00024dae041c7306d7ae55d2488e6f86aa96cdea155b007aefb561fba505141e8dbc717dc

ci/test/00_setup_env.sh

@@ -6,6 +6,12 @@
 
 export LC_ALL=C.UTF-8
 
+# The root dir.
+# The ci system copies this folder.
+# This is where the build is done (depends and dist).
+BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_ROOT_DIR
+
 echo "Setting specific values in env"
 if [ -n "${FILE_ENV}" ]; then
   set -o errexit;
@@ -17,9 +23,6 @@ export BUILD_TARGET=${BUILD_TARGET:-linux64}
 export PULL_REQUEST=${PULL_REQUEST:-false}
 export JOB_NUMBER=${JOB_NUMBER:-1}
 
-BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
-export BASE_ROOT_DIR
-
 echo "Fallback to default values in env (if not yet set)"
 # The number of parallel jobs to pass down to make and test_runner.py
 MAKEJOBS="-j$(nproc)"

ci/test/00_setup_env_native_tsan.sh

@@ -14,3 +14,7 @@ export GOAL="install"
 export BITCOIN_CONFIG="--enable-zmq --enable-reduce-exports --enable-crash-hooks --with-sanitizers=thread"
 export CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG -DARENA_DEBUG"
 export PYZMQ=true
+
+# xenial comes with old clang versions that can not parse the sanitizer suppressions files
+# Remove unparseable lines as a hacky workaround
+sed -i '/^implicit-/d' "${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan"

ci/test/04_install.sh

@@ -43,7 +43,7 @@ mkdir -p "${CCACHE_DIR}"
 export ASAN_OPTIONS="detect_stack_use_after_return=1"
 export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan"
-export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
+export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
 env | grep -E '^(CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_ADMIN="--cap-add SYS_ADMIN"

src/test/crypto_tests.cpp

@@ -190,14 +190,15 @@ static void TestHKDF_SHA256_32(const std::string &ikm_hex, const std::string &sa
     BOOST_CHECK(HexStr(out) == okm_check_hex);
 }
 
-static std::string LongTestString() {
+static std::string LongTestString()
+{
     std::string ret;
     for (int i = 0; i < 200000; i++) {
-        ret += (unsigned char)(i);
-        ret += (unsigned char)(i >> 4);
-        ret += (unsigned char)(i >> 8);
-        ret += (unsigned char)(i >> 12);
-        ret += (unsigned char)(i >> 16);
+        ret += (char)(i);
+        ret += (char)(i >> 4);
+        ret += (char)(i >> 8);
+        ret += (char)(i >> 12);
+        ret += (char)(i >> 16);
     }
     return ret;
 }

test/functional/test_framework/test_framework.py

@@ -14,6 +14,7 @@ import os
 import pdb
 import random
 import shutil
+import subprocess
 import sys
 import tempfile
 import time
@@ -152,6 +153,9 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
         except KeyError:
             self.log.exception("Key error")
             self.success = TestStatus.FAILED
+        except subprocess.CalledProcessError as e:
+            self.log.exception("Called Process failed with '{}'".format(e.output))
+            self.success = TestStatus.FAILED
         except Exception:
             self.log.exception("Unexpected exception caught during testing")
             self.success = TestStatus.FAILED

test/sanitizer_suppressions/ubsan

@@ -41,3 +41,45 @@ unsigned-integer-overflow:txmempool.cpp
 unsigned-integer-overflow:util/strencodings.cpp
 unsigned-integer-overflow:validation.cpp
 vptr:bls/bls.h
+
+implicit-integer-sign-change:*/include/c++/*/bits/*.h
+implicit-integer-sign-change:*/new_allocator.h
+implicit-integer-sign-change:/usr/include/boost/date_time/format_date_parser.hpp
+implicit-integer-sign-change:arith_uint256.cpp
+implicit-integer-sign-change:bech32.cpp
+implicit-integer-sign-change:bloom.cpp
+implicit-integer-sign-change:chain.*
+implicit-integer-sign-change:coins.h
+implicit-integer-sign-change:compat/stdin.cpp
+implicit-integer-sign-change:compressor.h
+implicit-integer-sign-change:crypto/*
+implicit-integer-sign-change:key.cpp
+implicit-integer-sign-change:noui.cpp
+implicit-integer-sign-change:prevector.h
+implicit-integer-sign-change:protocol.cpp
+implicit-integer-sign-change:script/bitcoinconsensus.cpp
+implicit-integer-sign-change:script/interpreter.cpp
+implicit-integer-sign-change:serialize.h
+implicit-integer-sign-change:test/arith_uint256_tests.cpp
+implicit-integer-sign-change:test/coins_tests.cpp
+implicit-integer-sign-change:test/pow_tests.cpp
+implicit-integer-sign-change:test/prevector_tests.cpp
+implicit-integer-sign-change:test/sighash_tests.cpp
+implicit-integer-sign-change:test/streams_tests.cpp
+implicit-integer-sign-change:test/transaction_tests.cpp
+implicit-integer-sign-change:txmempool.cpp
+implicit-integer-sign-change:util/strencodings.*
+implicit-integer-sign-change:validation.cpp
+implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp
+implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h
+implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp
+implicit-signed-integer-truncation:chain.h
+implicit-signed-integer-truncation:crypto/*
+implicit-signed-integer-truncation:cuckoocache.h
+implicit-signed-integer-truncation:leveldb/*
+implicit-signed-integer-truncation:streams.h
+implicit-signed-integer-truncation:test/arith_uint256_tests.cpp
+implicit-signed-integer-truncation:test/skiplist_tests.cpp
+implicit-signed-integer-truncation:torcontrol.cpp
+implicit-unsigned-integer-truncation:crypto/*
+implicit-unsigned-integer-truncation:leveldb/*