diff --git a/.travis.yml b/.travis.yml index 5a81f2049..70267409a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,21 @@ # errata: -# - A travis bug causes caches to trample eachother when using the same -# compiler key (which we don't use anyway). This is worked around for now by -# replacing the "compilers" with a build name prefixed by the no-op ":" -# command. See: https://github.com/travis-ci/travis-ci/issues/4393 # - sudo/dist/group are set so as to get Blue Box VMs, necessary for [loopback] # IPv6 support sudo: required dist: trusty -group: deprecated-2017Q3 os: linux language: generic +addons: + apt: + packages: + # Use more recent docker version + - docker-ce + # Can be removed if Travis ever upgrades to Bionic + - realpath + services: - docker @@ -20,9 +23,7 @@ cache: apt: true ccache: true directories: - - depends/built - - depends/sdk-sources - - $HOME/.ccache + - $HOME/cache env: global: @@ -30,78 +31,55 @@ env: - secure: "J0T+zrRBBzBnxoioCD378cRSJUF8Vms32QMmrENVVfuJl03oBQv0QxSatLEiD4PtRPrN00mJ8/k/HdqFQ0SN1yg6a00duOZ6C9jk6t/fExBO1LNhp3Z7lcaMUrNEyRN6sgfVeznDVTGj9jBQU2HNlwHk0UD9BNp/k+Bjhrtw7Njp9JTLZKBgxrXk0WZmfk0d75Q+9DajA1j2tSfWVpwzv6HMGfqIH1wsVEakBY+tnaE6EO3IdzdWHhf/iV+Tx0tL0Qz/6mejgSuoGLTJLmeTrceTTmsDlN3B95y+XfcYDs6EP7gpzN8Ffg/mLV8xJ2Myj7+b1o+smMJZE5tLuPIeAu0YsfkPNa/tNLotOlDxF72per0kXyNYbRf+JrpxiOu9pLCtdtd40V2SsiNhpQqHVy+b+wkkqo2syTHT+9UNkqEi3WFm6TqFuOeQNtDm6R0hRl/Talw/ZEeKs68z1Cf6VyBrZG5LqtOurtWlKwXCe/Tr1i2g4BEAcbywphtCCcfgMK9YSGQnjKkGk6k9gVEa2bneYLZz1RSh2OpXnIsIJOX7X0hNd5xtQDLgbpSUlKwVcEriWeGe12GCszKNX9gdEo2bHeq4xAJ6BE+ot745z9dc+nC2BJ6Su+I5L6zBg4m2FWn0TWhPvz/6JR26t6ls/E7m+hkwJTCHnIUiFmmzpEw=" # DOCKER_HUB_PASSWORD - secure: "RLzlMhfLqwSBrZqJOVOd61suXBn+HtUR3vOZfuFYF/Qmjjj5TE41+rObZmzc54hP/ZL+OH6blnibpvfDXlX+eN38ivFQfuxkJIGL68SJsEwNBRwW39Yw6Hl5RdI41MLCH7ByN15wifLp1JKBilHJ7XGMOUjI7P0yl7JjX8GBXUCtJbVLRugo80/yn+XQ1NdnlpbpYHNjMEQFWSODPa3pSK5McWvyQjDZDgS+IkdrZmIYJPMa7bmKH5I/edUPSmXQT905FgEwq9u8XR0SyBopli21EK9l6GkXIIvmDTYz5vT26Apvi2B4Aoazlklg+KNRUJuLGerpt6kbnU0gMSUChVkFfFhOk6GRSN3a/AUfD2FOudvMhet2QvlPHx+GYdEUr5XVo5HW42pHsqfD6eDtHd8VLTsHP0q4C8V85fNMv21lkkehy2ry8fx/RRy6x4O2wg2mua+79UkGKXp75gMKzWEcth34PCFCOu37l2F8R/ANnrQ52K/8vIQ88TtU2OpYX89fHjLojBxu+WKEBGZH2LRPsZBOUHeeO5C/xKDrhZU24ORnMW8wg66Qg5GIX1KI4a8yp73Mpues5hzpJ6wkMuRrQt40ymKndLCjv8KSd+5BfP6Or/KIrzDNYdZaasjk7JNi6rcZmm9d3fTAo+Ja/mjpUCIOo3SX14luzVCJIig=" - - MAKEJOBS=-j5 - - RUN_TESTS=false - - CHECK_DOC=0 - - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID - - CCACHE_SIZE=400M - - CCACHE_TEMPDIR=/tmp/.ccache-temp - - CCACHE_COMPRESS=1 - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - - SDK_URL=https://bitcoincore.org/depends-sources/sdks - - PYTHON_DEBUG=1 - - WINEDEBUG=fixme-all - DOCKER_BUILD=false matrix: -# ARM - - HOST=arm-linux-gnueabihf PPA="ppa:bitcoin/bitcoin" PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" -# Win32 - - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PPA="ppa:bitcoin/bitcoin" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.7 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j4" -# 32-bit + dash - - HOST=i686-pc-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="g++-multilib bc python3-zmq" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" PYZMQ=true -# Win64 - - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PPA="ppa:bitcoin/bitcoin" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.7 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j4" -# dashd - - HOST=x86_64-unknown-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="bc python3-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports" CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG" PYZMQ=true -# No wallet - - HOST=x86_64-unknown-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" -# Cross-Mac - - HOST=x86_64-apple-darwin11 PPA="ppa:bitcoin/bitcoin" PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy" -# dashd release build (for docker) - - HOST=x86_64-unknown-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="bc python3-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1" GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports" PYZMQ=true DOCKER_BUILD=true + - BUILD_TARGET=arm-linux + - BUILD_TARGET=win32 + - BUILD_TARGET=win64 + - BUILD_TARGET=linux32 + - BUILD_TARGET=linux64 + - BUILD_TARGET=linux64_nowallet + - BUILD_TARGET=linux64_release DOCKER_BUILD=true + - BUILD_TARGET=mac + +before_cache: + # Save builder image + - docker save dash-builder-$BUILD_TARGET-$TRAVIS_JOB_NUMBER $(docker history -q dash-builder-$BUILD_TARGET-$TRAVIS_JOB_NUMBER | grep -v \) | gzip -2 > $HOME/cache/docker/dash-builder-$BUILD_TARGET.tar.gz -before_install: - - git clone https://github.com/dashpay/dash_hash - - travis_retry sudo apt-get install python3-pip python3-dev - - travis_retry sudo add-apt-repository ppa:ubuntu-wine/ppa -y - - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") install: - - if [ "$PYZMQ" = "true" ]; then pip3 install pyzmq --user ; fi - - if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi - - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi - - cd dash_hash && python3 setup.py install --user && cd .. + # Our scripts try to be Travis agnostic + - export PULL_REQUEST="$TRAVIS_PULL_REQUEST" + - export JOB_NUMBER="$TRAVIS_JOB_NUMBER" + - export HOST_SRC_DIR=$TRAVIS_BUILD_DIR + - export HOST_CACHE_DIR=$HOME/cache + - source ./ci/matrix.sh + - mkdir -p $HOST_CACHE_DIR/docker && mkdir -p $HOST_CACHE_DIR/ccache && mkdir -p $HOST_CACHE_DIR/depends && mkdir -p $HOST_CACHE_DIR/sdk-sources + # Keep this as it makes caching related debugging easier + - ls -lah $HOST_CACHE_DIR && ls -lah $HOST_CACHE_DIR/depends && ls -lah $HOST_CACHE_DIR/ccache && ls -lah $HOST_CACHE_DIR/docker + # Load cached builder image + - if [ -f $HOST_CACHE_DIR/docker/dash-builder-$BUILD_TARGET.tar.gz ]; then zcat $HOST_CACHE_DIR/docker/dash-builder-$BUILD_TARGET.tar.gz | docker load || true; fi + - docker build --pull -t $BUILDER_IMAGE_NAME --build-arg=USER_ID=$UID --build-arg=GROUP_ID=$UID --build-arg=BUILD_TARGET=$BUILD_TARGET -f ci/Dockerfile.builder ci before_script: - - unset CC; unset CXX - - unset DISPLAY # Make sure stdout is in blocking mode. Otherwise builds will fail due to large writes to stdout # See https://github.com/travis-ci/travis-ci/issues/4704. If this gets fixed, this line can also be removed. - python3 -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' - #- if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi TODO reenable after all Bitcoin PRs have been merged and docs fully fixed - - mkdir -p depends/SDKs depends/sdk-sources - - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - - if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi - - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS script: - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1` - - if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi - - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - - BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" - - depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE - - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh - - mkdir build && cd build - - ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - - make distdir VERSION=$HOST - - cd dashcore-$HOST - - ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - - make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) - - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib - - if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi - - if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage; fi - - cd ../.. - - if [ "$DOCKER_BUILD" = "true" ]; then BUILD_DIR=build/dashcore-$HOST ./docker/build-docker.sh; fi + # Our scripts try to be Travis agnostic + - $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh; + # Skip Dash Core build if depends build take more than 20 mins. + - if [ $SECONDS -lt 1200 ]; then $DOCKER_RUN_IN_BUILDER ./ci/build_src.sh && export SRC_BUILT="true"; fi + - | + if [ "$SRC_BUILT" = "true" ]; then + travis_wait 30 $DOCKER_RUN_IN_BUILDER ./ci/test_unittests.sh; + $DOCKER_RUN_IN_BUILDER ./ci/test_integrationtests.sh; + if [ "$DOCKER_BUILD" = "true" ]; then BUILD_DIR=build-ci/dashcore-$BUILD_TARGET ./docker/build-docker.sh; fi; + else + # Let the build fail immediately. Caches are still uploaded even after failure. The next Travis run should then + # use the caches and thus be much faster, causing the build to succeed + false + fi; after_script: - echo $TRAVIS_COMMIT_RANGE - echo $TRAVIS_COMMIT_LOG diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..3df07ed03 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,93 @@ +// This Jenkinsfile will build a builder image and then run the actual build and tests inside this image +// It's very important to not execute any scripts outside of the builder container, as it's our protection against +// external developers bringing in harmful code into Jenkins. +// Jenkins will only run the build if this Jenkinsfile was not modified in an external pull request. Only branches +// which are part of the Dash repo will allow modification to the Jenkinsfile. + +def targets = [ + 'win32', + 'win64', + 'linux32', + 'linux64', + 'linux64_nowallet', + 'linux64_release', + 'mac', +] + +def tasks = [:] +for(int i = 0; i < targets.size(); i++) { + def target = targets[i] + + tasks["${target}"] = { + node { + def BUILD_NUMBER = sh(returnStdout: true, script: 'echo $BUILD_NUMBER').trim() + def BRANCH_NAME = sh(returnStdout: true, script: 'echo $BRANCH_NAME').trim() + def UID = sh(returnStdout: true, script: 'id -u').trim() + def HOME = sh(returnStdout: true, script: 'echo $HOME').trim() + def pwd = sh(returnStdout: true, script: 'pwd').trim() + + checkout scm + + // restore cache + try { + copyArtifacts(projectName: "dashpay-dash/${BRANCH_NAME}", optional: true, selector: lastSuccessful(), filter: "ci-cache-${target}.tar.gz") + } catch (Exception e) { + } + if (fileExists("ci-cache-${target}.tar.gz")) { + hasCache = true + echo "Using cache from dashpay-dash/${BRANCH_NAME}" + } else { + try { + copyArtifacts(projectName: 'dashpay-dash/develop', optional: true, selector: lastSuccessful(), filter: "ci-cache-${target}.tar.gz"); + } catch (Exception e) { + } + if (fileExists("ci-cache-${target}.tar.gz")) { + hasCache = true + echo "Using cache from dashpay-dash/develop" + } + } + + def env = [ + "BUILD_TARGET=${target}", + "PULL_REQUEST=false", + "JOB_NUMBER=${BUILD_NUMBER}", + ] + withEnv(env) { + def builderImageName="dash-builder-${target}" + + def builderImage + stage("${target}/builder-image") { + builderImage = docker.build("${builderImageName}", "--build-arg BUILD_TARGET=${target} ci -f ci/Dockerfile.builder") + } + + if (hasCache) { + sh "cd ${pwd} && tar xzfv ci-cache-${target}.tar.gz" + } else { + sh "mkdir -p ${pwd}/ci-cache-${target}" + } + + builderImage.inside("-t") { + stage("${target}/depends") { + sh './ci/build_depends.sh' + } + stage("${target}/build") { + sh './ci/build_src.sh' + } + stage("${target}/test") { + sh './ci/test_unittests.sh' + } + stage("${target}/test") { + sh './ci/test_integrationtests.sh' + } + } + + // archive cache + sh "tar czfv ci-cache-${target}.tar.gz ci-cache-${target}" + archiveArtifacts artifacts: "ci-cache-${target}.tar.gz", fingerprint: true + } + } + } +} + +parallel tasks + diff --git a/Jenkinsfile.gitian b/Jenkinsfile.gitian new file mode 100644 index 000000000..f7a0c212e --- /dev/null +++ b/Jenkinsfile.gitian @@ -0,0 +1,123 @@ +def targets = [ + 'linux', + 'win', + 'osx', +] + +def osslTarUrl = 'http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz' +def osslPatchUrl = 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch' +def SDK_URL='https://bitcoincore.org/depends-sources/sdks' +def OSX_SDK='10.11' +def proc = 4 +def mem = 2000 + +def repositoryUrl = "https://github.com/dashpay/dash.git" +def commit = "develop" + +def tasks = [:] +for(int i = 0; i < targets.size(); i++) { + def target = targets[i] + + tasks["${target}"] = { + node { + deleteDir() // cleanup workspace + + def pwd = sh(returnStdout: true, script: 'pwd').trim() + def dockerGid = sh(returnStdout: true, script: "stat -c '%g' /var/run/docker.sock").trim() + def BRANCH_NAME = sh(returnStdout: true, script: 'echo $BRANCH_NAME').trim() + def hasCache = false + + def gitianDescriptor + + stage("${target}/prepare") { + dir('dash') { + checkout scm + gitianDescriptor = readYaml file: "contrib/gitian-descriptors/gitian-${target}.yml" + } + dir('gitian-builder') { + git url: 'https://github.com/dashpay/gitian-builder.git' + } + sh "mkdir -p dashcore-binaries" + if (target == "osx") { + dir('gitian-builder') { + sh 'mkdir -p inputs' + sh "curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o inputs/MacOSX${OSX_SDK}.sdk.tar.gz" + } + } + + // restore cache + try { + copyArtifacts(projectName: "dashpay-dash-gitian-nightly/${BRANCH_NAME}", optional: true, selector: lastSuccessful(), filter: "cache-${gitianDescriptor.name}.tar.gz") + } catch (Exception e) { + } + if (fileExists("cache-${gitianDescriptor.name}.tar.gz")) { + hasCache = true + echo "Using cache from dashpay-dash-gitian-nightly/${BRANCH_NAME}" + } else { + try { + copyArtifacts(projectName: 'dashpay-dash-gitian-nightly/develop', optional: true, selector: lastSuccessful(), filter: "cache-${gitianDescriptor.name}.tar.gz"); + } catch (Exception e) { + } + if (fileExists("cache-${gitianDescriptor.name}.tar.gz")) { + hasCache = true + echo "Using cache from dashpay-dash-gitian-nightly/develop" + } + } + } + + def gitianImage + stage("${target}/builder-image") { + dir('dash') { + gitianImage = docker.build("dash-gitian:${env.BUILD_ID}", 'ci -f ci/Dockerfile.gitian-builder') + } + } + + gitianImage.inside("--group-add ${dockerGid} -t -v \"/var/run/docker.sock:/var/run/docker.sock\"") { + sh "mkdir -p gitian-builder/cache" + if (hasCache) { + sh "cd gitian-builder/cache && tar xzfv ../../cache-${gitianDescriptor.name}.tar.gz" + } + + stage("${target}/download") { + dir('gitian-builder') { + sh "mkdir -p inputs" + sh "cd inputs && curl -R -O ${osslPatchUrl}" + sh "cd inputs && curl -R -O ${osslTarUrl}" + sh "make -C ../dash/depends download SOURCES_PATH=`pwd`/cache/common" + } + } + stage("${target}/base-vm") { + dir('gitian-builder') { + sh "./bin/make-base-vm --suite trusty --arch amd64 --docker" + } + } + + stage("${target}/gbuild") { + dir('gitian-builder') { + // make sure an old version is not running + sh "docker rm -fv gitian-target || true" + + try { + sh """ + tail -F var/install.log & + tail -F var/build.log & + USE_DOCKER=1 ./bin/gbuild -j ${proc} -m ${mem} --commit dash=${commit} --url dash=${repositoryUrl} ../dash/contrib/gitian-descriptors/gitian-${target}.yml + """ + } finally { + // make sure it doesn't run forever + sh "docker rm -fv gitian-target || true" + } + sh "mv build/out/dashcore-* ../dashcore-binaries/" + sh "mv build/out/src/dashcore-* ../dashcore-binaries/" + } + archiveArtifacts artifacts: 'dashcore-binaries/*', fingerprint: true + } + + sh "cd gitian-builder/cache && tar czfv ../../cache-${gitianDescriptor.name}.tar.gz common ${gitianDescriptor.name}" + archiveArtifacts artifacts: "cache-${gitianDescriptor.name}.tar.gz", fingerprint: true + } + } + } +} + +parallel tasks diff --git a/ci/Dockerfile.builder b/ci/Dockerfile.builder new file mode 100644 index 000000000..66d282271 --- /dev/null +++ b/ci/Dockerfile.builder @@ -0,0 +1,58 @@ +FROM ubuntu:bionic + +# Build and base stuff +# (zlib1g-dev and libssl-dev are needed for the Qt host binary builds, but should not be used by target binaries) +RUN apt-get update && apt-get install -y \ + git \ + g++ \ + autotools-dev libtool m4 automake autoconf pkg-config \ + zlib1g-dev \ + libssl1.0-dev \ + curl \ + ccache \ + bsdmainutils \ + python3-pip python3-dev \ + && rm -rf /var/lib/apt/lists + +# Python stuff +RUN pip3 install pyzmq # really needed? + +# dash_hash +RUN git clone https://github.com/dashpay/dash_hash +RUN cd dash_hash && python3 setup.py install + +ARG USER_ID=1000 +ARG GROUP_ID=1000 + +# add user with specified (or default) user/group ids +ENV USER_ID ${USER_ID} +ENV GROUP_ID ${GROUP_ID} +RUN groupadd -g ${GROUP_ID} dash +RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /dash dash + +# Extra packages +ARG BUILD_TARGET=linux64 +ADD matrix.sh /tmp/matrix.sh +RUN . /tmp/matrix.sh && \ + if [ -n "$DPKG_ADD_ARCH" ]; then dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi && \ + if [ -n "$PACKAGES" ]; then apt-get update && apt-get install -y --no-install-recommends --no-upgrade $PACKAGES && rm -rf /var/lib/apt/lists; fi + +# Make sure std::thread and friends is available +# Will fail on non-win builds, but we ignore this +RUN \ + update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix; \ + update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix; \ + update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix; \ + update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; \ + exit 0 + +RUN mkdir /dash-src && \ + mkdir -p /cache/ccache && \ + mkdir /cache/depends && \ + mkdir /cache/sdk-sources && \ + chown $USER_ID:$GROUP_ID /dash-src && \ + chown $USER_ID:$GROUP_ID /cache && \ + chown $USER_ID:$GROUP_ID /cache -R +WORKDIR /dash-src + +USER dash diff --git a/ci/Dockerfile.gitian-builder b/ci/Dockerfile.gitian-builder new file mode 100644 index 000000000..79bd2a381 --- /dev/null +++ b/ci/Dockerfile.gitian-builder @@ -0,0 +1,17 @@ +FROM ubuntu:bionic + +RUN apt-get update && apt-get install -y \ + ruby curl make libltdl7 git apache2 apt-cacher-ng python-vm-builder ruby qemu-utils \ + && rm -rf /var/lib/apt/lists + +ARG USER_ID=1000 +ARG GROUP_ID=1000 + +# add user with specified (or default) user/group ids +ENV USER_ID ${USER_ID} +ENV GROUP_ID ${GROUP_ID} +RUN groupadd -g ${GROUP_ID} dash +RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /dash dash + +WORKDIR /dash +USER dash \ No newline at end of file diff --git a/ci/build_depends.sh b/ci/build_depends.sh new file mode 100755 index 000000000..d4e7cd0e5 --- /dev/null +++ b/ci/build_depends.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# This script is executed inside the builder image + +set -e + +source ./ci/matrix.sh + +unset CC; unset CXX +unset DISPLAY + +mkdir -p $CACHE_DIR/depends +mkdir -p $CACHE_DIR/sdk-sources + +ln -s $CACHE_DIR/depends depends/built +ln -s $CACHE_DIR/sdk-sources depends/sdk-sources + +mkdir -p depends/SDKs + +if [ -n "$OSX_SDK" ]; then + if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + fi + if [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz + fi +fi + +make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS diff --git a/ci/build_src.sh b/ci/build_src.sh new file mode 100755 index 000000000..2a659e7ef --- /dev/null +++ b/ci/build_src.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# This script is executed inside the builder image + +set -e + +source ./ci/matrix.sh + +unset CC; unset CXX +unset DISPLAY + +export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} +export CCACHE_SIZE=${CCACHE_SIZE:-400M} + +#if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi TODO reenable after all Bitcoin PRs have been merged and docs fully fixed + +depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE + +if [ -n "$USE_SHELL" ]; then + export CONFIG_SHELL="$USE_SHELL" +fi + +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BUILD_DIR/depends/$HOST --bindir=$OUT_DIR/bin --libdir=$OUT_DIR/lib" + +test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh + +rm -rf build-ci +mkdir build-ci +cd build-ci + +../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) +make distdir VERSION=$BUILD_TARGET + +cd dashcore-$BUILD_TARGET +./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) + +make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false ) diff --git a/ci/matrix.sh b/ci/matrix.sh new file mode 100755 index 000000000..15e918bc5 --- /dev/null +++ b/ci/matrix.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# This script is meant to be sourced into the actual build script. It contains the build matrix and will set all +# necessary environment variables for the request build target + +export BUILD_TARGET=${BUILD_TARGET:-linux64} +export PULL_REQUEST=${PULL_REQUEST:-false} +export JOB_NUMBER=${JOB_NUMBER:-1} + +export BUILDER_IMAGE_NAME="dash-builder-$BUILD_TARGET-$JOB_NUMBER" + +export HOST_SRC_DIR=${HOST_SRC_DIR:-$(pwd)} +export HOST_CACHE_DIR=${HOST_CACHE_DIR:-$(pwd)/ci-cache-$BUILD_TARGET} + +export SRC_DIR=${SRC_DIR:-$HOST_SRC_DIR} +export BUILD_DIR=$SRC_DIR +export OUT_DIR=$BUILD_DIR/out + +export CACHE_DIR=${CACHE_DIR:-$HOST_CACHE_DIR} +export CCACHE_DIR=$CACHE_DIR/ccache + +export DOCKER_RUN_VOLUME_ARGS="-v $HOST_SRC_DIR:$SRC_DIR -v $HOST_CACHE_DIR:$CACHE_DIR" +export DOCKER_RUN_ENV_ARGS="-e SRC_DIR=$SRC_DIR -e CACHE_DIR=$CACHE_DIR -e PULL_REQUEST=$PULL_REQUEST -e JOB_NUMBER=$JOB_NUMBER -e BUILD_TARGET=$BUILD_TARGET" +export DOCKER_RUN_ARGS="$DOCKER_RUN_VOLUME_ARGS $DOCKER_RUN_ENV_ARGS" +export DOCKER_RUN_IN_BUILDER="docker run -t --rm -w $SRC_DIR $DOCKER_RUN_ARGS $BUILDER_IMAGE_NAME" + +# Default values for targets +export GOAL="install" +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} +export PYTHON_DEBUG=1 +export MAKEJOBS="-j4" + +if [ "$BUILD_TARGET" = "arm-linux" ]; then + export HOST=arm-linux-gnueabihf + export PACKAGES="g++-arm-linux-gnueabihf" + export DEP_OPTS="NO_QT=1" + export CHECK_DOC=1 + export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +elif [ "$BUILD_TARGET" = "win32" ]; then + export HOST=i686-w64-mingw32 + export DPKG_ADD_ARCH="i386" + export DEP_OPTS="NO_QT=1" + export PACKAGES="python3 nsis g++-mingw-w64-i686 wine-stable wine32 bc" + export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" + export DIRECT_WINE_EXEC_TESTS=true + export RUN_TESTS=true +elif [ "$BUILD_TARGET" = "win64" ]; then + export HOST=x86_64-w64-mingw32 + export DPKG_ADD_ARCH="i386" + export DEP_OPTS="NO_QT=1" + export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-stable wine64 bc" + export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" + export DIRECT_WINE_EXEC_TESTS=true + export RUN_TESTS=true +elif [ "$BUILD_TARGET" = "linux32" ]; then + export HOST=i686-pc-linux-gnu + export PACKAGES="g++-multilib bc python3-zmq" + export DEP_OPTS="NO_QT=1" + export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" + export USE_SHELL="/bin/dash" + export PYZMQ=true + export RUN_TESTS=true +elif [ "$BUILD_TARGET" = "linux64" ]; then + export HOST=x86_64-unknown-linux-gnu + export PACKAGES="bc python3-zmq" + export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" + export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports" + export CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG" + export PYZMQ=true + export RUN_TESTS=true +elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then + export HOST=x86_64-unknown-linux-gnu + export PACKAGES="python3" + export DEP_OPTS="NO_WALLET=1" + export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" +elif [ "$BUILD_TARGET" = "linux64_release" ]; then + export HOST=x86_64-unknown-linux-gnu + export PACKAGES="bc python3-zmq" + export DEP_OPTS="NO_QT=1 NO_UPNP=1" + export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports" + export PYZMQ=true +elif [ "$BUILD_TARGET" = "mac" ]; then + export HOST=x86_64-apple-darwin11 + export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools" + export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" + export OSX_SDK=10.11 + export GOAL="deploy" +fi diff --git a/ci/test_integrationtests.sh b/ci/test_integrationtests.sh new file mode 100755 index 000000000..2223a98c8 --- /dev/null +++ b/ci/test_integrationtests.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# This script is executed inside the builder image + +set -e + +source ./ci/matrix.sh + +if [ "$RUN_TESTS" != "true" ]; then + echo "Skipping integration tests" + exit 0 +fi + +export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib + +cd build-ci/dashcore-$BUILD_TARGET + +./qa/pull-tester/rpc-tests.py --coverage diff --git a/ci/test_unittests.sh b/ci/test_unittests.sh new file mode 100755 index 000000000..f5af7e396 --- /dev/null +++ b/ci/test_unittests.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# This script is executed inside the builder image + +set -e + +source ./ci/matrix.sh + +if [ "$RUN_TESTS" != "true" ]; then + echo "Skipping unit tests" + exit 0 +fi + +# TODO this is not Travis agnostic +export BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID +export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib + +export WINEDEBUG=fixme-all +export BOOST_TEST_LOG_LEVEL=test_suite + +cd build-ci/dashcore-$BUILD_TARGET +if [ "$DIRECT_WINE_EXEC_TESTS" = "true" ]; then + # Inside Docker, binfmt isn't working so we can't trust in make invoking windows binaries correctly + wine ./src/test/test_dash.exe +else + make $MAKEJOBS check VERBOSE=1 +fi diff --git a/contrib/gitian-build.sh b/contrib/gitian-build.sh index 15392c177..b0d9b5ca4 100755 --- a/contrib/gitian-build.sh +++ b/contrib/gitian-build.sh @@ -28,6 +28,9 @@ scriptName=$(basename -- "$0") signProg="gpg --detach-sign" commitFiles=true +SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} +OSX_SDK=${OSX_SDK:-10.11} + # Help Message read -d '' usage <<- EOF Usage: $scriptName [-c|u|v|b|s|B|o|h|j|m|] signer version @@ -316,6 +319,13 @@ then echo "" echo "Compiling ${VERSION} Mac OSX" echo "" + + if [ -n "$OSX_SDK" ]; then + if [ ! -f inputs/MacOSX${OSX_SDK}.sdk.tar.gz ]; then + curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o inputs/MacOSX${OSX_SDK}.sdk.tar.gz + fi + fi + ./bin/gbuild -j ${proc} -m ${mem} --commit dash=${COMMIT} --url dash=${url} ../dash/contrib/gitian-descriptors/gitian-osx.yml ./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../dash/contrib/gitian-descriptors/gitian-osx.yml mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/dashcore-osx-unsigned.tar.gz diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index b13a0f1ad..602206d63 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -1,5 +1,5 @@ linux_CFLAGS=-pipe -linux_CXXFLAGS=$(linux_CFLAGS) +linux_CXXFLAGS=$(linux_CFLAGS) -static-libstdc++ linux_release_CFLAGS=-O2 linux_release_CXXFLAGS=$(linux_release_CFLAGS) diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index dbfb62fdc..83fc501a1 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -1,5 +1,5 @@ mingw32_CFLAGS=-pipe -mingw32_CXXFLAGS=$(mingw32_CFLAGS) +mingw32_CXXFLAGS=$(mingw32_CFLAGS) -static-libstdc++ mingw32_release_CFLAGS=-O2 mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS) diff --git a/depends/packages/native_biplist.mk b/depends/packages/native_biplist.mk index 3c6e8900f..373490aed 100644 --- a/depends/packages/native_biplist.mk +++ b/depends/packages/native_biplist.mk @@ -11,10 +11,10 @@ define $(package)_preprocess_cmds endef define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index 49f5829ac..e2d035d48 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -8,10 +8,10 @@ $(package)_install_libdir=$(build_prefix)/lib/python/dist-packages $(package)_dependencies=native_biplist define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 85a8a402b..9982a52a3 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -12,10 +12,10 @@ define $(package)_preprocess_cmds endef define $(package)_build_cmds - python setup.py build + python3 setup.py build endef define $(package)_stage_cmds mkdir -p $($(package)_install_libdir) && \ - python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) + python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir) endef diff --git a/src/Makefile.am b/src/Makefile.am index a54e81c1d..b015c2388 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -425,6 +425,7 @@ libdash_util_a_SOURCES = \ if GLIBC_BACK_COMPAT libdash_util_a_SOURCES += compat/glibc_compat.cpp +AM_LDFLAGS += -Wl,--wrap=log2f -Wl,--wrap=__divmoddi4 endif # cli: shared between dash-cli and dash-qt diff --git a/src/compat/glibc_compat.cpp b/src/compat/glibc_compat.cpp index f5f61a3ab..1df6e217f 100644 --- a/src/compat/glibc_compat.cpp +++ b/src/compat/glibc_compat.cpp @@ -7,6 +7,7 @@ #endif #include +#include #if defined(HAVE_SYS_SELECT_H) #include @@ -27,3 +28,47 @@ extern "C" FDELT_TYPE __fdelt_warn(FDELT_TYPE a) return a / __NFDBITS; } extern "C" FDELT_TYPE __fdelt_chk(FDELT_TYPE) __attribute__((weak, alias("__fdelt_warn"))); + +#if defined(__i386__) || defined(__arm__) + +extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp); + +extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp) +{ + int32_t c1 = 0, c2 = 0; + int64_t uu = u, vv = v; + int64_t w; + int64_t r; + + if (uu < 0) { + c1 = ~c1, c2 = ~c2, uu = -uu; + } + if (vv < 0) { + c1 = ~c1, vv = -vv; + } + + w = __udivmoddi4(uu, vv, (uint64_t*)&r); + if (c1) + w = -w; + if (c2) + r = -r; + + *rp = r; + return w; +} +#endif + +extern "C" float log2f_old(float x); +#ifdef __i386__ +__asm(".symver log2f_old,log2f@GLIBC_2.1"); +#elif defined(__amd64__) +__asm(".symver log2f_old,log2f@GLIBC_2.2.5"); +#elif defined(__arm__) +__asm(".symver log2f_old,log2f@GLIBC_2.4"); +#elif defined(__aarch64__) +__asm(".symver log2f_old,log2f@GLIBC_2.17"); +#endif +extern "C" float __wrap_log2f(float x) +{ + return log2f_old(x); +}