Move to in-docker CI builds and add Jenkins support (#2178)

* GCC-7 and glibc-2.27 compat code

* Statically link libstdc++ for GCC based builds

Makes sure binaries which are built on a newer build host still work
on older distros.

* Use python3 when installing MacOS native tools

* Move actual build logic out of Travis and upgrade to gcc-7

Travis will now simply call a few scripts which do the actual work.
These scripts will first create a "builder image" which contains the
necessary environment for the actual build. Then scripts are called
inside this builder image to do the build.

This should make us more independant from Travis and also allows us
to do local CI testing.

The build matrix is also moved out of .travis.yml and instead moved
into ci/matrix.sh. This script is sourced with only "BUILD_TARGET" being
set so that it internally can figure out which other environment
variables need to be set.

This commit also upgrades the used GCC version to 7. This is due to the
use of ubuntu:bionic as base image for the builder image.

* Add Jenkinsfiles for regular CI and nightly gitian builds

* Automatically download OSX SDK in gitian-build.sh

* Remove bogus "export MAKEJOBS=-j5"

* Forward cache/src dirs into builder container

Fixes caching issues on Travis.

* fix

* Fail build immediately when building depends took too long
This commit is contained in:
Alexander Block 2018-07-12 15:28:59 +02:00 committed by UdjinM6
parent 23dde9f12b
commit f3e380659a
18 changed files with 601 additions and 77 deletions

View File

@ -1,18 +1,21 @@
# errata:
# - A travis bug causes caches to trample eachother when using the same
# compiler key (which we don't use anyway). This is worked around for now by
# replacing the "compilers" with a build name prefixed by the no-op ":"
# command. See: https://github.com/travis-ci/travis-ci/issues/4393
# - sudo/dist/group are set so as to get Blue Box VMs, necessary for [loopback]
# IPv6 support
sudo: required
dist: trusty
group: deprecated-2017Q3
os: linux
language: generic
addons:
apt:
packages:
# Use more recent docker version
- docker-ce
# Can be removed if Travis ever upgrades to Bionic
- realpath
services:
- docker
@ -20,9 +23,7 @@ cache:
apt: true
ccache: true
directories:
- depends/built
- depends/sdk-sources
- $HOME/.ccache
- $HOME/cache
env:
global:
@ -30,78 +31,55 @@ env:
- secure: "J0T+zrRBBzBnxoioCD378cRSJUF8Vms32QMmrENVVfuJl03oBQv0QxSatLEiD4PtRPrN00mJ8/k/HdqFQ0SN1yg6a00duOZ6C9jk6t/fExBO1LNhp3Z7lcaMUrNEyRN6sgfVeznDVTGj9jBQU2HNlwHk0UD9BNp/k+Bjhrtw7Njp9JTLZKBgxrXk0WZmfk0d75Q+9DajA1j2tSfWVpwzv6HMGfqIH1wsVEakBY+tnaE6EO3IdzdWHhf/iV+Tx0tL0Qz/6mejgSuoGLTJLmeTrceTTmsDlN3B95y+XfcYDs6EP7gpzN8Ffg/mLV8xJ2Myj7+b1o+smMJZE5tLuPIeAu0YsfkPNa/tNLotOlDxF72per0kXyNYbRf+JrpxiOu9pLCtdtd40V2SsiNhpQqHVy+b+wkkqo2syTHT+9UNkqEi3WFm6TqFuOeQNtDm6R0hRl/Talw/ZEeKs68z1Cf6VyBrZG5LqtOurtWlKwXCe/Tr1i2g4BEAcbywphtCCcfgMK9YSGQnjKkGk6k9gVEa2bneYLZz1RSh2OpXnIsIJOX7X0hNd5xtQDLgbpSUlKwVcEriWeGe12GCszKNX9gdEo2bHeq4xAJ6BE+ot745z9dc+nC2BJ6Su+I5L6zBg4m2FWn0TWhPvz/6JR26t6ls/E7m+hkwJTCHnIUiFmmzpEw="
# DOCKER_HUB_PASSWORD
- secure: "RLzlMhfLqwSBrZqJOVOd61suXBn+HtUR3vOZfuFYF/Qmjjj5TE41+rObZmzc54hP/ZL+OH6blnibpvfDXlX+eN38ivFQfuxkJIGL68SJsEwNBRwW39Yw6Hl5RdI41MLCH7ByN15wifLp1JKBilHJ7XGMOUjI7P0yl7JjX8GBXUCtJbVLRugo80/yn+XQ1NdnlpbpYHNjMEQFWSODPa3pSK5McWvyQjDZDgS+IkdrZmIYJPMa7bmKH5I/edUPSmXQT905FgEwq9u8XR0SyBopli21EK9l6GkXIIvmDTYz5vT26Apvi2B4Aoazlklg+KNRUJuLGerpt6kbnU0gMSUChVkFfFhOk6GRSN3a/AUfD2FOudvMhet2QvlPHx+GYdEUr5XVo5HW42pHsqfD6eDtHd8VLTsHP0q4C8V85fNMv21lkkehy2ry8fx/RRy6x4O2wg2mua+79UkGKXp75gMKzWEcth34PCFCOu37l2F8R/ANnrQ52K/8vIQ88TtU2OpYX89fHjLojBxu+WKEBGZH2LRPsZBOUHeeO5C/xKDrhZU24ORnMW8wg66Qg5GIX1KI4a8yp73Mpues5hzpJ6wkMuRrQt40ymKndLCjv8KSd+5BfP6Or/KIrzDNYdZaasjk7JNi6rcZmm9d3fTAo+Ja/mjpUCIOo3SX14luzVCJIig="
- MAKEJOBS=-j5
- RUN_TESTS=false
- CHECK_DOC=0
- BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
- CCACHE_SIZE=400M
- CCACHE_TEMPDIR=/tmp/.ccache-temp
- CCACHE_COMPRESS=1
- BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
- SDK_URL=https://bitcoincore.org/depends-sources/sdks
- PYTHON_DEBUG=1
- WINEDEBUG=fixme-all
- DOCKER_BUILD=false
matrix:
# ARM
- HOST=arm-linux-gnueabihf PPA="ppa:bitcoin/bitcoin" PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
# Win32
- HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PPA="ppa:bitcoin/bitcoin" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.7 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j4"
# 32-bit + dash
- HOST=i686-pc-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="g++-multilib bc python3-zmq" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" PYZMQ=true
# Win64
- HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PPA="ppa:bitcoin/bitcoin" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.7 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j4"
# dashd
- HOST=x86_64-unknown-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="bc python3-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports" CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG" PYZMQ=true
# No wallet
- HOST=x86_64-unknown-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="python3" DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
# Cross-Mac
- HOST=x86_64-apple-darwin11 PPA="ppa:bitcoin/bitcoin" PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" OSX_SDK=10.11 GOAL="deploy"
# dashd release build (for docker)
- HOST=x86_64-unknown-linux-gnu PPA="ppa:bitcoin/bitcoin" PACKAGES="bc python3-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1" GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports" PYZMQ=true DOCKER_BUILD=true
- BUILD_TARGET=arm-linux
- BUILD_TARGET=win32
- BUILD_TARGET=win64
- BUILD_TARGET=linux32
- BUILD_TARGET=linux64
- BUILD_TARGET=linux64_nowallet
- BUILD_TARGET=linux64_release DOCKER_BUILD=true
- BUILD_TARGET=mac
before_cache:
# Save builder image
- docker save dash-builder-$BUILD_TARGET-$TRAVIS_JOB_NUMBER $(docker history -q dash-builder-$BUILD_TARGET-$TRAVIS_JOB_NUMBER | grep -v \<missing\>) | gzip -2 > $HOME/cache/docker/dash-builder-$BUILD_TARGET.tar.gz
before_install:
- git clone https://github.com/dashpay/dash_hash
- travis_retry sudo apt-get install python3-pip python3-dev
- travis_retry sudo add-apt-repository ppa:ubuntu-wine/ppa -y
- export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
install:
- if [ "$PYZMQ" = "true" ]; then pip3 install pyzmq --user ; fi
- if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi
- if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi
- if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi
- if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi
- cd dash_hash && python3 setup.py install --user && cd ..
# Our scripts try to be Travis agnostic
- export PULL_REQUEST="$TRAVIS_PULL_REQUEST"
- export JOB_NUMBER="$TRAVIS_JOB_NUMBER"
- export HOST_SRC_DIR=$TRAVIS_BUILD_DIR
- export HOST_CACHE_DIR=$HOME/cache
- source ./ci/matrix.sh
- mkdir -p $HOST_CACHE_DIR/docker && mkdir -p $HOST_CACHE_DIR/ccache && mkdir -p $HOST_CACHE_DIR/depends && mkdir -p $HOST_CACHE_DIR/sdk-sources
# Keep this as it makes caching related debugging easier
- ls -lah $HOST_CACHE_DIR && ls -lah $HOST_CACHE_DIR/depends && ls -lah $HOST_CACHE_DIR/ccache && ls -lah $HOST_CACHE_DIR/docker
# Load cached builder image
- if [ -f $HOST_CACHE_DIR/docker/dash-builder-$BUILD_TARGET.tar.gz ]; then zcat $HOST_CACHE_DIR/docker/dash-builder-$BUILD_TARGET.tar.gz | docker load || true; fi
- docker build --pull -t $BUILDER_IMAGE_NAME --build-arg=USER_ID=$UID --build-arg=GROUP_ID=$UID --build-arg=BUILD_TARGET=$BUILD_TARGET -f ci/Dockerfile.builder ci
before_script:
- unset CC; unset CXX
- unset DISPLAY
# Make sure stdout is in blocking mode. Otherwise builds will fail due to large writes to stdout
# See https://github.com/travis-ci/travis-ci/issues/4704. If this gets fixed, this line can also be removed.
- python3 -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
#- if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi TODO reenable after all Bitcoin PRs have been merged and docs fully fixed
- mkdir -p depends/SDKs depends/sdk-sources
- if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
- if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
- make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
script:
- export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
- if [ -n "$USE_SHELL" ]; then export CONFIG_SHELL="$USE_SHELL"; fi
- OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
- BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
- depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE
- test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
- mkdir build && cd build
- ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
- make distdir VERSION=$HOST
- cd dashcore-$HOST
- ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
- make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )
- export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib
- if [ "$RUN_TESTS" = "true" ]; then travis_wait 30 make $MAKEJOBS check VERBOSE=1; fi
- if [ "$RUN_TESTS" = "true" ]; then qa/pull-tester/rpc-tests.py --coverage; fi
- cd ../..
- if [ "$DOCKER_BUILD" = "true" ]; then BUILD_DIR=build/dashcore-$HOST ./docker/build-docker.sh; fi
# Our scripts try to be Travis agnostic
- $DOCKER_RUN_IN_BUILDER ./ci/build_depends.sh;
# Skip Dash Core build if depends build take more than 20 mins.
- if [ $SECONDS -lt 1200 ]; then $DOCKER_RUN_IN_BUILDER ./ci/build_src.sh && export SRC_BUILT="true"; fi
- |
if [ "$SRC_BUILT" = "true" ]; then
travis_wait 30 $DOCKER_RUN_IN_BUILDER ./ci/test_unittests.sh;
$DOCKER_RUN_IN_BUILDER ./ci/test_integrationtests.sh;
if [ "$DOCKER_BUILD" = "true" ]; then BUILD_DIR=build-ci/dashcore-$BUILD_TARGET ./docker/build-docker.sh; fi;
else
# Let the build fail immediately. Caches are still uploaded even after failure. The next Travis run should then
# use the caches and thus be much faster, causing the build to succeed
false
fi;
after_script:
- echo $TRAVIS_COMMIT_RANGE
- echo $TRAVIS_COMMIT_LOG

93
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,93 @@
// This Jenkinsfile will build a builder image and then run the actual build and tests inside this image
// It's very important to not execute any scripts outside of the builder container, as it's our protection against
// external developers bringing in harmful code into Jenkins.
// Jenkins will only run the build if this Jenkinsfile was not modified in an external pull request. Only branches
// which are part of the Dash repo will allow modification to the Jenkinsfile.
def targets = [
'win32',
'win64',
'linux32',
'linux64',
'linux64_nowallet',
'linux64_release',
'mac',
]
def tasks = [:]
for(int i = 0; i < targets.size(); i++) {
def target = targets[i]
tasks["${target}"] = {
node {
def BUILD_NUMBER = sh(returnStdout: true, script: 'echo $BUILD_NUMBER').trim()
def BRANCH_NAME = sh(returnStdout: true, script: 'echo $BRANCH_NAME').trim()
def UID = sh(returnStdout: true, script: 'id -u').trim()
def HOME = sh(returnStdout: true, script: 'echo $HOME').trim()
def pwd = sh(returnStdout: true, script: 'pwd').trim()
checkout scm
// restore cache
try {
copyArtifacts(projectName: "dashpay-dash/${BRANCH_NAME}", optional: true, selector: lastSuccessful(), filter: "ci-cache-${target}.tar.gz")
} catch (Exception e) {
}
if (fileExists("ci-cache-${target}.tar.gz")) {
hasCache = true
echo "Using cache from dashpay-dash/${BRANCH_NAME}"
} else {
try {
copyArtifacts(projectName: 'dashpay-dash/develop', optional: true, selector: lastSuccessful(), filter: "ci-cache-${target}.tar.gz");
} catch (Exception e) {
}
if (fileExists("ci-cache-${target}.tar.gz")) {
hasCache = true
echo "Using cache from dashpay-dash/develop"
}
}
def env = [
"BUILD_TARGET=${target}",
"PULL_REQUEST=false",
"JOB_NUMBER=${BUILD_NUMBER}",
]
withEnv(env) {
def builderImageName="dash-builder-${target}"
def builderImage
stage("${target}/builder-image") {
builderImage = docker.build("${builderImageName}", "--build-arg BUILD_TARGET=${target} ci -f ci/Dockerfile.builder")
}
if (hasCache) {
sh "cd ${pwd} && tar xzfv ci-cache-${target}.tar.gz"
} else {
sh "mkdir -p ${pwd}/ci-cache-${target}"
}
builderImage.inside("-t") {
stage("${target}/depends") {
sh './ci/build_depends.sh'
}
stage("${target}/build") {
sh './ci/build_src.sh'
}
stage("${target}/test") {
sh './ci/test_unittests.sh'
}
stage("${target}/test") {
sh './ci/test_integrationtests.sh'
}
}
// archive cache
sh "tar czfv ci-cache-${target}.tar.gz ci-cache-${target}"
archiveArtifacts artifacts: "ci-cache-${target}.tar.gz", fingerprint: true
}
}
}
}
parallel tasks

123
Jenkinsfile.gitian Normal file
View File

@ -0,0 +1,123 @@
def targets = [
'linux',
'win',
'osx',
]
def osslTarUrl = 'http://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz'
def osslPatchUrl = 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch'
def SDK_URL='https://bitcoincore.org/depends-sources/sdks'
def OSX_SDK='10.11'
def proc = 4
def mem = 2000
def repositoryUrl = "https://github.com/dashpay/dash.git"
def commit = "develop"
def tasks = [:]
for(int i = 0; i < targets.size(); i++) {
def target = targets[i]
tasks["${target}"] = {
node {
deleteDir() // cleanup workspace
def pwd = sh(returnStdout: true, script: 'pwd').trim()
def dockerGid = sh(returnStdout: true, script: "stat -c '%g' /var/run/docker.sock").trim()
def BRANCH_NAME = sh(returnStdout: true, script: 'echo $BRANCH_NAME').trim()
def hasCache = false
def gitianDescriptor
stage("${target}/prepare") {
dir('dash') {
checkout scm
gitianDescriptor = readYaml file: "contrib/gitian-descriptors/gitian-${target}.yml"
}
dir('gitian-builder') {
git url: 'https://github.com/dashpay/gitian-builder.git'
}
sh "mkdir -p dashcore-binaries"
if (target == "osx") {
dir('gitian-builder') {
sh 'mkdir -p inputs'
sh "curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o inputs/MacOSX${OSX_SDK}.sdk.tar.gz"
}
}
// restore cache
try {
copyArtifacts(projectName: "dashpay-dash-gitian-nightly/${BRANCH_NAME}", optional: true, selector: lastSuccessful(), filter: "cache-${gitianDescriptor.name}.tar.gz")
} catch (Exception e) {
}
if (fileExists("cache-${gitianDescriptor.name}.tar.gz")) {
hasCache = true
echo "Using cache from dashpay-dash-gitian-nightly/${BRANCH_NAME}"
} else {
try {
copyArtifacts(projectName: 'dashpay-dash-gitian-nightly/develop', optional: true, selector: lastSuccessful(), filter: "cache-${gitianDescriptor.name}.tar.gz");
} catch (Exception e) {
}
if (fileExists("cache-${gitianDescriptor.name}.tar.gz")) {
hasCache = true
echo "Using cache from dashpay-dash-gitian-nightly/develop"
}
}
}
def gitianImage
stage("${target}/builder-image") {
dir('dash') {
gitianImage = docker.build("dash-gitian:${env.BUILD_ID}", 'ci -f ci/Dockerfile.gitian-builder')
}
}
gitianImage.inside("--group-add ${dockerGid} -t -v \"/var/run/docker.sock:/var/run/docker.sock\"") {
sh "mkdir -p gitian-builder/cache"
if (hasCache) {
sh "cd gitian-builder/cache && tar xzfv ../../cache-${gitianDescriptor.name}.tar.gz"
}
stage("${target}/download") {
dir('gitian-builder') {
sh "mkdir -p inputs"
sh "cd inputs && curl -R -O ${osslPatchUrl}"
sh "cd inputs && curl -R -O ${osslTarUrl}"
sh "make -C ../dash/depends download SOURCES_PATH=`pwd`/cache/common"
}
}
stage("${target}/base-vm") {
dir('gitian-builder') {
sh "./bin/make-base-vm --suite trusty --arch amd64 --docker"
}
}
stage("${target}/gbuild") {
dir('gitian-builder') {
// make sure an old version is not running
sh "docker rm -fv gitian-target || true"
try {
sh """
tail -F var/install.log &
tail -F var/build.log &
USE_DOCKER=1 ./bin/gbuild -j ${proc} -m ${mem} --commit dash=${commit} --url dash=${repositoryUrl} ../dash/contrib/gitian-descriptors/gitian-${target}.yml
"""
} finally {
// make sure it doesn't run forever
sh "docker rm -fv gitian-target || true"
}
sh "mv build/out/dashcore-* ../dashcore-binaries/"
sh "mv build/out/src/dashcore-* ../dashcore-binaries/"
}
archiveArtifacts artifacts: 'dashcore-binaries/*', fingerprint: true
}
sh "cd gitian-builder/cache && tar czfv ../../cache-${gitianDescriptor.name}.tar.gz common ${gitianDescriptor.name}"
archiveArtifacts artifacts: "cache-${gitianDescriptor.name}.tar.gz", fingerprint: true
}
}
}
}
parallel tasks

58
ci/Dockerfile.builder Normal file
View File

@ -0,0 +1,58 @@
FROM ubuntu:bionic
# Build and base stuff
# (zlib1g-dev and libssl-dev are needed for the Qt host binary builds, but should not be used by target binaries)
RUN apt-get update && apt-get install -y \
git \
g++ \
autotools-dev libtool m4 automake autoconf pkg-config \
zlib1g-dev \
libssl1.0-dev \
curl \
ccache \
bsdmainutils \
python3-pip python3-dev \
&& rm -rf /var/lib/apt/lists
# Python stuff
RUN pip3 install pyzmq # really needed?
# dash_hash
RUN git clone https://github.com/dashpay/dash_hash
RUN cd dash_hash && python3 setup.py install
ARG USER_ID=1000
ARG GROUP_ID=1000
# add user with specified (or default) user/group ids
ENV USER_ID ${USER_ID}
ENV GROUP_ID ${GROUP_ID}
RUN groupadd -g ${GROUP_ID} dash
RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /dash dash
# Extra packages
ARG BUILD_TARGET=linux64
ADD matrix.sh /tmp/matrix.sh
RUN . /tmp/matrix.sh && \
if [ -n "$DPKG_ADD_ARCH" ]; then dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi && \
if [ -n "$PACKAGES" ]; then apt-get update && apt-get install -y --no-install-recommends --no-upgrade $PACKAGES && rm -rf /var/lib/apt/lists; fi
# Make sure std::thread and friends is available
# Will fail on non-win builds, but we ignore this
RUN \
update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix; \
update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix; \
update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix; \
update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; \
exit 0
RUN mkdir /dash-src && \
mkdir -p /cache/ccache && \
mkdir /cache/depends && \
mkdir /cache/sdk-sources && \
chown $USER_ID:$GROUP_ID /dash-src && \
chown $USER_ID:$GROUP_ID /cache && \
chown $USER_ID:$GROUP_ID /cache -R
WORKDIR /dash-src
USER dash

View File

@ -0,0 +1,17 @@
FROM ubuntu:bionic
RUN apt-get update && apt-get install -y \
ruby curl make libltdl7 git apache2 apt-cacher-ng python-vm-builder ruby qemu-utils \
&& rm -rf /var/lib/apt/lists
ARG USER_ID=1000
ARG GROUP_ID=1000
# add user with specified (or default) user/group ids
ENV USER_ID ${USER_ID}
ENV GROUP_ID ${GROUP_ID}
RUN groupadd -g ${GROUP_ID} dash
RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /dash dash
WORKDIR /dash
USER dash

29
ci/build_depends.sh Executable file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env bash
# This script is executed inside the builder image
set -e
source ./ci/matrix.sh
unset CC; unset CXX
unset DISPLAY
mkdir -p $CACHE_DIR/depends
mkdir -p $CACHE_DIR/sdk-sources
ln -s $CACHE_DIR/depends depends/built
ln -s $CACHE_DIR/sdk-sources depends/sdk-sources
mkdir -p depends/SDKs
if [ -n "$OSX_SDK" ]; then
if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
fi
if [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
fi
fi
make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS

37
ci/build_src.sh Executable file
View File

@ -0,0 +1,37 @@
#!/usr/bin/env bash
# This script is executed inside the builder image
set -e
source ./ci/matrix.sh
unset CC; unset CXX
unset DISPLAY
export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
export CCACHE_SIZE=${CCACHE_SIZE:-400M}
#if [ "$CHECK_DOC" = 1 ]; then contrib/devtools/check-doc.py; fi TODO reenable after all Bitcoin PRs have been merged and docs fully fixed
depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE
if [ -n "$USE_SHELL" ]; then
export CONFIG_SHELL="$USE_SHELL"
fi
BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BUILD_DIR/depends/$HOST --bindir=$OUT_DIR/bin --libdir=$OUT_DIR/lib"
test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh
rm -rf build-ci
mkdir build-ci
cd build-ci
../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
make distdir VERSION=$BUILD_TARGET
cd dashcore-$BUILD_TARGET
./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && make $GOAL V=1 ; false )

88
ci/matrix.sh Executable file
View File

@ -0,0 +1,88 @@
#!/usr/bin/env bash
# This script is meant to be sourced into the actual build script. It contains the build matrix and will set all
# necessary environment variables for the request build target
export BUILD_TARGET=${BUILD_TARGET:-linux64}
export PULL_REQUEST=${PULL_REQUEST:-false}
export JOB_NUMBER=${JOB_NUMBER:-1}
export BUILDER_IMAGE_NAME="dash-builder-$BUILD_TARGET-$JOB_NUMBER"
export HOST_SRC_DIR=${HOST_SRC_DIR:-$(pwd)}
export HOST_CACHE_DIR=${HOST_CACHE_DIR:-$(pwd)/ci-cache-$BUILD_TARGET}
export SRC_DIR=${SRC_DIR:-$HOST_SRC_DIR}
export BUILD_DIR=$SRC_DIR
export OUT_DIR=$BUILD_DIR/out
export CACHE_DIR=${CACHE_DIR:-$HOST_CACHE_DIR}
export CCACHE_DIR=$CACHE_DIR/ccache
export DOCKER_RUN_VOLUME_ARGS="-v $HOST_SRC_DIR:$SRC_DIR -v $HOST_CACHE_DIR:$CACHE_DIR"
export DOCKER_RUN_ENV_ARGS="-e SRC_DIR=$SRC_DIR -e CACHE_DIR=$CACHE_DIR -e PULL_REQUEST=$PULL_REQUEST -e JOB_NUMBER=$JOB_NUMBER -e BUILD_TARGET=$BUILD_TARGET"
export DOCKER_RUN_ARGS="$DOCKER_RUN_VOLUME_ARGS $DOCKER_RUN_ENV_ARGS"
export DOCKER_RUN_IN_BUILDER="docker run -t --rm -w $SRC_DIR $DOCKER_RUN_ARGS $BUILDER_IMAGE_NAME"
# Default values for targets
export GOAL="install"
export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
export PYTHON_DEBUG=1
export MAKEJOBS="-j4"
if [ "$BUILD_TARGET" = "arm-linux" ]; then
export HOST=arm-linux-gnueabihf
export PACKAGES="g++-arm-linux-gnueabihf"
export DEP_OPTS="NO_QT=1"
export CHECK_DOC=1
export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
elif [ "$BUILD_TARGET" = "win32" ]; then
export HOST=i686-w64-mingw32
export DPKG_ADD_ARCH="i386"
export DEP_OPTS="NO_QT=1"
export PACKAGES="python3 nsis g++-mingw-w64-i686 wine-stable wine32 bc"
export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports"
export DIRECT_WINE_EXEC_TESTS=true
export RUN_TESTS=true
elif [ "$BUILD_TARGET" = "win64" ]; then
export HOST=x86_64-w64-mingw32
export DPKG_ADD_ARCH="i386"
export DEP_OPTS="NO_QT=1"
export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-stable wine64 bc"
export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports"
export DIRECT_WINE_EXEC_TESTS=true
export RUN_TESTS=true
elif [ "$BUILD_TARGET" = "linux32" ]; then
export HOST=i686-pc-linux-gnu
export PACKAGES="g++-multilib bc python3-zmq"
export DEP_OPTS="NO_QT=1"
export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++"
export USE_SHELL="/bin/dash"
export PYZMQ=true
export RUN_TESTS=true
elif [ "$BUILD_TARGET" = "linux64" ]; then
export HOST=x86_64-unknown-linux-gnu
export PACKAGES="bc python3-zmq"
export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1"
export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports"
export CPPFLAGS="-DDEBUG_LOCKORDER -DENABLE_DASH_DEBUG"
export PYZMQ=true
export RUN_TESTS=true
elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then
export HOST=x86_64-unknown-linux-gnu
export PACKAGES="python3"
export DEP_OPTS="NO_WALLET=1"
export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports"
elif [ "$BUILD_TARGET" = "linux64_release" ]; then
export HOST=x86_64-unknown-linux-gnu
export PACKAGES="bc python3-zmq"
export DEP_OPTS="NO_QT=1 NO_UPNP=1"
export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports"
export PYZMQ=true
elif [ "$BUILD_TARGET" = "mac" ]; then
export HOST=x86_64-apple-darwin11
export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools"
export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports"
export OSX_SDK=10.11
export GOAL="deploy"
fi

18
ci/test_integrationtests.sh Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
# This script is executed inside the builder image
set -e
source ./ci/matrix.sh
if [ "$RUN_TESTS" != "true" ]; then
echo "Skipping integration tests"
exit 0
fi
export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib
cd build-ci/dashcore-$BUILD_TARGET
./qa/pull-tester/rpc-tests.py --coverage

27
ci/test_unittests.sh Executable file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env bash
# This script is executed inside the builder image
set -e
source ./ci/matrix.sh
if [ "$RUN_TESTS" != "true" ]; then
echo "Skipping unit tests"
exit 0
fi
# TODO this is not Travis agnostic
export BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib
export WINEDEBUG=fixme-all
export BOOST_TEST_LOG_LEVEL=test_suite
cd build-ci/dashcore-$BUILD_TARGET
if [ "$DIRECT_WINE_EXEC_TESTS" = "true" ]; then
# Inside Docker, binfmt isn't working so we can't trust in make invoking windows binaries correctly
wine ./src/test/test_dash.exe
else
make $MAKEJOBS check VERBOSE=1
fi

View File

@ -28,6 +28,9 @@ scriptName=$(basename -- "$0")
signProg="gpg --detach-sign"
commitFiles=true
SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
OSX_SDK=${OSX_SDK:-10.11}
# Help Message
read -d '' usage <<- EOF
Usage: $scriptName [-c|u|v|b|s|B|o|h|j|m|] signer version
@ -316,6 +319,13 @@ then
echo ""
echo "Compiling ${VERSION} Mac OSX"
echo ""
if [ -n "$OSX_SDK" ]; then
if [ ! -f inputs/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o inputs/MacOSX${OSX_SDK}.sdk.tar.gz
fi
fi
./bin/gbuild -j ${proc} -m ${mem} --commit dash=${COMMIT} --url dash=${url} ../dash/contrib/gitian-descriptors/gitian-osx.yml
./bin/gsign -p "$signProg" --signer "$SIGNER" --release ${VERSION}-osx-unsigned --destination ../gitian.sigs/ ../dash/contrib/gitian-descriptors/gitian-osx.yml
mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/dashcore-osx-unsigned.tar.gz

View File

@ -1,5 +1,5 @@
linux_CFLAGS=-pipe
linux_CXXFLAGS=$(linux_CFLAGS)
linux_CXXFLAGS=$(linux_CFLAGS) -static-libstdc++
linux_release_CFLAGS=-O2
linux_release_CXXFLAGS=$(linux_release_CFLAGS)

View File

@ -1,5 +1,5 @@
mingw32_CFLAGS=-pipe
mingw32_CXXFLAGS=$(mingw32_CFLAGS)
mingw32_CXXFLAGS=$(mingw32_CFLAGS) -static-libstdc++
mingw32_release_CFLAGS=-O2
mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS)

View File

@ -11,10 +11,10 @@ define $(package)_preprocess_cmds
endef
define $(package)_build_cmds
python setup.py build
python3 setup.py build
endef
define $(package)_stage_cmds
mkdir -p $($(package)_install_libdir) && \
python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
endef

View File

@ -8,10 +8,10 @@ $(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
$(package)_dependencies=native_biplist
define $(package)_build_cmds
python setup.py build
python3 setup.py build
endef
define $(package)_stage_cmds
mkdir -p $($(package)_install_libdir) && \
python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
endef

View File

@ -12,10 +12,10 @@ define $(package)_preprocess_cmds
endef
define $(package)_build_cmds
python setup.py build
python3 setup.py build
endef
define $(package)_stage_cmds
mkdir -p $($(package)_install_libdir) && \
python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
python3 setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
endef

View File

@ -425,6 +425,7 @@ libdash_util_a_SOURCES = \
if GLIBC_BACK_COMPAT
libdash_util_a_SOURCES += compat/glibc_compat.cpp
AM_LDFLAGS += -Wl,--wrap=log2f -Wl,--wrap=__divmoddi4
endif
# cli: shared between dash-cli and dash-qt

View File

@ -7,6 +7,7 @@
#endif
#include <cstddef>
#include <cstdint>
#if defined(HAVE_SYS_SELECT_H)
#include <sys/select.h>
@ -27,3 +28,47 @@ extern "C" FDELT_TYPE __fdelt_warn(FDELT_TYPE a)
return a / __NFDBITS;
}
extern "C" FDELT_TYPE __fdelt_chk(FDELT_TYPE) __attribute__((weak, alias("__fdelt_warn")));
#if defined(__i386__) || defined(__arm__)
extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp);
extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp)
{
int32_t c1 = 0, c2 = 0;
int64_t uu = u, vv = v;
int64_t w;
int64_t r;
if (uu < 0) {
c1 = ~c1, c2 = ~c2, uu = -uu;
}
if (vv < 0) {
c1 = ~c1, vv = -vv;
}
w = __udivmoddi4(uu, vv, (uint64_t*)&r);
if (c1)
w = -w;
if (c2)
r = -r;
*rp = r;
return w;
}
#endif
extern "C" float log2f_old(float x);
#ifdef __i386__
__asm(".symver log2f_old,log2f@GLIBC_2.1");
#elif defined(__amd64__)
__asm(".symver log2f_old,log2f@GLIBC_2.2.5");
#elif defined(__arm__)
__asm(".symver log2f_old,log2f@GLIBC_2.4");
#elif defined(__aarch64__)
__asm(".symver log2f_old,log2f@GLIBC_2.17");
#endif
extern "C" float __wrap_log2f(float x)
{
return log2f_old(x);
}