diff --git a/gitlab-ci.yml b/gitlab-ci.yml new file mode 100644 index 000000000..cafe64017 --- /dev/null +++ b/gitlab-ci.yml @@ -0,0 +1,159 @@ +image: "ubuntu:bionic" + +variables: + DOCKER_DRIVER: overlay2 + +cache: + # Cache by branch/tag and job name + # Gitlab can't use caches from parent pipelines when doing the first build in a PR, so we use artifacts to copy + # caches into PRs + key: ${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}${CI_EXTERNAL_PULL_REQUEST_IID} + paths: + - $CI_PROJECT_DIR/cache + +stages: + - build + +.build_template: &build_template + stage: build + before_script: + - export BUILD_TARGET="$CI_JOB_NAME" + - echo BUILD_TARGET=$BUILD_TARGET + - source ./ci/matrix.sh + + # The ubuntu base image has apt configured to delete caches after each invocation, which is something that is not desirable for us + - rm /etc/apt/apt.conf.d/docker-clean + - apt-get update + - apt-get install -y wget unzip + + # Init cache + - export CACHE_DIR=$CI_PROJECT_DIR/cache + - mkdir -p $CACHE_DIR + - | + if [ "$CI_COMMIT_REF_SLUG" != "develop" -a "$CI_COMMIT_TAG" == "" ]; then + if [ ! -d $CACHE_DIR/ccache ]; then + echo "Downloading cache from develop branch" + mkdir cache-artifact + cd cache-artifact + if wget --quiet -O cache-artifact.zip https://gitlab.com/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/-/jobs/artifacts/develop/download?job=$CI_JOB_NAME; then + unzip -q cache-artifact.zip + rm cache-artifact.zip + mv cache-artifact/* $CACHE_DIR/ || true + else + echo "Failed to download cache" + fi + cd .. + rm -rf cache-artifact + else + echo "Not touching cache (was initialized from previous build)" + fi + else + echo "Not touching cache (building develop branch or tag)" + fi + # Create missing cache dirs + - mkdir -p $CACHE_DIR/ccache && mkdir -p $CACHE_DIR/depends && mkdir -p $CACHE_DIR/sdk-sources && mkdir -p $CACHE_DIR/apt + # Keep this as it makes caching related debugging easier + - ls -lah $CACHE_DIR && ls -lah $CACHE_DIR/depends && ls -lah $CACHE_DIR/ccache && ls -lah $CACHE_DIR/apt + - mv $CACHE_DIR/apt/* /var/cache/apt/archives/ || true + + # Install base packages + - apt-get dist-upgrade -y + - apt-get install -y git g++ autotools-dev libtool m4 automake autoconf pkg-config zlib1g-dev libssl1.0-dev curl ccache bsdmainutils cmake + - apt-get install -y python3 python3-dev python3-pip + + # jinja2 is needed for combine_logs.py + - pip3 install jinja2 + + # Setup some environment variables + - | + if [ "$CI_EXTERNAL_PULL_REQUEST_IID" != "" ]; then + export PULL_REQUEST="true" + else + # CI_EXTERNAL_PULL_REQUEST_IID is false every time until https://gitlab.com/gitlab-org/gitlab/issues/5667 is done + # Until then, we're using https://github.com/brndnmtthws/labhub atm to mirror Github pull requests as branches into Gitlab, + # which allows us to use Gitlab CI for Github. The following check detects such mirrored branches. + if [[ $CI_COMMIT_REF_NAME =~ ^pr-[^/]*/[^/]*/[^/]*/[^/]*$ ]]; then + export PULL_REQUEST="true" + # CI_COMMIT_BEFORE_SHA is also invalid until #5667 is implemented, so we need to figure it out by ourself + git fetch origin develop + export CI_COMMIT_BEFORE_SHA="$(git merge-base origin/develop HEAD)" + else + export PULL_REQUEST="false" + fi + fi + - export COMMIT_RANGE="$CI_COMMIT_BEFORE_SHA..$CI_COMMIT_SHA" + - export JOB_NUMBER="$CI_JOB_ID" + - export HOST_SRC_DIR=$CI_PROJECT_DIR + - echo PULL_REQUEST=$PULL_REQUEST COMMIT_RANGE=$COMMIT_RANGE HOST_SRC_DIR=$HOST_SRC_DIR CACHE_DIR=$CACHE_DIR + - echo "Commit log:" && git log --format=fuller -1 + + # Build neobytes_hash + - git clone https://github.com/neobytes-project/neobytes_hash + - cd neobytesh_hash && python3 setup.py install + + # Install build target specific packages + - echo PACKAGES=$PACKAGES + - if [ -n "$DPKG_ADD_ARCH" ]; then dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi + - if [ -n "$PACKAGES" ]; then apt-get update && apt-get install -y --no-install-recommends --no-upgrade $PACKAGES; fi + + # Move apt packages into cache + - mv /var/cache/apt/archives/* $CACHE_DIR/apt/ || true + + # Make mingw use correct threading libraries + - update-alternatives --set i686-w64-mingw32-gcc /usr/bin/i686-w64-mingw32-gcc-posix || true + - update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix || true + - update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix || true + - update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix || true + + script: + - export BUILD_TARGET="$CI_JOB_NAME" + - cd $CI_PROJECT_DIR + - ./ci/build_depends.sh + - ./ci/build_src.sh + - ./ci/test_unittests.sh + - ./ci/test_integrationtests.sh --extended --exclude pruning,dbcrash + + after_script: + # Copy all cache files into cache-artifact so that they get uploaded. We only do this for develop so that artifacts + # stay minimal for PRs and branches (we never need them) + - mkdir -p $CI_PROJECT_DIR/cache-artifact + - mkdir -p $CI_PROJECT_DIR/testlogs + - | + if [ "$CI_COMMIT_REF_SLUG" = "develop" ]; then + cp -ra $CI_PROJECT_DIR/cache/* $CI_PROJECT_DIR/cache-artifact/ + fi + + # We're actually only interested in the develop branch creating the cache artifact, but there is no way to control this + # until https://gitlab.com/gitlab-org/gitlab-foss/issues/25478 gets implemented. Until then, we use an expiration time of + # 3 days and rely on daily builds to refresh the cache artifacts. We also keep non-develop artifacts at minimum size + artifacts: + name: cache-artifact + when: always + paths: + - $CI_PROJECT_DIR/cache-artifact + - $CI_PROJECT_DIR/testlogs + expire_in: 3 days + +arm-linux: + <<: *build_template + +win32: + <<: *build_template + +win64: + <<: *build_template + +linux32: + <<: *build_template + +linux64: + <<: *build_template + +linux64_nowallet: + <<: *build_template + +linux64_release: + <<: *build_template + +mac: + <<: *build_template