From 35c76705d13727f01aa8507fbdd1d0584bd9ecef Mon Sep 17 00:00:00 2001 From: pasta Date: Sun, 4 Aug 2024 02:32:54 -0500 Subject: [PATCH 1/5] feat: implement basic Github Actions based CI, which reuses underlying logic from GitLab CI --- .github/workflows/build.yml | 176 ++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..85a5ce2629 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,176 @@ +name: CI + +on: + push: + pull_request_target: + +permissions: + contents: read + packages: write + +env: + DOCKER_DRIVER: overlay2 + FAST_MODE: false + +jobs: + builder-image: + name: Builder Image + runs-on: ubuntu-20.04 + outputs: + image-tag: ${{ steps.prepare.outputs.image-tag }} + repo-name: ${{ steps.prepare.outputs.repo-name }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Prepare + id: prepare + run: | + BRANCH_NAME=$(echo "${GITHUB_REF##*/}" | tr '[:upper:]' '[:lower:]') + REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + echo "::set-output name=image-tag::${BRANCH_NAME}" + echo "::set-output name=repo-name::${REPO_NAME}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: ./contrib/containers/ci + file: ./contrib/containers/ci/Dockerfile + push: true + tags: | + ghcr.io/${{ steps.prepare.outputs.repo-name }}/builder:${{ steps.prepare.outputs.image-tag }} + ghcr.io/${{ steps.prepare.outputs.repo-name }}/builder:latest + cache-from: type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo-name }}/builder:latest + cache-to: type=inline + + build-depends: + name: Build Dependencies + needs: builder-image + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - build_target: arm-linux + host: arm-linux-gnueabihf + container: + image: ghcr.io/${{ needs.builder-image.outputs.repo-name }}/builder:${{ needs.builder-image.outputs.image-tag }} + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + depends/built + depends/${{ matrix.host }} + depends/sdk-sources + # We don't care about no specific key as depends system will handle that for us + key: ${{ runner.os }}-depends-${{ matrix.host }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-depends-${{ matrix.host }}-${{ github.sha }} + ${{ runner.os }}-depends-${{ matrix.host }} + ${{ runner.os }}-depends + + - name: Build dependencies + run: make -j$(nproc) -C depends HOST=${{ matrix.host }} + + build: + name: Build + needs: [builder-image, build-depends] + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - build_target: arm-linux + host: arm-linux-gnueabihf + dep_opts: DEBUG=1 + container: + image: ghcr.io/${{ needs.builder-image.outputs.repo-name }}/builder:${{ needs.builder-image.outputs.image-tag }} + options: --user root + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Restore Cache dependencies + uses: actions/cache/restore@v4 + with: + path: | + depends/built + depends/${{ matrix.host }} + depends/sdk-sources + # We don't care about no specific key as depends system will handle that for us + key: ${{ runner.os }}-depends-${{ matrix.host }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-depends-${{ matrix.host }}-${{ github.sha }} + ${{ runner.os }}-depends-${{ matrix.host }} + ${{ runner.os }}-depends + + - name: CCache + uses: actions/cache@v4 + with: + path: | + /cache + key: ${{ runner.os }}-${{ matrix.host }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ matrix.host }}-${{ github.sha }} + ${{ runner.os }}-${{ matrix.host }} + ${{ runner.os }} + + - name: Build source and run tests + run: | + git config --global --add safe.directory "$PWD" + CCACHE_SIZE="400M" + CACHE_DIR="/cache" + mkdir /output + BASE_OUTDIR="/output" + BUILD_TARGET="${{ matrix.build_target }}" + source ./ci/dash/matrix.sh + ./ci/dash/build_src.sh + ./ci/dash/test_unittests.sh + shell: bash + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + /output + + +# Come back to this later and implement tests :) +# test: +# name: Test +# needs: [builder-image, build] +# runs-on: ubuntu-20.04 +# container: +# image: ghcr.io/${{ needs.builder-image.outputs.repo-name }}/builder:${{ needs.builder-image.outputs.image-tag }} +# options: --user root +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 +# +# - name: Download build artifacts +# uses: actions/download-artifact@v4 +# with: +# name: build-artifacts +# path: src/ +# +## - name: Setup environment +## run: | +## echo "BUILD_TARGET=${{ needs.build.matrix.build_target }}" +## source ./ci/dash/matrix.sh +# +# - name: Run integration tests +# run: ./ci/dash/test_integrationtests.sh --extended --exclude feature_pruning,feature_dbcrash From d1addb27aa147b59bd54331317b009977a4844ac Mon Sep 17 00:00:00 2001 From: pasta Date: Sun, 4 Aug 2024 02:42:06 -0500 Subject: [PATCH 2/5] fix: change fallback download path to be an s3 link which includes a few packages Reasonably enough I guess, it appears that a number of our dependencies block github runner IP addresses? the blobs stored at the s3 link were downloadable from the normal location when accessed locally, but returned a 404 from github runners. These blobs were also not present in the bitcoincore depends-sources anyhow. --- depends/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/depends/Makefile b/depends/Makefile index 67d4fae792..69ceb518cc 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -41,7 +41,7 @@ NO_ZMQ ?= NO_UPNP ?= NO_NATPMP ?= MULTIPROCESS ?= -FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources +FALLBACK_DOWNLOAD_PATH ?= http://dash-depends-sources.s3-website-us-west-2.amazonaws.com BUILD = $(shell ./config.guess) HOST ?= $(BUILD) From ed8ffa7fb46ece8f593811e59699b2aed7228fd8 Mon Sep 17 00:00:00 2001 From: pasta Date: Sun, 4 Aug 2024 02:43:00 -0500 Subject: [PATCH 3/5] feat: have cppcheck linter respect CACHE_DIR env variable --- test/lint/lint-cppcheck-dash.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/lint/lint-cppcheck-dash.sh b/test/lint/lint-cppcheck-dash.sh index 3c6b9ddd93..202be43d14 100755 --- a/test/lint/lint-cppcheck-dash.sh +++ b/test/lint/lint-cppcheck-dash.sh @@ -80,10 +80,15 @@ ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") FILES_REGEXP=$(join_array "|" "${FILES[@]}") SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -CPPCHECK_DIR=$SCRIPT_DIR/.cppcheck/ +# Check if CACHE_DIR is set and non-empty, otherwise use default .cppcheck/ directory +if [[ -n "$CACHE_DIR" ]]; then + CPPCHECK_DIR=$CACHE_DIR/cppcheck/ +else + CPPCHECK_DIR=$SCRIPT_DIR/.cppcheck/ +fi if [ ! -d $CPPCHECK_DIR ] then - mkdir $CPPCHECK_DIR + mkdir -p $CPPCHECK_DIR fi WARNINGS=$(echo "${FILES}" | \ xargs cppcheck --enable=all --inline-suppr --suppress=missingIncludeSystem --cppcheck-build-dir=$CPPCHECK_DIR -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DENABLE_WALLET -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -DUSE_EPOLL -DCHAR_BIT=8 -I src/ -q 2>&1 | sort -u | \ From 4dbdecdd1e466a5a25681a1e382dc75190580754 Mon Sep 17 00:00:00 2001 From: pasta Date: Sun, 4 Aug 2024 03:10:22 -0500 Subject: [PATCH 4/5] refactor: rename builder-image -> build-image and builder as image name to dashcore-ci-runner --- .github/workflows/build.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85a5ce2629..b46f6e0162 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,8 +13,8 @@ env: FAST_MODE: false jobs: - builder-image: - name: Builder Image + build-image: + name: Build Image runs-on: ubuntu-20.04 outputs: image-tag: ${{ steps.prepare.outputs.image-tag }} @@ -48,14 +48,14 @@ jobs: file: ./contrib/containers/ci/Dockerfile push: true tags: | - ghcr.io/${{ steps.prepare.outputs.repo-name }}/builder:${{ steps.prepare.outputs.image-tag }} - ghcr.io/${{ steps.prepare.outputs.repo-name }}/builder:latest - cache-from: type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo-name }}/builder:latest + ghcr.io/${{ steps.prepare.outputs.repo-name }}/dashcore-ci-runner:${{ steps.prepare.outputs.image-tag }} + ghcr.io/${{ steps.prepare.outputs.repo-name }}/dashcore-ci-runner:latest + cache-from: type=registry,ref=ghcr.io/${{ steps.prepare.outputs.repo-name }}/dashcore-ci-runner:latest cache-to: type=inline build-depends: name: Build Dependencies - needs: builder-image + needs: build-image runs-on: ubuntu-20.04 strategy: matrix: @@ -63,7 +63,7 @@ jobs: - build_target: arm-linux host: arm-linux-gnueabihf container: - image: ghcr.io/${{ needs.builder-image.outputs.repo-name }}/builder:${{ needs.builder-image.outputs.image-tag }} + image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root steps: - name: Checkout code @@ -88,7 +88,7 @@ jobs: build: name: Build - needs: [builder-image, build-depends] + needs: [build-image, build-depends] runs-on: ubuntu-20.04 strategy: matrix: @@ -97,7 +97,7 @@ jobs: host: arm-linux-gnueabihf dep_opts: DEBUG=1 container: - image: ghcr.io/${{ needs.builder-image.outputs.repo-name }}/builder:${{ needs.builder-image.outputs.image-tag }} + image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root steps: - name: Checkout code @@ -152,10 +152,10 @@ jobs: # Come back to this later and implement tests :) # test: # name: Test -# needs: [builder-image, build] +# needs: [build-image, build] # runs-on: ubuntu-20.04 # container: -# image: ghcr.io/${{ needs.builder-image.outputs.repo-name }}/builder:${{ needs.builder-image.outputs.image-tag }} +# image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} # options: --user root # steps: # - name: Checkout code From acf1315270b483c0f3e08948fa0295f99d9e836c Mon Sep 17 00:00:00 2001 From: MacroFake Date: Mon, 9 May 2022 15:08:28 +0200 Subject: [PATCH 5/5] Merge bitcoin/bitcoin#25091: test: Remove extended lint (cppcheck) BACKPORT NOTICE: we keep and maintain cppcheck linter as lint-cppcheck-dash.sh efae252f3072da598160670691757a0d60b9beb4 test: Remove extended lint (cppcheck) (laanwj) Pull request description: These are unreferenced in the CI and documentation, and have been since 2019 (see #17549). I'm not sure the cppcheck is worthwhile. It takes a long time to run (I think this is why it isn't in the normal lints), and right now it only appears to find implicit constructors. The list of exceptions is out of date. But if anyone wants to bring it back at any time in the future they can do so from git history (and port it to Python). ACKs for top commit: fanquake: ACK efae252f3072da598160670691757a0d60b9beb4 Tree-SHA512: 1a770b5d20ff1199d0d6bc471ae3d2c3438f0f0b169ce8d2fe73480daf8d3a7146c066b799afc90aa7898982c5fee79c1daca10e16e2bff0a7b38850aedd55b2 --- ci/dash/build_src.sh | 1 - test/lint/extended-lint-all.sh | 26 --------- test/lint/extended-lint-cppcheck.sh | 86 ----------------------------- 3 files changed, 113 deletions(-) delete mode 100755 test/lint/extended-lint-all.sh delete mode 100755 test/lint/extended-lint-cppcheck.sh diff --git a/ci/dash/build_src.sh b/ci/dash/build_src.sh index 8cf4116c7c..c0696d49df 100755 --- a/ci/dash/build_src.sh +++ b/ci/dash/build_src.sh @@ -26,7 +26,6 @@ if [ "$CHECK_DOC" = 1 ]; then #test/lint/check-doc.py # Run all linters test/lint/lint-all.sh - test/lint/extended-lint-all.sh fi ccache --zero-stats --max-size=$CCACHE_SIZE diff --git a/test/lint/extended-lint-all.sh b/test/lint/extended-lint-all.sh deleted file mode 100755 index be5d9db4a9..0000000000 --- a/test/lint/extended-lint-all.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/extended-lint-*.sh files, and fails if -# any exit with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -for f in "${SCRIPTDIR}"/extended-lint-*.sh; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - exit 1 - fi - fi -done diff --git a/test/lint/extended-lint-cppcheck.sh b/test/lint/extended-lint-cppcheck.sh deleted file mode 100755 index 3b66d3d433..0000000000 --- a/test/lint/extended-lint-cppcheck.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2019-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# - -export LC_ALL=C - -ENABLED_CHECKS=( - "Class '.*' has a constructor with 1 argument that is not explicit." - "Struct '.*' has a constructor with 1 argument that is not explicit." -) - -IGNORED_WARNINGS=( - "src/arith_uint256.h:.* Class 'arith_uint256' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint < 256 >' has a constructor with 1 argument that is not explicit." - "src/arith_uint256.h:.* Class 'base_uint' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewBacked' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCache' has a constructor with 1 argument that is not explicit." - "src/coins.h:.* Class 'CCoinsViewCursor' has a constructor with 1 argument that is not explicit." - "src/net.h:.* Class 'CNetMessage' has a constructor with 1 argument that is not explicit." - "src/policy/feerate.h:.* Class 'CFeeRate' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'const_reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'iterator' has a constructor with 1 argument that is not explicit." - "src/prevector.h:.* Class 'reverse_iterator' has a constructor with 1 argument that is not explicit." - "src/primitives/block.h:.* Class 'CBlock' has a constructor with 1 argument that is not explicit." - "src/primitives/transaction.h:.* Class 'CTransaction' has a constructor with 1 argument that is not explicit." - "src/protocol.h:.* Class 'CMessageHeader' has a constructor with 1 argument that is not explicit." - "src/qt/guiutil.h:.* Class 'ItemDelegate' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'RPCResults' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/rpc/util.h:.* style: Struct 'UniValueType' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'AddressDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ComboDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'ConstPubkeyProvider' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'PKHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'RawDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/descriptor.cpp:.* Class 'SHDescriptor' has a constructor with 1 argument that is not explicit." - "src/script/script.h:.* Class 'CScript' has a constructor with 1 argument that is not explicit." - "src/script/standard.h:.* Class 'CScriptID' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const CRPCCommand >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const char >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const std :: vector >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span < const uint8_t >' has a constructor with 1 argument that is not explicit." - "src/span.h:.* Class 'Span' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < RNGState >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/secure.h:.* Struct 'secure_allocator < unsigned char >' has a constructor with 1 argument that is not explicit." - "src/support/allocators/zeroafterfree.h:.* Struct 'zero_after_free_allocator < char >' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'FailingCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'MemoryCheck' has a constructor with 1 argument that is not explicit." - "src/test/checkqueue_tests.cpp:.* Struct 'UniqueCheck' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedFileProvider' has a constructor with 1 argument that is not explicit." - "src/test/fuzz/util.h:.* Class 'FuzzedAutoFileProvider' has a constructor with 1 argument that is not explicit." - "src/wallet/db.h:.* Class 'BerkeleyEnvironment' has a constructor with 1 argument that is not explicit." -) - -if ! command -v cppcheck > /dev/null; then - echo "Skipping cppcheck linting since cppcheck is not installed. Install by running \"apt install cppcheck\"" - exit 0 -fi - -function join_array { - local IFS="$1" - shift - echo "$*" -} - -ENABLED_CHECKS_REGEXP=$(join_array "|" "${ENABLED_CHECKS[@]}") -IGNORED_WARNINGS_REGEXP=$(join_array "|" "${IGNORED_WARNINGS[@]}") -WARNINGS=$(git ls-files -- "*.cpp" "*.h" ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/leveldb/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" | \ - xargs cppcheck --enable=all -j "$(getconf _NPROCESSORS_ONLN)" --language=c++ --std=c++17 --template=gcc -D__cplusplus -DCLIENT_VERSION_BUILD -DCLIENT_VERSION_IS_RELEASE -DCLIENT_VERSION_MAJOR -DCLIENT_VERSION_MINOR -DCOPYRIGHT_YEAR -DDEBUG -I src/ -q 2>&1 | sort -u | \ - grep -E "${ENABLED_CHECKS_REGEXP}" | \ - grep -vE "${IGNORED_WARNINGS_REGEXP}") -if [[ ${WARNINGS} != "" ]]; then - echo "${WARNINGS}" - echo - echo "Advice not applicable in this specific case? Add an exception by updating" - echo "IGNORED_WARNINGS in $0" - # Uncomment to enforce the developer note policy "By default, declare single-argument constructors `explicit`" - # exit 1 -fi -exit 0