From 38b8344ea5b4d383f631fd824be8d01d1c50027d Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com> Date: Tue, 27 Jun 2023 14:50:42 +0000 Subject: [PATCH 1/3] contrib: create Guix container with interactive abilities --- contrib/containers/guix/Dockerfile | 91 ++++++++++++++++++++++ contrib/containers/guix/docker-compose.yml | 16 ++++ contrib/containers/guix/motd.txt | 4 + contrib/containers/guix/scripts/entrypoint | 15 ++++ contrib/containers/guix/scripts/guix-check | 17 ++++ contrib/containers/guix/scripts/guix-start | 20 +++++ 6 files changed, 163 insertions(+) create mode 100644 contrib/containers/guix/Dockerfile create mode 100644 contrib/containers/guix/docker-compose.yml create mode 100644 contrib/containers/guix/motd.txt create mode 100755 contrib/containers/guix/scripts/entrypoint create mode 100755 contrib/containers/guix/scripts/guix-check create mode 100755 contrib/containers/guix/scripts/guix-start diff --git a/contrib/containers/guix/Dockerfile b/contrib/containers/guix/Dockerfile new file mode 100644 index 0000000000..3ec737582d --- /dev/null +++ b/contrib/containers/guix/Dockerfile @@ -0,0 +1,91 @@ +# Note: Using 'docker compose up' will leave you hanging, you need +# to use 'docker compose run guix_ubuntu' to drop into an +# interactive shell + +FROM ubuntu:focal + +SHELL ["/bin/bash", "-c"] + +RUN apt-get update && \ + apt-get install -y --no-install-recommends --no-upgrade \ + build-essential \ + bzip2 \ + ca-certificates \ + curl \ + git \ + locales \ + netbase \ + sudo \ + wget \ + xz-utils && \ + rm -rf /var/lib/apt/lists/* + +ARG guix_download_path=ftp://ftp.gnu.org/gnu/guix +ARG guix_version=1.4.0 +ARG guix_checksum_aarch64=72d807392889919940b7ec9632c45a259555e6b0942ea7bfd131101e08ebfcf4 +ARG guix_checksum_x86_64=236ca7c9c5958b1f396c2924fcc5bc9d6fdebcb1b4cf3c7c6d46d4bf660ed9c9 +ARG builder_count=32 + +ENV PATH /usr/local/bin:/usr/local/guix/current/bin:$PATH + +# Application Setup +# https://guix.gnu.org/manual/en/html_node/Application-Setup.html +ENV GUIX_LOCPATH="/usr/local/guix/profile" \ + LC_ALL="C" + +RUN guix_file_name=guix-binary-${guix_version}.$(uname -m)-linux.tar.xz && \ + eval "guix_checksum=\${guix_checksum_$(uname -m)}" && \ + cd /tmp && \ + wget -q -O "$guix_file_name" "${guix_download_path}/${guix_file_name}" && \ + echo "${guix_checksum} ${guix_file_name}" | sha256sum -c && \ + tar xJf "$guix_file_name" && \ + mv var/guix /var/ && \ + mv gnu / && \ + mkdir -p /usr/local/guix && \ + ln -sf /var/guix/profiles/per-user/root/current-guix /usr/local/guix/current && \ + ln -sf /var/guix/profiles/per-user/root/guix-profile /usr/local/guix/profile && \ + chmod 1777 /tmp /var/tmp && \ + source /usr/local/guix/current/etc/profile + +RUN touch /etc/nsswitch.conf + +RUN guix archive --authorize < /usr/local/guix/current/share/guix/ci.guix.gnu.org.pub && \ + guix archive --authorize < /usr/local/guix/current/share/guix/bordeaux.guix.gnu.org.pub + +# Build Environment Setup +# https://guix.gnu.org/manual/en/html_node/Build-Environment-Setup.html +RUN groupadd --system guixbuild && \ + for i in $(seq -w 1 ${builder_count}); do \ + useradd -g guixbuild -G guixbuild \ + -d /var/empty -s $(which nologin) \ + -c "Guix build user ${i}" --system \ + "guixbuilder${i}" ; \ + done + +# Create unprivileged user +ARG USER_ID=1000 \ + GROUP_ID=1000 \ + USERNAME=ubuntu +RUN groupadd -g ${GROUP_ID} ${USERNAME} && \ + useradd -u ${USER_ID} -g ${USERNAME} -s /bin/bash -m -d /home/${USERNAME} ${USERNAME} + +# Grant it passwordless admin permissions +RUN usermod -aG sudo ${USERNAME} && \ + echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# Copy required files to container +COPY ./motd.txt /etc/motd +COPY ./scripts/entrypoint /usr/local/bin/entrypoint +COPY ./scripts/guix-check /usr/local/bin/guix-check +COPY ./scripts/guix-start /usr/local/bin/guix-start + +# Create directory for mounting and grant necessary permissions +RUN mkdir -p /src/dash && \ + chown -R ${USER_ID}:${GROUP_ID} /src +WORKDIR "/src/dash" + +# Switch to unprivileged context +USER ${USERNAME} + +# Set entrypoint to copied file +ENTRYPOINT ["/usr/local/bin/entrypoint"] diff --git a/contrib/containers/guix/docker-compose.yml b/contrib/containers/guix/docker-compose.yml new file mode 100644 index 0000000000..3bead9181b --- /dev/null +++ b/contrib/containers/guix/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.9" +services: + guix_ubuntu: + build: + context: '.' + dockerfile: './Dockerfile' + args: + USER_ID: 1000 # set this to $(id -u) of the host + GROUP_ID: 1000 # set this to $(id -g) of the host + container_name: guix_ubuntu + tty: true + stdin_open: true + privileged: true + network_mode: host + volumes: + - "../../..:/src/dash:rw" diff --git a/contrib/containers/guix/motd.txt b/contrib/containers/guix/motd.txt new file mode 100644 index 0000000000..448da60592 --- /dev/null +++ b/contrib/containers/guix/motd.txt @@ -0,0 +1,4 @@ +##################################################### +To get started, run 'guix-start' and then calculate +hashes using 'guix-check' +##################################################### diff --git a/contrib/containers/guix/scripts/entrypoint b/contrib/containers/guix/scripts/entrypoint new file mode 100755 index 0000000000..f84cf42270 --- /dev/null +++ b/contrib/containers/guix/scripts/entrypoint @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -eo pipefail + +# Read instructions +cat /etc/motd + +# Start the Guix daemon +sudo env PATH=${PATH} guix-daemon \ + --build-users-group='guixbuild' \ + --substitute-urls='https://bordeaux.guix.gnu.org https://ci.guix.gnu.org' < /dev/null 2>&1 | + sudo tee /var/log/guix.log > /dev/null & + +# Hand over control +exec bash diff --git a/contrib/containers/guix/scripts/guix-check b/contrib/containers/guix/scripts/guix-check new file mode 100755 index 0000000000..e70170066a --- /dev/null +++ b/contrib/containers/guix/scripts/guix-check @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -eo pipefail + +cd /src/dash +COMMIT_ID=$(git rev-parse --short=12 HEAD) + +printf "Binaries:\n" +( \ +SRC_PATH_PREFIX=guix-build-${COMMIT_ID}/distsrc- && \ +sha256sum ${SRC_PATH_PREFIX}*/src/dash{d,-cli,-tx,-wallet}{,.exe} && \ +sha256sum ${SRC_PATH_PREFIX}*/src/qt/dash-qt{,.exe} && \ +sha256sum ${SRC_PATH_PREFIX}*/src/test/test_dash{,.exe} \ +) | sort -k 2 + +printf "Archives:\n" +find guix-build-"${COMMIT_ID}"/output -type f | grep -v SHA256 | xargs sha256sum | sort -k 2 diff --git a/contrib/containers/guix/scripts/guix-start b/contrib/containers/guix/scripts/guix-start new file mode 100755 index 0000000000..0e1e1eaf49 --- /dev/null +++ b/contrib/containers/guix/scripts/guix-start @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -eo pipefail + +XCODE_VERSION="12.1" +XCODE_RELEASE="12A7403" +XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers" + +# Check if macOS SDK is present, if not, download it +if [ ! -d "/src/dash/depends/SDKs/${XCODE_ARCHIVE}" ] +then + mkdir -p /src/dash/depends/SDKs + curl -L https://bitcoincore.org/depends-sources/sdks/${XCODE_ARCHIVE}.tar.gz | tar -xz -C /src/dash/depends/SDKs +fi + +cd /src/dash +git status >> /dev/null +git config --global --add safe.directory /src/dash + +./contrib/guix/guix-build From ddb38f42dae5d85d046f1008a9a6390142226bbb Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com> Date: Sun, 25 Jun 2023 03:11:04 +0530 Subject: [PATCH 2/3] contrib: move context to repository root, use additional context for copy --- contrib/containers/guix/Dockerfile | 8 ++++---- contrib/containers/guix/docker-compose.yml | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/contrib/containers/guix/Dockerfile b/contrib/containers/guix/Dockerfile index 3ec737582d..12b531a715 100644 --- a/contrib/containers/guix/Dockerfile +++ b/contrib/containers/guix/Dockerfile @@ -74,10 +74,10 @@ RUN usermod -aG sudo ${USERNAME} && \ echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers # Copy required files to container -COPY ./motd.txt /etc/motd -COPY ./scripts/entrypoint /usr/local/bin/entrypoint -COPY ./scripts/guix-check /usr/local/bin/guix-check -COPY ./scripts/guix-start /usr/local/bin/guix-start +COPY --from=docker_root ./motd.txt /etc/motd +COPY --from=docker_root ./scripts/entrypoint /usr/local/bin/entrypoint +COPY --from=docker_root ./scripts/guix-check /usr/local/bin/guix-check +COPY --from=docker_root ./scripts/guix-start /usr/local/bin/guix-start # Create directory for mounting and grant necessary permissions RUN mkdir -p /src/dash && \ diff --git a/contrib/containers/guix/docker-compose.yml b/contrib/containers/guix/docker-compose.yml index 3bead9181b..dc90916531 100644 --- a/contrib/containers/guix/docker-compose.yml +++ b/contrib/containers/guix/docker-compose.yml @@ -2,8 +2,10 @@ version: "3.9" services: guix_ubuntu: build: - context: '.' - dockerfile: './Dockerfile' + context: '../../..' + additional_contexts: + - docker_root=. + dockerfile: './contrib/containers/guix/Dockerfile' args: USER_ID: 1000 # set this to $(id -u) of the host GROUP_ID: 1000 # set this to $(id -g) of the host From 7399ea5e23e41e151b56b00a930467b9d6070ce6 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com> Date: Mon, 26 Jun 2023 00:52:27 +0530 Subject: [PATCH 3/3] contrib: switch to our Guix container instead of the bundled one --- .github/workflows/guix-build.yml | 34 ++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/guix-build.yml b/.github/workflows/guix-build.yml index 9580f74a7f..9e5328e213 100644 --- a/.github/workflows/guix-build.yml +++ b/.github/workflows/guix-build.yml @@ -17,10 +17,12 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Hash Dockerfile + - name: Commit variables id: dockerfile run: | - echo "::set-output name=hash::$(sha256sum ./contrib/guix/Dockerfile | cut -d ' ' -f1)" + echo "::set-output name=hash::$(sha256sum ./contrib/containers/guix/Dockerfile | cut -d ' ' -f1)" + echo "::set-output name=host_user_id::$(id -u)" + echo "::set-output name=host_group_id::$(id -g)" - name: Cache Docker layers uses: actions/cache@v2 @@ -33,25 +35,27 @@ jobs: - name: Build Docker image uses: docker/build-push-action@v2 with: - context: . - file: ./contrib/guix/Dockerfile + context: ${{ github.workspace }} + build-args: | + USER_ID=${{ steps.dockerfile.outputs.host_user_id }} + GROUP_ID=${{ steps.dockerfile.outputs.host_group_id }} + build-contexts: | + docker_root=${{ github.workspace }}/contrib/containers/guix + file: ./contrib/containers/guix/Dockerfile load: true - tags: alpine-guix:latest + tags: guix_ubuntu:latest cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - name: Run Guix build run: | - export ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=32' && \ - mkdir -p depends/SDKs && \ - mkdir -p /tmp/guix-store && \ - curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs && \ - docker run --privileged -d --name guix-daemon --rm -v ${{ github.workspace }}:/dash -w /dash alpine-guix:latest && \ - docker exec guix-daemon bash -c "\ - chmod 777 /dash/depends && \ - git config --global --add safe.directory /dash && \ - cd /dash && \ - contrib/guix/guix-build" + docker run --privileged -d --rm -t \ + --name guix-daemon \ + -e ADDITIONAL_GUIX_COMMON_FLAGS='--max-jobs=32' \ + -v ${{ github.workspace }}:/src/dash \ + -w /src/dash \ + guix_ubuntu:latest && \ + docker exec guix-daemon bash -c "/usr/local/bin/guix-start" - name: Ensure build passes run: |