diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27fc871dd1..9ee27e55e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,26 +40,11 @@ builder-image: needs: - builder-image image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG - variables: - SDK_URL: https://bitcoincore.org/depends-sources/sdks - XCODE_VERSION: "15.0" - XCODE_BUILD_ID: 15A240d before_script: - - echo HOST=$HOST - | - if [ "$HOST" = "x86_64-apple-darwin" ]; then - mkdir -p depends/SDKs - mkdir -p depends/sdk-sources - OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" - OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}" - if [ ! -f "$OSX_SDK_PATH" ]; then - echo "Downloading MacOS SDK" - curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" - fi - if [ -f "$OSX_SDK_PATH" ]; then - echo "Extracting MacOS SDK" - tar -C depends/SDKs -xf "$OSX_SDK_PATH" - fi + echo HOST=${HOST} + if [[ "${HOST}" == "x86_64-apple-darwin" ]]; then + ./contrib/containers/guix/scripts/setup-sdk fi script: - make -j$(nproc) -C depends HOST=$HOST $DEP_OPTS diff --git a/ci/dash/build_depends.sh b/ci/dash/build_depends.sh index 6ad5803ae5..3f1ff978eb 100755 --- a/ci/dash/build_depends.sh +++ b/ci/dash/build_depends.sh @@ -20,17 +20,8 @@ mkdir -p $CACHE_DIR/sdk-sources ln -s $CACHE_DIR/depends ${DEPENDS_DIR}/built ln -s $CACHE_DIR/sdk-sources ${DEPENDS_DIR}/sdk-sources -mkdir -p ${DEPENDS_DIR}/SDKs - -if [ -n "$XCODE_VERSION" ]; then - OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz" - OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" - if [ ! -f "$OSX_SDK_PATH" ]; then - curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" - fi - if [ -f "$OSX_SDK_PATH" ]; then - tar -C ${DEPENDS_DIR}/SDKs -xf "$OSX_SDK_PATH" - fi +if [[ "${HOST}" == "x86_64-apple-darwin" ]]; then + ./contrib/containers/guix/scripts/setup-sdk fi make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index 29bb92fff2..85df2c3105 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -7,14 +7,17 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" # (zlib1g-dev is needed for the Qt host binary builds, but should not be used by target binaries) ENV APT_ARGS="-y --no-install-recommends --no-upgrade" - -# Install packages for i386; disabled on aarch64 and arm64 hosts -RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || dpkg --add-architecture i386) -RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || (apt-get update && apt-get install $APT_ARGS \ - g++-multilib \ - wine32) && rm -rf /var/lib/apt/lists/*) - -RUN apt-get update && apt-get install $APT_ARGS \ +# Install packages for i386 on amd64 hosts, then install common packages +RUN set -ex; \ + apt-get update && \ + if [ "$(dpkg --print-architecture)" = "amd64" ]; then \ + dpkg --add-architecture i386 && \ + apt-get update && \ + apt-get install $APT_ARGS \ + g++-multilib \ + wine32; \ + fi; \ + apt-get install $APT_ARGS \ autotools-dev \ automake \ autoconf \ @@ -23,13 +26,11 @@ RUN apt-get update && apt-get install $APT_ARGS \ bsdmainutils \ curl \ ccache \ - clang \ cmake \ g++ \ gettext \ git \ - libc++-dev \ - libc++abi-dev \ + gnupg \ libtool \ libxcb-icccm4 \ libxcb-image0 \ @@ -42,11 +43,38 @@ RUN apt-get update && apt-get install $APT_ARGS \ libxcb-xinerama0 \ libxcb-xkb1 \ libxkbcommon-x11-0 \ - wget \ + lsb-release \ + software-properties-common \ unzip \ + wget \ m4 \ pkg-config \ - zlib1g-dev + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install Clang+LLVM and set it as default +# We don't need all packages but the default set doesn't include some +# packages we want so we will need to install some of them manually. +ARG LLVM_VERSION=16 +RUN set -ex; \ + echo "Installing LLVM and Clang ${LLVM_VERSION}..."; \ + curl -sL https://apt.llvm.org/llvm.sh | bash -s "${LLVM_VERSION}"; \ + echo "Installing additional packages..."; \ + apt-get update && apt-get install $APT_ARGS \ + "clang-format-${LLVM_VERSION}" \ + "clang-tidy-${LLVM_VERSION}" \ + "libc++-${LLVM_VERSION}-dev" \ + "libc++abi-${LLVM_VERSION}-dev" \ + "libclang-rt-${LLVM_VERSION}-dev"; \ + rm -rf /var/lib/apt/lists/*; \ + echo "Setting defaults..."; \ + lldbUpdAltArgs="update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${LLVM_VERSION} 100"; \ + for binName in clang clang++ clang-format clang-tidy clangd ld.lld lldb lldb-server; do \ + lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/${binName}-${LLVM_VERSION}"; \ + done; \ + sh -c "${lldbUpdAltArgs}"; +# LD_LIBRARY_PATH is empty by default, this is the first entry +ENV LD_LIBRARY_PATH="/usr/lib/llvm-${LLVM_VERSION}/lib" # Python setup # PYTHON_VERSION should match the value in .python-version @@ -61,10 +89,11 @@ RUN apt-get update && apt-get install $APT_ARGS \ libreadline-dev \ libsqlite3-dev \ libssl-dev \ - llvm \ make \ tk-dev \ - xz-utils + xz-utils \ + && rm -rf /var/lib/apt/lists/* + ENV PYENV_ROOT="/usr/local/pyenv" ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}" RUN curl https://pyenv.run | bash \ @@ -82,19 +111,38 @@ RUN pip3 install \ pyzmq==22.3.0 \ vulture==2.3 -# dash_hash ARG DASH_HASH_VERSION=1.4.0 -RUN git clone --depth 1 --no-tags --branch=${DASH_HASH_VERSION} https://github.com/dashpay/dash_hash -RUN cd dash_hash && pip3 install -r requirements.txt . +RUN set -ex; \ + cd /tmp; \ + git clone --depth 1 --no-tags --branch=${DASH_HASH_VERSION} https://github.com/dashpay/dash_hash; \ + cd dash_hash && pip3 install -r requirements.txt .; \ + cd .. && rm -rf dash_hash +ARG CPPCHECK_VERSION=2.13.0 +RUN set -ex; \ + curl -fL "https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz" -o /tmp/cppcheck.tar.gz; \ + mkdir -p /opt/cppcheck && tar -xzf /tmp/cppcheck.tar.gz -C /opt/cppcheck --strip-components=1 && rm /tmp/cppcheck.tar.gz; \ + cd /opt/cppcheck; \ + mkdir build && cd build && cmake .. && cmake --build . -j "$(( $(nproc) - 1 ))"; \ + mkdir /usr/local/share/Cppcheck && ln -s /opt/cppcheck/cfg/ /usr/local/share/Cppcheck/cfg; \ + rm -rf /tmp/cppcheck.tar.gz +ENV PATH="/opt/cppcheck/build/bin:${PATH}" + +ARG SHELLCHECK_VERSION=v0.7.1 +RUN set -ex; \ + curl -fL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" -o /tmp/shellcheck.tar.xz; \ + mkdir -p /opt/shellcheck && tar -xf /tmp/shellcheck.tar.xz -C /opt/shellcheck --strip-components=1 && rm /tmp/shellcheck.tar.xz +ENV PATH="/opt/shellcheck:${PATH}" + +# Add user with specified (or default) user/group ids and setup configuration files ARG USER_ID=1000 ARG GROUP_ID=1000 - -# add user with specified (or default) user/group ids -ENV USER_ID="${USER_ID}" -ENV GROUP_ID="${GROUP_ID}" -RUN groupadd -g ${GROUP_ID} dash -RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /home/dash dash +RUN set -ex; \ + groupadd -g ${GROUP_ID} dash; \ + useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /home/dash dash; \ + mkdir -p /home/dash/.config/gdb; \ + echo "add-auto-load-safe-path /usr/lib/llvm-${LLVM_VERSION}/lib" | tee /home/dash/.config/gdb/gdbinit; \ + chown ${USER_ID}:${GROUP_ID} -R /home/dash # Packages needed for all target builds RUN apt-get update && apt-get install $APT_ARGS \ @@ -111,17 +159,8 @@ RUN apt-get update && apt-get install $APT_ARGS \ valgrind \ wine-stable \ wine64 \ - xorriso - -ARG CPPCHECK_VERSION=2.13.0 -RUN curl -sL "https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz" | tar -xvzf - --directory /tmp/ -RUN cd /tmp/cppcheck-${CPPCHECK_VERSION} && mkdir build && cd build && cmake .. && cmake --build . -j 8 -ENV PATH="/tmp/cppcheck-${CPPCHECK_VERSION}/build/bin:${PATH}" -RUN mkdir /usr/local/share/Cppcheck && ln -s /tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ /usr/local/share/Cppcheck/cfg - -ARG SHELLCHECK_VERSION=v0.7.1 -RUN curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ -ENV PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" + xorriso \ + && rm -rf /var/lib/apt/lists/* # This is a hack. It is needed because gcc-multilib and g++-multilib are conflicting with g++-arm-linux-gnueabihf. This is # due to gcc-multilib installing the following symbolic link, which is needed for -m32 support. However, this causes @@ -135,20 +174,6 @@ RUN \ update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; \ exit 0 -ARG LLVM_VERSION=16 -# Setup Clang+LLVM support -RUN apt-get update && apt-get install $APT_ARGS \ - lsb-release \ - software-properties-common \ - gnupg \ - && rm -rf /var/lib/apt/lists/* - -RUN cd /tmp && \ - wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - /tmp/llvm.sh ${LLVM_VERSION} && \ - rm -rf /tmp/llvm.sh - RUN \ mkdir -p /src/dash && \ mkdir -p /cache/ccache && \ diff --git a/contrib/containers/develop/docker-compose.yml b/contrib/containers/develop/docker-compose.yml index 95241c0a56..9238c97183 100644 --- a/contrib/containers/develop/docker-compose.yml +++ b/contrib/containers/develop/docker-compose.yml @@ -1,17 +1,18 @@ -version: "3.9" services: container: entrypoint: /bin/bash build: context: '..' dockerfile: './develop/Dockerfile' - tty: true # Equivalent to -t - stdin_open: true # Equivalent to -i ports: - "9998:9998" # Mainnet Ports - "9999:9999" - "19998:19998" # Testnet Ports - "19999:19999" + security_opt: + - seccomp:unconfined + stdin_open: true # Equivalent to -i + tty: true # Equivalent to -t # A note about volumes: # diff --git a/contrib/containers/guix/Dockerfile b/contrib/containers/guix/Dockerfile index 4b09a24c12..861678502c 100644 --- a/contrib/containers/guix/Dockerfile +++ b/contrib/containers/guix/Dockerfile @@ -18,7 +18,11 @@ RUN apt-get update && \ sudo \ wget \ xz-utils && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/*; \ + targetLocale="en_US.UTF-8"; \ + locale-gen ${targetLocale} && \ + update-locale LC_ALL=${targetLocale} && \ + update-locale LANG=${targetLocale}; ARG guix_download_path=ftp://ftp.gnu.org/gnu/guix ARG guix_version=1.4.0 @@ -30,8 +34,7 @@ ENV PATH="/usr/local/bin:/usr/local/guix/current/bin:$PATH" # Application Setup # https://guix.gnu.org/manual/en/html_node/Application-Setup.html -ENV GUIX_LOCPATH="/usr/local/guix/profile" \ - LC_ALL="en_US.UTF-8" +ENV GUIX_LOCPATH="/usr/local/guix/profile" RUN guix_file_name=guix-binary-${guix_version}.$(uname -m)-linux.tar.xz && \ eval "guix_checksum=\${guix_checksum_$(uname -m)}" && \ @@ -74,18 +77,19 @@ RUN usermod -aG sudo ${USERNAME} && \ echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers # Copy required files to container -COPY --from=docker_root ./motd.txt /etc/motd -COPY --from=docker_root ./scripts/entrypoint /usr/local/bin/entrypoint -COPY --from=docker_root ./scripts/guix-check /usr/local/bin/guix-check -COPY --from=docker_root ./scripts/guix-start /usr/local/bin/guix-start +COPY --from=docker_root ./motd.txt /etc/motd +COPY --from=docker_root ./scripts/entrypoint /usr/local/bin/entrypoint +COPY --from=docker_root ./scripts/guix-check /usr/local/bin/guix-check +COPY --from=docker_root ./scripts/guix-start /usr/local/bin/guix-start +COPY --from=docker_root ./scripts/setup-sdk /usr/local/bin/setup-sdk # Create directories for mounting to save/restore cache and grant necessary permissions RUN mkdir -p \ /home/${USERNAME}/.cache \ - /src/dash/depends/{built,sources,work} && \ + /src/dash/depends/{built,sources,work}; \ chown -R ${USER_ID}:${GROUP_ID} \ /home/${USERNAME}/.cache \ - /src + /src; WORKDIR "/src/dash" diff --git a/contrib/containers/guix/docker-compose.yml b/contrib/containers/guix/docker-compose.yml index dc90916531..b4f6861a08 100644 --- a/contrib/containers/guix/docker-compose.yml +++ b/contrib/containers/guix/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.9" services: guix_ubuntu: build: diff --git a/contrib/containers/guix/scripts/guix-start b/contrib/containers/guix/scripts/guix-start index 4d0c6f6dba..50264c42d0 100755 --- a/contrib/containers/guix/scripts/guix-start +++ b/contrib/containers/guix/scripts/guix-start @@ -9,19 +9,10 @@ if [[ ! -d "${WORKSPACE_PATH}" || ! "${WORKSPACE_PATH}" = /* || ! -f "${WORKSPAC exit 1 fi -XCODE_VERSION="15.0" -XCODE_RELEASE="15A240d" -XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers" -XCODE_SOURCE="${XCODE_SOURCE:-https://bitcoincore.org/depends-sources/sdks}" - export SDK_PATH="${SDK_PATH:-${WORKSPACE_PATH}/depends/SDKs}" +export SDK_SRCS="${SDK_PATH:-${WORKSPACE_PATH}/depends/sdk-sources}" -# Check if macOS SDK is present, if not, download it -if [[ ! -d "${SDK_PATH}/${XCODE_ARCHIVE}" ]]; then - echo "Preparing macOS SDK..." - mkdir -p "${SDK_PATH}" - curl -L "${XCODE_SOURCE}/${XCODE_ARCHIVE}.tar.gz" | tar -xz -C "${SDK_PATH}" -fi +./contrib/containers/guix/scripts/setup-sdk # Add safe.directory option only when WORKSPACE_PATH was specified via cmd-line arguments (happens in CI) if [[ -n "${1}" ]]; then diff --git a/contrib/containers/guix/scripts/setup-sdk b/contrib/containers/guix/scripts/setup-sdk new file mode 100755 index 0000000000..4550aeed64 --- /dev/null +++ b/contrib/containers/guix/scripts/setup-sdk @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright (c) 2024 The Dash Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +set -eo pipefail + +SDK_URL="${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}" +SDK_PATH="${SDK_PATH:-depends/SDKs}" +SDK_SRCS="${SDK_SOURCES:-depends/sdk-sources}" +XCODE_VERSION="${XCODE_VERSION:-15.0}" +XCODE_RELEASE="${XCODE_RELEASE:-15A240d}" +XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers" +XCODE_AR_PATH="${SDK_SRCS}/${XCODE_ARCHIVE}.tar.gz" + +if [ ! -d "${SDK_PATH}/${XCODE_ARCHIVE}" ]; then + if [ ! -f "${XCODE_AR_PATH}" ]; then + echo "Downloading macOS SDK..." + mkdir -p "${SDK_SRCS}" + curl --location --fail "${SDK_URL}/${XCODE_ARCHIVE}.tar.gz" -o "${XCODE_AR_PATH}" + fi + echo "Extracting macOS SDK..." + mkdir -p "${SDK_PATH}" + tar -C "${SDK_PATH}" -xf "${XCODE_AR_PATH}" +fi