merge bitcoin#21851: support cross-compiling for arm64-apple-darwin

2024-12-24 19:42:46 +01:00 · 2021-05-03 14:45:08 +08:00 · 2021-05-03 14:45:08 +08:00 · 66e1541808
commit 66e1541808
parent be2eb53c57
16 changed files with 64 additions and 50 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -42,8 +42,8 @@ builder-image:
  image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
  variables:
    SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    XCODE_VERSION: "12.1"
-    XCODE_BUILD_ID: 12A7403
+    XCODE_VERSION: "12.2"
+    XCODE_BUILD_ID: 12B45b
    MAKEJOBS: -j4
  before_script:
    - echo HOST=$HOST
--- a/ci/test/00_setup_env_mac.sh
+++ b/ci/test/00_setup_env_mac.sh
@ -9,8 +9,8 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_macos_cross
 export HOST=x86_64-apple-darwin
 export PACKAGES="cmake libcap-dev libz-dev libbz2-dev python3-dev python3-setuptools"
-export XCODE_VERSION=12.1
-export XCODE_BUILD_ID=12A7403
+export XCODE_VERSION=12.2
+export XCODE_BUILD_ID=12B45b
 export RUN_UNIT_TESTS=false
 export RUN_INTEGRATION_TESTS=false
 export GOAL="all deploy"
--- a/contrib/containers/guix/scripts/guix-start
+++ b/contrib/containers/guix/scripts/guix-start
@ -13,8 +13,8 @@ if [[ ! -d "$WORKSPACE_PATH" ]]; then
    exit 1
 fi

-XCODE_VERSION="12.1"
-XCODE_RELEASE="12A7403"
+XCODE_VERSION="12.2"
+XCODE_RELEASE="12B45b"
 XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers"

 # Check if macOS SDK is present, if not, download it
--- a/contrib/devtools/security-check.py
+++ b/contrib/devtools/security-check.py
@ -206,12 +206,9 @@ BASE_PE = [
 ]

 BASE_MACHO = [
-    ('PIE', check_PIE),
    ('NOUNDEFS', check_MACHO_NOUNDEFS),
-    ('NX', check_NX),
    ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS),
    ('Canary', check_MACHO_Canary),
-    ('CONTROL_FLOW', check_MACHO_control_flow),
 ]

 CHECKS = {
@ -226,7 +223,10 @@ CHECKS = {
        lief.ARCHITECTURES.X86: BASE_PE,
    },
    lief.EXE_FORMATS.MACHO: {
-        lief.ARCHITECTURES.X86: BASE_MACHO,
+        lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE),
+                                              ('NX', check_NX),
+                                              ('CONTROL_FLOW', check_MACHO_control_flow)],
+        lief.ARCHITECTURES.ARM64: BASE_MACHO,
    }
 }

--- a/contrib/devtools/symbol-check.py
+++ b/contrib/devtools/symbol-check.py
@ -217,7 +217,7 @@ def check_MACHO_min_os(binary) -> bool:
    return False

 def check_MACHO_sdk(binary) -> bool:
-    if binary.build_version.sdk == [10, 15, 6]:
+    if binary.build_version.sdk == [11, 0, 0]:
        return True
    return False

--- a/contrib/devtools/test-security-check.py
+++ b/contrib/devtools/test-security-check.py
@ -116,21 +116,34 @@ class TestSecurityChecks(unittest.TestCase):
        executable = 'test1'
        cc = determine_wellknown_cmd('CC', 'clang')
        write_testcode(source)
+        arch = get_arch(cc, source, executable)

+        if arch == lief.ARCHITECTURES.X86:
            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW'))
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW'))
            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW'))
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW'))
            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW'))
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW'))
            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
-            (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW'))
+                (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW'))
            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
                (1, executable+': failed PIE CONTROL_FLOW'))
            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
                (1, executable+': failed PIE'))
            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
                (0, ''))
+        else:
+            # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']),
+                (1, executable+': failed LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']),
+                (0, ''))
+

        clean_files(source, executable)

--- a/contrib/gitian-build.py
+++ b/contrib/gitian-build.py
@ -74,8 +74,8 @@ def build():

    if args.macos:
        print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'])
-        subprocess.check_output(["echo 'be17f48fd0b08fb4dcd229f55a6ae48d9f781d210839b4ea313ef17dd12d6ea5 inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'])
+        subprocess.check_output(["echo 'df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619 inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
        subprocess.check_call(['bin/gbuild', '--fetch-tags',  '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
        subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)
@ -218,7 +218,7 @@ def main():
    args.macos = 'm' in args.os

    # Disable for MacOS if no SDK found
-    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'):
+    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'):
        print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
        args.macos = False

--- a/contrib/gitian-descriptors/gitian-osx.yml
+++ b/contrib/gitian-descriptors/gitian-osx.yml
@ -33,7 +33,7 @@ remotes:
 - "url": "https://github.com/dashpay/dash.git"
  "dir": "dash"
 files:
- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz"
+- "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
  set -e -o pipefail

@ -108,7 +108,7 @@ script: |
  BASEPREFIX="${PWD}/depends"

  mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz

  # Build dependencies for each host
  for i in $HOSTS; do
--- a/contrib/guix/Dockerfile
+++ b/contrib/guix/Dockerfile
@ -59,5 +59,5 @@ RUN mkdir base_cache sources SDKs
 WORKDIR /dash

 RUN mkdir -p depends/SDKs && \
-    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs
+    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs

--- a/contrib/guix/README.md
+++ b/contrib/guix/README.md
@ -224,7 +224,7 @@ details.

  _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu
  powerpc64-linux-gnu powerpc64le-linux-gnu
-  x86\_64-w64-mingw32 x86\_64-apple-darwin")_
+  x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_

 * _**SOURCES_PATH**_

@ -249,7 +249,7 @@ details.
  Set the path where _extracted_ SDKs can be found. This is passed through to
  the depends tree. Note that this is should be set to the _parent_ directory of
  the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of
-  `$HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers`).
+  `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`).

  The path that this environment variable points to **must be a directory**, and
  **NOT a symlink to a directory**.
--- a/contrib/guix/guix-build
+++ b/contrib/guix/guix-build
@ -76,7 +76,7 @@ mkdir -p "$VERSION_BASE"
 # Default to building for all supported HOSTs (overridable by environment)
 export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu
                       x86_64-w64-mingw32
-                       x86_64-apple-darwin}"
+                       x86_64-apple-darwin arm64-apple-darwin}"

 # Usage: distsrc_for_host HOST
 #
--- a/contrib/guix/guix-codesign
+++ b/contrib/guix/guix-codesign
@ -91,7 +91,7 @@ fi
 ################

 # Default to building for all supported HOSTs (overridable by environment)
-export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin}"
+export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}"

 # Usage: distsrc_for_host HOST
 #
--- a/contrib/guix/manifest.scm
+++ b/contrib/guix/manifest.scm
@ -605,7 +605,7 @@ parse, modify and abstract ELF, PE and MachO formats.")
        ;; Build tools
        gnu-make
        libtool
-        autoconf
+        autoconf-2.71
        automake
        pkg-config
        bison
--- a/contrib/macdeploy/README.md
+++ b/contrib/macdeploy/README.md
@ -13,13 +13,13 @@ When complete, it will have produced `Dash-Qt.dmg`.
 ### Step 1: Obtaining `Xcode.app`

 Our current macOS SDK
-(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be
+(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be
 extracted from
-[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
+[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
 Alternatively, after logging in to your account go to 'Downloads', then 'More'
-and look for [`Xcode_12.1`](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
+and look for [`Xcode_12.2`](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
 An Apple ID and cookies enabled for the hostname are needed to download this.
-The `sha256sum` of the archive should be `612443b1894b39368a596ea1607f30cbb0481ad44d5e29c75edb71a6d2cf050f`.
+The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.

 After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
 archive. This makes the SDK less-trivial to extract on non-macOS machines. One
@ -30,25 +30,25 @@ approach (tested on Debian Buster) is outlined below:
 apt install cpio
 git clone https://github.com/bitcoin-core/apple-sdk-tools.git

-# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current
+# Unpack Xcode_12.2.xip and place the resulting Xcode.app in your current
 # working directory
-python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.2.xip | cpio -d -i
 ```

 On macOS the process is more straightforward:

 ```bash
-xip -x Xcode_12.1.xip
+xip -x Xcode_12.2.xip
 ```

-### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
+### Step 2: Generating `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`

-To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run
+To generate `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`, run
 the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
 previous stage) as the first argument.

 ```bash
-# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from
+# Generate a Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz from
 # the supplied Xcode.app
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
@ -79,7 +79,7 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done

 To complicate things further, all builds must target an Apple SDK. These SDKs are free to
 download, but not redistributable. To obtain it, register for an Apple Developer Account,
-then download [Xcode_12.1](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
+then download [Xcode_12.2](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).

 This file is many gigabytes in size, but most (but not all) of what we need is
 contained only in a single directory:
--- a/depends/README.md
+++ b/depends/README.md
@ -26,6 +26,7 @@ Common `host-platform-triplets` for cross compilation are:
 - `x86_64-pc-linux-gnu` for x86 Linux
 - `x86_64-w64-mingw32` for Win64
 - `x86_64-apple-darwin` for macOS
+- `arm64-apple-darwin` for ARM macOS
 - `arm-linux-gnueabihf` for Linux ARM 32 bit
 - `aarch64-linux-gnu` for Linux ARM 64 bit
 - `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian)
--- a/depends/hosts/darwin.mk
+++ b/depends/hosts/darwin.mk
@ -1,7 +1,7 @@
 OSX_MIN_VERSION=10.15
-OSX_SDK_VERSION=10.15.6
-XCODE_VERSION=12.1
-XCODE_BUILD_ID=12A7403
+OSX_SDK_VERSION=11.0
+XCODE_VERSION=12.2
+XCODE_BUILD_ID=12B45b
 LD64_VERSION=609

 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers