merge bitcoin#21851: support cross-compiling for arm64-apple-darwin

.gitlab-ci.yml

@@ -42,8 +42,8 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    XCODE_VERSION: "12.1"
+    XCODE_VERSION: "12.2"
-    XCODE_BUILD_ID: 12A7403
+    XCODE_BUILD_ID: 12B45b
     MAKEJOBS: -j4
   before_script:
     - echo HOST=$HOST

ci/test/00_setup_env_mac.sh

@@ -9,8 +9,8 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_macos_cross
 export HOST=x86_64-apple-darwin
 export PACKAGES="cmake libcap-dev libz-dev libbz2-dev python3-dev python3-setuptools"
-export XCODE_VERSION=12.1
+export XCODE_VERSION=12.2
-export XCODE_BUILD_ID=12A7403
+export XCODE_BUILD_ID=12B45b
 export RUN_UNIT_TESTS=false
 export RUN_INTEGRATION_TESTS=false
 export GOAL="all deploy"

contrib/containers/guix/scripts/guix-start

@@ -13,8 +13,8 @@ if [[ ! -d "$WORKSPACE_PATH" ]]; then
     exit 1
 fi
 
-XCODE_VERSION="12.1"
+XCODE_VERSION="12.2"
-XCODE_RELEASE="12A7403"
+XCODE_RELEASE="12B45b"
 XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers"
 
 # Check if macOS SDK is present, if not, download it

contrib/devtools/security-check.py

@@ -206,12 +206,9 @@ BASE_PE = [
 ]
 
 BASE_MACHO = [
-    ('PIE', check_PIE),
     ('NOUNDEFS', check_MACHO_NOUNDEFS),
-    ('NX', check_NX),
     ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS),
     ('Canary', check_MACHO_Canary),
-    ('CONTROL_FLOW', check_MACHO_control_flow),
 ]
 
 CHECKS = {
@@ -226,7 +223,10 @@ CHECKS = {
         lief.ARCHITECTURES.X86: BASE_PE,
     },
     lief.EXE_FORMATS.MACHO: {
-        lief.ARCHITECTURES.X86: BASE_MACHO,
+        lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE),
+                                              ('NX', check_NX),
+                                              ('CONTROL_FLOW', check_MACHO_control_flow)],
+        lief.ARCHITECTURES.ARM64: BASE_MACHO,
     }
 }
 

contrib/devtools/symbol-check.py

@@ -217,7 +217,7 @@ def check_MACHO_min_os(binary) -> bool:
     return False
 
 def check_MACHO_sdk(binary) -> bool:
-    if binary.build_version.sdk == [10, 15, 6]:
+    if binary.build_version.sdk == [11, 0, 0]:
         return True
     return False
 

contrib/devtools/test-security-check.py

@@ -116,21 +116,34 @@ class TestSecurityChecks(unittest.TestCase):
         executable = 'test1'
         cc = determine_wellknown_cmd('CC', 'clang')
         write_testcode(source)
+        arch = get_arch(cc, source, executable)
+
+        if arch == lief.ARCHITECTURES.X86:
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
+                (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
+                (1, executable+': failed PIE CONTROL_FLOW'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
+                (1, executable+': failed PIE'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
+                (0, ''))
+        else:
+            # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']),
+                (1, executable+': failed NOUNDEFS LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']),
+                (1, executable+': failed LAZY_BINDINGS'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']),
+                (0, ''))
 
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']),
-            (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']),
-            (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']),
-            (1, executable+': failed PIE CONTROL_FLOW'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
-            (1, executable+': failed PIE'))
-        self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']),
-            (0, ''))
 
         clean_files(source, executable)
 

contrib/gitian-build.py

@@ -74,8 +74,8 @@ def build():
 
     if args.macos:
         print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'])
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'])
-        subprocess.check_output(["echo 'be17f48fd0b08fb4dcd229f55a6ae48d9f781d210839b4ea313ef17dd12d6ea5 inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_output(["echo 'df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619 inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
         subprocess.check_call(['bin/gbuild', '--fetch-tags',  '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)
@@ -218,7 +218,7 @@ def main():
     args.macos = 'm' in args.os
 
     # Disable for MacOS if no SDK found
-    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'):
+    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz'):
         print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
         args.macos = False
 

contrib/gitian-descriptors/gitian-osx.yml

@@ -33,7 +33,7 @@ remotes:
 - "url": "https://github.com/dashpay/dash.git"
   "dir": "dash"
 files:
-- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz"
+- "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
@@ -108,7 +108,7 @@ script: |
   BASEPREFIX="${PWD}/depends"
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do

contrib/guix/Dockerfile

@@ -59,5 +59,5 @@ RUN mkdir base_cache sources SDKs
 WORKDIR /dash
 
 RUN mkdir -p depends/SDKs && \
-    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs
+    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs
 

contrib/guix/README.md

@@ -224,7 +224,7 @@ details.
 
   _(defaults to "x86\_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu
   powerpc64-linux-gnu powerpc64le-linux-gnu
-  x86\_64-w64-mingw32 x86\_64-apple-darwin")_
+  x86\_64-w64-mingw32 x86\_64-apple-darwin arm64-apple-darwin")_
 
 * _**SOURCES_PATH**_
 
@@ -249,7 +249,7 @@ details.
   Set the path where _extracted_ SDKs can be found. This is passed through to
   the depends tree. Note that this is should be set to the _parent_ directory of
   the actual SDK (e.g. `SDK_PATH=$HOME/Downloads/macOS-SDKs` instead of
-  `$HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers`).
+  `$HOME/Downloads/macOS-SDKs/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers`).
 
   The path that this environment variable points to **must be a directory**, and
   **NOT a symlink to a directory**.

contrib/guix/guix-build

@@ -76,7 +76,7 @@ mkdir -p "$VERSION_BASE"
 # Default to building for all supported HOSTs (overridable by environment)
 export HOSTS="${HOSTS:-x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv64-linux-gnu
                        x86_64-w64-mingw32
-                       x86_64-apple-darwin}"
+                       x86_64-apple-darwin arm64-apple-darwin}"
 
 # Usage: distsrc_for_host HOST
 #

contrib/guix/guix-codesign

@@ -91,7 +91,7 @@ fi
 ################
 
 # Default to building for all supported HOSTs (overridable by environment)
-export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin}"
+export HOSTS="${HOSTS:-x86_64-w64-mingw32 x86_64-apple-darwin arm64-apple-darwin}"
 
 # Usage: distsrc_for_host HOST
 #

contrib/guix/manifest.scm

@@ -605,7 +605,7 @@ parse, modify and abstract ELF, PE and MachO formats.")
         ;; Build tools
         gnu-make
         libtool
-        autoconf
+        autoconf-2.71
         automake
         pkg-config
         bison

contrib/macdeploy/README.md

@@ -13,13 +13,13 @@ When complete, it will have produced `Dash-Qt.dmg`.
 ### Step 1: Obtaining `Xcode.app`
 
 Our current macOS SDK
-(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be
+(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be
 extracted from
-[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
+[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
 Alternatively, after logging in to your account go to 'Downloads', then 'More'
-and look for [`Xcode_12.1`](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
+and look for [`Xcode_12.2`](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
 An Apple ID and cookies enabled for the hostname are needed to download this.
-The `sha256sum` of the archive should be `612443b1894b39368a596ea1607f30cbb0481ad44d5e29c75edb71a6d2cf050f`.
+The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
 
 After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
 archive. This makes the SDK less-trivial to extract on non-macOS machines. One
@@ -30,25 +30,25 @@ approach (tested on Debian Buster) is outlined below:
 apt install cpio
 git clone https://github.com/bitcoin-core/apple-sdk-tools.git
 
-# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current
+# Unpack Xcode_12.2.xip and place the resulting Xcode.app in your current
 # working directory
-python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.2.xip | cpio -d -i
 ```
 
 On macOS the process is more straightforward:
 
 ```bash
-xip -x Xcode_12.1.xip
+xip -x Xcode_12.2.xip
 ```
 
-### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
+### Step 2: Generating `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
 
-To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run
+To generate `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`, run
 the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
 previous stage) as the first argument.
 
 ```bash
-# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from
+# Generate a Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz from
 # the supplied Xcode.app
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
@@ -79,7 +79,7 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done
 
 To complicate things further, all builds must target an Apple SDK. These SDKs are free to
 download, but not redistributable. To obtain it, register for an Apple Developer Account,
-then download [Xcode_12.1](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
+then download [Xcode_12.2](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
 
 This file is many gigabytes in size, but most (but not all) of what we need is
 contained only in a single directory:

depends/README.md

@@ -26,6 +26,7 @@ Common `host-platform-triplets` for cross compilation are:
 - `x86_64-pc-linux-gnu` for x86 Linux
 - `x86_64-w64-mingw32` for Win64
 - `x86_64-apple-darwin` for macOS
+- `arm64-apple-darwin` for ARM macOS
 - `arm-linux-gnueabihf` for Linux ARM 32 bit
 - `aarch64-linux-gnu` for Linux ARM 64 bit
 - `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian)

depends/hosts/darwin.mk

@@ -1,7 +1,7 @@
 OSX_MIN_VERSION=10.15
-OSX_SDK_VERSION=10.15.6
+OSX_SDK_VERSION=11.0
-XCODE_VERSION=12.1
+XCODE_VERSION=12.2
-XCODE_BUILD_ID=12A7403
+XCODE_BUILD_ID=12B45b
 LD64_VERSION=609
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers