merge #19240: macOS toolchain simplification and bump

.gitlab-ci.yml

@@ -37,19 +37,25 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    OSX_SDK: "10.14"
+    XCODE_VERSION: 11.3.1
+    XCODE_BUILD_ID: 11C505
     MAKEJOBS: -j4
   before_script:
     - echo HOST=$HOST
     - |
       if [ "$HOST" = "x86_64-apple-darwin16" ]; then
-        echo "Downloading MacOS SDK"
         mkdir -p depends/SDKs
         mkdir -p depends/sdk-sources
-        if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+        OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-          curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+        OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
+        if [ ! -f "$OSX_SDK_PATH" ]; then
+          echo "Downloading MacOS SDK"
+          curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
+        fi
+        if [ -f "$OSX_SDK_PATH" ]; then
+          echo "Extracting MacOS SDK"
+          tar -C depends/SDKs -xf "$OSX_SDK_PATH"
         fi
-        tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
       fi
   script:
     - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS

.travis.yml

@@ -296,7 +296,8 @@ after_success:
       env: >-
         HOST=x86_64-apple-darwin16
         PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
-        OSX_SDK=10.14
+        export XCODE_VERSION=11.3.1
+        export XCODE_BUILD_ID=11C505
         RUN_UNIT_TESTS=false
         RUN_FUNCTIONAL_TESTS=false
         GOAL="deploy"

.travis/test_05_before_script.sh

@@ -7,14 +7,16 @@
 export LC_ALL=C.UTF-8
 
 DOCKER_EXEC echo \> \$HOME/.bitcoin  # Make sure default datadir does not exist and is never read by creating a dummy file
+OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
+OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
 
 mkdir -p depends/SDKs depends/sdk-sources
 
-if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then
-  curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
 fi
-if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then
-  tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  DOCKER_EXEC tar -C "depends/SDKs" -xf "$OSX_SDK_PATH"
 fi
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)

ci/build_depends.sh

@@ -19,12 +19,14 @@ ln -s $CACHE_DIR/sdk-sources depends/sdk-sources
 
 mkdir -p depends/SDKs
 
-if [ -n "$OSX_SDK" ]; then
+if [ -n "$XCODE_VERSION" ]; then
-  if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-    curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
+  if [ ! -f "$OSX_SDK_PATH" ]; then
+    curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
   fi
-  if [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  if [ -f "$OSX_SDK_PATH" ]; then
-    tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+    tar -C depends/SDKs -xf "$OSX_SDK_PATH"
   fi
 fi
 

ci/matrix.sh

@@ -78,7 +78,8 @@ elif [ "$BUILD_TARGET" = "linux64_release" ]; then
 elif [ "$BUILD_TARGET" = "mac" ]; then
   export HOST=x86_64-apple-darwin16
   export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --disable-miner --enable-werror"
-  export OSX_SDK=10.14
+  export XCODE_VERSION=11.3.1
+  export XCODE_BUILD_ID=11C505
   export GOAL="all deploy"
   export RUN_UNITTESTS=false
   export RUN_INTEGRATIONTESTS=false

contrib/gitian-build.py

@@ -71,8 +71,8 @@ def build():
 
     if args.macos:
         print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/MacOSX10.14.sdk.tar.gz'])
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'])
-        subprocess.check_output(["echo '328aff47e28c17093d59a72712a9b2e62cd8a8b87bbe03f91abb32960b413f0f inputs/MacOSX10.14.sdk.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_output(["echo '436df6dfc7073365d12f8ef6c1fdb060777c720602cc67c2dcf9a59d94290e38 inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
         subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)

contrib/gitian-descriptors/gitian-osx.yml

@@ -31,7 +31,7 @@ remotes:
 - "url": "https://github.com/dashpay/dash.git"
   "dir": "dash"
 files:
-- "MacOSX10.14.sdk.tar.gz"
+- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
@@ -107,7 +107,7 @@ script: |
   BASEPREFIX=`pwd`/depends
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do

contrib/macdeploy/README.md

@@ -14,51 +14,44 @@ When complete, it will have produced `Dash-Qt.dmg`.
 
 ## SDK Extraction
 
-`Xcode.app` is packaged in a `.xip` archive.
+### Step 1: Obtaining `Xcode.app`
-This makes the SDK less-trivial to extract on non-macOS machines.
+
-One approach (tested on Debian Buster) is outlined below:
+Our current macOS SDK
+(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be
+extracted from
+[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
+An Apple ID is needed to download this.
+
+After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
+archive. This makes the SDK less-trivial to extract on non-macOS machines. One
+approach (tested on Debian Buster) is outlined below:
 
 ```bash
+# Install/clone tools needed for extracting Xcode.app
+apt install cpio
+git clone https://github.com/bitcoin-core/apple-sdk-tools.git
 
-apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make
+# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current
-
+# working directory
-git clone https://github.com/tpoechtrager/xar
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i
-pushd xar/xar
-./configure
-make
-make install
-popd
-
-git clone https://github.com/NiklasRosenstein/pbzx
-pushd pbzx
-clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib
-popd
-
-xar -xf Xcode_10.2.1.xip -C .
-
-./pbzx/pbzx -n Content | cpio -i
-
-find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \;
 ```
 
-on macOS the process is more straightforward:
+On macOS the process is more straightforward:
 
 ```bash
-xip -x Xcode_10.2.1.xip
+xip -x Xcode_11.3.1.xip
-tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk
 ```
 
-Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from
+### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
-[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg).
+
-The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First
+To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run
-ensure the DMG file is in the current directory, and then run the script. You
+the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
-may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the
+previous stage) as the first argument.
-directory) when you've confirmed the extraction succeeded.
 
 ```bash
-apt-get install p7zip-full sleuthkit
+# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from
-contrib/macdeploy/extract-osx-sdk.sh
+# the supplied Xcode.app
-rm -rf 5.hfs MacOSX10.11.sdk
+./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
 
 ## Deterministic macOS DMG Notes
@@ -87,13 +80,13 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done
 
 To complicate things further, all builds must target an Apple SDK. These SDKs are free to
 download, but not redistributable. To obtain it, register for an Apple Developer Account,
-then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip).
+then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
 
 This file is many gigabytes in size, but most (but not all) of what we need is
 contained only in a single directory:
 
 ```bash
-Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk
+Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
 ```
 
 See the SDK Extraction notes above for how to obtain it.
@@ -130,7 +123,6 @@ deterministic. Here's how it works:
   of a tarball, which also contains all of the tools that have been previously (deterministically)
   built in order to create a final DMG.
 - The Apple keyholder uses this unsigned app to create a detached signature, using the
-  script that is also included there. Detached signatures are available.
+  script that is also included there. Detached signatures are available
 - Builders feed the unsigned app + detached signature back into Gitian. It uses the
   pre-built tools to recombine the pieces into a deterministic DMG.
-

contrib/macdeploy/extract-osx-sdk.sh

@@ -8,7 +8,7 @@ set -e
 
 INPUTFILE="Xcode_7.3.1.dmg"
 HFSFILENAME="5.hfs"
-SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk"
+SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
 
 7z x "${INPUTFILE}" "${HFSFILENAME}"
 SDKNAME="$(basename "${SDKDIR}")"

contrib/macdeploy/gen-sdk

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+import argparse
+import plistlib
+import pathlib
+import sys
+import tarfile
+import gzip
+import os
+import contextlib
+
+@contextlib.contextmanager
+def cd(path):
+    """Context manager that restores PWD even if an exception was raised."""
+    old_pwd = os.getcwd()
+    os.chdir(str(path))
+    try:
+        yield
+    finally:
+        os.chdir(old_pwd)
+
+def run():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1)
+    parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False)
+
+    args = parser.parse_args()
+
+    xcode_app = pathlib.Path(args.xcode_app[0]).resolve()
+    assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app)
+
+    xcode_app_plist = xcode_app.joinpath("Contents/version.plist")
+    with xcode_app_plist.open('rb') as fp:
+        pl = plistlib.load(fp)
+        xcode_version = pl['CFBundleShortVersionString']
+        xcode_build_id = pl['ProductBuildVersion']
+    print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id))
+
+    sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk")
+    sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist")
+    with sdk_plist.open('rb') as fp:
+        pl = plistlib.load(fp)
+        sdk_version = pl['ProductVersion']
+        sdk_build_id = pl['ProductBuildVersion']
+    print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id))
+
+    out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)
+
+    xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1")
+    assert xcode_libcxx_dir.is_dir()
+
+    if args.out_sdktgz:
+        out_sdktgz_path = pathlib.Path(args.out_sdktgz_path)
+    else:
+        # Construct our own out_sdktgz if not specified on the command line
+        out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name))
+
+    def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir):
+        """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files'
+        names
+
+        e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking:
+
+            tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir")
+
+        would result in the following members being added to tarfp:
+
+            foo/bar/             -> corresponding to /root/bazdir
+            foo/bar/qux          -> corresponding to /root/bazdir/qux
+
+        """
+        def change_tarinfo_base(tarinfo):
+            if tarinfo.name and tarinfo.name.startswith("./"):
+                tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name))
+            if tarinfo.linkname and tarinfo.linkname.startswith("./"):
+                tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname))
+            return tarinfo
+        with cd(dir_to_add):
+            tarfp.add(".", recursive=True, filter=change_tarinfo_base)
+
+    print("Creating output .tar.gz file...")
+    with out_sdktgz_path.open("wb") as fp:
+        with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf:
+            with tarfile.open(mode="w", fileobj=gzf) as tarfp:
+                print("Adding MacOSX SDK {} files...".format(sdk_version))
+                tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
+                print("Adding libc++ headers...")
+                tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name))
+    print("Done! Find the resulting gzipped tarball at:")
+    print(out_sdktgz_path.resolve())
+
+if __name__ == '__main__':
+    run()

depends/hosts/darwin.mk

@@ -1,8 +1,12 @@
 OSX_MIN_VERSION=10.12
-OSX_SDK_VERSION=10.14
+OSX_SDK_VERSION=10.15.1
-OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk
+XCODE_VERSION=11.3.1
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK)
+XCODE_BUILD_ID=11C505
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++
+LD64_VERSION=530
+
+OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
+darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION)
+darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION)
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)

depends/packages/native_cctools.mk

@@ -1,14 +1,14 @@
 package=native_cctools
-$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f
+$(package)_version=4da2f3b485bcf4cef526f30c0b8c0bcda99cdbb4
 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04
+$(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec
 $(package)_build_subdir=cctools
-$(package)_clang_version=6.0.1
+$(package)_clang_version=8.0.0
 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
 $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0
+$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
 
 $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef
 $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
@@ -72,7 +72,5 @@ define $(package)_stage_cmds
   cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\
   cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
   cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
-  cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
+  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil
-  if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \
-  if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi
 endef

doc/build-cross.md

@@ -19,8 +19,8 @@ Additionally, the Mac OSX SDK must be downloaded and extracted manually:
 ```bash
 $ mkdir -p depends/sdk-sources
 $ mkdir -p depends/SDKs
-$ curl https://bitcoincore.org/depends-sources/sdks/MacOSX10.14.sdk.tar.gz -o depends/sdk-sources/MacOSX10.14.sdk.tar.gz
+$ curl https://bitcoincore.org/depends-sources/sdks/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz -o depends/sdk-sources/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
-$ tar -C depends/SDKs -xf depends/sdk-sources/MacOSX10.14.sdk.tar.gz
+$ tar -C depends/SDKs -xf depends/sdk-sources/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
 ```
 
 When building the dependencies, as described in [build-generic](build-generic.md), use