Merge pull request #4378 from kittywhiskers/darwinqol_2

build: gitian, linting and macOS-specific QoL improvements (part 2)

.gitlab-ci.yml

@@ -37,19 +37,25 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    OSX_SDK: "10.14"
+    XCODE_VERSION: 11.3.1
+    XCODE_BUILD_ID: 11C505
     MAKEJOBS: -j4
   before_script:
     - echo HOST=$HOST
     - |
-      if [ "$HOST" = "x86_64-apple-darwin16" ]; then
+      if [ "$HOST" = "x86_64-apple-darwin18" ]; then
-        echo "Downloading MacOS SDK"
         mkdir -p depends/SDKs
         mkdir -p depends/sdk-sources
-        if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+        OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-          curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+        OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
+        if [ ! -f "$OSX_SDK_PATH" ]; then
+          echo "Downloading MacOS SDK"
+          curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
+        fi
+        if [ -f "$OSX_SDK_PATH" ]; then
+          echo "Extracting MacOS SDK"
+          tar -C depends/SDKs -xf "$OSX_SDK_PATH"
         fi
-        tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
       fi
   script:
     - make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS
@@ -180,12 +186,12 @@ x86_64-unknown-linux-gnu-release:
     HOST: x86_64-unknown-linux-gnu
     DEP_OPTS: "NO_UPNP=1"
 
-x86_64-apple-darwin16:
+x86_64-apple-darwin18:
   extends:
     - .build-depends-template
     - .skip-in-fast-mode-template
   variables:
-    HOST: x86_64-apple-darwin16
+    HOST: x86_64-apple-darwin18
 
 ###
 
@@ -251,7 +257,7 @@ mac-build:
     - .build-template
     - .skip-in-fast-mode-template
   needs:
-    - x86_64-apple-darwin16
+    - x86_64-apple-darwin18
   variables:
     BUILD_TARGET: mac
 

.travis.yml

@@ -294,9 +294,10 @@ after_success:
     - stage: test
       name: 'macOS 10.12  [GOAL: deploy]'
       env: >-
-        HOST=x86_64-apple-darwin16
+        HOST=x86_64-apple-darwin18
         PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
-        OSX_SDK=10.14
+        export XCODE_VERSION=11.3.1
+        export XCODE_BUILD_ID=11C505
         RUN_UNIT_TESTS=false
         RUN_FUNCTIONAL_TESTS=false
         GOAL="deploy"

.travis/lint_04_install.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -9,3 +9,4 @@ export LC_ALL=C
 travis_retry pip install codespell==1.13.0
 travis_retry pip install flake8==3.5.0
 travis_retry pip install vulture==0.29
+travis_retry pip3 install yq

.travis/test_05_before_script.sh

@@ -7,14 +7,16 @@
 export LC_ALL=C.UTF-8
 
 DOCKER_EXEC echo \> \$HOME/.bitcoin  # Make sure default datadir does not exist and is never read by creating a dummy file
+OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
+OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
 
 mkdir -p depends/SDKs depends/sdk-sources
 
-if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then
-  curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
 fi
-if [ -n "$OSX_SDK" -a -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then
-  tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  DOCKER_EXEC tar -C "depends/SDKs" -xf "$OSX_SDK_PATH"
 fi
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)

CMakeLists.txt

@@ -20,7 +20,7 @@ include_directories(
 if(UNIX AND NOT APPLE)
     set(DEPENDS_PREFIX depends/x86_64-pc-linux-gnu)
 elseif(APPLE)
-    set(DEPENDS_PREFIX depends/x86_64-apple-darwin16)
+    set(DEPENDS_PREFIX depends/x86_64-apple-darwin18)
 elseif(WIN32)
     set(DEPENDS_PREFIX depends/x86_64-w64-mingw32)
 endif()

ci/Dockerfile.builder

@@ -18,6 +18,7 @@ RUN pip3 install jinja2
 RUN pip3 install flake8==3.5.0
 RUN pip3 install codespell==1.13.0
 RUN pip3 install vulture==0.29
+RUN pip3 install yq
 
 # dash_hash
 RUN git clone https://github.com/dashpay/dash_hash

ci/build_depends.sh

@@ -19,12 +19,14 @@ ln -s $CACHE_DIR/sdk-sources depends/sdk-sources
 
 mkdir -p depends/SDKs
 
-if [ -n "$OSX_SDK" ]; then
+if [ -n "$XCODE_VERSION" ]; then
-  if [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-    curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
+  if [ ! -f "$OSX_SDK_PATH" ]; then
+    curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
   fi
-  if [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
+  if [ -f "$OSX_SDK_PATH" ]; then
-    tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+    tar -C depends/SDKs -xf "$OSX_SDK_PATH"
   fi
 fi
 

ci/matrix.sh

@@ -76,9 +76,10 @@ elif [ "$BUILD_TARGET" = "linux64_release" ]; then
   export BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports"
   export PYZMQ=true
 elif [ "$BUILD_TARGET" = "mac" ]; then
-  export HOST=x86_64-apple-darwin16
+  export HOST=x86_64-apple-darwin18
   export BITCOIN_CONFIG="--enable-gui --enable-reduce-exports --disable-miner --enable-werror"
-  export OSX_SDK=10.14
+  export XCODE_VERSION=11.3.1
+  export XCODE_BUILD_ID=11C505
   export GOAL="all deploy"
   export RUN_UNITTESTS=false
   export RUN_INTEGRATIONTESTS=false

contrib/gitian-build.py

@@ -71,8 +71,8 @@ def build():
 
     if args.macos:
         print('\nCompiling ' + args.version + ' MacOS')
-        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/MacOSX10.14.sdk.tar.gz'])
+        subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/depends-sources/sdks/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'])
-        subprocess.check_output(["echo '328aff47e28c17093d59a72712a9b2e62cd8a8b87bbe03f91abb32960b413f0f inputs/MacOSX10.14.sdk.tar.gz' | sha256sum -c"], shell=True)
+        subprocess.check_output(["echo '436df6dfc7073365d12f8ef6c1fdb060777c720602cc67c2dcf9a59d94290e38 inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz' | sha256sum -c"], shell=True)
         subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'dash='+args.commit, '--url', 'dash='+args.url, '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx-unsigned', '--destination', '../gitian.sigs/', '../dash/contrib/gitian-descriptors/gitian-osx.yml'])
         subprocess.check_call('mv build/out/dashcore-*-osx-unsigned.tar.gz inputs/', shell=True)

contrib/gitian-descriptors/gitian-linux.yml

@@ -50,7 +50,7 @@ script: |
   export GZIP="-9n"
   export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
   export TZ="UTC"
-  export BUILD_DIR=`pwd`
+  export BUILD_DIR="$PWD"
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
@@ -128,7 +128,7 @@ script: |
   cat << EOF > ${WRAP_DIR}/${prog}
   #!/usr/bin/env bash
   # GCCVERSION=${GCCVERSION}
-  REAL="`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1`"
+  REAL="$(which -a ${prog}-8 | grep -v ${WRAP_DIR}/${prog} | head -1)"
   for var in "\$@"
   do
     if [ "\$var" = "-m32" ]; then
@@ -144,7 +144,7 @@ script: |
   done
 
   cd dash
-  BASEPREFIX=`pwd`/depends
+  BASEPREFIX="${PWD}/depends"
   # Build dependencies for each host
   for i in $HOSTS; do
     EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
@@ -163,10 +163,11 @@ script: |
 
   # Create the release tarball using (arbitrarily) the first host
   ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/
+  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
   make dist
-  SOURCEDIST=`echo dashcore-*.tar.gz`
+  SOURCEDIST=$(echo dashcore-*.tar.gz)
-  DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
+  DISTNAME=${SOURCEDIST/%.tar.gz}
+
   # Correct tar file order
   mkdir -p temp
   pushd temp
@@ -183,7 +184,7 @@ script: |
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir -p distsrc-${i}
     cd distsrc-${i}
-    INSTALLPATH=`pwd`/installed/${DISTNAME}
+    INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
     tar --strip-components=1 -xf ../$SOURCEDIST
 

contrib/gitian-descriptors/gitian-osx-signer.yml

@@ -16,7 +16,7 @@ script: |
 
   WRAP_DIR=$HOME/wrapped
   mkdir -p ${WRAP_DIR}
-  export PATH=`pwd`:$PATH
+  export PATH="$PWD":$PATH
   FAKETIME_PROGS="dmg genisoimage"
 
   # Create global faketime wrappers

contrib/gitian-descriptors/gitian-osx.yml

@@ -31,12 +31,12 @@ remotes:
 - "url": "https://github.com/dashpay/dash.git"
   "dir": "dash"
 files:
-- "MacOSX10.14.sdk.tar.gz"
+- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-apple-darwin16"
+  HOSTS="x86_64-apple-darwin18"
   CONFIGFLAGS="--enable-reduce-exports --disable-miner --disable-bench --disable-gui-tests GENISOIMAGE=$WRAP_DIR/genisoimage --enable-crash-hooks"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="ar ranlib date dmg genisoimage"
@@ -46,7 +46,7 @@ script: |
   export GZIP="-9n"
   export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
   export TZ="UTC"
-  export BUILD_DIR=`pwd`
+  export BUILD_DIR="$PWD"
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
@@ -104,10 +104,10 @@ script: |
   export PATH=${WRAP_DIR}:${PATH}
 
   cd dash
-  BASEPREFIX=`pwd`/depends
+  BASEPREFIX="${PWD}/depends"
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do
@@ -122,10 +122,10 @@ script: |
 
   # Create the release tarball using (arbitrarily) the first host
   ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/
+  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
   make dist
-  SOURCEDIST=`echo dashcore-*.tar.gz`
+  SOURCEDIST=$(echo dashcore-*.tar.gz)
-  DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
+  DISTNAME=${SOURCEDIST/%.tar.gz}
 
   # Correct tar file order
   mkdir -p temp
@@ -143,7 +143,7 @@ script: |
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir -p distsrc-${i}
     cd distsrc-${i}
-    INSTALLPATH=`pwd`/installed/${DISTNAME}
+    INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
     tar --strip-components=1 -xf ../$SOURCEDIST
 
@@ -186,8 +186,8 @@ script: |
   done
   mkdir -p $OUTDIR/src
   mv $SOURCEDIST $OUTDIR/src
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin18.tar.gz ${OUTDIR}/${DISTNAME}-osx64.tar.gz
-  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin16-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
+  mv ${OUTDIR}/${DISTNAME}-x86_64-apple-darwin18-debug.tar.gz ${OUTDIR}/${DISTNAME}-osx64-debug.tar.gz
 
   # Compress ccache (otherwise the assert file will get too huge)
   if [ "$CCACHE_DIR" != "" ]; then

contrib/gitian-descriptors/gitian-win-signer.yml

@@ -19,7 +19,7 @@ files:
 script: |
   set -e -o pipefail
 
-  BUILD_DIR=`pwd`
+  BUILD_DIR="$PWD"
   SIGDIR=${BUILD_DIR}/signature/win
   UNSIGNED_DIR=${BUILD_DIR}/unsigned
 
@@ -35,7 +35,7 @@ script: |
   ./configure --without-gsf --without-curl --disable-dependency-tracking
   make
   find ${UNSIGNED_DIR} -name "*-unsigned.exe" | while read i; do
-    INFILE="`basename "${i}"`"
+    INFILE="$(basename "${i}")"
-    OUTFILE="`echo "${INFILE}" | sed s/-unsigned//`"
+    OUTFILE="${INFILE/%-unsigned}"
     ./osslsigncode attach-signature -in "${i}" -out "${OUTDIR}/${OUTFILE}" -sigin "${SIGDIR}/${INFILE}.pem"
   done

contrib/gitian-descriptors/gitian-win.yml

@@ -43,7 +43,7 @@ script: |
   export GZIP="-9n"
   export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
   export TZ="UTC"
-  export BUILD_DIR=`pwd`
+  export BUILD_DIR="$PWD"
   mkdir -p ${WRAP_DIR}
   if test -n "$GBUILD_CACHE_ENABLED"; then
     export SOURCES_PATH=${GBUILD_COMMON_CACHE}
@@ -137,7 +137,7 @@ script: |
   export PATH=${WRAP_DIR}:${PATH}
 
   cd dash
-  BASEPREFIX=`pwd`/depends
+  BASEPREFIX="${PWD}/depends"
   # Build dependencies for each host
   for i in $HOSTS; do
     make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
@@ -152,10 +152,10 @@ script: |
 
   # Create the release tarball using (arbitrarily) the first host
   ./autogen.sh
-  CONFIG_SITE=${BASEPREFIX}/`echo "${HOSTS}" | awk '{print $1;}'`/share/config.site ./configure --prefix=/
+  CONFIG_SITE=${BASEPREFIX}/$(echo "${HOSTS}" | awk '{print $1;}')/share/config.site ./configure --prefix=/
   make dist
-  SOURCEDIST=`echo dashcore-*.tar.gz`
+  SOURCEDIST=$(echo dashcore-*.tar.gz)
-  DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
+  DISTNAME=${SOURCEDIST/%.tar.gz}
 
   # Correct tar file order
   mkdir -p temp
@@ -175,7 +175,7 @@ script: |
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir -p distsrc-${i}
     cd distsrc-${i}
-    INSTALLPATH=`pwd`/installed/${DISTNAME}
+    INSTALLPATH="${PWD}/installed/${DISTNAME}"
     mkdir -p ${INSTALLPATH}
     tar --strip-components=1 -xf ../$SOURCEDIST
 

contrib/macdeploy/README.md

@@ -14,51 +14,44 @@ When complete, it will have produced `Dash-Qt.dmg`.
 
 ## SDK Extraction
 
-`Xcode.app` is packaged in a `.xip` archive.
+### Step 1: Obtaining `Xcode.app`
-This makes the SDK less-trivial to extract on non-macOS machines.
+
-One approach (tested on Debian Buster) is outlined below:
+Our current macOS SDK
+(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be
+extracted from
+[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
+An Apple ID is needed to download this.
+
+After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
+archive. This makes the SDK less-trivial to extract on non-macOS machines. One
+approach (tested on Debian Buster) is outlined below:
 
 ```bash
+# Install/clone tools needed for extracting Xcode.app
+apt install cpio
+git clone https://github.com/bitcoin-core/apple-sdk-tools.git
 
-apt install clang cpio git liblzma-dev libxml2-dev libssl-dev make
+# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current
-
+# working directory
-git clone https://github.com/tpoechtrager/xar
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i
-pushd xar/xar
-./configure
-make
-make install
-popd
-
-git clone https://github.com/NiklasRosenstein/pbzx
-pushd pbzx
-clang -llzma -lxar pbzx.c -o pbzx -Wl,-rpath=/usr/local/lib
-popd
-
-xar -xf Xcode_10.2.1.xip -C .
-
-./pbzx/pbzx -n Content | cpio -i
-
-find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \;
 ```
 
-on macOS the process is more straightforward:
+On macOS the process is more straightforward:
 
 ```bash
-xip -x Xcode_10.2.1.xip
+xip -x Xcode_11.3.1.xip
-tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk
 ```
 
-Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from
+### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
-[Xcode 7.3.1 dmg](https://developer.apple.com/devcenter/download.action?path=/Developer_Tools/Xcode_7.3.1/Xcode_7.3.1.dmg).
+
-The script [`extract-osx-sdk.sh`](./extract-osx-sdk.sh) automates this. First
+To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run
-ensure the DMG file is in the current directory, and then run the script. You
+the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
-may wish to delete the `intermediate 5.hfs` file and `MacOSX10.11.sdk` (the
+previous stage) as the first argument.
-directory) when you've confirmed the extraction succeeded.
 
 ```bash
-apt-get install p7zip-full sleuthkit
+# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from
-contrib/macdeploy/extract-osx-sdk.sh
+# the supplied Xcode.app
-rm -rf 5.hfs MacOSX10.11.sdk
+./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
 
 ## Deterministic macOS DMG Notes
@@ -87,13 +80,13 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done
 
 To complicate things further, all builds must target an Apple SDK. These SDKs are free to
 download, but not redistributable. To obtain it, register for an Apple Developer Account,
-then download [Xcode 10.2.1](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip).
+then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
 
 This file is many gigabytes in size, but most (but not all) of what we need is
 contained only in a single directory:
 
 ```bash
-Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk
+Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
 ```
 
 See the SDK Extraction notes above for how to obtain it.
@@ -130,7 +123,6 @@ deterministic. Here's how it works:
   of a tarball, which also contains all of the tools that have been previously (deterministically)
   built in order to create a final DMG.
 - The Apple keyholder uses this unsigned app to create a detached signature, using the
-  script that is also included there. Detached signatures are available.
+  script that is also included there. Detached signatures are available
 - Builders feed the unsigned app + detached signature back into Gitian. It uses the
   pre-built tools to recombine the pieces into a deterministic DMG.
-

contrib/macdeploy/extract-osx-sdk.sh

@@ -8,7 +8,7 @@ set -e
 
 INPUTFILE="Xcode_7.3.1.dmg"
 HFSFILENAME="5.hfs"
-SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.14.sdk"
+SDKDIR="Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
 
 7z x "${INPUTFILE}" "${HFSFILENAME}"
 SDKNAME="$(basename "${SDKDIR}")"

contrib/macdeploy/gen-sdk

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+import argparse
+import plistlib
+import pathlib
+import sys
+import tarfile
+import gzip
+import os
+import contextlib
+
+@contextlib.contextmanager
+def cd(path):
+    """Context manager that restores PWD even if an exception was raised."""
+    old_pwd = os.getcwd()
+    os.chdir(str(path))
+    try:
+        yield
+    finally:
+        os.chdir(old_pwd)
+
+def run():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1)
+    parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False)
+
+    args = parser.parse_args()
+
+    xcode_app = pathlib.Path(args.xcode_app[0]).resolve()
+    assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app)
+
+    xcode_app_plist = xcode_app.joinpath("Contents/version.plist")
+    with xcode_app_plist.open('rb') as fp:
+        pl = plistlib.load(fp)
+        xcode_version = pl['CFBundleShortVersionString']
+        xcode_build_id = pl['ProductBuildVersion']
+    print("Found Xcode (version: {xcode_version}, build id: {xcode_build_id})".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id))
+
+    sdk_dir = xcode_app.joinpath("Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk")
+    sdk_plist = sdk_dir.joinpath("System/Library/CoreServices/SystemVersion.plist")
+    with sdk_plist.open('rb') as fp:
+        pl = plistlib.load(fp)
+        sdk_version = pl['ProductVersion']
+        sdk_build_id = pl['ProductBuildVersion']
+    print("Found MacOSX SDK (version: {sdk_version}, build id: {sdk_build_id})".format(sdk_version=sdk_version, sdk_build_id=sdk_build_id))
+
+    out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id)
+
+    xcode_libcxx_dir = xcode_app.joinpath("Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1")
+    assert xcode_libcxx_dir.is_dir()
+
+    if args.out_sdktgz:
+        out_sdktgz_path = pathlib.Path(args.out_sdktgz_path)
+    else:
+        # Construct our own out_sdktgz if not specified on the command line
+        out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name))
+
+    def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir):
+        """Add all files in dir_to_add to tarfp, but prepent MEMBERPREFIX to the files'
+        names
+
+        e.g. if the only file under /root/bazdir is /root/bazdir/qux, invoking:
+
+            tarfp_add_with_base_change(tarfp, "foo/bar", "/root/bazdir")
+
+        would result in the following members being added to tarfp:
+
+            foo/bar/             -> corresponding to /root/bazdir
+            foo/bar/qux          -> corresponding to /root/bazdir/qux
+
+        """
+        def change_tarinfo_base(tarinfo):
+            if tarinfo.name and tarinfo.name.startswith("./"):
+                tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name))
+            if tarinfo.linkname and tarinfo.linkname.startswith("./"):
+                tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname))
+            return tarinfo
+        with cd(dir_to_add):
+            tarfp.add(".", recursive=True, filter=change_tarinfo_base)
+
+    print("Creating output .tar.gz file...")
+    with out_sdktgz_path.open("wb") as fp:
+        with gzip.GzipFile(fileobj=fp, compresslevel=9, mtime=0) as gzf:
+            with tarfile.open(mode="w", fileobj=gzf) as tarfp:
+                print("Adding MacOSX SDK {} files...".format(sdk_version))
+                tarfp_add_with_base_change(tarfp, sdk_dir, out_name)
+                print("Adding libc++ headers...")
+                tarfp_add_with_base_change(tarfp, xcode_libcxx_dir, "{}/usr/include/c++/v1".format(out_name))
+    print("Done! Find the resulting gzipped tarball at:")
+    print(out_sdktgz_path.resolve())
+
+if __name__ == '__main__':
+    run()

depends/Makefile

@@ -116,6 +116,13 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
+ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# Make sure that cache is invalidated when switching between system and
+# depends-managed, pinned clang
+build_id_string+=system_clang
+$(host_arch)_$(host_os)_id_string+=system_clang
+endif
+
 qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages)
 wallet_packages_$(NO_WALLET) = $(wallet_packages)
 upnp_packages_$(NO_UPNP) = $(upnp_packages)
@@ -131,11 +138,11 @@ all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
 
+$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils)
 $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
 
 include funcs.mk
 
-toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
 final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
 $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
@@ -146,15 +153,39 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
 	$(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); )
 	$(AT)touch $@
 
+# $PATH is not preserved between ./configure and make by convention. Its
+# modification and overriding at ./configure time is (as I understand it)
+# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros,
+# which will expand the program names to their full absolute paths. The notable
+# exception is command line overriding: ./configure CC=clang, which skips the
+# program name expansion step, and works because the user implicitly indicates
+# with CC=clang that clang will be available in $PATH at all times, and is most
+# likely part of the user's system.
+#
+# Therefore, when we "seed the autoconf cache"/"override well-known program
+# vars" by setting AR=<blah> in our config.site, either one of two things needs
+# to be true for the build system to work correctly:
+#
+#   1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the
+#      tool needs to be available in $PATH at all times.
+#
+#   2. If the tool is _**not**_ expected to be available in $PATH at all times
+#      (such as is the case for our native_cctools binutils tools), it needs to
+#      be referred to by its absolute path, such as would be output by the
+#      AC_PATH_{PROG,TOOL} macros.
+#
+# Minor note: it is also okay to refer to tools by their absolute path even if
+# we expect them to be available in $PATH at all times, more specificity does
+# not hurt.
 $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
 	$(AT)@mkdir -p $(@D)
 	$(AT)sed -e 's|@HOST@|$(host)|' \
-            -e 's|@CC@|$(toolchain_path)$(host_CC)|' \
+            -e 's|@CC@|$(host_CC)|' \
-            -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
+            -e 's|@CXX@|$(host_CXX)|' \
-            -e 's|@AR@|$(toolchain_path)$(host_AR)|' \
+            -e 's|@AR@|$(host_AR)|' \
-            -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \
+            -e 's|@RANLIB@|$(host_RANLIB)|' \
-            -e 's|@NM@|$(toolchain_path)$(host_NM)|' \
+            -e 's|@NM@|$(host_NM)|' \
-            -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \
+            -e 's|@STRIP@|$(host_STRIP)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \

depends/README.md

@@ -21,7 +21,7 @@ created. To use it for Dash:
 Common `host-platform-triplets` for cross compilation are:
 
 - `x86_64-w64-mingw32` for Win64
-- `x86_64-apple-darwin16` for macOS
+- `x86_64-apple-darwin18` for macOS
 - `arm-linux-gnueabihf` for Linux ARM 32 bit
 - `aarch64-linux-gnu` for Linux ARM 64 bit
 - `riscv32-linux-gnu` for Linux RISC-V 32 bit
@@ -52,19 +52,34 @@ RISC-V known issue: gcc-7.3.0 and gcc-7.3.1 result in a broken `test_dash` execu
 this is apparently fixed in gcc-8.1.0.
 
 Dependency Options:
-The following can be set when running make: make FOO=bar
+The following can be set when running make: `make FOO=bar`
-
-    SOURCES_PATH: downloaded sources will be placed here
-    BASE_CACHE: built packages will be placed here
-    SDK_PATH: Path where sdk's can be found (used by macOS)
-    FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up
-    NO_QT: Don't download/build/cache qt and its dependencies
-    NO_WALLET: Don't download/build/cache libs needed to enable the wallet
-    NO_UPNP: Don't download/build/cache packages needed for enabling upnp
-    DEBUG: disable some optimizations and enable more runtime checking
-    HOST_ID_SALT: Optional salt to use when generating host package ids
-    BUILD_ID_SALT: Optional salt to use when generating build package ids
 
+<dl>
+<dt>SOURCES_PATH</dt>
+<dd>downloaded sources will be placed here</dd>
+<dt>BASE_CACHE</dt>
+<dd>built packages will be placed here</dd>
+<dt>SDK_PATH</dt>
+<dd>Path where sdk's can be found (used by macOS)</dd>
+<dt>FALLBACK_DOWNLOAD_PATH</dt>
+<dd>If a source file can't be fetched, try here before giving up</dd>
+<dt>NO_QT</dt>
+<dd>Don't download/build/cache qt and its dependencies</dd>
+<dt>NO_WALLET</dt>
+<dd>Don't download/build/cache libs needed to enable the wallet</dd>
+<dt>NO_UPNP</dt>
+<dd>Don't download/build/cache packages needed for enabling upnp</dd>
+<dt>DEBUG</dt>
+<dd>disable some optimizations and enable more runtime checking</dd>
+<dt>HOST_ID_SALT</dt>
+<dd>Optional salt to use when generating host package ids</dd>
+<dt>BUILD_ID_SALT</dt>
+<dd>Optional salt to use when generating build package ids</dd>
+<dt>FORCE_USE_SYSTEM_CLANG</dt>
+<dd>(EXPERTS ONLY) When cross-compiling for macOS, use clang found in the
+system's <code>$PATH</code> rather than the default prebuilt release of clang
+from llvm.org</dd>
+</dl>
 If some packages are not built, for example `make NO_WALLET=1`, the appropriate
 options will be passed to Dash Core's configure. In this case, `--disable-wallet`.
 

depends/builders/darwin.mk

@@ -19,4 +19,5 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool)
 darwin_OTOOL:=$(shell xcrun -f otool)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+darwin_native_binutils=
 darwin_native_toolchain=

depends/config.site.in

@@ -1,4 +1,14 @@
-depends_prefix="`dirname ${ac_site_file}`/.."
+# shellcheck shell=sh disable=SC2034 # Many variables set will be used in
+                                     # ./configure but shellcheck doesn't know
+                                     # that, hence: disable=SC2034
+
+true  # Dummy command because shellcheck treats all directives before first
+      # command as file-wide, and we only want to disable for one line.
+      #
+      # See: https://github.com/koalaman/shellcheck/wiki/Directive
+
+# shellcheck disable=SC2154
+depends_prefix="$(cd "$(dirname ${ac_site_file})/.." && pwd)"
 
 cross_compiling=maybe
 host_alias=@HOST@
@@ -43,7 +53,7 @@ if test x@host_os@ = xdarwin; then
 fi
 
 PATH=$depends_prefix/native/bin:$PATH
-PKG_CONFIG="`which pkg-config` --static"
+PKG_CONFIG="$(which pkg-config) --static"
 
 # These two need to remain exported because pkg-config does not see them
 # otherwise. That means they must be unexported at the end of configure.ac to
@@ -61,7 +71,7 @@ fi
 if test -n "@CXX@" -a -z "${CXX}"; then
   CXX="@CXX@"
 fi
-PYTHONPATH=$depends_prefix/native/lib/python3/dist-packages:$PYTHONPATH
+PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PATH_SEPARATOR}}${PYTHONPATH}"
 
 if test -n "@AR@"; then
   AR=@AR@

depends/funcs.mk

@@ -1,17 +1,23 @@
 define int_vars
 #Set defaults for vars which may be overridden per-package
-$(1)_cc=$($($(1)_type)_CC)
+$(1)_cc=$$($$($(1)_type)_CC)
-$(1)_cxx=$($($(1)_type)_CXX)
+$(1)_cxx=$$($$($(1)_type)_CXX)
-$(1)_objc=$($($(1)_type)_OBJC)
+$(1)_objc=$$($$($(1)_type)_OBJC)
-$(1)_objcxx=$($($(1)_type)_OBJCXX)
+$(1)_objcxx=$$($$($(1)_type)_OBJCXX)
-$(1)_ar=$($($(1)_type)_AR)
+$(1)_ar=$$($$($(1)_type)_AR)
-$(1)_ranlib=$($($(1)_type)_RANLIB)
+$(1)_ranlib=$$($$($(1)_type)_RANLIB)
-$(1)_libtool=$($($(1)_type)_LIBTOOL)
+$(1)_libtool=$$($$($(1)_type)_LIBTOOL)
-$(1)_nm=$($($(1)_type)_NM)
+$(1)_nm=$$($$($(1)_type)_NM)
-$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS)
+$(1)_cflags=$$($$($(1)_type)_CFLAGS) \
-$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS)
+            $$($$($(1)_type)_$$(release_type)_CFLAGS)
-$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib
+$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \
-$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include
+              $$($$($(1)_type)_$$(release_type)_CXXFLAGS)
+$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \
+             $$($$($(1)_type)_$$(release_type)_LDFLAGS) \
+             -L$$($($(1)_type)_prefix)/lib
+$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \
+              $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \
+              -I$$($$($(1)_type)_prefix)/include
 $(1)_recipe_hash:=
 endef
 
@@ -43,7 +49,7 @@ endef
 
 define int_get_build_id
 $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
-$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
+$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies)))
 $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
 $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string))
 $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
@@ -253,4 +259,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
 $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))
 
 #special exception: if a toolchain package exists, all non-native packages depend on it
-$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))
+$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) ))

depends/hosts/darwin.mk

@@ -1,8 +1,119 @@
-OSX_MIN_VERSION=10.12
+OSX_MIN_VERSION=10.14
-OSX_SDK_VERSION=10.14
+OSX_SDK_VERSION=10.15.1
-OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk
+XCODE_VERSION=11.3.1
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK)
+XCODE_BUILD_ID=11C505
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++
+LD64_VERSION=530
+
+OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
+
+darwin_native_binutils=native_cctools
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
+# from llvm.org
+
+# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG
+# is empty
+darwin_native_toolchain=native_cctools
+
+clang_prog=$(build_prefix)/bin/clang
+clangxx_prog=$(clang_prog)++
+
+clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version)
+else
+# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
+# system
+
+darwin_native_toolchain=
+
+# We can't just use $(shell command -v clang) because GNU Make handles builtins
+# in a special way and doesn't know that `command` is a POSIX-standard builtin
+# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90.
+# At the time of writing, GNU Make v4.2.1 is still being used in supported
+# distro releases.
+#
+# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html
+clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang")
+clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++")
+
+clang_resource_dir=$(shell clang -print-resource-dir)
+endif
+
+cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL
+
+# Make-only lowercase function
+lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+# For well-known tools provided by cctools, make sure that their well-known
+# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
+# would.
+$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL))))
+
+# Flag explanations:
+#
+#     -mlinker-version
+#
+#         Ensures that modern linker features are enabled. See here for more
+#         details: https://github.com/bitcoin/bitcoin/pull/19407.
+#
+#     -B$(build_prefix)/bin
+#
+#         Explicitly point to our binaries (e.g. cctools) so that they are
+#         ensured to be found and preferred over other possibilities.
+#
+#     -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1
+#
+#         Forces clang to use the libc++ headers from our SDK and completely
+#         forget about the libc++ headers from the standard directories
+#
+#         TODO: Once we start requiring a clang version that has the
+#         -stdlib++-isystem<directory> flag first introduced here:
+#         https://reviews.llvm.org/D64089, we should use that instead. Read the
+#         differential summary there for more details.
+#
+#     -Xclang -*system<path_a> \
+#     -Xclang -*system<path_b> \
+#     -Xclang -*system<path_c> ...
+#
+#         Adds path_a, path_b, and path_c to the bottom of clang's list of
+#         include search paths. This is used to explicitly specify the list of
+#         system include search paths and its ordering, rather than rely on
+#         clang's autodetection routine. This routine has been shown to:
+#             1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1
+#                when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034)
+#             2. Fail to pickup C headers in $SYSROOT/usr/include when
+#                C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9)
+#
+#         Talking directly to cc1 with -Xclang here grants us access to specify
+#         more granular categories for these system include search paths, and we
+#         can use the correct categories that these search paths would have been
+#         placed in if the autodetection routine had worked correctly. (see:
+#         https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment)
+#
+#         Furthermore, it places these search paths after any "non-Xclang"
+#         specified search paths. This prevents any additional clang options or
+#         environment variables from coming after or in between these system
+#         include search paths, as that would be wrong in general but would also
+#         break #include_next's.
+#
+darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+              -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+              -u LIBRARY_PATH \
+            $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+              -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+              --sysroot=$(OSX_SDK) \
+              -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+              -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
+darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+               -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+               -u LIBRARY_PATH \
+             $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+               -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+               --sysroot=$(OSX_SDK) \
+               -stdlib=libc++ -nostdinc++ \
+               -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \
+               -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+               -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
@@ -12,5 +123,3 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
-
-darwin_native_toolchain=native_cctools

depends/packages/bls-dash.mk

@@ -7,6 +7,7 @@ $(package)_build_subdir=build
 $(package)_sha256_hash=276c8573104e5f18bb5b9fd3ffd49585dda5ba5f6de2de74759dda8ca5a9deac
 $(package)_dependencies=gmp cmake
 $(package)_patches=gcc_alignment_cast.patch
+$(package)_darwin_triplet=x86_64-apple-darwin18
 
 $(package)_relic_version=3a23142be0a5510a3aa93cd6c76fc59d3fc732a5
 $(package)_relic_download_path=https://github.com/relic-toolkit/relic/archive
@@ -46,8 +47,9 @@ define $(package)_set_vars
   $(package)_config_opts_debug=-DDEBUG=ON -DCMAKE_BUILD_TYPE=Debug
 
   ifneq ($(darwin_native_toolchain),)
-    $(package)_config_opts_darwin+= -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_ar)"
+    $(package)_config_opts_darwin+= -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_darwin_triplet)-ar"
-    $(package)_config_opts_darwin+= -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)"
+    $(package)_config_opts_darwin+= -DCMAKE_LINKER="$(host_prefix)/native/bin/$($(package)_darwin_triplet)-ld"
+    $(package)_config_opts_darwin+= -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_darwin_triplet)-ranlib"
   endif
 
   $(package)_cppflags+=-UBLSALLOC_SODIUM

depends/packages/native_cctools.mk

@@ -1,14 +1,16 @@
 package=native_cctools
-$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f
+$(package)_version=4da2f3b485bcf4cef526f30c0b8c0bcda99cdbb4
 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04
+$(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec
 $(package)_build_subdir=cctools
-$(package)_clang_version=6.0.1
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+$(package)_clang_version=8.0.0
 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
 $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0
+$(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
+endif
 
 $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef
 $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
@@ -16,15 +18,25 @@ $(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz
 $(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz
 $(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
 
-$(package)_extra_sources=$($(package)_clang_file_name)
+$(package)_extra_sources=$($(package)_libtapi_file_name)
-$(package)_extra_sources += $($(package)_libtapi_file_name)
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+$(package)_extra_sources += $($(package)_clang_file_name)
+endif
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_fetch_cmds
 $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
 $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
 $(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
 endef
+else
+define $(package)_fetch_cmds
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
+endef
+endif
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_extract_cmds
   mkdir -p $($(package)_extract_dir) && \
   echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
@@ -38,12 +50,28 @@ define $(package)_extract_cmds
   rm -f toolchain/lib/libc++abi.so* && \
   tar --no-same-owner --strip-components=1 -xf $($(package)_source)
 endef
+else
+define $(package)_extract_cmds
+  mkdir -p $($(package)_extract_dir) && \
+  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  mkdir -p libtapi && \
+  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
+  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
+endef
+endif
 
 define $(package)_set_vars
   $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir)
   $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
+  ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
   $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang
   $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++
+  else
+  $(package)_cc=clang
+  $(package)_cxx=clang++
+  endif
 endef
 
 define $(package)_preprocess_cmds
@@ -60,6 +88,7 @@ define $(package)_build_cmds
   $(MAKE)
 endef
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_stage_cmds
   $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
   mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
@@ -72,7 +101,13 @@ define $(package)_stage_cmds
   cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\
   cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
   cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
-  cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
+  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil
-  if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \
-  if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi
 endef
+else
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
+  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
+  cd $($(package)_extract_dir) && \
+  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/
+endef
+endif

doc/build-cross.md

@@ -19,20 +19,20 @@ Additionally, the Mac OSX SDK must be downloaded and extracted manually:
 ```bash
 $ mkdir -p depends/sdk-sources
 $ mkdir -p depends/SDKs
-$ curl https://bitcoincore.org/depends-sources/sdks/MacOSX10.14.sdk.tar.gz -o depends/sdk-sources/MacOSX10.14.sdk.tar.gz
+$ curl https://bitcoincore.org/depends-sources/sdks/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz -o depends/sdk-sources/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
-$ tar -C depends/SDKs -xf depends/sdk-sources/MacOSX10.14.sdk.tar.gz
+$ tar -C depends/SDKs -xf depends/sdk-sources/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
 ```
 
 When building the dependencies, as described in [build-generic](build-generic.md), use
 
 ```bash
-$ make HOST=x86_64-apple-darwin16 -j4
+$ make HOST=x86_64-apple-darwin18 -j4
 ```
 
 When building Dash Core, use
 
 ```bash
-$ ./configure --prefix=`pwd`/depends/x86_64-apple-darwin16
+$ ./configure --prefix=`pwd`/depends/x86_64-apple-darwin18
 ```
 
 Windows 64bit Cross-compilation

doc/build-generic.md

@@ -48,7 +48,7 @@ Please replace `<host>` with your local system's `host-platform-triplet`. The fo
 - `i686-pc-linux-gnu` for Linux32
 - `x86_64-pc-linux-gnu` for Linux64
 - `x86_64-w64-mingw32` for Win64
-- `x86_64-apple-darwin16` for MacOSX
+- `x86_64-apple-darwin18` for MacOSX
 - `arm-linux-gnueabihf` for Linux ARM 32 bit
 - `aarch64-linux-gnu` for Linux ARM 64 bit
 

share/qt/Info.plist.in

@@ -3,7 +3,7 @@
 <plist version="0.9">
 <dict>
   <key>LSMinimumSystemVersion</key>
-  <string>10.12.0</string>
+  <string>10.14.0</string>
 
   <key>LSArchitecturePriority</key>
   <array>

src/random.cpp

@@ -13,7 +13,7 @@
 #endif
 #include <logging.h>  // for LogPrint()
 #include <sync.h>     // for WAIT_LOCK
-#include <util/time.h> // for GetTime()
+#include <util/time.h> // for GetTimeMicros()
 
 #include <stdlib.h>
 #include <chrono>
@@ -248,14 +248,11 @@ void GetOSRand(unsigned char *ent32)
     // Silence a compiler warning about unused function.
     (void)GetDevURandom;
 #elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX)
-    // We need a fallback for OSX < 10.12
+    /* getentropy() is available on macOS 10.12 and later.
-    if (&getentropy != nullptr) {
+     */
     if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) {
         RandFailure();
     }
-    } else {
-        GetDevURandom(ent32);
-    }
     // Silence a compiler warning about unused function.
     (void)GetDevURandom;
 #elif defined(HAVE_SYSCTL_ARND)

test/lint/lint-shell.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2018 The Bitcoin Core developers
+# Copyright (c) 2018-2019 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
@@ -16,11 +16,6 @@ if [ "$TRAVIS" = "true" ]; then
   unset LC_ALL
 fi
 
-if ! command -v shellcheck > /dev/null; then
-    echo "Skipping shell linting since shellcheck is not installed."
-    exit 0
-fi
-
 # Disabled warnings:
 disabled=(
     SC1087 # Use braces when expanding arrays, e.g. ${array[idx]} (or ${var}[.. to quiet).
@@ -46,5 +41,40 @@ disabled=(
     SC2230 # which is non-standard. Use builtin 'command -v' instead.
     SC2236 # Don't force -n instead of ! -z.
 )
-shellcheck -e "$(IFS=","; echo "${disabled[*]}")" \
+disabled_gitian=(
-    $(git ls-files -- "*.sh" | grep -vE 'src/(secp256k1|univalue)/')
+    SC2094 # Make sure not to read and write the same file in the same pipeline.
+    SC2129 # Consider using { cmd1; cmd2; } >> file instead of individual redirects.
+    SC2230 # which is non-standard. Use builtin 'command -v' instead.
+)
+
+EXIT_CODE=0
+
+if ! command -v shellcheck > /dev/null; then
+    echo "Skipping shell linting since shellcheck is not installed."
+    exit $EXIT_CODE
+fi
+
+EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")"
+SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}')  # Check shellcheck directive used for sourced files
+if ! shellcheck "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then
+    EXIT_CODE=1
+fi
+
+if ! command -v yq > /dev/null; then
+    echo "Skipping Gitian desriptor scripts checking since yq is not installed."
+    exit $EXIT_CODE
+fi
+
+EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")"
+for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml')
+do
+    echo
+    echo "$descriptor"
+    # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script.
+    SCRIPT=$'#!/bin/bash\n'$(yq -r .script "$descriptor")
+    if ! echo "$SCRIPT" | shellcheck "$EXCLUDE_GITIAN" -; then
+        EXIT_CODE=1
+    fi
+done
+
+exit $EXIT_CODE