merge #17919: Allow building with system clang

depends/Makefile

@@ -116,6 +116,11 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
+ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+build_id_string+=system_clang
+$(host_arch)_$(host_os)_id_string+=system_clang
+endif
+
 qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages)
 wallet_packages_$(NO_WALLET) = $(wallet_packages)
 upnp_packages_$(NO_UPNP) = $(upnp_packages)
@@ -131,11 +136,17 @@ all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
 
+$(host_arch)_$(host_os)_native_binutils?=$($(host_os)_native_binutils)
 $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
 
 include funcs.mk
 
+binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin)
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
+else
+toolchain_path=
+endif
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
 final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
 $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
@@ -151,10 +162,10 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
 	$(AT)sed -e 's|@HOST@|$(host)|' \
             -e 's|@CC@|$(toolchain_path)$(host_CC)|' \
             -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
-            -e 's|@AR@|$(toolchain_path)$(host_AR)|' \
+            -e 's|@AR@|$(binutils_path)$(host_AR)|' \
-            -e 's|@RANLIB@|$(toolchain_path)$(host_RANLIB)|' \
+            -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \
-            -e 's|@NM@|$(toolchain_path)$(host_NM)|' \
+            -e 's|@NM@|$(binutils_path)$(host_NM)|' \
-            -e 's|@STRIP@|$(toolchain_path)$(host_STRIP)|' \
+            -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \

depends/README.md

@@ -52,19 +52,34 @@ RISC-V known issue: gcc-7.3.0 and gcc-7.3.1 result in a broken `test_dash` execu
 this is apparently fixed in gcc-8.1.0.
 
 Dependency Options:
-The following can be set when running make: make FOO=bar
+The following can be set when running make: `make FOO=bar`
-
-    SOURCES_PATH: downloaded sources will be placed here
-    BASE_CACHE: built packages will be placed here
-    SDK_PATH: Path where sdk's can be found (used by macOS)
-    FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up
-    NO_QT: Don't download/build/cache qt and its dependencies
-    NO_WALLET: Don't download/build/cache libs needed to enable the wallet
-    NO_UPNP: Don't download/build/cache packages needed for enabling upnp
-    DEBUG: disable some optimizations and enable more runtime checking
-    HOST_ID_SALT: Optional salt to use when generating host package ids
-    BUILD_ID_SALT: Optional salt to use when generating build package ids
 
+<dl>
+<dt>SOURCES_PATH</dt>
+<dd>downloaded sources will be placed here</dd>
+<dt>BASE_CACHE</dt>
+<dd>built packages will be placed here</dd>
+<dt>SDK_PATH</dt>
+<dd>Path where sdk's can be found (used by macOS)</dd>
+<dt>FALLBACK_DOWNLOAD_PATH</dt>
+<dd>If a source file can't be fetched, try here before giving up</dd>
+<dt>NO_QT</dt>
+<dd>Don't download/build/cache qt and its dependencies</dd>
+<dt>NO_WALLET</dt>
+<dd>Don't download/build/cache libs needed to enable the wallet</dd>
+<dt>NO_UPNP</dt>
+<dd>Don't download/build/cache packages needed for enabling upnp</dd>
+<dt>DEBUG</dt>
+<dd>disable some optimizations and enable more runtime checking</dd>
+<dt>HOST_ID_SALT</dt>
+<dd>Optional salt to use when generating host package ids</dd>
+<dt>BUILD_ID_SALT</dt>
+<dd>Optional salt to use when generating build package ids</dd>
+<dt>FORCE_USE_SYSTEM_CLANG</dt>
+<dd>(EXPERTS ONLY) When cross-compiling for macOS, use clang found in the
+system's <code>$PATH</code> rather than the default prebuilt release of clang
+from llvm.org</dd>
+</dl>
 If some packages are not built, for example `make NO_WALLET=1`, the appropriate
 options will be passed to Dash Core's configure. In this case, `--disable-wallet`.
 

depends/builders/darwin.mk

@@ -19,4 +19,5 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool)
 darwin_OTOOL:=$(shell xcrun -f otool)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+darwin_native_binutils=
 darwin_native_toolchain=

depends/funcs.mk

@@ -43,7 +43,7 @@ endef
 
 define int_get_build_id
 $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
-$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
+$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($($(1)_type)_native_binutils) $($(1)_dependencies)))
 $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
 $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id_string))
 $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
@@ -253,4 +253,4 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
 $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))
 
 #special exception: if a toolchain package exists, all non-native packages depend on it
-$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))
+$(foreach package,$(packages),$(eval $($(package)_unpacked): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) $($($(host_arch)_$(host_os)_native_binutils)_cached) ))

depends/hosts/darwin.mk

@@ -6,10 +6,30 @@ LD64_VERSION=530
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
-# When cross-compiling for Darwin using Clang, -mlinker-version must be passed to
+# Flag explanations:
-# ensure that modern linker features are enabled.
+#
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION)
+#     -mlinker-version
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION)
+#
+#         Ensures that modern linker features are enabled. See here for more
+#         details: https://github.com/bitcoin/bitcoin/pull/19407.
+#
+#     -B$(build_prefix)/bin
+#
+#         Explicitly point to our binaries (e.g. cctools) so that they are
+#         ensured to be found and preferred over other possibilities.
+#
+#     -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#
+#         Forces clang to use the libc++ headers from our SDK and completely
+#         forget about the libc++ headers from the standard directories
+#
+#         TODO: Once we start requiring a clang version that has the
+#         -stdlib++-isystem<directory> flag first introduced here:
+#         https://reviews.llvm.org/D64089, we should use that instead. Read the
+#         differential summary there for more details.
+#
+darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin
+darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
@@ -20,4 +40,9 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
 
+darwin_native_binutils=native_cctools
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 darwin_native_toolchain=native_cctools
+else
+darwin_native_toolchain=
+endif

depends/packages/native_cctools.mk

@@ -4,11 +4,13 @@ $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
 $(package)_sha256_hash=a2d491c0981cef72fee2b833598f20f42a6c44a7614a61c439bda93d56446fec
 $(package)_build_subdir=cctools
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 $(package)_clang_version=8.0.0
 $(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
 $(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
 $(package)_clang_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
+endif
 
 $(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef
 $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
@@ -16,15 +18,25 @@ $(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz
 $(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz
 $(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
 
-$(package)_extra_sources=$($(package)_clang_file_name)
+$(package)_extra_sources=$($(package)_libtapi_file_name)
-$(package)_extra_sources += $($(package)_libtapi_file_name)
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+$(package)_extra_sources += $($(package)_clang_file_name)
+endif
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_fetch_cmds
 $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
 $(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
 $(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
 endef
+else
+define $(package)_fetch_cmds
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
+endef
+endif
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_extract_cmds
   mkdir -p $($(package)_extract_dir) && \
   echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
@@ -38,12 +50,28 @@ define $(package)_extract_cmds
   rm -f toolchain/lib/libc++abi.so* && \
   tar --no-same-owner --strip-components=1 -xf $($(package)_source)
 endef
+else
+define $(package)_extract_cmds
+  mkdir -p $($(package)_extract_dir) && \
+  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  mkdir -p libtapi && \
+  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
+  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
+endef
+endif
 
 define $(package)_set_vars
   $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir)
   $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
+  ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
   $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang
   $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++
+  else
+  $(package)_cc=clang
+  $(package)_cxx=clang++
+  endif
 endef
 
 define $(package)_preprocess_cmds
@@ -60,6 +88,7 @@ define $(package)_build_cmds
   $(MAKE)
 endef
 
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 define $(package)_stage_cmds
   $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
   mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
@@ -74,3 +103,11 @@ define $(package)_stage_cmds
   cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
   cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil
 endef
+else
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
+  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
+  cd $($(package)_extract_dir) && \
+  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/
+endef
+endif