From 7260597fc67b12d5410d462d28cc21a63844d0f1 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kittywhiskers@users.noreply.github.com> Date: Thu, 20 Aug 2020 17:30:30 -0400 Subject: [PATCH] partial merge #19683: Pin clang search paths for darwin host Skipped https://github.com/bitcoin/bitcoin/pull/19683/commits/80331107416b8a6cb487ee1c89a39c6a8bced27b --- depends/Makefile | 44 ++++++++++++++----- depends/funcs.mk | 30 ++++++++----- depends/hosts/darwin.mk | 97 ++++++++++++++++++++++++++++++++++++----- 3 files changed, 137 insertions(+), 34 deletions(-) diff --git a/depends/Makefile b/depends/Makefile index 3e8453019c..6d3b1d852d 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -117,6 +117,8 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null) $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null) ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +# Make sure that cache is invalidated when switching between system and +# depends-managed, pinned clang build_id_string+=system_clang $(host_arch)_$(host_os)_id_string+=system_clang endif @@ -141,12 +143,6 @@ $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain) include funcs.mk -binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin) -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin) -else -toolchain_path= -endif final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in) final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)) $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) @@ -157,15 +153,39 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages) $(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); ) $(AT)touch $@ +# $PATH is not preserved between ./configure and make by convention. Its +# modification and overriding at ./configure time is (as I understand it) +# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros, +# which will expand the program names to their full absolute paths. The notable +# exception is command line overriding: ./configure CC=clang, which skips the +# program name expansion step, and works because the user implicitly indicates +# with CC=clang that clang will be available in $PATH at all times, and is most +# likely part of the user's system. +# +# Therefore, when we "seed the autoconf cache"/"override well-known program +# vars" by setting AR= in our config.site, either one of two things needs +# to be true for the build system to work correctly: +# +# 1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the +# tool needs to be available in $PATH at all times. +# +# 2. If the tool is _**not**_ expected to be available in $PATH at all times +# (such as is the case for our native_cctools binutils tools), it needs to +# be referred to by its absolute path, such as would be output by the +# AC_PATH_{PROG,TOOL} macros. +# +# Minor note: it is also okay to refer to tools by their absolute path even if +# we expect them to be available in $PATH at all times, more specificity does +# not hurt. $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id) $(AT)@mkdir -p $(@D) $(AT)sed -e 's|@HOST@|$(host)|' \ - -e 's|@CC@|$(toolchain_path)$(host_CC)|' \ - -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \ - -e 's|@AR@|$(binutils_path)$(host_AR)|' \ - -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \ - -e 's|@NM@|$(binutils_path)$(host_NM)|' \ - -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \ + -e 's|@CC@|$(host_CC)|' \ + -e 's|@CXX@|$(host_CXX)|' \ + -e 's|@AR@|$(host_AR)|' \ + -e 's|@RANLIB@|$(host_RANLIB)|' \ + -e 's|@NM@|$(host_NM)|' \ + -e 's|@STRIP@|$(host_STRIP)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ diff --git a/depends/funcs.mk b/depends/funcs.mk index 7d731c6982..32032b9cb7 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -1,17 +1,23 @@ define int_vars #Set defaults for vars which may be overridden per-package -$(1)_cc=$($($(1)_type)_CC) -$(1)_cxx=$($($(1)_type)_CXX) -$(1)_objc=$($($(1)_type)_OBJC) -$(1)_objcxx=$($($(1)_type)_OBJCXX) -$(1)_ar=$($($(1)_type)_AR) -$(1)_ranlib=$($($(1)_type)_RANLIB) -$(1)_libtool=$($($(1)_type)_LIBTOOL) -$(1)_nm=$($($(1)_type)_NM) -$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS) -$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS) -$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib -$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include +$(1)_cc=$$($$($(1)_type)_CC) +$(1)_cxx=$$($$($(1)_type)_CXX) +$(1)_objc=$$($$($(1)_type)_OBJC) +$(1)_objcxx=$$($$($(1)_type)_OBJCXX) +$(1)_ar=$$($$($(1)_type)_AR) +$(1)_ranlib=$$($$($(1)_type)_RANLIB) +$(1)_libtool=$$($$($(1)_type)_LIBTOOL) +$(1)_nm=$$($$($(1)_type)_NM) +$(1)_cflags=$$($$($(1)_type)_CFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CFLAGS) +$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CXXFLAGS) +$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \ + $$($$($(1)_type)_$$(release_type)_LDFLAGS) \ + -L$$($($(1)_type)_prefix)/lib +$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \ + $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \ + -I$$($$($(1)_type)_prefix)/include $(1)_recipe_hash:= endef diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 3a11d1493e..ee4a45154b 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -6,6 +6,49 @@ LD64_VERSION=530 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers +darwin_native_binutils=native_cctools + +ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) +# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang +# from llvm.org + +# The native_cctools package is what provides clang when FORCE_USE_SYSTEM_CLANG +# is empty +darwin_native_toolchain=native_cctools + +clang_prog=$(build_prefix)/bin/clang +clangxx_prog=$(clang_prog)++ + +clang_resource_dir=$(build_prefix)/lib/clang/$(native_cctools_clang_version) +else +# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's +# system + +darwin_native_toolchain= + +# We can't just use $(shell command -v clang) because GNU Make handles builtins +# in a special way and doesn't know that `command` is a POSIX-standard builtin +# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90. +# At the time of writing, GNU Make v4.2.1 is still being used in supported +# distro releases. +# +# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html +clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") +clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") + +clang_resource_dir=$(shell clang -print-resource-dir) +endif + +cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL + +# Make-only lowercase function +lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) + +# For well-known tools provided by cctools, make sure that their well-known +# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG} +# would. +$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL)))) + # Flag explanations: # # -mlinker-version @@ -18,7 +61,7 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with- # Explicitly point to our binaries (e.g. cctools) so that they are # ensured to be found and preferred over other possibilities. # -# -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1 +# -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 # # Forces clang to use the libc++ headers from our SDK and completely # forget about the libc++ headers from the standard directories @@ -28,8 +71,49 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with- # https://reviews.llvm.org/D64089, we should use that instead. Read the # differential summary there for more details. # -darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1 +# -Xclang -*system \ +# -Xclang -*system \ +# -Xclang -*system ... +# +# Adds path_a, path_b, and path_c to the bottom of clang's list of +# include search paths. This is used to explicitly specify the list of +# system include search paths and its ordering, rather than rely on +# clang's autodetection routine. This routine has been shown to: +# 1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1 +# when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034) +# 2. Fail to pickup C headers in $SYSROOT/usr/include when +# C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9) +# +# Talking directly to cc1 with -Xclang here grants us access to specify +# more granular categories for these system include search paths, and we +# can use the correct categories that these search paths would have been +# placed in if the autodetection routine had worked correctly. (see: +# https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment) +# +# Furthermore, it places these search paths after any "non-Xclang" +# specified search paths. This prevents any additional clang options or +# environment variables from coming after or in between these system +# include search paths, as that would be wrong in general but would also +# break #include_next's. +# +darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + --sysroot=$(OSX_SDK) \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include +darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ + -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ + -u LIBRARY_PATH \ + $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ + -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ + --sysroot=$(OSX_SDK) \ + -stdlib=libc++ -nostdinc++ \ + -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \ + -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) @@ -39,10 +123,3 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) - -darwin_native_binutils=native_cctools -ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) -darwin_native_toolchain=native_cctools -else -darwin_native_toolchain= -endif