Merge bitcoin/bitcoin#23611: build: add LTO option to depends

094772656d71b3f5022ae292094e878da035de9e build: support LTO in depends (fanquake)

Pull request description:

  This adds an `LTO` option to depends, i.e `make -C depends LTO=1`, which passes `-flto` when building packages (not currently qt), and automatically configures with `--enable-lto` when doing a build using a `CONFIG_SITE`.

  The following tables comapres the size (in bytes) of the stripped `x86_64` Linux binaries produced with master and this PR (full depends build):

  | Binary | stripped master | stripped LTO=1 | saving |
  | -------- | ----------------: | -------------: | --------: |
  | bitcoin-cli | 1178632 | 469872 | 60% |
  | bitcoin-tx  | 2710584 | 1866504 | 31% |
  | bitcoin-util | 952880 | 240104 | 74% |
  | bitcoin-wallet | 7992888 | 5365984 | 32% |
  | bitcoind | 13421336 | 11868592 | 12% |
  | bitcoin-qt | 37680496 | 31640976 | 16% |

ACKs for top commit:
  laanwj:
    Tested ACK 094772656d71b3f5022ae292094e878da035de9e

Tree-SHA512: 6b8483ea490e57a153105ad8c38b25fb1af5d55b1af22db398c7c2573612aaf71b4d2b4cf09c18fd6331b1358dba01641eeaa03e5018a925392e1937118d984a

depends/Makefile

@@ -42,6 +42,7 @@ NO_UPNP ?=
 NO_USDT ?=
 NO_NATPMP ?=
 MULTIPROCESS ?=
+LTO ?=
 FALLBACK_DOWNLOAD_PATH ?= http://dash-depends-sources.s3-website-us-west-2.amazonaws.com
 
 BUILD = $(shell ./config.guess)
@@ -140,8 +141,8 @@ include packages/packages.mk
 #     2. Before including packages/*.mk (excluding packages/packages.mk), since
 #        they rely on the build_id variables
 #
-build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
+build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
-$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
+$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))')
 
 qrencode_packages_$(NO_QR) = $(qrencode_$(host_os)_packages)
 
@@ -239,6 +240,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@no_usdt@|$(NO_USDT)|' \
             -e 's|@no_natpmp@|$(NO_NATPMP)|' \
             -e 's|@multiprocess@|$(MULTIPROCESS)|' \
+            -e 's|@lto@|$(LTO)|' \
             -e 's|@debug@|$(DEBUG)|' \
             $< > $@
 	touch $@

depends/README.md

@@ -120,6 +120,7 @@ The following can be set when running make: `make FOO=bar`
 - `LOG`: Use file-based logging for individual packages. During a package build its log file
   resides in the `depends` directory, and the log file is printed out automatically in case
   of build error. After successful build log files are moved along with package archives
+- `LTO`: Use LTO when building packages.
 
 If some packages are not built, for example `make NO_WALLET=1`, the appropriate
 options will be passed to Dash Core's configure. In this case, `--disable-wallet`.

depends/config.site.in

@@ -78,6 +78,10 @@ if test "@host_os@" = darwin; then
   BREW=no
 fi
 
+if test -z "$enable_lto" && test -n "@lto@"; then
+  enable_lto=yes
+fi
+
 PATH="${depends_prefix}/native/bin:${PATH}"
 PKG_CONFIG="$(which pkg-config) --static"
 

depends/gen_id

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Usage: env [ CC=... ] [ CXX=... ] [ AR=... ] [ RANLIB=... ] [ STRIP=... ] \
-#            [ DEBUG=... ] ./build-id [ID_SALT]...
+#            [ DEBUG=... ] [ LTO=... ] ./build-id [ID_SALT]...
 #
 # Prints to stdout a SHA256 hash representing the current toolset, used by
 # depends/Makefile as a build id for caching purposes (detecting when the
@@ -63,6 +63,10 @@
     env | grep '^STRIP_'
     echo "END STRIP"
 
+    echo "BEGIN LTO"
+    echo "LTO=${LTO}"
+    echo "END LTO"
+
     echo "END ALL"
 ) | if [ -n "$DEBUG" ] && command -v tee > /dev/null 2>&1; then
         # When debugging and `tee` is available, output the preimage to stderr

depends/hosts/android.mk

@@ -5,6 +5,12 @@ else
 android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++
 android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang
 endif
+
+ifneq ($(LTO),)
+android_CFLAGS += -flto
+android_LDFLAGS += -flto
+endif
+
 android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar
 android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib
 

depends/hosts/darwin.mk

@@ -113,6 +113,12 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
                -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 
 darwin_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+darwin_CFLAGS += -flto
+darwin_LDFLAGS += -flto
+endif
+
 darwin_CXXFLAGS=$(darwin_CFLAGS)
 
 darwin_release_CFLAGS=-O2

depends/hosts/freebsd.mk

@@ -1,4 +1,10 @@
 freebsd_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+freebsd_CFLAGS += -flto
+freebsd_LDFLAGS += -flto
+endif
+
 freebsd_CXXFLAGS=$(freebsd_CFLAGS)
 
 freebsd_release_CFLAGS=-O2

depends/hosts/linux.mk

@@ -1,4 +1,10 @@
 linux_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+linux_CFLAGS += -flto
+linux_LDFLAGS += -flto
+endif
+
 linux_CXXFLAGS=$(linux_CFLAGS)
 
 linux_release_CFLAGS=-O2

depends/hosts/mingw32.mk

@@ -3,6 +3,12 @@ mingw32_CXX := $(host)-g++-posix
 endif
 
 mingw32_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+mingw32_CFLAGS += -flto
+mingw32_LDFLAGS += -flto
+endif
+
 mingw32_CXXFLAGS=$(mingw32_CFLAGS)
 
 mingw32_release_CFLAGS=-O2

depends/hosts/netbsd.mk

@@ -1,4 +1,10 @@
 netbsd_CFLAGS=-pipe
+
+ifneq ($(LTO),)
+netbsd_CFLAGS += -flto
+netbsd_LDFLAGS += -flto
+endif
+
 netbsd_CXXFLAGS=$(netbsd_CFLAGS)
 
 netbsd_release_CFLAGS=-O2

depends/hosts/openbsd.mk

@@ -1,6 +1,11 @@
 openbsd_CFLAGS=-pipe
 openbsd_CXXFLAGS=$(openbsd_CFLAGS)
 
+ifneq ($(LTO),)
+openbsd_CFLAGS += -flto
+openbsd_LDFLAGS += -flto
+endif
+
 openbsd_release_CFLAGS=-O2
 openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS)