From 098d0fd430c3c7839da5dca899866fb470886bed Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 21 May 2020 15:10:52 +0800 Subject: [PATCH] Merge #18677: Multiprocess build support e2bab2aa162ae38b2bf8195b577c982402fbee9d multiprocess: add multiprocess travis configuration (Russell Yanofsky) 603fd6a2e708c04ef6c9880f89d0a4cbaa6fc7c5 depends: add MULTIPROCESS depends option (Russell Yanofsky) 5d1377b52bfcd4edf8553aaf332bfeb92fc554cc build: multiprocess autotools changes (Russell Yanofsky) Pull request description: This PR is part of the [process separation project](https://github.com/bitcoin/bitcoin/projects/10). --- This PR consists of build changes only. It adds an `--enable-multiprocess` autoconf option (off by default and marked experimental), that builds new `bitcoin-node` and `bitcoin-gui` binaries. These currently function the same as existing `bitcoind` and `bitcoin-qt` binaries, but are extended in #10102 with IPC features to execute node, wallet, and gui functions in separate processes. In addition to adding the `--enable-multiprocess` config flag, it also adds a depends package and autoconf rules to build with the [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library, and it adds new travis configuration to exercise the build code and run functional tests with the new binaries. The changes in this PR were originally part of #10102 but were moved into #16367 to be able to develop and review the multiprocess build changes independently of the code changes. #16367 was briefly merged and then reverted in #18588. Only change since #16367 has been dropping the `native_boost.mk` depends package which was pointed out to be no longer necessary in https://github.com/bitcoin/bitcoin/pull/16367#issuecomment-596484337 and https://github.com/bitcoin/bitcoin/pull/18588#pullrequestreview-391765649 ACKs for top commit: practicalswift: ACK e2bab2aa162ae38b2bf8195b577c982402fbee9d Sjors: tACK e2bab2aa162ae38b2bf8195b577c982402fbee9d on macOS 10.15.4 hebasto: ACK e2bab2aa162ae38b2bf8195b577c982402fbee9d, tested on Linux Mint 19.3 (x86_64): Tree-SHA512: b5a76eab5abf63d9d8b6d628cbdff4cc1888eef15cafa0a5d56369e2f9d02595fed623f4b74b2cf2830c42c05a774f0943e700f9c768a82d9d348cad199e135c --- .gitignore | 2 + .gitlab-ci.yml | 28 +++++++++ .travis.yml | 5 ++ ci/dash/test_integrationtests.sh | 2 +- ci/test/00_setup_env.sh | 1 + ci/test/00_setup_env_native_multiprocess.sh | 14 +++++ configure.ac | 63 +++++++++++++++++++++ depends/Makefile | 8 +++ depends/README.md | 1 + depends/config.site.in | 7 +++ depends/funcs.mk | 11 +++- depends/hosts/android.mk | 1 + depends/hosts/darwin.mk | 2 + depends/hosts/linux.mk | 1 + depends/hosts/mingw32.mk | 2 + depends/packages/capnp.mk | 18 ++++++ depends/packages/libmultiprocess.mk | 18 ++++++ depends/packages/native_capnp.mk | 18 ++++++ depends/packages/native_libmultiprocess.mk | 18 ++++++ depends/packages/packages.mk | 3 + doc/multiprocess.md | 35 ++++++++++++ src/Makefile.am | 61 ++++++++++---------- src/Makefile.qt.include | 43 ++++++++++---- 23 files changed, 316 insertions(+), 46 deletions(-) create mode 100644 ci/test/00_setup_env_native_multiprocess.sh create mode 100644 depends/packages/capnp.mk create mode 100644 depends/packages/libmultiprocess.mk create mode 100644 depends/packages/native_capnp.mk create mode 100644 depends/packages/native_libmultiprocess.mk create mode 100644 doc/multiprocess.md diff --git a/.gitignore b/.gitignore index ff575f2a83..54855b71f7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ reset-files.bash src/dash src/dashd src/dash-cli +src/dash-gui +src/dash-node src/dash-tx src/dash-wallet src/test/fuzz/* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65c54348a3..0752a77f02 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -204,6 +204,16 @@ x86_64-pc-linux-gnu-nowallet: HOST: x86_64-pc-linux-gnu DEP_OPTS: "NO_WALLET=1" +# TODO: # Skip on non-depends, non-multiprocess PRs +# if: type != pull_request OR commit_message =~ /depends:|multiprocess:/ +x86_64-pc-linux-gnu-multiprocess: + extends: + - .build-depends-template + - .skip-in-fast-mode-template + variables: + HOST: x86_64-pc-linux-gnu + DEP_OPTS: "MULTIPROCESS=1" + x86_64-apple-darwin: extends: - .build-depends-template @@ -299,6 +309,15 @@ linux64_nowallet-build: variables: BUILD_TARGET: linux64_nowallet +linux64_multiprocess-build: + extends: + - .build-template + - .skip-in-fast-mode-template + needs: + - x86_64-pc-linux-gnu-multiprocess + variables: + BUILD_TARGET: linux64_multiprocess + #linux64_valgrind-build: # extends: # - .build-template @@ -362,6 +381,15 @@ linux64_ubsan-test: variables: BUILD_TARGET: linux64_ubsan +linux64_multiprocess-test: + extends: + - .test-template + - .skip-in-fast-mode-template + needs: + - linux64_multiprocess-build + variables: + BUILD_TARGET: linux64_multiprocess + #linux64_valgrind-test: # extends: # - .test-template diff --git a/.travis.yml b/.travis.yml index 07af403a57..32c6728b1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -250,6 +250,11 @@ after_success: env: >- FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" + - stage: test + name: 'x86_64 Linux [GOAL: install] [focal] [multiprocess]' + env: >- + FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh" + - stage: test name: 'x86_64 Linux [GOAL: install] [focal] [no wallet]' env: >- diff --git a/ci/dash/test_integrationtests.sh b/ci/dash/test_integrationtests.sh index 707fbd9fde..1d674503ac 100755 --- a/ci/dash/test_integrationtests.sh +++ b/ci/dash/test_integrationtests.sh @@ -42,7 +42,7 @@ echo "Using socketevents mode: $SOCKETEVENTS" EXTRA_ARGS="--dashd-arg=-socketevents=$SOCKETEVENTS" set +e -LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS +LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS RESULT=$? set -e diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e94162dde2..03947c26cb 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -39,6 +39,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:-true} export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} +export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export RUN_SYMBOL_TESTS=${RUN_SYMBOL_TESTS:-true} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh new file mode 100644 index 0000000000..48a268c373 --- /dev/null +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CONTAINER_NAME=ci_native_multiprocess +export PACKAGES="cmake python3" +export DEP_OPTS="MULTIPROCESS=1" +export GOAL="install" +export BITCOIN_CONFIG="" +export TEST_RUNNER_ENV="BITCOIND=dash-node" diff --git a/configure.ac b/configure.ac index 88205b6990..8d06f8d2fb 100644 --- a/configure.ac +++ b/configure.ac @@ -256,6 +256,24 @@ AC_ARG_ENABLE([zmq], [use_zmq=$enableval], [use_zmq=yes]) +AC_ARG_WITH([libmultiprocess], + [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto], + [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])], + [with_libmultiprocess=$withval], + [with_libmultiprocess=auto]) + +AC_ARG_WITH([mpgen], + [AS_HELP_STRING([--with-mpgen=yes|no|auto|PREFIX], + [Build with libmultiprocess codegen tool. Useful to specify different libmultiprocess host system library and build system codegen tool prefixes when cross-compiling (default is host system libmultiprocess prefix)])], + [with_mpgen=$withval], + [with_mpgen=auto]) + +AC_ARG_ENABLE([multiprocess], + [AS_HELP_STRING([--enable-multiprocess], + [build multiprocess dash-node, dash-wallet, and dash-gui executables in addition to monolithic dashd and dash-qt executables. Requires libmultiprocess library. Experimental (default is no)])], + [enable_multiprocess=$enableval], + [enable_multiprocess=no]) + AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [do not install man pages (default is to install)])],, @@ -1530,6 +1548,50 @@ dnl check if libgmp is present AC_CHECK_HEADER([gmp.h],, AC_MSG_ERROR(libgmp headers missing)) AC_CHECK_LIB([gmp], [__gmpz_init],GMP_LIBS=-lgmp, AC_MSG_ERROR(libgmp missing)) +dnl libmultiprocess library check + +libmultiprocess_found=no +if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then + if test "x$use_pkgconfig" = xyes; then + m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [ + libmultiprocess_found=yes; + libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`; + ], [true])]) + fi +elif test "x$with_libmultiprocess" != xno; then + AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no]) +fi +AC_SUBST(LIBMULTIPROCESS_CFLAGS) +AC_SUBST(LIBMULTIPROCESS_LIBS) + +dnl Enable multiprocess check + +if test "x$enable_multiprocess" = xyes; then + if test "x$libmultiprocess_found" != xyes; then + AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.]) + fi + build_multiprocess=yes +elif test "x$enable_multiprocess" = xauto; then + build_multiprocess=$libmultiprocess_found +else + build_multiprocess=no +fi + +AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes]) +AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes]) + +dnl codegen tools check + +if test x$build_multiprocess != xno; then + if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then + MPGEN_PREFIX="$libmultiprocess_prefix" + elif test "x$with_mpgen" != xno; then + MPGEN_PREFIX="$with_mpgen"; + fi + AC_SUBST(MPGEN_PREFIX) +fi + AC_MSG_CHECKING([whether to build dashd]) AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes]) AC_MSG_RESULT($build_bitcoind) @@ -1853,6 +1915,7 @@ esac echo echo "Options used to compile and link:" +echo " multiprocess = $build_multiprocess" echo " with wallet = $enable_wallet" echo " with gui / qt = $bitcoin_enable_qt" if test "x$enable_wallet" != "xno"; then diff --git a/depends/Makefile b/depends/Makefile index 07300bca5a..823be933a7 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -40,6 +40,7 @@ NO_WALLET ?= NO_ZMQ ?= NO_UPNP ?= NO_NATPMP ?= +MULTIPROCESS ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources BUILD = $(shell ./config.guess) @@ -152,6 +153,7 @@ upnp_packages_$(NO_UPNP) = $(upnp_packages) natpmp_packages_$(NO_NATPMP) = $(natpmp_packages) zmq_packages_$(NO_ZMQ) = $(zmq_packages) +multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) @@ -160,6 +162,11 @@ ifneq ($(zmq_packages_),) packages += $(zmq_packages) endif +ifeq ($(multiprocess_packages_),) +packages += $(multiprocess_packages) +native_packages += $(multiprocess_native_packages) +endif + all_packages = $(packages) $(native_packages) meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk @@ -227,6 +234,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_sqlite@|$(NO_SQLITE)|' \ -e 's|@no_upnp@|$(NO_UPNP)|' \ -e 's|@no_natpmp@|$(NO_NATPMP)|' \ + -e 's|@multiprocess@|$(MULTIPROCESS)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ touch $@ diff --git a/depends/README.md b/depends/README.md index 9f29498cdb..633fc14d99 100644 --- a/depends/README.md +++ b/depends/README.md @@ -107,6 +107,7 @@ The following can be set when running make: `make FOO=bar` - `NO_SQLITE`: Don't download/build/cache SQLite - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP - `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP +- `MULTIPROCESS`: build libmultiprocess (experimental, requires cmake) - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids - `BUILD_ID_SALT`: Optional salt to use when generating build package ids diff --git a/depends/config.site.in b/depends/config.site.in index c2c22ea364..a20e54068e 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -26,6 +26,9 @@ fi if test -z "$with_qt_bindir" && test -z "@no_qt@"; then with_qt_bindir="${depends_prefix}/native/bin" fi +if test -z "$with_mpgen" && test -n "@multiprocess@"; then + with_mpgen="${depends_prefix}/native" +fi if test -z "$with_qrencode" && test -n "@no_qr@"; then with_qrencode=no @@ -43,6 +46,10 @@ if test -z "$with_sqlite" && test -n "@no_sqlite@"; then with_sqlite=no fi +if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then + enable_multiprocess=yes +fi + if test -z "$with_miniupnpc" && test -n "@no_upnp@"; then with_miniupnpc=no fi diff --git a/depends/funcs.mk b/depends/funcs.mk index ccef22f72f..766f9b6cab 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -138,11 +138,11 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$( $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" - ifneq ($($(1)_nm),) $(1)_autoconf += NM="$$($(1)_nm)" endif @@ -164,6 +164,15 @@ endif ifneq ($($(1)_ldflags),) $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)" endif + +$(1)_cmake=cmake -DCMAKE_INSTALL_PREFIX=$($($(1)_type)_prefix) +ifneq ($($(1)_type),build) +ifneq ($(host),$(build)) +$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) -DCMAKE_SYSROOT=$(host_prefix) +$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) -DCMAKE_C_COMPILER=$(firstword $($($(1)_type)_CC)) -DCMAKE_C_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CC))" +$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) -DCMAKE_CXX_COMPILER=$(firstword $($($(1)_type)_CXX)) -DCMAKE_CXX_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CXX))" +endif +endif endef define int_add_cmds diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index 969ec2a1cb..eabd84bbbe 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -9,3 +9,4 @@ android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib endif +android_cmake_system=Android diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 1f09b15f1a..e470d2d285 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -120,3 +120,5 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) + +darwin_cmake_system=Darwin diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index 99434c90d5..a95f934b66 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -29,3 +29,4 @@ i686_linux_CXX=$(default_host_CXX) -m32 x86_64_linux_CC=$(default_host_CC) -m64 x86_64_linux_CXX=$(default_host_CXX) -m64 endif +linux_cmake_system=Linux diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 83fc501a19..226925c47a 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -8,3 +8,5 @@ mingw32_debug_CFLAGS=-O1 mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC + +mingw_cmake_system=Windows diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk new file mode 100644 index 0000000000..abeb26545f --- /dev/null +++ b/depends/packages/capnp.mk @@ -0,0 +1,18 @@ +package=capnp +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) + +define $(package)_config_cmds + $($(package)_autoconf) --with-external-capnp +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk new file mode 100644 index 0000000000..3e5cf5f160 --- /dev/null +++ b/depends/packages/libmultiprocess.mk @@ -0,0 +1,18 @@ +package=libmultiprocess +$(package)_version=$(native_$(package)_version) +$(package)_download_path=$(native_$(package)_download_path) +$(package)_file_name=$(native_$(package)_file_name) +$(package)_sha256_hash=$(native_$(package)_sha256_hash) +$(package)_dependencies=native_$(package) boost capnp + +define $(package)_config_cmds + $($(package)_cmake) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/native_capnp.mk b/depends/packages/native_capnp.mk new file mode 100644 index 0000000000..ed5a6deee2 --- /dev/null +++ b/depends/packages/native_capnp.mk @@ -0,0 +1,18 @@ +package=native_capnp +$(package)_version=0.7.0 +$(package)_download_path=https://capnproto.org/ +$(package)_download_file=capnproto-c++-$($(package)_version).tar.gz +$(package)_file_name=capnproto-cxx-$($(package)_version).tar.gz +$(package)_sha256_hash=c9a4c0bd88123064d483ab46ecee777f14d933359e23bff6fb4f4dbd28b4cd41 + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk new file mode 100644 index 0000000000..c50fdc3f6b --- /dev/null +++ b/depends/packages/native_libmultiprocess.mk @@ -0,0 +1,18 @@ +package=native_libmultiprocess +$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0 +$(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4 +$(package)_dependencies=native_capnp + +define $(package)_config_cmds + $($(package)_cmake) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 03c4cedbd6..e53729a7c8 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -19,6 +19,9 @@ zmq_packages=zeromq upnp_packages=miniupnpc natpmp_packages=libnatpmp +multiprocess_packages = libmultiprocess capnp +multiprocess_native_packages = native_libmultiprocess native_capnp + darwin_native_packages = native_ds_store native_mac_alias $(host_arch)_$(host_os)_native_packages += native_b2 diff --git a/doc/multiprocess.md b/doc/multiprocess.md new file mode 100644 index 0000000000..9c46e68d4f --- /dev/null +++ b/doc/multiprocess.md @@ -0,0 +1,35 @@ +# Multiprocess Dash + +On unix systems, the `--enable-multiprocess` build option can be passed to `./configure` to build new `dash-node`, `dash-wallet`, and `dash-gui` executables alongside existing `dashd` and `dash-qt` executables. + +`dash-node` is a drop-in replacement for `dashd`, and `dash-gui` is a drop-in replacement for `dash-qt`, and there are no differences in use or external behavior between the new and old executables. But internally (after backporting [bitcoin#10102](https://github.com/bitcoin/bitcoin/pull/10102)), `dash-gui` will spawn a `dash-node` process to run P2P and RPC code, communicating with it across a socket pair, and `dash-node` will spawn `dash-wallet` to run wallet code, also communicating over a socket pair. This will let node, wallet, and GUI code run in separate address spaces for better isolation, and allow future improvements like being able to start and stop components independently on different machines and environments. + +## Next steps + +Specific next steps after backporting [bitcoin#10102](https://github.com/bitcoin/bitcoin/pull/10102) will be: + +- [ ] Adding `-ipcbind` and `-ipcconnect` options to `dash-node`, `dash-wallet`, and `dash-gui` executables so they can listen and connect to TCP ports and unix socket paths. This will allow separate processes to be started and stopped any time and connect to each other. +- [ ] Adding `-server` and `-rpcbind` options to the `dash-wallet` executable so wallet processes can handle RPC requests directly without going through the node. +- [ ] Supporting windows, not just unix systems. The existing socket code is already cross-platform, so the only windows-specific code that needs to be written is code spawning a process and passing a socket descriptor. This can be implemented with `CreateProcess` and `WSADuplicateSocket`. Example: https://memset.wordpress.com/2010/10/13/win32-api-passing-socket-with-ipc-method/. +- [ ] Adding sandbox features, restricting subprocess access to resources and data. See [https://eklitzke.org/multiprocess-bitcoin](https://eklitzke.org/multiprocess-bitcoin). + +## Debugging + +After backporting [bitcoin#10102](https://github.com/bitcoin/bitcoin/pull/10102), the `-debug=ipc` command line option can be used to see requests and responses between processes. + +## Installation + +The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get starting using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make: + +``` +cd +make -C depends NO_QT=1 MULTIPROCESS=1 +./configure --prefix=$PWD/depends/x86_64-pc-linux-gnu +make +src/dash-node -regtest -printtoconsole -debug=ipc +DASHD=dash-node test/functional/test_runner.py +``` + +The configure script will pick up settings and library locations from the depends directory, so there is no need to pass `--enable-multiprocess` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). + +Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `./configure --enable-multiprocess` without using the depends system. The configure script will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess#installation) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. diff --git a/src/Makefile.am b/src/Makefile.am index 792c884f08..2cd6ec442b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -110,6 +110,10 @@ if BUILD_BITCOIND bin_PROGRAMS += dashd endif +if BUILD_BITCOIN_NODE + bin_PROGRAMS += dash-node +endif + if BUILD_BITCOIN_CLI bin_PROGRAMS += dash-cli endif @@ -796,22 +800,21 @@ libbitcoin_cli_a_SOURCES = \ nodist_libbitcoin_util_a_SOURCES = $(srcdir)/obj/build.h # -# dashd binary # -dashd_SOURCES = bitcoind.cpp -dashd_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -dashd_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -dashd_LDFLAGS = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) +# dashd & dash-node binaries # +bitcoin_daemon_sources = bitcoind.cpp +bitcoin_bin_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +bitcoin_bin_cxxflags = $(AM_CXXFLAGS) $(PIE_FLAGS) +bitcoin_bin_ldflags = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) if TARGET_WINDOWS -dashd_SOURCES += dashd-res.rc +bitcoin_daemon_sources += dashd-res.rc endif -dashd_LDADD = \ - $(LIBBITCOIN_SERVER) \ +bitcoin_bin_ldadd = \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ - $(LIBUNIVALUE) \ $(LIBBITCOIN_UTIL) \ + $(LIBUNIVALUE) \ $(LIBBITCOIN_ZMQ) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ @@ -821,7 +824,19 @@ dashd_LDADD = \ $(LIBMEMENV) \ $(LIBSECP256K1) -dashd_LDADD += $(BACKTRACE_LIB) $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) +bitcoin_bin_ldadd += $(BACKTRACE_LIB) $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) + +dashd_SOURCES = $(bitcoin_daemon_sources) +dashd_CPPFLAGS = $(bitcoin_bin_cppflags) +dashd_CXXFLAGS = $(bitcoin_bin_cxxflags) +dashd_LDFLAGS = $(bitcoin_bin_ldflags) +dashd_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) + +dash_node_SOURCES = $(bitcoin_daemon_sources) +dash_node_CPPFLAGS = $(bitcoin_bin_cppflags) +dash_node_CXXFLAGS = $(bitcoin_bin_cxxflags) +dash_node_LDFLAGS = $(bitcoin_bin_ldflags) +dash_node_LDADD = $(LIBBITCOIN_SERVER) $(bitcoin_bin_ldadd) # dash-cli binary # dash_cli_SOURCES = bitcoin-cli.cpp @@ -865,32 +880,14 @@ dash_tx_LDADD += $(BACKTRACE_LIB) $(BOOST_LIBS) $(GMP_LIBS) # dash-wallet binary # dash_wallet_SOURCES = bitcoin-wallet.cpp -dash_wallet_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -dash_wallet_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -dash_wallet_LDFLAGS = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) +dash_wallet_CPPFLAGS = $(bitcoin_bin_cppflags) +dash_wallet_CXXFLAGS = $(bitcoin_bin_cxxflags) +dash_wallet_LDFLAGS = $(bitcoin_bin_ldflags) +dash_wallet_LDADD = $(LIBBITCOIN_WALLET_TOOL) $(bitcoin_bin_ldadd) if TARGET_WINDOWS dash_wallet_SOURCES += dash-wallet-res.rc endif - -# Libraries below may be listed more than once to resolve circular dependencies (see -# https://eli.thegreenplace.net/2013/07/09/library-order-in-static-linking#circular-dependency) -dash_wallet_LDADD = \ - $(LIBBITCOIN_WALLET_TOOL) \ - $(LIBBITCOIN_WALLET) \ - $(LIBBITCOIN_COMMON) \ - $(LIBBITCOIN_CONSENSUS) \ - $(LIBBITCOIN_UTIL) \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBBITCOIN_ZMQ) \ - $(LIBDASHBLS) \ - $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ - $(LIBMEMENV) \ - $(LIBSECP256K1) \ - $(LIBUNIVALUE) - -dash_wallet_LDADD += $(BACKTRACE_LIB) $(BOOST_LIBS) $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(ZMQ_LIBS) $(GMP_LIBS) # # dashconsensus library # diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 280d56991c..893d49c559 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -4,6 +4,11 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. bin_PROGRAMS += qt/dash-qt + +if BUILD_BITCOIN_GUI + bin_PROGRAMS += dash-gui +endif + EXTRA_LIBRARIES += qt/libbitcoinqt.a # dash qt core # @@ -371,29 +376,43 @@ QT_FORMS_H=$(join $(dir $(QT_FORMS_UI)),$(addprefix ui_, $(notdir $(QT_FORMS_UI: # Most files will depend on the forms and moc files as includes. Generate them # before anything else. $(QT_MOC): $(QT_FORMS_H) -$(qt_libbitcoinqt_a_OBJECTS) $(qt_dash_qt_OBJECTS) : | $(QT_MOC) +$(qt_libbitcoinqt_a_OBJECTS) $(qt_dash_qt_OBJECTS) $(dash_gui_OBJECTS): | $(QT_MOC) -# dash-qt binary # -qt_dash_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ +# dash-qt binary and dash-gui binaries # +bitcoin_qt_cppflags = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ $(QT_INCLUDES) $(QR_CFLAGS) -qt_dash_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) +bitcoin_qt_cxxflags = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) -qt_dash_qt_SOURCES = qt/main.cpp +bitcoin_qt_sources = qt/main.cpp if TARGET_WINDOWS - qt_dash_qt_SOURCES += $(BITCOIN_RC) + bitcoin_qt_sources += $(BITCOIN_RC) endif -qt_dash_qt_LDADD = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) +bitcoin_qt_ldadd = qt/libbitcoinqt.a $(LIBBITCOIN_SERVER) if ENABLE_WALLET -qt_dash_qt_LDADD += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) +bitcoin_qt_ldadd += $(LIBBITCOIN_UTIL) $(LIBBITCOIN_WALLET) endif if ENABLE_ZMQ -qt_dash_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) +bitcoin_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -qt_dash_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBDASHBLS) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ +bitcoin_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBDASHBLS) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BACKTRACE_LIB) $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS) -qt_dash_qt_LDFLAGS = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -qt_dash_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX +bitcoin_qt_ldflags = $(LDFLAGS_WRAP_EXCEPTIONS) $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) +bitcoin_qt_libtoolflags = $(AM_LIBTOOLFLAGS) --tag CXX + +qt_dash_qt_CPPFLAGS = $(bitcoin_qt_cppflags) +qt_dash_qt_CXXFLAGS = $(bitcoin_qt_cxxflags) +qt_dash_qt_SOURCES = $(bitcoin_qt_sources) +qt_dash_qt_LDADD = $(bitcoin_qt_ldadd) +qt_dash_qt_LDFLAGS = $(bitcoin_qt_ldflags) +qt_dash_qt_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) + +dash_gui_CPPFLAGS = $(bitcoin_qt_cppflags) +dash_gui_CXXFLAGS = $(bitcoin_qt_cxxflags) +dash_gui_SOURCES = $(bitcoin_qt_sources) +dash_gui_LDADD = $(bitcoin_qt_ldadd) +dash_gui_LDFLAGS = $(bitcoin_qt_ldflags) +dash_gui_LIBTOOLFLAGS = $(bitcoin_qt_libtoolflags) #locale/foo.ts -> locale/foo.qm QT_QM=$(QT_TS:.ts=.qm)