Merge bitcoin#18677: Multiprocess build support

e2bab2a multiprocess: add multiprocess travis configuration (Russell Yanofsky) 603fd6a depends: add MULTIPROCESS depends option (Russell Yanofsky) 5d1377b build: multiprocess autotools changes (Russell Yanofsky) Pull request description: This PR is part of the [process separation project](https://github.com/bitcoin/bitcoin/projects/10). --- This PR consists of build changes only. It adds an `--enable-multiprocess` autoconf option (off by default and marked experimental), that builds new `bitcoin-node` and `bitcoin-gui` binaries. These currently function the same as existing `bitcoind` and `bitcoin-qt` binaries, but are extended in bitcoin#10102 with IPC features to execute node, wallet, and gui functions in separate processes. In addition to adding the `--enable-multiprocess` config flag, it also adds a depends package and autoconf rules to build with the [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) library, and it adds new travis configuration to exercise the build code and run functional tests with the new binaries. The changes in this PR were originally part of bitcoin#10102 but were moved into bitcoin#16367 to be able to develop and review the multiprocess build changes independently of the code changes. bitcoin#16367 was briefly merged and then reverted in bitcoin#18588. Only change since bitcoin#16367 has been dropping the `native_boost.mk` depends package which was pointed out to be no longer necessary in bitcoin#16367 (comment) and bitcoin#18588 (review) ACKs for top commit: practicalswift: ACK e2bab2a Sjors: tACK e2bab2a on macOS 10.15.4 hebasto: ACK e2bab2a, tested on Linux Mint 19.3 (x86_64): Tree-SHA512: b5a76eab5abf63d9d8b6d628cbdff4cc1888eef15cafa0a5d56369e2f9d02595fed623f4b74b2cf2830c42c05a774f0943e700f9c768a82d9d348cad199e135c
dashpay · Jan 16, 2024 · 098d0fd · 098d0fd
1 parent 9d46455
commit 098d0fd
Show file tree

Hide file tree

Showing 23 changed files with 316 additions and 46 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,8 @@ reset-files.bash
 src/dash
 src/dashd
 src/dash-cli
+src/dash-gui
+src/dash-node
 src/dash-tx
 src/dash-wallet
 src/test/fuzz/*

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -204,6 +204,16 @@ x86_64-pc-linux-gnu-nowallet:
     HOST: x86_64-pc-linux-gnu
     DEP_OPTS: "NO_WALLET=1"
 
+# TODO: # Skip on non-depends, non-multiprocess PRs
+# if: type != pull_request OR commit_message =~ /depends:|multiprocess:/
+x86_64-pc-linux-gnu-multiprocess:
+  extends:
+    - .build-depends-template
+    - .skip-in-fast-mode-template
+  variables:
+    HOST: x86_64-pc-linux-gnu
+    DEP_OPTS: "MULTIPROCESS=1"
+
 x86_64-apple-darwin:
   extends:
     - .build-depends-template
@@ -299,6 +309,15 @@ linux64_nowallet-build:
   variables:
     BUILD_TARGET: linux64_nowallet
 
+linux64_multiprocess-build:
+  extends:
+    - .build-template
+    - .skip-in-fast-mode-template
+  needs:
+    - x86_64-pc-linux-gnu-multiprocess
+  variables:
+    BUILD_TARGET: linux64_multiprocess
+
 #linux64_valgrind-build:
 #  extends:
 #    - .build-template
@@ -362,6 +381,15 @@ linux64_ubsan-test:
   variables:
     BUILD_TARGET: linux64_ubsan
 
+linux64_multiprocess-test:
+  extends:
+    - .test-template
+    - .skip-in-fast-mode-template
+  needs:
+    - linux64_multiprocess-build
+  variables:
+    BUILD_TARGET: linux64_multiprocess
+
 #linux64_valgrind-test:
 #  extends:
 #    - .test-template

diff --git a/.travis.yml b/.travis.yml
@@ -250,6 +250,11 @@ after_success:
       env: >-
         FILE_ENV="./ci/test/00_setup_env_native_fuzz_with_valgrind.sh"
 
+    - stage: test
+      name: 'x86_64 Linux [GOAL: install]  [focal]  [multiprocess]'
+      env: >-
+        FILE_ENV="./ci/test/00_setup_env_native_multiprocess.sh"
+
     - stage: test
       name: 'x86_64 Linux  [GOAL: install]  [focal]  [no wallet]'
       env: >-

diff --git a/ci/dash/test_integrationtests.sh b/ci/dash/test_integrationtests.sh
@@ -42,7 +42,7 @@ echo "Using socketevents mode: $SOCKETEVENTS"
 EXTRA_ARGS="--dashd-arg=-socketevents=$SOCKETEVENTS"
 
 set +e
-LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS
+LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS
 RESULT=$?
 set -e
 

diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
@@ -39,6 +39,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
 export RUN_INTEGRATION_TESTS=${RUN_INTEGRATION_TESTS:-true}
 export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false}
+export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-}
 export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
 export RUN_SYMBOL_TESTS=${RUN_SYMBOL_TESTS:-true}
 export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed}

diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export CONTAINER_NAME=ci_native_multiprocess
+export PACKAGES="cmake python3"
+export DEP_OPTS="MULTIPROCESS=1"
+export GOAL="install"
+export BITCOIN_CONFIG=""
+export TEST_RUNNER_ENV="BITCOIND=dash-node"
diff --git a/configure.ac b/configure.ac
@@ -256,6 +256,24 @@ AC_ARG_ENABLE([zmq],
   [use_zmq=$enableval],
   [use_zmq=yes])
 
+AC_ARG_WITH([libmultiprocess],
+  [AS_HELP_STRING([--with-libmultiprocess=yes|no|auto],
+  [Build with libmultiprocess library. (default: auto, i.e. detect with pkg-config)])],
+  [with_libmultiprocess=$withval],
+  [with_libmultiprocess=auto])
+
+AC_ARG_WITH([mpgen],
+  [AS_HELP_STRING([--with-mpgen=yes|no|auto|PREFIX],
+  [Build with libmultiprocess codegen tool. Useful to specify different libmultiprocess host system library and build system codegen tool prefixes when cross-compiling (default is host system libmultiprocess prefix)])],
+  [with_mpgen=$withval],
+  [with_mpgen=auto])
+
+AC_ARG_ENABLE([multiprocess],
+  [AS_HELP_STRING([--enable-multiprocess],
+  [build multiprocess dash-node, dash-wallet, and dash-gui executables in addition to monolithic dashd and dash-qt executables. Requires libmultiprocess library. Experimental (default is no)])],
+  [enable_multiprocess=$enableval],
+  [enable_multiprocess=no])
+
 AC_ARG_ENABLE(man,
     [AS_HELP_STRING([--disable-man],
                     [do not install man pages (default is to install)])],,
@@ -1530,6 +1548,50 @@ dnl check if libgmp is present
 AC_CHECK_HEADER([gmp.h],, AC_MSG_ERROR(libgmp headers missing))
 AC_CHECK_LIB([gmp],  [__gmpz_init],GMP_LIBS=-lgmp, AC_MSG_ERROR(libgmp missing))
 
+dnl libmultiprocess library check
+
+libmultiprocess_found=no
+if test "x$with_libmultiprocess" = xyes || test "x$with_libmultiprocess" = xauto; then
+  if test "x$use_pkgconfig" = xyes; then
+    m4_ifdef([PKG_CHECK_MODULES], [PKG_CHECK_MODULES([LIBMULTIPROCESS], [libmultiprocess], [
+       libmultiprocess_found=yes;
+       libmultiprocess_prefix=`$PKG_CONFIG --variable=prefix libmultiprocess`;
+    ], [true])])
+  fi
+elif test "x$with_libmultiprocess" != xno; then
+  AC_MSG_ERROR([--with-libmultiprocess=$with_libmultiprocess value is not yes, auto, or no])
+fi
+AC_SUBST(LIBMULTIPROCESS_CFLAGS)
+AC_SUBST(LIBMULTIPROCESS_LIBS)
+
+dnl Enable multiprocess check
+
+if test "x$enable_multiprocess" = xyes; then
+  if test "x$libmultiprocess_found" != xyes; then
+    AC_MSG_ERROR([--enable-multiprocess=yes option specified but libmultiprocess library was not found. May need to install libmultiprocess library, or specify install path with PKG_CONFIG_PATH environment variable. Running 'pkg-config --debug libmultiprocess' may be helpful for debugging.])
+  fi
+  build_multiprocess=yes
+elif test "x$enable_multiprocess" = xauto; then
+  build_multiprocess=$libmultiprocess_found
+else
+  build_multiprocess=no
+fi
+
+AM_CONDITIONAL([BUILD_MULTIPROCESS],[test "x$build_multiprocess" = xyes])
+AM_CONDITIONAL([BUILD_BITCOIN_NODE], [test "x$build_multiprocess" = xyes])
+AM_CONDITIONAL([BUILD_BITCOIN_GUI], [test "x$build_multiprocess" = xyes])
+
+dnl codegen tools check
+
+if test x$build_multiprocess != xno; then
+  if test "x$with_mpgen" = xyes || test "x$with_mpgen" = xauto; then
+    MPGEN_PREFIX="$libmultiprocess_prefix"
+  elif test "x$with_mpgen" != xno; then
+    MPGEN_PREFIX="$with_mpgen";
+  fi
+  AC_SUBST(MPGEN_PREFIX)
+fi
+
 AC_MSG_CHECKING([whether to build dashd])
 AM_CONDITIONAL([BUILD_BITCOIND], [test x$build_bitcoind = xyes])
 AC_MSG_RESULT($build_bitcoind)
@@ -1853,6 +1915,7 @@ esac
 
 echo
 echo "Options used to compile and link:"
+echo "  multiprocess  = $build_multiprocess"
 echo "  with wallet         = $enable_wallet"
 echo "  with gui / qt       = $bitcoin_enable_qt"
 if test "x$enable_wallet" != "xno"; then

diff --git a/depends/Makefile b/depends/Makefile
@@ -40,6 +40,7 @@ NO_WALLET ?=
 NO_ZMQ ?=
 NO_UPNP ?=
 NO_NATPMP ?=
+MULTIPROCESS ?=
 FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
 
 BUILD = $(shell ./config.guess)
@@ -152,6 +153,7 @@ upnp_packages_$(NO_UPNP) = $(upnp_packages)
 natpmp_packages_$(NO_NATPMP) = $(natpmp_packages)
 
 zmq_packages_$(NO_ZMQ) = $(zmq_packages)
+multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages)
 
 packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
@@ -160,6 +162,11 @@ ifneq ($(zmq_packages_),)
 packages += $(zmq_packages)
 endif
 
+ifeq ($(multiprocess_packages_),)
+packages += $(multiprocess_packages)
+native_packages += $(multiprocess_native_packages)
+endif
+
 all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
@@ -227,6 +234,7 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@no_sqlite@|$(NO_SQLITE)|' \
             -e 's|@no_upnp@|$(NO_UPNP)|' \
             -e 's|@no_natpmp@|$(NO_NATPMP)|' \
+            -e 's|@multiprocess@|$(MULTIPROCESS)|' \
             -e 's|@debug@|$(DEBUG)|' \
             $< > $@
 	touch $@

diff --git a/depends/README.md b/depends/README.md
@@ -107,6 +107,7 @@ The following can be set when running make: `make FOO=bar`
 - `NO_SQLITE`: Don't download/build/cache SQLite
 - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP
 - `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP
+- `MULTIPROCESS`: build libmultiprocess (experimental, requires cmake)
 - `DEBUG`: Disable some optimizations and enable more runtime checking
 - `HOST_ID_SALT`: Optional salt to use when generating host package ids
 - `BUILD_ID_SALT`: Optional salt to use when generating build package ids

diff --git a/depends/config.site.in b/depends/config.site.in
@@ -26,6 +26,9 @@ fi
 if test -z "$with_qt_bindir" && test -z "@no_qt@"; then
   with_qt_bindir="${depends_prefix}/native/bin"
 fi
+if test -z "$with_mpgen" && test -n "@multiprocess@"; then
+  with_mpgen="${depends_prefix}/native"
+fi
 
 if test -z "$with_qrencode" && test -n "@no_qr@"; then
   with_qrencode=no
@@ -43,6 +46,10 @@ if test -z "$with_sqlite" && test -n "@no_sqlite@"; then
   with_sqlite=no
 fi
 
+if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then
+  enable_multiprocess=yes
+fi
+
 if test -z "$with_miniupnpc" && test -n "@no_upnp@"; then
   with_miniupnpc=no
 fi

diff --git a/depends/funcs.mk b/depends/funcs.mk
@@ -138,11 +138,11 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$(
 
 $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig
 $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig
+$(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake
 $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH)
 $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH)
 $(1)_stage_env+=PATH=$(build_prefix)/bin:$(PATH)
 $(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"
-
 ifneq ($($(1)_nm),)
 $(1)_autoconf += NM="$$($(1)_nm)"
 endif
@@ -164,6 +164,15 @@ endif
 ifneq ($($(1)_ldflags),)
 $(1)_autoconf += LDFLAGS="$$($(1)_ldflags)"
 endif
+
+$(1)_cmake=cmake -DCMAKE_INSTALL_PREFIX=$($($(1)_type)_prefix)
+ifneq ($($(1)_type),build)
+ifneq ($(host),$(build))
+$(1)_cmake += -DCMAKE_SYSTEM_NAME=$($(host_os)_cmake_system) -DCMAKE_SYSROOT=$(host_prefix)
+$(1)_cmake += -DCMAKE_C_COMPILER_TARGET=$(host) -DCMAKE_C_COMPILER=$(firstword $($($(1)_type)_CC)) -DCMAKE_C_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CC))"
+$(1)_cmake += -DCMAKE_CXX_COMPILER_TARGET=$(host) -DCMAKE_CXX_COMPILER=$(firstword $($($(1)_type)_CXX)) -DCMAKE_CXX_FLAGS="$(wordlist 2,1000,$($($(1)_type)_CXX))"
+endif
+endif
 endef
 
 define int_add_cmds

diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk
@@ -9,3 +9,4 @@ android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++
 android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang
 android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib
 endif
+android_cmake_system=Android
diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk
@@ -120,3 +120,5 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
+
+darwin_cmake_system=Darwin
diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk
@@ -29,3 +29,4 @@ i686_linux_CXX=$(default_host_CXX) -m32
 x86_64_linux_CC=$(default_host_CC) -m64
 x86_64_linux_CXX=$(default_host_CXX) -m64
 endif
+linux_cmake_system=Linux
diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk
@@ -8,3 +8,5 @@ mingw32_debug_CFLAGS=-O1
 mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS)
 
 mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
+
+mingw_cmake_system=Windows
diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk
@@ -0,0 +1,18 @@
+package=capnp
+$(package)_version=$(native_$(package)_version)
+$(package)_download_path=$(native_$(package)_download_path)
+$(package)_file_name=$(native_$(package)_file_name)
+$(package)_sha256_hash=$(native_$(package)_sha256_hash)
+$(package)_dependencies=native_$(package)
+
+define $(package)_config_cmds
+  $($(package)_autoconf) --with-external-capnp
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef
diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk
@@ -0,0 +1,18 @@
+package=libmultiprocess
+$(package)_version=$(native_$(package)_version)
+$(package)_download_path=$(native_$(package)_download_path)
+$(package)_file_name=$(native_$(package)_file_name)
+$(package)_sha256_hash=$(native_$(package)_sha256_hash)
+$(package)_dependencies=native_$(package) boost capnp
+
+define $(package)_config_cmds
+  $($(package)_cmake)
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef
diff --git a/depends/packages/native_capnp.mk b/depends/packages/native_capnp.mk
@@ -0,0 +1,18 @@
+package=native_capnp
+$(package)_version=0.7.0
+$(package)_download_path=https://capnproto.org/
+$(package)_download_file=capnproto-c++-$($(package)_version).tar.gz
+$(package)_file_name=capnproto-cxx-$($(package)_version).tar.gz
+$(package)_sha256_hash=c9a4c0bd88123064d483ab46ecee777f14d933359e23bff6fb4f4dbd28b4cd41
+
+define $(package)_config_cmds
+  $($(package)_autoconf)
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef
diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk
@@ -0,0 +1,18 @@
+package=native_libmultiprocess
+$(package)_version=5741d750a04e644a03336090d8979c6d033e32c0
+$(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive
+$(package)_file_name=$($(package)_version).tar.gz
+$(package)_sha256_hash=ac848db49a6ed53e423c62d54bd87f1f08cbb0326254a8667e10bbfe5bf032a4
+$(package)_dependencies=native_capnp
+
+define $(package)_config_cmds
+  $($(package)_cmake)
+endef
+
+define $(package)_build_cmds
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef
diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk
@@ -19,6 +19,9 @@ zmq_packages=zeromq
 upnp_packages=miniupnpc
 natpmp_packages=libnatpmp
 
+multiprocess_packages = libmultiprocess capnp
+multiprocess_native_packages = native_libmultiprocess native_capnp
+
 darwin_native_packages = native_ds_store native_mac_alias
 
 $(host_arch)_$(host_os)_native_packages += native_b2
Original file line number	Diff line number	Diff line change
Expand Up		@@ -120,3 +120,5 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)

		darwin_debug_CFLAGS=-O1
		darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)

		darwin_cmake_system=Darwin
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,3 +8,5 @@ mingw32_debug_CFLAGS=-O1
		mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS)

		mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC

		mingw_cmake_system=Windows