build: WIP Automatic Builds

depends/Makefile

@@ -126,6 +126,8 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
 ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# Make sure that cache is invalidated when switching between system and
+# depends-managed, pinned clang
 build_id_string+=system_clang
 $(host_arch)_$(host_os)_id_string+=system_clang
 endif
@@ -163,12 +165,6 @@ $(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
 
 include funcs.mk
 
-binutils_path=$($($(host_arch)_$(host_os)_native_binutils)_prefixbin)
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-toolchain_path=$($($(host_arch)_$(host_os)_native_toolchain)_prefixbin)
-else
-toolchain_path=
-endif
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)
 final_build_id+=$(shell echo -n "$(final_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH))
 $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
@@ -179,15 +175,39 @@ $(host_prefix)/.stamp_$(final_build_id): $(native_packages) $(packages)
 	$(AT)cd $(@D); $(foreach package,$^, tar xf $($(package)_cached); )
 	$(AT)touch $@
 
+# $PATH is not preserved between ./configure and make by convention. Its
+# modification and overriding at ./configure time is (as I understand it)
+# supposed to be captured by the AC_{PROG_{,OBJ}CXX,PATH_{PROG,TOOL}} macros,
+# which will expand the program names to their full absolute paths. The notable
+# exception is command line overriding: ./configure CC=clang, which skips the
+# program name expansion step, and works because the user implicitly indicates
+# with CC=clang that clang will be available in $PATH at all times, and is most
+# likely part of the user's system.
+#
+# Therefore, when we "seed the autoconf cache"/"override well-known program
+# vars" by setting AR=<blah> in our config.site, either one of two things needs
+# to be true for the build system to work correctly:
+#
+#   1. If we refer to the program by name (e.g. AR=riscv64-gnu-linux-ar), the
+#      tool needs to be available in $PATH at all times.
+#
+#   2. If the tool is _**not**_ expected to be available in $PATH at all times
+#      (such as is the case for our native_cctools binutils tools), it needs to
+#      be referred to by its absolute path, such as would be output by the
+#      AC_PATH_{PROG,TOOL} macros.
+#
+# Minor note: it is also okay to refer to tools by their absolute path even if
+# we expect them to be available in $PATH at all times, more specificity does
+# not hurt.
 $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_build_id)
 	$(AT)@mkdir -p $(@D)
 	$(AT)sed -e 's|@HOST@|$(host)|' \
-            -e 's|@CC@|$(toolchain_path)$(host_CC)|' \
-            -e 's|@CXX@|$(toolchain_path)$(host_CXX)|' \
-            -e 's|@AR@|$(binutils_path)$(host_AR)|' \
-            -e 's|@RANLIB@|$(binutils_path)$(host_RANLIB)|' \
-            -e 's|@NM@|$(binutils_path)$(host_NM)|' \
-            -e 's|@STRIP@|$(binutils_path)$(host_STRIP)|' \
+            -e 's|@CC@|$(host_CC)|' \
+            -e 's|@CXX@|$(host_CXX)|' \
+            -e 's|@AR@|$(host_AR)|' \
+            -e 's|@RANLIB@|$(host_RANLIB)|' \
+            -e 's|@NM@|$(host_NM)|' \
+            -e 's|@STRIP@|$(host_STRIP)|' \
             -e 's|@build_os@|$(build_os)|' \
             -e 's|@host_os@|$(host_os)|' \
             -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \

depends/builders/darwin.mk

@@ -1,5 +1,5 @@
-build_darwin_CC:=$(shell xcrun -f clang) --sysroot $(shell xcrun --show-sdk-path)
-build_darwin_CXX:=$(shell xcrun -f clang++) --sysroot $(shell xcrun --show-sdk-path)
+build_darwin_CC:=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path)
+build_darwin_CXX:=$(shell xcrun -f clang++) -isysroot$(shell xcrun --show-sdk-path)
 build_darwin_AR:=$(shell xcrun -f ar)
 build_darwin_RANLIB:=$(shell xcrun -f ranlib)
 build_darwin_STRIP:=$(shell xcrun -f strip)
@@ -10,8 +10,8 @@ build_darwin_SHA256SUM=shasum -a 256
 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
 
 #darwin host on darwin builder. overrides darwin host preferences.
-darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(shell xcrun --show-sdk-path)
-darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ --sysroot $(shell xcrun --show-sdk-path)
+darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path)
+darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path)
 darwin_AR:=$(shell xcrun -f ar)
 darwin_RANLIB:=$(shell xcrun -f ranlib)
 darwin_STRIP:=$(shell xcrun -f strip)

depends/config.site.in

@@ -1,67 +1,67 @@
-depends_prefix="`dirname ${ac_site_file}`/.."
+depends_prefix="`dirname "$ac_site_file"`/.."
 
 cross_compiling=maybe
-host_alias=@HOST@
-ac_tool_prefix=${host_alias}-
+host_alias="@HOST@"
+ac_tool_prefix="${host_alias}-"
 
-if test -z $with_boost; then
-  with_boost=$depends_prefix
+if test -z "$with_boost"; then
+  with_boost="$depends_prefix"
 fi
-if test -z $with_qt_plugindir; then
-  with_qt_plugindir=$depends_prefix/plugins
+if test -z "$with_qt_plugindir"; then
+  with_qt_plugindir="${depends_prefix}/plugins"
 fi
-if test -z $with_qt_translationdir; then
-  with_qt_translationdir=$depends_prefix/translations
+if test -z "$with_qt_translationdir"; then
+  with_qt_translationdir="${depends_prefix}/translations"
 fi
-if test -z $with_qt_bindir && test -z "@no_qt@"; then
-  with_qt_bindir=$depends_prefix/native/bin
+if test -z "$with_qt_bindir" && test -z "@no_qt@"; then
+  with_qt_bindir="${depends_prefix}/native/bin"
 fi
-if test -z $with_mpgen && test -n "@multiprocess@"; then
-  with_mpgen=$depends_prefix/native
+if test -z "$with_mpgen" && test -n "@multiprocess@"; then
+  with_mpgen="${depends_prefix}/native"
 fi
 
-if test -z $with_qrencode && test -n "@no_qr@"; then
+if test -z "$with_qrencode" && test -n "@no_qr@"; then
   with_qrencode=no
 fi
 
-if test -z $enable_wallet && test -n "@no_wallet@"; then
+if test -z "$enable_wallet" && test -n "@no_wallet@"; then
   enable_wallet=no
 fi
 
-if test -z $enable_multiprocess && test -n "@multiprocess@"; then
+if test -z "$enable_multiprocess" && test -n "@multiprocess@"; then
   enable_multiprocess=yes
 fi
 
-if test -z $with_miniupnpc && test -n "@no_upnp@"; then
+if test -z "$with_miniupnpc" && test -n "@no_upnp@"; then
   with_miniupnpc=no
 fi
 
-if test -z $with_gui && test -n "@no_qt@"; then
+if test -z "$with_gui" && test -n "@no_qt@"; then
   with_gui=no
 fi
 
-if test -z $enable_zmq && test -n "@no_zmq@"; then
+if test -z "$enable_zmq" && test -n "@no_zmq@"; then
   enable_zmq=no
 fi
 
-if test x@host_os@ = xdarwin; then
+if test "x@host_os@" = xdarwin; then
   BREW=no
   PORT=no
 fi
 
-PATH=$depends_prefix/native/bin:$PATH
-PKG_CONFIG="`which pkg-config` --static"
+PATH="${depends_prefix}/native/bin:${PATH}"
+PKG_CONFIG="$(which pkg-config) --static"
 
 # These two need to remain exported because pkg-config does not see them
 # otherwise. That means they must be unexported at the end of configure.ac to
 # avoid ruining the cache. Sigh.
-export PKG_CONFIG_PATH=$depends_prefix/share/pkgconfig:$depends_prefix/lib/pkgconfig
+export PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig"
 if test -z "@allow_host_packages@"; then
-  export PKG_CONFIG_LIBDIR=$depends_prefix/lib/pkgconfig
+  export PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig"
 fi
 
-CPPFLAGS="-I$depends_prefix/include/ $CPPFLAGS"
-LDFLAGS="-L$depends_prefix/lib $LDFLAGS"
+CPPFLAGS="-I${depends_prefix}/include/ ${CPPFLAGS}"
+LDFLAGS="-L${depends_prefix}/lib ${LDFLAGS}"
 
 if test -n "@CC@" -a -z "${CC}"; then
   CC="@CC@"
@@ -72,33 +72,33 @@ fi
 PYTHONPATH=$depends_prefix/native/lib/python3/dist-packages:$PYTHONPATH
 
 if test -n "@AR@"; then
-  AR=@AR@
-  ac_cv_path_ac_pt_AR=${AR}
+  AR="@AR@"
+  ac_cv_path_ac_pt_AR="${AR}"
 fi
 
 if test -n "@RANLIB@"; then
-  RANLIB=@RANLIB@
-  ac_cv_path_ac_pt_RANLIB=${RANLIB}
+  RANLIB="@RANLIB@"
+  ac_cv_path_ac_pt_RANLIB="${RANLIB}"
 fi
 
 if test -n "@NM@"; then
-  NM=@NM@
-  ac_cv_path_ac_pt_NM=${NM}
+  NM="@NM@"
+  ac_cv_path_ac_pt_NM="${NM}"
 fi
 
 if test -n "@debug@"; then
   enable_reduce_exports=no
 fi
 
 if test -n "@CFLAGS@"; then
-  CFLAGS="@CFLAGS@ $CFLAGS"
+  CFLAGS="@CFLAGS@ ${CFLAGS}"
 fi
 if test -n "@CXXFLAGS@"; then
-  CXXFLAGS="@CXXFLAGS@ $CXXFLAGS"
+  CXXFLAGS="@CXXFLAGS@ ${CXXFLAGS}"
 fi
 if test -n "@CPPFLAGS@"; then
-  CPPFLAGS="@CPPFLAGS@ $CPPFLAGS"
+  CPPFLAGS="@CPPFLAGS@ ${CPPFLAGS}"
 fi
 if test -n "@LDFLAGS@"; then
-  LDFLAGS="@LDFLAGS@ $LDFLAGS"
+  LDFLAGS="@LDFLAGS@ ${LDFLAGS}"
 fi

depends/funcs.mk

@@ -1,17 +1,23 @@
 define int_vars
 #Set defaults for vars which may be overridden per-package
-$(1)_cc=$($($(1)_type)_CC)
-$(1)_cxx=$($($(1)_type)_CXX)
-$(1)_objc=$($($(1)_type)_OBJC)
-$(1)_objcxx=$($($(1)_type)_OBJCXX)
-$(1)_ar=$($($(1)_type)_AR)
-$(1)_ranlib=$($($(1)_type)_RANLIB)
-$(1)_libtool=$($($(1)_type)_LIBTOOL)
-$(1)_nm=$($($(1)_type)_NM)
-$(1)_cflags=$($($(1)_type)_CFLAGS) $($($(1)_type)_$(release_type)_CFLAGS)
-$(1)_cxxflags=$($($(1)_type)_CXXFLAGS) $($($(1)_type)_$(release_type)_CXXFLAGS)
-$(1)_ldflags=$($($(1)_type)_LDFLAGS) $($($(1)_type)_$(release_type)_LDFLAGS) -L$($($(1)_type)_prefix)/lib
-$(1)_cppflags=$($($(1)_type)_CPPFLAGS) $($($(1)_type)_$(release_type)_CPPFLAGS) -I$($($(1)_type)_prefix)/include
+$(1)_cc=$$($$($(1)_type)_CC)
+$(1)_cxx=$$($$($(1)_type)_CXX)
+$(1)_objc=$$($$($(1)_type)_OBJC)
+$(1)_objcxx=$$($$($(1)_type)_OBJCXX)
+$(1)_ar=$$($$($(1)_type)_AR)
+$(1)_ranlib=$$($$($(1)_type)_RANLIB)
+$(1)_libtool=$$($$($(1)_type)_LIBTOOL)
+$(1)_nm=$$($$($(1)_type)_NM)
+$(1)_cflags=$$($$($(1)_type)_CFLAGS) \
+            $$($$($(1)_type)_$$(release_type)_CFLAGS)
+$(1)_cxxflags=$$($$($(1)_type)_CXXFLAGS) \
+              $$($$($(1)_type)_$$(release_type)_CXXFLAGS)
+$(1)_ldflags=$$($$($(1)_type)_LDFLAGS) \
+             $$($$($(1)_type)_$$(release_type)_LDFLAGS) \
+             -L$$($($(1)_type)_prefix)/lib
+$(1)_cppflags=$$($$($(1)_type)_CPPFLAGS) \
+              $$($$($(1)_type)_$$(release_type)_CPPFLAGS) \
+              -I$$($$($(1)_type)_prefix)/include
 $(1)_recipe_hash:=
 endef
 

depends/hosts/darwin.mk

@@ -6,6 +6,48 @@ LD64_VERSION=530
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
+darwin_native_binutils=native_cctools
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
+# from llvm.org
+
+# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty
+darwin_native_toolchain=native_cctools
+
+clang_prog=$(build_prefix)/bin/clang
+clangxx_prog=$(clang_prog)++
+
+clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version)
+else
+# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
+# system
+
+darwin_native_toolchain=
+
+# We can't just use $(shell command -v clang) because GNU Make handles builtins
+# in a special way and doesn't know that `command` is a POSIX-standard builtin
+# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90.
+# At the time of writing, GNU Make v4.2.1 is still being used in supported
+# distro releases.
+#
+# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html
+clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang")
+clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++")
+
+clang_resource_dir=$(shell clang -print-resource-dir)
+endif
+
+cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL
+
+# Make-only lowercase function
+lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+# For well-known tools provided by cctools, make sure that their well-known
+# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
+# would.
+$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL))))
+
 # Flag explanations:
 #
 #     -mlinker-version
@@ -18,18 +60,54 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-
 #         Explicitly point to our binaries (e.g. cctools) so that they are
 #         ensured to be found and preferred over other possibilities.
 #
-#     -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+#     -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1
 #
 #         Forces clang to use the libc++ headers from our SDK and completely
 #         forget about the libc++ headers from the standard directories
 #
-#         TODO: Once we start requiring a clang version that has the
-#         -stdlib++-isystem<directory> flag first introduced here:
-#         https://reviews.llvm.org/D64089, we should use that instead. Read the
-#         differential summary there for more details.
+#     -Xclang -*system<path_a> \
+#     -Xclang -*system<path_b> \
+#     -Xclang -*system<path_c> ...
+#
+#         Adds path_a, path_b, and path_c to the bottom of clang's list of
+#         include search paths. This is used to explicitly specify the list of
+#         system include search paths and its ordering, rather than rely on
+#         clang's autodetection routine. This routine has been shown to:
+#             1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1
+#                when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034)
+#             2. Fail to pickup C headers in $SYSROOT/usr/include when
+#                C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9)
+#
+#         Talking directly to cc1 with -Xclang here grants us access to specify
+#         more granular categories for these system include search paths, and we
+#         can use the correct categories that these search paths would have been
+#         placed in if the autodetection routine had worked correctly. (see:
+#         https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment)
+#
+#         Furthermore, it places these search paths after any "non-Xclang"
+#         specified search paths. This prevents any additional clang options or
+#         environment variables from coming after or in between these system
+#         include search paths, as that would be wrong in general but would also
+#         break #include_next's.
 #
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++ -mlinker-version=$(LD64_VERSION) -B$(build_prefix)/bin -nostdinc++ -isystem $(OSX_SDK)/usr/include/c++/v1
+darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+              -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+              -u LIBRARY_PATH \
+            $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+              -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+              -isysroot$(OSX_SDK) \
+              -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+              -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
+darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
+               -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
+               -u LIBRARY_PATH \
+             $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+               -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+               -isysroot$(OSX_SDK) \
+               -stdlib=libc++ \
+               -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \
+               -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+               -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
@@ -40,11 +118,4 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
 
-darwin_native_binutils=native_cctools
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-darwin_native_toolchain=native_cctools
-else
-darwin_native_toolchain=
-endif
-
 darwin_cmake_system=Darwin