Fix cross platform builds and additional refinements (#2044)

* Fix for darwin cross-compiles, cleanup * Allow en_US.UTF-8 locale in shell scripts * Fix shell lints * Add pre-reqs * Fix windows builds * Fix for darwin cross-compiles, cleanup * Allow en_US.UTF-8 locale in shell scripts * Fix shell lints * Add pre-reqs * Fix windows builds * Enable building on aarch64 * Add arm as well as aarch64 * Set build-dev CI default to release * Remove enable debug in build, fix bug in build-release * Remove log * Fixes to pass debug arg to dockerfiles * Remove clean deps and clean conf from dockerfiles * Fix typo * Rename docker variable --------- Co-authored-by: Niven <[email protected]>
DeFiCh · Jun 12, 2023 · 3708c2d · 3708c2d
1 parent 3df3351
commit 3708c2d
Show file tree

Hide file tree

Showing 22 changed files with 170 additions and 2,691 deletions.
diff --git a/.github/workflows/build-dev.yaml b/.github/workflows/build-dev.yaml
@@ -14,6 +14,7 @@ on:
 env:
   BUILD_VERSION: latest # Computed
   DOCKER_HUB_USER: defi
+  MAKE_DEBUG: 0
 
 jobs:
   linux-x64:

diff --git a/configure.ac b/configure.ac
@@ -503,6 +503,9 @@ case $host in
      AC_CHECK_LIB([iphlpapi],      [main],, AC_MSG_ERROR(libiphlpapi missing))
      AC_CHECK_LIB([crypt32],      [main],, AC_MSG_ERROR(libcrypt32 missing))
      AC_CHECK_LIB([bcrypt],      [main],, AC_MSG_ERROR(libbcrypt missing))
+     AC_CHECK_LIB([ncrypt],      [main],, AC_MSG_ERROR(libncrypt missing))
+     AC_CHECK_LIB([schannel],      [main],, AC_MSG_ERROR(libschannel missing))
+     AC_CHECK_LIB([ntdll],      [main],, AC_MSG_ERROR(libntdll missing))
 
      # -static is interpreted by libtool, where it has a different meaning.
      # In libtool-speak, it's -all-static.

diff --git a/contrib/dockerfiles/aarch64-linux-gnu.dockerfile b/contrib/dockerfiles/aarch64-linux-gnu.dockerfile
@@ -3,6 +3,7 @@ ARG TARGET=aarch64-linux-gnu
 # -----------
 FROM --platform=linux/amd64 ubuntu:latest as builder
 ARG TARGET
+ARG DEBUG
 LABEL org.defichain.name="defichain-builder"
 LABEL org.defichain.arch=${TARGET}
 
@@ -16,8 +17,8 @@ RUN ./make.sh ci-setup-deps
 RUN ./make.sh ci-setup-deps-target
 
 COPY . .
-RUN ./make.sh clean-depends && ./make.sh build-deps
-RUN ./make.sh clean-conf && ./make.sh build-conf 
+RUN ./make.sh build-deps
+RUN MAKE_DEBUG=${DEBUG} ./make.sh build-conf
 RUN ./make.sh build-make
 
 RUN mkdir /app && cd build/ && \

diff --git a/contrib/dockerfiles/arm-linux-gnueabihf.dockerfile b/contrib/dockerfiles/arm-linux-gnueabihf.dockerfile
@@ -3,6 +3,7 @@ ARG TARGET=arm-linux-gnueabihf
 # -----------
 FROM --platform=linux/amd64 ubuntu:latest as builder
 ARG TARGET
+ARG DEBUG
 LABEL org.defichain.name="defichain-builder"
 LABEL org.defichain.arch=${TARGET}
 
@@ -16,8 +17,8 @@ RUN ./make.sh ci-setup-deps
 RUN ./make.sh ci-setup-deps-target
 
 COPY . .
-RUN ./make.sh clean-depends && ./make.sh build-deps
-RUN ./make.sh clean-conf && ./make.sh build-conf 
+RUN ./make.sh build-deps
+RUN MAKE_DEBUG=${DEBUG} ./make.sh build-conf
 RUN ./make.sh build-make
 
 RUN mkdir /app && cd build/ && \

diff --git a/contrib/dockerfiles/noarch.dockerfile b/contrib/dockerfiles/noarch.dockerfile
@@ -4,8 +4,9 @@
 ARG TARGET=unknown
 
 # -----------
-FROM --platform=linux/amd64 ubuntu:latest as builder
+FROM ubuntu:latest as builder
 ARG TARGET
+ARG DEBUG
 LABEL org.defichain.name="defichain-builder"
 LABEL org.defichain.arch=${TARGET}
 
@@ -19,8 +20,8 @@ RUN ./make.sh ci-setup-deps
 RUN ./make.sh ci-setup-deps-target
 
 COPY . .
-RUN ./make.sh clean-depends && ./make.sh build-deps
-RUN ./make.sh clean-conf && ./make.sh build-conf 
+RUN ./make.sh build-deps
+RUN MAKE_DEBUG=${DEBUG} ./make.sh build-conf
 RUN ./make.sh build-make
 
 RUN mkdir /app && cd build/ && \

diff --git a/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile b/contrib/dockerfiles/x86_64-pc-linux-gnu.dockerfile
@@ -3,6 +3,7 @@ ARG TARGET=x86_64-pc-linux-gnu
 # -----------
 FROM --platform=linux/amd64 ubuntu:latest as builder
 ARG TARGET
+ARG DEBUG
 LABEL org.defichain.name="defichain-builder"
 LABEL org.defichain.arch=${TARGET}
 
@@ -16,8 +17,8 @@ RUN ./make.sh ci-setup-deps
 RUN ./make.sh ci-setup-deps-target
 
 COPY . .
-RUN ./make.sh clean-depends && ./make.sh build-deps
-RUN ./make.sh clean-conf && ./make.sh build-conf 
+RUN ./make.sh build-deps
+RUN MAKE_DEBUG=${DEBUG} ./make.sh build-conf
 RUN ./make.sh build-make
 
 RUN mkdir /app && cd build/ && \

diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk
@@ -9,11 +9,17 @@ OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-
 SILENCED_WARNINGS=-Wno-unused-command-line-argument -Wno-deprecated-non-prototype -Wno-unused-but-set-variable -Wno-unused-parameter -Wno-unused-variable
 DARWIN_SHAREDCC_FLAGS=-target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -mlinker-version=$(LD64_VERSION) $(SILENCED_WARNINGS)
 
+# There are some blockers to cleanly using just the flags and not 
+# overriding the CC/CXX with flags above, due to how depends system
+# has been built on Bitoin. For now, we follow the BTC way
+# until we can resolve the specific blockers, however we also
+# add them to the flags, so downstream nested compilations can 
+# use them properly.
 darwin_CC=clang $(DARWIN_SHAREDCC_FLAGS)
-darwin_CXX=clang++ -stdlib=libc++ $(DARWIN_SHAREDCC_FLAGS)
+darwin_CXX=clang++ -stdlib=libc++ -std=c++17 $(DARWIN_SHAREDCC_FLAGS)
 
-darwin_CFLAGS=-pipe
-darwin_CXXFLAGS=$(darwin_CFLAGS)
+darwin_CFLAGS=-pipe $(DARWIN_SHAREDCC_FLAGS)
+darwin_CXXFLAGS=$(darwin_CFLAGS) -stdlib=libc++ -std=c++17
 
 darwin_release_CFLAGS=-O2
 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)

diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk
@@ -1,5 +1,5 @@
 linux_CFLAGS=-pipe
-linux_CXXFLAGS=$(linux_CFLAGS)
+linux_CXXFLAGS=$(linux_CFLAGS) -std=c++17
 
 linux_release_CFLAGS=-O2
 linux_release_CXXFLAGS=$(linux_release_CFLAGS)

diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk
@@ -1,5 +1,5 @@
-mingw32_CFLAGS=-pipe
-mingw32_CXXFLAGS=$(mingw32_CFLAGS)
+mingw32_CFLAGS=-pipe 
+mingw32_CXXFLAGS=$(mingw32_CFLAGS) -std=c++17
 
 mingw32_release_CFLAGS=-O2
 mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS)

diff --git a/depends/packages/boost.mk b/depends/packages/boost.mk
@@ -27,7 +27,6 @@ $(package)_archiver_$(host_os)=$($(package)_ar)
 $(package)_toolset_darwin=darwin
 $(package)_archiver_darwin=$($(package)_libtool)
 $(package)_config_libraries=filesystem,system,test
-$(package)_cxxflags=-std=c++17 -fvisibility=hidden
 $(package)_cxxflags_linux=-fPIC
 endef
 

diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk
@@ -6,9 +6,16 @@ $(package)_sha256_hash=4a1359b6a79738b375b39ae05852712a77ff24d7ef2a498e99d35de78
 $(package)_build_subdir=cctools
 $(package)_clang_version=15.0.6
 $(package)_clang_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_clang_version)
-$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-18.04.tar.xz
-$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-18.04.tar.xz
-$(package)_clang_sha256_hash=38bc7f5563642e73e69ac5626724e206d6d539fbef653541b34cae0ba9c3f036
+$(package)_clang_file_name=clang-llvm-$($(package)_clang_version).tar.xz
+# x64 host
+$(package)_clang_download_file__x86_64=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-18.04.tar.xz
+$(package)_clang_sha256_hash__x86_64=38bc7f5563642e73e69ac5626724e206d6d539fbef653541b34cae0ba9c3f036
+# aarch64 host
+$(package)_clang_download_file__aarch64=clang+llvm-$($(package)_clang_version)-aarch64-linux-gnu.tar.xz
+$(package)_clang_sha256_hash__aarch64=8ca4d68cf103da8331ca3f35fe23d940c1b78fb7f0d4763c1c059e352f5d1bec
+# We duplicate this with arm, since config.guess for darwin returns arm
+$(package)_clang_download_file__arm=clang+llvm-$($(package)_clang_version)-aarch64-linux-gnu.tar.xz
+$(package)_clang_sha256_hash__arm=8ca4d68cf103da8331ca3f35fe23d940c1b78fb7f0d4763c1c059e352f5d1bec
 
 $(package)_libtapi_version=b7b5bdbfda9e8062d405b48da3b811afad98ae76
 $(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
@@ -21,14 +28,14 @@ $(package)_extra_sources += $($(package)_libtapi_file_name)
 
 define $(package)_fetch_cmds
 $(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file__$(build_arch)),$($(package)_clang_file_name),$($(package)_clang_sha256_hash__$(build_arch))) && \
 $(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
 endef
 
 define $(package)_extract_cmds
   mkdir -p $($(package)_extract_dir) && \
   echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_clang_sha256_hash)  $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
+  echo "$($(package)_clang_sha256_hash__$(build_arch))  $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
   echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
   $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
   mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \