From f84516f86cde562abf0282d5cd0d18b21bf9acf3 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Sat, 6 Aug 2022 07:52:03 +0200 Subject: [PATCH] mac build: force using source dist for most of our python dependencies We compile from tar.gz, instead of using pre-built binary wheels from PyPI. (or if the dep is pure-python, use tar.gz instead of "source-only" wheel) ----- Some unorganised things below for future reference. ``` $ dsymutil -dump-debug-map dist1/hid.cpython-39-darwin.so warning: (x86_64) /private/var/folders/1n/zc14m3td0rg4nt0ftklmm7z00000gn/T/pip-install-bm88zvc1/hidapi_cd307bc31ab34252b77d11d6d7212fc5/build/temp.macosx-10.9-x86_64-3.9/hid.o unable to open object file: No such file or directory warning: (x86_64) /private/var/folders/1n/zc14m3td0rg4nt0ftklmm7z00000gn/T/pip-install-bm88zvc1/hidapi_cd307bc31ab34252b77d11d6d7212fc5/build/temp.macosx-10.9-x86_64-3.9/hidapi/mac/hid.o unable to open object file: No such file or directory --- triple: 'x86_64-apple-darwin' binary-path: 'dist1/hid.cpython-39-darwin.so' ... ``` ``` $ nm -pa dist1/hid.cpython-39-darwin.so ``` - https://stackoverflow.com/questions/10044697/where-how-does-apples-gcc-store-dwarf-inside-an-executable - https://github.com/pypa/pip/issues/6505 - https://github.com/pypa/pip/issues/7808#issuecomment-770275723 - https://github.com/NixOS/nixpkgs/pull/91272 - https://github.com/cython/cython/pull/1576 - https://github.com/cython/cython/blob/9d2ba1611b28999663ab71657f4938b0ba92fe07/Cython/Compiler/ModuleNode.py#L913 --- contrib/build-linux/appimage/make_appimage.sh | 12 ++++++--- contrib/osx/make_osx | 27 ++++++++++++++++--- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/contrib/build-linux/appimage/make_appimage.sh b/contrib/build-linux/appimage/make_appimage.sh index 84529539b9a8..0fcc5329dcc9 100755 --- a/contrib/build-linux/appimage/make_appimage.sh +++ b/contrib/build-linux/appimage/make_appimage.sh @@ -101,16 +101,20 @@ info "preparing electrum-locale." info "Installing build dependencies." +# note: re pip installing from PyPI, +# we prefer compiling C extensions ourselves, instead of using binary wheels, +# hence "--no-binary :all:" flags. However, we specifically allow +# - PyQt5, as it's harder to build from source +# - cryptography, as it's harder to build from source +# - the whole of "requirements-build-base.txt", which includes pip and friends, as it also includes "wheel", +# and I am not quite sure how to break the circular dependence there (I guess we could introduce +# "requirements-build-base-base.txt" with just wheel in it...) "$python" -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \ --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-base.txt" "$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \ --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt" info "installing electrum and its dependencies." -# note: we prefer compiling C extensions ourselves, instead of using binary wheels, -# hence "--no-binary :all:" flags. However, we specifically allow -# - PyQt5, as it's harder to build from source -# - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04 "$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \ --cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements.txt" "$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \ diff --git a/contrib/osx/make_osx b/contrib/osx/make_osx index d4b17ef016e9..1712323e9b33 100755 --- a/contrib/osx/make_osx +++ b/contrib/osx/make_osx @@ -4,6 +4,7 @@ set -e # Parameterize PYTHON_VERSION=3.9.11 +PY_VER_MAJOR="3.9" # as it appears in fs paths PACKAGE=Electrum GIT_REPO=https://github.com/spesmilo/electrum @@ -93,13 +94,23 @@ source $VENV_DIR/bin/activate # don't add debug info to compiled C files (e.g. when pip calls setuptools/wheel calls gcc) # see https://github.com/pypa/pip/issues/6505#issuecomment-526613584 +# note: this does not seem sufficient when cython is involved (although it is on linux, just not on mac... weird.) +# see additional "strip" pass on built files later in the file. export CFLAGS="-g0" info "Installing build dependencies" +# note: re pip installing from PyPI, +# we prefer compiling C extensions ourselves, instead of using binary wheels, +# hence "--no-binary :all:" flags. However, we specifically allow +# - PyQt5, as it's harder to build from source +# - cryptography, as it's harder to build from source +# - the whole of "requirements-build-base.txt", which includes pip and friends, as it also includes "wheel", +# and I am not quite sure how to break the circular dependence there (I guess we could introduce +# "requirements-build-base-base.txt" with just wheel in it...) python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \ -Ir ./contrib/deterministic-build/requirements-build-base.txt \ || fail "Could not install build dependencies (base)" -python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \ +python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \ -Ir ./contrib/deterministic-build/requirements-build-mac.txt \ || fail "Could not install build dependencies (mac)" @@ -192,17 +203,20 @@ cp "$PROJECT_ROOT"/electrum/libusb-1.0.dylib "$CONTRIB"/osx info "Installing requirements..." -python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \ +python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: \ + --no-warn-script-location \ -Ir ./contrib/deterministic-build/requirements.txt \ || fail "Could not install requirements" info "Installing hardware wallet requirements..." -python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \ +python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary cryptography \ + --no-warn-script-location \ -Ir ./contrib/deterministic-build/requirements-hw.txt \ || fail "Could not install hardware wallet requirements" info "Installing dependencies specific to binaries..." -python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \ +python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography \ + --no-warn-script-location \ -Ir ./contrib/deterministic-build/requirements-binaries-mac.txt \ || fail "Could not install dependencies specific to binaries" @@ -210,6 +224,11 @@ info "Building $PACKAGE..." python3 -m pip install --no-build-isolation --no-dependencies \ --no-warn-script-location . > /dev/null || fail "Could not build $PACKAGE" +# strip debug symbols of some compiled libs +# - hidapi (hid.cpython-39-darwin.so) in particular is not reproducible without this +find "$VENV_DIR/lib/python$PY_VER_MAJOR/site-packages/" -type f -name '*.so' -print0 \ + | xargs -0 -t strip -x + info "Faking timestamps..." find . -exec touch -t '200101220000' {} + || true