From 57bfbdaf0257a66f364baf9231dff91be5bd3c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Br=C3=A9nainn=20Woodsend?= Date: Fri, 22 Nov 2024 19:53:41 +0000 Subject: [PATCH] ci(macos): Avoid linking against homebrew Homebrew binaries are always compiled for exactly the version they're installed on making them very un-portable. When a wheel is "repaired" by cibuildwheel, delocate-wheel pulls in _psycopg's dependencies (libpq.dylib, libssl.dylib and libcrypto.dylib) which, on a GitHub Actions macOS 14 runner, are provided by Homebrew and are therefore only macOS >= 14 compatible. The resultant wheel is therefore incompatible with all but the latest macOS versions. Build all dependencies from source so that we can set the deployment target to something sensible. Fixes #1753. --- .github/workflows/packages.yml | 26 ++++----- .github/workflows/tests.yml | 1 + scripts/build/build_libpq.sh | 102 ++++++++++++++++++++++++++------- 3 files changed, 93 insertions(+), 36 deletions(-) diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 0659f0dc3..5a1758ba4 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -5,10 +5,12 @@ on: env: PIP_BREAK_SYSTEM_PACKAGES: "1" + LIBPQ_VERSION: "16.0" + OPENSSL_VERSION: "1.1.1w" jobs: build-sdist: - if: true + if: false strategy: fail-fast: false matrix: @@ -58,16 +60,12 @@ jobs: build-linux: if: true - env: - LIBPQ_VERSION: "16.0" - OPENSSL_VERSION: "1.1.1w" - strategy: fail-fast: false matrix: platform: [manylinux, musllinux] - arch: [x86_64, i686, aarch64, ppc64le] - pyver: [cp38, cp39, cp310, cp311, cp312, cp313] + arch: [x86_64] #, i686, aarch64, ppc64le] + pyver: [cp38] #, cp39, cp310, cp311, cp312, cp313] runs-on: ubuntu-latest steps: @@ -140,18 +138,12 @@ jobs: matrix: # These archs require an Apple M1 runner: [arm64, universal2] arch: [x86_64] - pyver: [cp38, cp39, cp310, cp311, cp312, cp313] + pyver: [cp38, cp313] macver: ["12"] include: - arch: arm64 pyver: cp310 macver: "14" - - arch: arm64 - pyver: cp311 - macver: "14" - - arch: arm64 - pyver: cp312 - macver: "14" - arch: arm64 pyver: cp313 macver: "14" @@ -160,6 +152,9 @@ jobs: - name: Checkout repos uses: actions/checkout@v4 + - name: Build dependencies + run: ./scripts/build/build_libpq.sh + - name: Build wheels uses: pypa/cibuildwheel@v2.21.2 env: @@ -170,12 +165,11 @@ jobs: export PYTHONPATH={project} && python -c "import tests; tests.unittest.main(defaultTest='tests.test_suite')" CIBW_ENVIRONMENT: >- - MACOSX_DEPLOYMENT_TARGET=${{ matrix.macver }}.0 PG_VERSION=16 PACKAGE_NAME=psycopg2-binary PSYCOPG2_TESTDB=postgres PSYCOPG2_TEST_FAST=1 - PATH="/usr/local/opt/postgresql@${PG_VERSION}/bin:$PATH" + PATH="/tmp/libpq.build/bin:$PATH" - name: Upload artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ec1475bbf..f29aac9cd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,7 @@ on: jobs: tests: + if: false name: Unit tests run runs-on: ubuntu-latest diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 8a99fbe82..5e5250662 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Build a modern version of libpq and depending libs from source on Centos 5 +# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS set -euo pipefail set -x @@ -12,30 +12,66 @@ postgres_version="${LIBPQ_VERSION}" # last release: https://www.openssl.org/source/ openssl_version="${OPENSSL_VERSION}" +# last release: https://kerberos.org/dist/ +krb5_version="1.21.3" + # last release: https://openldap.org/software/download/ -ldap_version="2.6.3" +ldap_version="2.6.8" # last release: https://github.com/cyrusimap/cyrus-sasl/releases sasl_version="2.1.28" export LIBPQ_BUILD_PREFIX=${LIBPQ_BUILD_PREFIX:-/tmp/libpq.build} -if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.so" ]]; then +case "$(uname)" in + Darwin) + ID=macos + library_suffix=dylib + ;; + + Linux) + source /etc/os-release + library_suffix=so + ;; + + *) + echo "$0: unexpected Operating system: '$(uname)'" >&2 + exit 1 + ;; +esac + +if [[ -f "${LIBPQ_BUILD_PREFIX}/lib/libpq.${library_suffix}" ]]; then echo "libpq already available: build skipped" >&2 exit 0 fi -source /etc/os-release - case "$ID" in centos) yum update -y yum install -y zlib-devel krb5-devel pam-devel + curl="$(which curl)" ;; alpine) apk upgrade apk add --no-cache zlib-dev krb5-dev linux-pam-dev openldap-dev openssl-dev + curl="$(which curl)" + ;; + + macos) + brew install automake + # If available, libpq seemingly insists on linking against homebrew's + # openssl no matter what so remove it. Since homebrew's curl depends on + # it, force use of system curl. + brew uninstall --force --ignore-dependencies openssl + curl="/usr/bin/curl" + # The deployment target should be <= to that of the oldest supported Python version. + # e.g. https://www.python.org/downloads/release/python-380/ + if [ "$(uname -m)" == "x86_64" ]; then + export MACOSX_DEPLOYMENT_TARGET=10.9 + else + export MACOSX_DEPLOYMENT_TARGET=11.0 + fi ;; *) @@ -44,12 +80,12 @@ case "$ID" in ;; esac -if [ "$ID" == "centos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openssl if needed openssl_tag="OpenSSL_${openssl_version//./_}" openssl_dir="openssl-${openssl_tag}" - if [ ! -d "${openssl_dir}" ]; then curl -sL \ + if [ ! -d "${openssl_dir}" ]; then "$curl" -sL \ https://github.com/openssl/openssl/archive/${openssl_tag}.tar.gz \ | tar xzf - @@ -70,7 +106,31 @@ if [ "$ID" == "centos" ]; then fi -if [ "$ID" == "centos" ]; then +if [ "$ID" == "macos" ]; then + + # Build kerberos if needed + krb5_dir="krb5-${krb5_version}/src" + if [ ! -d "${krb5_dir}" ]; then + "$curl" -sL \ + curl -sL "https://kerberos.org/dist/krb5/$(echo 1.21.3 | grep -oE '\d+\.\d+')/krb5-${krb5_version}.tar.gz" \ + | tar xzf - + + cd "${krb5_dir}" + + ./configure --prefix=${LIBPQ_BUILD_PREFIX} \ + CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib + make + else + cd "${krb5_dir}" + fi + + make install + cd ../.. + +fi + + +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build libsasl2 if needed # The system package (cyrus-sasl-devel) causes an amazing error on i686: @@ -79,14 +139,14 @@ if [ "$ID" == "centos" ]; then sasl_tag="cyrus-sasl-${sasl_version}" sasl_dir="cyrus-sasl-${sasl_tag}" if [ ! -d "${sasl_dir}" ]; then - curl -sL \ + "$curl" -sL \ https://github.com/cyrusimap/cyrus-sasl/archive/${sasl_tag}.tar.gz \ | tar xzf - cd "${sasl_dir}" autoreconf -i - ./configure --prefix=${LIBPQ_BUILD_PREFIX} \ + ./configure --prefix=${LIBPQ_BUILD_PREFIX} --disable-macos-framework \ CPPFLAGS=-I${LIBPQ_BUILD_PREFIX}/include/ LDFLAGS=-L${LIBPQ_BUILD_PREFIX}/lib make else @@ -102,13 +162,13 @@ if [ "$ID" == "centos" ]; then fi -if [ "$ID" == "centos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then # Build openldap if needed ldap_tag="${ldap_version}" ldap_dir="openldap-${ldap_tag}" if [ ! -d "${ldap_dir}" ]; then - curl -sL \ + "$curl" -sL \ https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-${ldap_tag}.tgz \ | tar xzf - @@ -129,7 +189,7 @@ if [ "$ID" == "centos" ]; then make -C libraries/liblber/ install make -C libraries/libldap/ install make -C include/ install - chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.so* + chmod +x ${LIBPQ_BUILD_PREFIX}/lib/{libldap,liblber}*.${library_suffix}* cd .. fi @@ -139,17 +199,19 @@ fi postgres_tag="REL_${postgres_version//./_}" postgres_dir="postgres-${postgres_tag}" if [ ! -d "${postgres_dir}" ]; then - curl -sL \ + "$curl" -sL \ https://github.com/postgres/postgres/archive/${postgres_tag}.tar.gz \ | tar xzf - cd "${postgres_dir}" - # Match the default unix socket dir default with what defined on Ubuntu and - # Red Hat, which seems the most common location - sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\ -'|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \ - src/include/pg_config_manual.h + if [ "$ID" != "macos" ]; then + # Match the default unix socket dir default with what defined on Ubuntu and + # Red Hat, which seems the most common location + sed -i 's|#define DEFAULT_PGSOCKET_DIR .*'\ + '|#define DEFAULT_PGSOCKET_DIR "/var/run/postgresql"|' \ + src/include/pg_config_manual.h + fi # Often needed, but currently set by the workflow # export LD_LIBRARY_PATH="${LIBPQ_BUILD_PREFIX}/lib" @@ -171,4 +233,4 @@ make -C src/bin/pg_config install make -C src/include install cd .. -find ${LIBPQ_BUILD_PREFIX} -name \*.so.\* -type f -exec strip --strip-unneeded {} \; +find ${LIBPQ_BUILD_PREFIX} -name \*.${library_suffix}.\* -type f -exec strip --strip-unneeded {} \;