diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template index 8b3742f93..8cb707e5a 100644 --- a/.github/docker/Dockerfile-manylinux.template +++ b/.github/docker/Dockerfile-manylinux.template @@ -6,7 +6,10 @@ # # Copy this template file and replace: # - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28 -# - `{{ ARCH }}` with a valid arch, ex: x86_64, i686 +# - `{{ ARCH }}` architecture, ex: x86_64, i686 +# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10" +# - `{{ EXTRA_PRE }}` extra commands before installation +# - `{{ EXTRA_POST }}` extra commands after installation # Remove the `.template` suffix from the copy. # # Examples @@ -14,30 +17,28 @@ # # Generate dockerfile: # -# .github/docker/gen_dockerfile.sh manylinux 2014 i686 > \ -# .github/docker/dockerfiles/Dockerfile-manylinux2014_i686 +# .github/docker/gen_dockerfile.sh manylinux 2014 x86_64 > \ +# .github/docker/dockerfiles/Dockerfile-manylinux2014_x86_64 # # Build: # # cd dlite # cd to DLite root directory -# docker build -t dlite-manylinux2014_i686 \ -# -f .github/docker/dockerfiles/Dockerfile-manylinux2014_i686 . +# docker build -t dlite-manylinux2014_x86_64 \ +# -f .github/docker/dockerfiles/Dockerfile-manylinux2014_x86_64 . # # Run (for debugging): # # docker run --rm -it \ # --volume $PWD:/io \ # --user $(id -u):$(id -g) \ -# dlite-manylinux2014_i686 \ +# dlite-manylinux2014_x86_64 \ # /bin/bash # # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest -ARG PY_MINORS="8 9 10 11 12" - -COPY requirements_full.txt requirements_dev.txt /tmp/ +COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/ {{ EXTRA_PRE }} @@ -53,7 +54,9 @@ RUN \ libcurl-devel \ libxslt-devel \ libxml2-devel \ - libffi-devel && \ + libffi-devel \ + rust \ + cargo && \ # hdf5-devel is needed for building the hdf5 plugin yum install -y hdf5-devel || true && \ # Unpack static libraries @@ -77,7 +80,14 @@ RUN \ -e '/scikit-image/d' \ -e '/ncempy/d' \ -e '/h5py/d' \ - -i /tmp/requirements_full.txt && \ + -e '/pymongo/d' \ + -e '/redis/d' \ + -e '/minio/d' \ + -i /tmp/requirements_full.txt + +ARG PY_MINORS="{{ PY_MINORS }}" + +RUN \ # Install required Python packages mkdir -p /ci/pip_cache && \ for minor in ${PY_MINORS}; do \ @@ -85,7 +95,7 @@ RUN \ python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ -U setuptools wheel && \ python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ - oldest-supported-numpy \ + -r /tmp/requirements.txt \ -r /tmp/requirements_dev.txt \ -r /tmp/requirements_full.txt; \ done && \ diff --git a/.github/docker/Dockerfile-manylinux_x_y.template b/.github/docker/Dockerfile-manylinux_x_y.template index a3c355af0..099c142f7 100644 --- a/.github/docker/Dockerfile-manylinux_x_y.template +++ b/.github/docker/Dockerfile-manylinux_x_y.template @@ -5,8 +5,11 @@ # Usage: # # Copy this template file and replace: -# - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28 -# - `{{ ARCH }}` with a valid arch, ex: x86_64, i686 +# - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28, _2_34 +# - `{{ ARCH }}` architecture, ex: x86_64, i686 +# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10 11 12 13" +# - `{{ EXTRA_PRE }}` extra commands before installation +# - `{{ EXTRA_POST }}` extra commands after installation # Remove the `.template` suffix from the copy. # # Examples @@ -14,32 +17,30 @@ # # Generate dockerfile: # -# .github/docker/gen_dockerfile.sh manylinux _2_24 x86_64 > \ -# .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64 +# .github/docker/gen_dockerfile.sh manylinux _2_34 x86_64 > \ +# .github/docker/dockerfiles/Dockerfile-manylinux_2_34_x86_64 # # Build: # # cd dlite # cd to DLite root directory -# docker build -t dlite-manylinux \ -# -f .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64 . +# docker build -t dlite-manylinux_2_34_x86_64 \ +# -f .github/docker/dockerfiles/Dockerfile-manylinux_2_34_x86_64 . # # Run (for debugging): # # docker run --rm -it \ # --volume $PWD:/io \ # --user $(id -u):$(id -g) \ -# dlite-manylinux_2_24_x86_64 \ +# dlite-manylinux_2_34_x86_64 \ # /bin/bash # # Reference: https://github.com/pypa/manylinux FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest -ARG PY_MINORS="8 9 10 11 12" - {{ EXTRA_PRE }} -COPY requirements_full.txt requirements_dev.txt /tmp/ +COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/ # Enable rpmfusion for additional packages RUN \ @@ -58,21 +59,25 @@ RUN \ # It's necessary to be in /opt/_internal because the internal libraries # exist here. cd /opt/_internal && \ - tar -Jxvf static-libs-for-embedding-only.tar.xz && \ + tar -Jxvf static-libs-for-embedding-only.tar.xz # Filter out challenging packages from requirements_full.txt #sed \ # -e '/pyarrow/d' \ # -e '/tables/d' \ # -e '/scikit-image/d' \ # -i /tmp/requirements_full.txt && \ - # Install required Python packages + # Install required Python packages + +ARG PY_MINORS="{{ PY_MINORS }}" + +RUN \ mkdir -p /ci/pip_cache && \ for minor in ${PY_MINORS}; do \ python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \ python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ -U setuptools wheel && \ python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ - oldest-supported-numpy \ + -r /tmp/requirements.txt \ -r /tmp/requirements_dev.txt \ -r /tmp/requirements_full.txt; \ done && \ diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template index 136a730c8..308bd9199 100644 --- a/.github/docker/Dockerfile-musllinux.template +++ b/.github/docker/Dockerfile-musllinux.template @@ -5,7 +5,10 @@ # # Copy this template file and replace: # - `{{ TYPE }}` with a valid musllinux type, e.g. '_1_1' or '_1_2' -# - `{{ ARCH }}` with a valid arch, e.g., x86_64 or i686 +# - `{{ ARCH }}` architecture, ex: x86_64, i686 +# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10 11 12" +# - `{{ EXTRA_PRE }}` extra commands before installation +# - `{{ EXTRA_POST }}` extra commands after installation # Remove the `.template` suffix from the copy. # # Examples @@ -32,9 +35,7 @@ # FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest -ARG PY_MINORS="8 9 10 11 12" - -COPY requirements_full.txt requirements_dev.txt /tmp/ +COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/ {{ EXTRA_PRE }} @@ -50,13 +51,16 @@ RUN \ hdf5-dev \ openssl-dev \ openblas-dev \ - swig && \ + swig \ + rust \ + cargo && \ # Unpack static libraries # It's necessary to be in /opt/_internal because the internal libraries # exist here. cd /opt/_internal && \ tar -Jxvf static-libs-for-embedding-only.tar.xz && \ # Filter out challenging packages from requirements_full.txt + #-e 's/^pydantic.*/pydantic<2/' \ sed \ -e '/pandas/d' \ -e '/pyarrow/d' \ @@ -67,8 +71,16 @@ RUN \ -e '/scikit-image/d' \ -e '/ncempy/d' \ -e '/h5py/d' \ + -e '/pymongo/d' \ + -e '/fortran-language-server/d' \ -i /tmp/requirements_full.txt && \ - #-e 's/^pydantic.*/pydantic<2/' \ + sed \ + -e '/mongomock/d' \ + -i /tmp/requirements_dev.txt + +ARG PY_MINORS="{{ PY_MINORS }}" + +RUN \ # Install required Python packages mkdir -p /ci/pip_cache && \ for minor in ${PY_MINORS}; do \ @@ -76,7 +88,7 @@ RUN \ python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ -U setuptools wheel && \ python3.${minor} -m pip install --cache-dir=/ci/pip_cache \ - oldest-supported-numpy \ + -r /tmp/requirements.txt \ -r /tmp/requirements_dev.txt \ -r /tmp/requirements_full.txt; \ done && \ diff --git a/.github/docker/gen_dockerfile.sh b/.github/docker/gen_dockerfile.sh index ba382a792..d54cf98dc 100755 --- a/.github/docker/gen_dockerfile.sh +++ b/.github/docker/gen_dockerfile.sh @@ -3,42 +3,46 @@ # # Usage: # -# gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH +# gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH [PY_MINORS] # # Arguments: -# SYSTEM: system type. Ex: manylinux, musllinux -# SYSTEM_TYPE: Ex: 2010, 2014, _2_24, _2_28 (manylinux), _1_1, _1_2 (musllinux) -# ARCH: Ex: x86_64, i686 +# SYSTEM: system. Ex: manylinux, musllinux +# SYSTEM_TYPE: System type. Ex: 2010, 2014, _2_24, _2_28 (manylinux), _1_1, _1_2 (musllinux) +# ARCH: Architecture. Ex: x86_64, i686 +# PY_MINORS: Optional list of Python3 minor version numbers. Default: "8 9 10 11 12" set -eu -if [[ $# -ne 3 ]]; then - echo "Usage: gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH" +if [[ $# -lt 3 ]]; then + echo "Usage: gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH [PY_MINORS]" exit 1 fi SYSTEM=$1 SYSTEM_TYPE=$2 ARCH=$3 +PY_MINORS="${4:=8 9 10 11 12}" -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd -P ) +REL_DIR=${SCRIPT_DIR##*/dlite/} if [ $SYSTEM = "manylinux" -a ${SYSTEM_TYPE:0:1} = "_" ]; then - template=${SCRIPT_DIR}/Dockerfile-${SYSTEM}_x_y.template + template=${REL_DIR}/Dockerfile-${SYSTEM}_x_y.template else - template=${SCRIPT_DIR}/Dockerfile-${SYSTEM}.template + template=${REL_DIR}/Dockerfile-${SYSTEM}.template fi EXTRA_PRE="" EXTRA_POST="" if [ ${SYSTEM_TYPE} == "2010" ]; then - EXTRA_PRE="COPY ${SCRIPT_DIR}/pgdg-91_${ARCH}.repo /etc/yum.repos.d/pgdg-91.repo" + EXTRA_PRE="COPY ${REL_DIR}/pgdg-91_${ARCH}.repo /etc/yum.repos.d/pgdg-91.repo" EXTRA_POST="ENV PATH=\$PATH:/usr/pgsql-9.1/bin" fi sed \ -e "s|{{ ARCH }}|${ARCH}|" \ -e "s|{{ TYPE }}|${SYSTEM_TYPE}|" \ + -e "s|{{ PY_MINORS }}|${PY_MINORS}|" \ -e "s|{{ EXTRA_PRE }}|${EXTRA_PRE}|" \ -e "s|{{ EXTRA_POST }}|${EXTRA_POST}|" \ $template diff --git a/.github/workflows/ci_build_wheels.yml b/.github/workflows/ci_build_wheels.yml index 9bbad55db..0fc45e6c2 100644 --- a/.github/workflows/ci_build_wheels.yml +++ b/.github/workflows/ci_build_wheels.yml @@ -17,62 +17,60 @@ jobs: fail-fast: false matrix: include: - # 32-bit linux - - os: ubuntu-20.04 - system_type: ["manylinux", "2010"] - arch: i686 - py_minors: 8,9 - # Python 3.12 often fails with segfault during shutdown for - # manylinux2014_i686 - should be debugged - - os: ubuntu-20.04 - system_type: ["manylinux", "2014"] - arch: i686 - py_minors: 8,11 - # Python 3.12 fails since cibuildwheel still depends on distutils - # for musllinux + + # 64-bit manylinux - os: ubuntu-20.04 - system_type: ["musllinux", "_1_1"] - arch: i686 - py_minors: 8,11 - # Python 3.12 fails since cibuildwheel still depends on distutils - # for musllinux + system_type: ["manylinux", "_2_34"] + arch: x86_64 + py_minors: 8,12 + #- os: ubuntu-20.04 + # system_type: ["manylinux", "_2_28"] + # arch: x86_64 + # py_minors: 8,12 + #- os: ubuntu-20.04 + # system_type: ["manylinux", "2014"] + # arch: x86_64 + # py_minors: 8,12 + #- os: ubuntu-20.04 + # system_type: ["manylinux", "2010"] + # arch: x86_64 + # py_minors: 8,9 + + # 64-bit musllinux - os: ubuntu-20.04 system_type: ["musllinux", "_1_2"] - arch: i686 + arch: x86_64 py_minors: 8,11 + # Python 3.12 fails since cibuildwheel still depends on distutils + # for musllinux + #- os: ubuntu-20.04 + # system_type: ["musllinux", "_1_1"] + # arch: x86_64 + # py_minors: 8,11 - # 64-bit linux - - os: ubuntu-20.04 - system_type: ["manylinux", "2010"] - arch: x86_64 - py_minors: 8,9 + # 32-bit manylinux - os: ubuntu-20.04 system_type: ["manylinux", "2014"] - arch: x86_64 - py_minors: 8,12 - - os: ubuntu-20.04 - system_type: ["manylinux", "_2_28"] - arch: x86_64 + arch: i686 py_minors: 8,12 - # Python 3.12 fails since cibuildwheel still depends on distutils - # for musllinux - - os: ubuntu-20.04 - system_type: ["musllinux", "_1_1"] - arch: x86_64 - py_minors: 8,11 - # Python 3.12 fails since cibuildwheel still depends on distutils - # for musllinux + # Python 3.12 fails since cibuildwheel still depends on distutils + # for musllinux + #- os: ubuntu-20.04 + # system_type: ["manylinux", "2010"] + # arch: i686 + # py_minors: 8,9 + + # 32-bit musllinux - os: ubuntu-20.04 system_type: ["musllinux", "_1_2"] - arch: x86_64 + arch: i686 py_minors: 8,11 - - # See issue #220: https://github.com/SINTEF/dlite/issues/220 - # # 32-bit Windows - # - os: windows-2019 - # system_type: ["win32", ""] - # arch: "" - # py_minors: 7,10 + # Python 3.12 fails since cibuildwheel still depends on distutils + # for musllinux + #- os: ubuntu-20.04 + # system_type: ["musllinux", "_1_1"] + # arch: i686 + # py_minors: 8,11 # 64-bit Windows - os: windows-2019 @@ -80,8 +78,15 @@ jobs: arch: amd64 py_minors: 8,12 + # 32-bit Windows + # See issue #220: https://github.com/SINTEF/dlite/issues/220 + # - os: windows-2019 + # system_type: ["win32", ""] + # arch: "" + # py_minors: 7,10 + + # 64-bit (Intel) macOS # See issue #221: https://github.com/SINTEF/dlite/issues/221 - # # 64-bit (Intel) macOS # - os: macos-10.15 # system_type: ["macosx", ""] # arch: x86_64 diff --git a/.github/workflows/container_builds_weekly.yml b/.github/workflows/container_builds_weekly.yml index 23dc892e7..4ee21e857 100644 --- a/.github/workflows/container_builds_weekly.yml +++ b/.github/workflows/container_builds_weekly.yml @@ -22,45 +22,61 @@ jobs: fail-fast: false matrix: include: - # 64-bit linux + # 64-bit manylinux - system: "manylinux" - type: "2010" - arch: "x86_64" - py_minors: "8 9" - - system: "manylinux" - type: "2014" - arch: "x86_64" - py_minors: "8 9 10 11 12" - - system: "manylinux" - type: "_2_28" + type: "_2_34" arch: "x86_64" + #py_minors: "8 9 10 11 12 13" py_minors: "8 9 10 11 12" - - system: "musllinux" - type: "_1_1" - arch: "x86_64" - py_minors: "8 9 10 11" + #- system: "manylinux" + # type: "_2_28" + # arch: "x86_64" + # py_minors: "8 9 10 11 12" + #- system: "manylinux" + # type: "2014" + # arch: "x86_64" + # py_minors: "8 9 10 11 12" + #- system: "manylinux" + # type: "2010" + # arch: "x86_64" + # py_minors: "8 9" + + # 64-bit musllinux - system: "musllinux" type: "_1_2" arch: "x86_64" + #py_minors: "8 9 10 11 12 13" py_minors: "8 9 10 11" + #- system: "musllinux" + # type: "_1_1" + # arch: "x86_64" + # py_minors: "8 9 10 11" + #- system: "musllinux" + # type: "_1_2" + # arch: "x86_64" + # py_minors: "8 9 10 11" - # 32-bit linux - - system: "manylinux" - type: "2010" - arch: "i686" - py_minors: "8 9" + # 32-bit manylinux - system: "manylinux" type: "2014" arch: "i686" + #py_minors: "8 9 10 11 12 13" py_minors: "8 9 10 11 12" - - system: "musllinux" - type: "_1_1" - arch: "i686" - py_minors: "8 9 10 11" + #- system: "manylinux" + # type: "2010" + # arch: "i686" + # py_minors: "8 9" + + # 32-bit musllinux - system: "musllinux" type: "_1_2" arch: "i686" + #py_minors: "8 9 10 11 12 13" py_minors: "8 9 10 11" + #- system: "musllinux" + # type: "_1_1" + # arch: "i686" + # py_minors: "8 9 10 11" steps: - name: Checkout repository @@ -95,7 +111,7 @@ jobs: if: env.BUILD_NEW_IMAGE == 'true' run: | mkdir -p docker_build_wheel - bash .github/docker/gen_dockerfile.sh ${{ matrix.system }} ${{ matrix.type }} ${{ matrix.arch }} \ + bash .github/docker/gen_dockerfile.sh ${{ matrix.system }} ${{ matrix.type }} ${{ matrix.arch }} "${{ matrix.py_minors }}" \ > docker_build_wheel/Dockerfile-${{ matrix.system }}${{ matrix.type }}_${{ matrix.arch }} - name: Login to GitHub Container Registry diff --git a/requirements_full.txt b/requirements_full.txt index f05ca3bba..e34db4b04 100644 --- a/requirements_full.txt +++ b/requirements_full.txt @@ -22,7 +22,7 @@ pymongo>=4.4.0,<5 # Utilities pydantic>=1.10.0,<2.9.3 -typing_extensions>=4.1,<5 +typing_extensions>=4.1,<5; python_version<'3.9' fortran-language-server>=1.12.0,<1.13 # For TEM demo diff --git a/src/config-paths.h.in b/src/config-paths.h.in index 28f9ee936..33e9237ab 100644 --- a/src/config-paths.h.in +++ b/src/config-paths.h.in @@ -25,9 +25,17 @@ /* dlite-pyembed.c does not imports config.h to avoid conflicts with Python .h Include configurations needed by dlite-pyembed.c here */ +#ifndef HAVE_SETENV #cmakedefine HAVE_SETENV +#endif + +#ifndef HAVE__PUTENV_S #cmakedefine HAVE__PUTENV_S +#endif + +#ifndef HAVE_UNSETENV #cmakedefine HAVE_UNSETENV +#endif