Swig container (#1020)

Updated the weekly build of docker containers: - change to relative path in the COPY statements of the Dockerfiles generated with gen_dockerfile.sh - reduce number of build containers to 4: manylinux/musllinux, i686/x86_64 - parametrised python versions - installed rust and cargo on musllinux and manylinux2014 - updated ci_build_wheels to match the reduced number of containers that are build - install SWIG with pip instead of via the linux distribution to ensure a consistent swig version - avoid that split_url() and split_meta_uri() returns and extra None argument with SWIG 4.3.0 This PR combines PR #1017 and #1019. It should be followed up by a PR adding support for Python 3.13 and more included dependencies like pydantic, paramiko, pandas, ... (except for manylinux2014)
SINTEF · Nov 28, 2024 · fae504a · fae504a
1 parent 33cbf4e
commit fae504a
Show file tree

Hide file tree

Showing 15 changed files with 222 additions and 178 deletions.
diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template
@@ -1,12 +1,17 @@
 # -*- Mode: Dockerfile -*-
-# Template Dockerfile for building Python pypi package.  Going to be
-# superseeded by Dockerfile-manylinux_x_y.template
+# Template Dockerfile for manylinux 2010 and 2014 for building Python pypi package.
+# Is superseeded by the Dockerfile-manylinux_x_y.template, but will continue to be used to support i686 architectures.
+#
+# On these old systems we do not install requirements_full.txt
 #
 # Usage:
 #
 # Copy this template file and replace:
 # - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28
-# - `{{ ARCH }}` with a valid arch, ex:  x86_64, i686
+# - `{{ ARCH }}` architecture, ex:  x86_64, i686
+# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10"
+# - `{{ EXTRA_PRE }}` extra commands before installation
+# - `{{ EXTRA_POST }}` extra commands after installation
 # Remove the `.template` suffix from the copy.
 #
 # Examples
@@ -35,9 +40,7 @@
 # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based
 FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="8 9 10 11 12"
-
-COPY requirements_full.txt requirements_dev.txt /tmp/
+COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/
 
 {{ EXTRA_PRE }}
 
@@ -48,7 +51,6 @@ RUN \
     https://mirrors.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm --eval %{centos_ver}).noarch.rpm \
     https://mirrors.rpmfusion.org/nonfree/el/rpmfusion-nonfree-release-$(rpm --eval %{centos_ver}).noarch.rpm && \
   yum install -y \
-    swig \
     redland-devel \
     rasqal-devel \
     libcurl-devel \
@@ -65,30 +67,20 @@ RUN \
   # Filter out challenging packages from requirements
   sed \
     -e '/mongomock/d' \
-    -i  /tmp/requirements_dev.txt && \
-  sed \
-    -e '/pandas/d' \
-    -e '/pyarrow/d' \
-    -e '/tables/d' \
-    -e '/openpyxl/d' \
-    -e '/jinja2/d' \
-    -e '/pydantic/d' \
-    -e '/fortran-language-server/d' \
-    -e '/matplotlib/d' \
-    -e '/scikit-image/d' \
-    -e '/ncempy/d' \
-    -e '/h5py/d' \
-    -i /tmp/requirements_full.txt && \
+    -i  /tmp/requirements_dev.txt
+
+ARG PY_MINORS="{{ PY_MINORS }}"
+
+RUN \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
       -U setuptools wheel && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
-      oldest-supported-numpy \
-      -r /tmp/requirements_dev.txt \
-      -r /tmp/requirements_full.txt; \
+      -r /tmp/requirements.txt \
+      -r /tmp/requirements_dev.txt; \
   done && \
   rm -rf /ci/pip_cache
 

diff --git a/.github/docker/Dockerfile-manylinux_x_y.template b/.github/docker/Dockerfile-manylinux_x_y.template
@@ -5,41 +5,42 @@
 # Usage:
 #
 # Copy this template file and replace:
-# - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28
-# - `{{ ARCH }}` with a valid arch, ex:  x86_64, i686
+# - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28, _2_34
+# - `{{ ARCH }}` architecture, ex:  x86_64, i686
+# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10 11 12 13"
+# - `{{ EXTRA_PRE }}` extra commands before installation
+# - `{{ EXTRA_POST }}` extra commands after installation
 # Remove the `.template` suffix from the copy.
 #
 # Examples
 # --------
 #
 # Generate dockerfile:
 #
-#     .github/docker/gen_dockerfile.sh manylinux _2_24 x86_64 > \
-#         .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64
+#     .github/docker/gen_dockerfile.sh manylinux _2_34 x86_64 > \
+#         .github/docker/dockerfiles/Dockerfile-manylinux_2_34_x86_64
 #
 # Build:
 #
 #     cd dlite  # cd to DLite root directory
-#     docker build -t dlite-manylinux \
-#         -f .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64 .
+#     docker build -t dlite-manylinux_2_34_x86_64 \
+#         -f .github/docker/dockerfiles/Dockerfile-manylinux_2_34_x86_64 .
 #
 # Run (for debugging):
 #
 #     docker run --rm -it \
 #         --volume $PWD:/io \
 #         --user $(id -u):$(id -g) \
-#         dlite-manylinux_2_24_x86_64 \
+#         dlite-manylinux_2_34_x86_64 \
 #         /bin/bash
 #
 
 # Reference: https://github.com/pypa/manylinux
 FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="8 9 10 11 12"
-
 {{ EXTRA_PRE }}
 
-COPY requirements_full.txt requirements_dev.txt /tmp/
+COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/
 
 # Enable rpmfusion for additional packages
 RUN \
@@ -50,7 +51,6 @@ RUN \
   dnf install -y \
     redland-devel \
     rasqal-devel \
-    swig \
     libcurl-devel \
     libxslt-devel \
     libxml2-devel \
@@ -59,21 +59,25 @@ RUN \
   # It's necessary to be in /opt/_internal because the internal libraries
   # exist here.
   cd /opt/_internal && \
-  tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+  tar -Jxvf static-libs-for-embedding-only.tar.xz
   # Filter out challenging packages from requirements_full.txt
   #sed \
   #  -e '/pyarrow/d' \
   #  -e '/tables/d' \
   #  -e '/scikit-image/d' \
   #  -i /tmp/requirements_full.txt && \
-  # Install required Python packages
+    # Install required Python packages
+
+ARG PY_MINORS="{{ PY_MINORS }}"
+
+RUN \
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
       -U setuptools wheel && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
-      oldest-supported-numpy \
+      -r /tmp/requirements.txt \
       -r /tmp/requirements_dev.txt \
       -r /tmp/requirements_full.txt; \
   done && \

diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template
@@ -5,7 +5,10 @@
 #
 # Copy this template file and replace:
 # - `{{ TYPE }}` with a valid musllinux type, e.g. '_1_1' or '_1_2'
-# - `{{ ARCH }}` with a valid arch, e.g., x86_64 or i686
+# - `{{ ARCH }}` architecture, ex:  x86_64, i686
+# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10 11 12"
+# - `{{ EXTRA_PRE }}` extra commands before installation
+# - `{{ EXTRA_POST }}` extra commands after installation
 # Remove the `.template` suffix from the copy.
 #
 # Examples
@@ -32,9 +35,7 @@
 #
 FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="8 9 10 11 12"
-
-COPY requirements_full.txt requirements_dev.txt /tmp/
+COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/
 
 {{ EXTRA_PRE }}
 
@@ -50,13 +51,16 @@ RUN \
     hdf5-dev \
     openssl-dev \
     openblas-dev \
-    swig && \
+    swig \
+    rust \
+    cargo && \
   # Unpack static libraries
   # It's necessary to be in /opt/_internal because the internal libraries
   # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
   # Filter out challenging packages from requirements_full.txt
+    #-e 's/^pydantic.*/pydantic<2/' \
   sed \
     -e '/pandas/d' \
     -e '/pyarrow/d' \
@@ -67,16 +71,24 @@ RUN \
     -e '/scikit-image/d' \
     -e '/ncempy/d' \
     -e '/h5py/d' \
+    -e '/pymongo/d' \
+    -e '/fortran-language-server/d' \
     -i /tmp/requirements_full.txt && \
-    #-e 's/^pydantic.*/pydantic<2/' \
+  sed \
+    -e '/mongomock/d' \
+    -i /tmp/requirements_dev.txt
+
+ARG PY_MINORS="{{ PY_MINORS }}"
+
+RUN \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
       -U setuptools wheel && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
-      oldest-supported-numpy \
+      -r /tmp/requirements.txt \
       -r /tmp/requirements_dev.txt \
       -r /tmp/requirements_full.txt; \
   done && \

diff --git a/.github/docker/gen_dockerfile.sh b/.github/docker/gen_dockerfile.sh
@@ -3,42 +3,50 @@
 #
 # Usage:
 #
-#     gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH
+#     gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH [PY_MINORS]
 #
 # Arguments:
-#   SYSTEM: system type. Ex: manylinux, musllinux
-#   SYSTEM_TYPE: Ex: 2010, 2014, _2_24, _2_28 (manylinux), _1_1, _1_2 (musllinux)
-#   ARCH: Ex: x86_64, i686
+#   SYSTEM: system. Ex: manylinux, musllinux
+#   SYSTEM_TYPE: System type. Ex: 2010, 2014, _2_24, _2_28 (manylinux), _1_1, _1_2 (musllinux)
+#   ARCH: Architecture. Ex: x86_64, i686
+#   PY_MINORS: Optional list of Python3 minor version numbers. Default: "8 9 10 11 12"
 set -eu
 
-if [[ $# -ne 3 ]]; then
-    echo "Usage: gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH"
+if [[ $# -lt 3 ]]; then
+    echo "Usage: gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH [PY_MINORS]"
     exit 1
 fi
 SYSTEM=$1
 SYSTEM_TYPE=$2
 ARCH=$3
+if [[ $# -ge 4 ]]; then
+    PY_MINORS="$4"
+else
+    PY_MINORS="8 9 10 11 12"
+fi
 
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd -P )
+REL_DIR=${SCRIPT_DIR##*/dlite/}
 
 
 if [ $SYSTEM = "manylinux" -a ${SYSTEM_TYPE:0:1} = "_" ]; then
-    template=${SCRIPT_DIR}/Dockerfile-${SYSTEM}_x_y.template
+    template=${REL_DIR}/Dockerfile-${SYSTEM}_x_y.template
 else
-    template=${SCRIPT_DIR}/Dockerfile-${SYSTEM}.template
+    template=${REL_DIR}/Dockerfile-${SYSTEM}.template
 fi
 
 EXTRA_PRE=""
 EXTRA_POST=""
 if [ ${SYSTEM_TYPE} == "2010" ]; then
-    EXTRA_PRE="COPY ${SCRIPT_DIR}/pgdg-91_${ARCH}.repo /etc/yum.repos.d/pgdg-91.repo"
+    EXTRA_PRE="COPY ${REL_DIR}/pgdg-91_${ARCH}.repo /etc/yum.repos.d/pgdg-91.repo"
     EXTRA_POST="ENV PATH=\$PATH:/usr/pgsql-9.1/bin"
 fi
 
 sed \
     -e "s|{{ ARCH }}|${ARCH}|" \
     -e "s|{{ TYPE }}|${SYSTEM_TYPE}|" \
+    -e "s|{{ PY_MINORS }}|${PY_MINORS}|" \
     -e "s|{{ EXTRA_PRE }}|${EXTRA_PRE}|" \
     -e "s|{{ EXTRA_POST }}|${EXTRA_POST}|" \
     $template
diff --git a/.github/workflows/cd_docs.yml b/.github/workflows/cd_docs.yml
@@ -46,7 +46,6 @@ jobs:
             libxml2-dev \
             libxslt-dev \
             libhdf5-dev \
-            swig4.0 \
             doxygen \
             graphviz \
             python3 \
@@ -56,19 +55,20 @@ jobs:
             python3-yaml
           #python3 -m pip install psycopg2-binary==2.9.5
 
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -U setuptools wheel
+          pip install -r requirements.txt -r requirements_doc.txt -r requirements_dev.txt
+
       - name: Available version of installed programs
         run: |
           python --version
           swig -version
+          cmake -version
           doxygen --version
           dot -V
 
-      - name: Install Python dependencies
-        run: |
-          pip install --upgrade pip
-          pip install -U setuptools wheel
-          pip install -r requirements.txt -r requirements_doc.txt
-
       - name: Run CMAKE
         run: |
           mkdir -p build