Merge branch 'gen_dockerfile' into swig-container

SINTEF · Nov 27, 2024 · 3aa59db · 3aa59db
2 parents 51e3943 + 39de9e3
commit 3aa59db
Show file tree

Hide file tree

Showing 8 changed files with 173 additions and 113 deletions.
diff --git a/.github/docker/Dockerfile-manylinux.template b/.github/docker/Dockerfile-manylinux.template
@@ -6,38 +6,39 @@
 #
 # Copy this template file and replace:
 # - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28
-# - `{{ ARCH }}` with a valid arch, ex:  x86_64, i686
+# - `{{ ARCH }}` architecture, ex:  x86_64, i686
+# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10"
+# - `{{ EXTRA_PRE }}` extra commands before installation
+# - `{{ EXTRA_POST }}` extra commands after installation
 # Remove the `.template` suffix from the copy.
 #
 # Examples
 # --------
 #
 # Generate dockerfile:
 #
-#     .github/docker/gen_dockerfile.sh manylinux 2014 i686 > \
-#         .github/docker/dockerfiles/Dockerfile-manylinux2014_i686
+#     .github/docker/gen_dockerfile.sh manylinux 2014 x86_64 > \
+#         .github/docker/dockerfiles/Dockerfile-manylinux2014_x86_64
 #
 # Build:
 #
 #     cd dlite  # cd to DLite root directory
-#     docker build -t dlite-manylinux2014_i686 \
-#         -f .github/docker/dockerfiles/Dockerfile-manylinux2014_i686 .
+#     docker build -t dlite-manylinux2014_x86_64 \
+#         -f .github/docker/dockerfiles/Dockerfile-manylinux2014_x86_64 .
 #
 # Run (for debugging):
 #
 #     docker run --rm -it \
 #         --volume $PWD:/io \
 #         --user $(id -u):$(id -g) \
-#         dlite-manylinux2014_i686 \
+#         dlite-manylinux2014_x86_64 \
 #         /bin/bash
 #
 
 # Reference: https://github.com/pypa/manylinux#manylinux2014-centos-7-based
 FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="8 9 10 11 12"
-
-COPY requirements_full.txt requirements_dev.txt /tmp/
+COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/
 
 {{ EXTRA_PRE }}
 
@@ -53,7 +54,9 @@ RUN \
     libcurl-devel \
     libxslt-devel \
     libxml2-devel \
-    libffi-devel && \
+    libffi-devel \
+    rust \
+    cargo && \
   # hdf5-devel is needed for building the hdf5 plugin
   yum install -y hdf5-devel || true && \
   # Unpack static libraries
@@ -77,15 +80,22 @@ RUN \
     -e '/scikit-image/d' \
     -e '/ncempy/d' \
     -e '/h5py/d' \
-    -i /tmp/requirements_full.txt && \
+    -e '/pymongo/d' \
+    -e '/redis/d' \
+    -e '/minio/d' \
+    -i /tmp/requirements_full.txt
+
+ARG PY_MINORS="{{ PY_MINORS }}"
+
+RUN \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
       -U setuptools wheel && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
-      oldest-supported-numpy \
+      -r /tmp/requirements.txt \
       -r /tmp/requirements_dev.txt \
       -r /tmp/requirements_full.txt; \
   done && \

diff --git a/.github/docker/Dockerfile-manylinux_x_y.template b/.github/docker/Dockerfile-manylinux_x_y.template
@@ -5,41 +5,42 @@
 # Usage:
 #
 # Copy this template file and replace:
-# - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28
-# - `{{ ARCH }}` with a valid arch, ex:  x86_64, i686
+# - `{{ TYPE }}` major glibc version, ex: _2_24, _2_28, _2_34
+# - `{{ ARCH }}` architecture, ex:  x86_64, i686
+# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10 11 12 13"
+# - `{{ EXTRA_PRE }}` extra commands before installation
+# - `{{ EXTRA_POST }}` extra commands after installation
 # Remove the `.template` suffix from the copy.
 #
 # Examples
 # --------
 #
 # Generate dockerfile:
 #
-#     .github/docker/gen_dockerfile.sh manylinux _2_24 x86_64 > \
-#         .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64
+#     .github/docker/gen_dockerfile.sh manylinux _2_34 x86_64 > \
+#         .github/docker/dockerfiles/Dockerfile-manylinux_2_34_x86_64
 #
 # Build:
 #
 #     cd dlite  # cd to DLite root directory
-#     docker build -t dlite-manylinux \
-#         -f .github/docker/dockerfiles/Dockerfile-manylinux_2_24_x86_64 .
+#     docker build -t dlite-manylinux_2_34_x86_64 \
+#         -f .github/docker/dockerfiles/Dockerfile-manylinux_2_34_x86_64 .
 #
 # Run (for debugging):
 #
 #     docker run --rm -it \
 #         --volume $PWD:/io \
 #         --user $(id -u):$(id -g) \
-#         dlite-manylinux_2_24_x86_64 \
+#         dlite-manylinux_2_34_x86_64 \
 #         /bin/bash
 #
 
 # Reference: https://github.com/pypa/manylinux
 FROM quay.io/pypa/manylinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="8 9 10 11 12"
-
 {{ EXTRA_PRE }}
 
-COPY requirements_full.txt requirements_dev.txt /tmp/
+COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/
 
 # Enable rpmfusion for additional packages
 RUN \
@@ -58,21 +59,25 @@ RUN \
   # It's necessary to be in /opt/_internal because the internal libraries
   # exist here.
   cd /opt/_internal && \
-  tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+  tar -Jxvf static-libs-for-embedding-only.tar.xz
   # Filter out challenging packages from requirements_full.txt
   #sed \
   #  -e '/pyarrow/d' \
   #  -e '/tables/d' \
   #  -e '/scikit-image/d' \
   #  -i /tmp/requirements_full.txt && \
-  # Install required Python packages
+    # Install required Python packages
+
+ARG PY_MINORS="{{ PY_MINORS }}"
+
+RUN \
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
       -U setuptools wheel && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
-      oldest-supported-numpy \
+      -r /tmp/requirements.txt \
       -r /tmp/requirements_dev.txt \
       -r /tmp/requirements_full.txt; \
   done && \

diff --git a/.github/docker/Dockerfile-musllinux.template b/.github/docker/Dockerfile-musllinux.template
@@ -5,7 +5,10 @@
 #
 # Copy this template file and replace:
 # - `{{ TYPE }}` with a valid musllinux type, e.g. '_1_1' or '_1_2'
-# - `{{ ARCH }}` with a valid arch, e.g., x86_64 or i686
+# - `{{ ARCH }}` architecture, ex:  x86_64, i686
+# - `{{ PY_MINORS }}` minor Python3 version numbers, ex: "8 9 10 11 12"
+# - `{{ EXTRA_PRE }}` extra commands before installation
+# - `{{ EXTRA_POST }}` extra commands after installation
 # Remove the `.template` suffix from the copy.
 #
 # Examples
@@ -32,9 +35,7 @@
 #
 FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest
 
-ARG PY_MINORS="8 9 10 11 12"
-
-COPY requirements_full.txt requirements_dev.txt /tmp/
+COPY requirements.txt requirements_full.txt requirements_dev.txt /tmp/
 
 {{ EXTRA_PRE }}
 
@@ -50,13 +51,16 @@ RUN \
     hdf5-dev \
     openssl-dev \
     openblas-dev \
-    swig && \
+    swig \
+    rust \
+    cargo && \
   # Unpack static libraries
   # It's necessary to be in /opt/_internal because the internal libraries
   # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
   # Filter out challenging packages from requirements_full.txt
+    #-e 's/^pydantic.*/pydantic<2/' \
   sed \
     -e '/pandas/d' \
     -e '/pyarrow/d' \
@@ -67,16 +71,24 @@ RUN \
     -e '/scikit-image/d' \
     -e '/ncempy/d' \
     -e '/h5py/d' \
+    -e '/pymongo/d' \
+    -e '/fortran-language-server/d' \
     -i /tmp/requirements_full.txt && \
-    #-e 's/^pydantic.*/pydantic<2/' \
+  sed \
+    -e '/mongomock/d' \
+    -i /tmp/requirements_dev.txt
+
+ARG PY_MINORS="{{ PY_MINORS }}"
+
+RUN \
   # Install required Python packages
   mkdir -p /ci/pip_cache && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache -U pip && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
       -U setuptools wheel && \
     python3.${minor} -m pip install --cache-dir=/ci/pip_cache \
-      oldest-supported-numpy \
+      -r /tmp/requirements.txt \
       -r /tmp/requirements_dev.txt \
       -r /tmp/requirements_full.txt; \
   done && \

diff --git a/.github/docker/gen_dockerfile.sh b/.github/docker/gen_dockerfile.sh
@@ -3,42 +3,46 @@
 #
 # Usage:
 #
-#     gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH
+#     gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH [PY_MINORS]
 #
 # Arguments:
-#   SYSTEM: system type. Ex: manylinux, musllinux
-#   SYSTEM_TYPE: Ex: 2010, 2014, _2_24, _2_28 (manylinux), _1_1, _1_2 (musllinux)
-#   ARCH: Ex: x86_64, i686
+#   SYSTEM: system. Ex: manylinux, musllinux
+#   SYSTEM_TYPE: System type. Ex: 2010, 2014, _2_24, _2_28 (manylinux), _1_1, _1_2 (musllinux)
+#   ARCH: Architecture. Ex: x86_64, i686
+#   PY_MINORS: Optional list of Python3 minor version numbers. Default: "8 9 10 11 12"
 set -eu
 
-if [[ $# -ne 3 ]]; then
-    echo "Usage: gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH"
+if [[ $# -lt 3 ]]; then
+    echo "Usage: gen_dockerfile.sh SYSTEM SYSTEM_TYPE ARCH [PY_MINORS]"
     exit 1
 fi
 SYSTEM=$1
 SYSTEM_TYPE=$2
 ARCH=$3
+PY_MINORS="${4:=8 9 10 11 12}"
 
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd -P )
+REL_DIR=${SCRIPT_DIR##*/dlite/}
 
 
 if [ $SYSTEM = "manylinux" -a ${SYSTEM_TYPE:0:1} = "_" ]; then
-    template=${SCRIPT_DIR}/Dockerfile-${SYSTEM}_x_y.template
+    template=${REL_DIR}/Dockerfile-${SYSTEM}_x_y.template
 else
-    template=${SCRIPT_DIR}/Dockerfile-${SYSTEM}.template
+    template=${REL_DIR}/Dockerfile-${SYSTEM}.template
 fi
 
 EXTRA_PRE=""
 EXTRA_POST=""
 if [ ${SYSTEM_TYPE} == "2010" ]; then
-    EXTRA_PRE="COPY ${SCRIPT_DIR}/pgdg-91_${ARCH}.repo /etc/yum.repos.d/pgdg-91.repo"
+    EXTRA_PRE="COPY ${REL_DIR}/pgdg-91_${ARCH}.repo /etc/yum.repos.d/pgdg-91.repo"
     EXTRA_POST="ENV PATH=\$PATH:/usr/pgsql-9.1/bin"
 fi
 
 sed \
     -e "s|{{ ARCH }}|${ARCH}|" \
     -e "s|{{ TYPE }}|${SYSTEM_TYPE}|" \
+    -e "s|{{ PY_MINORS }}|${PY_MINORS}|" \
     -e "s|{{ EXTRA_PRE }}|${EXTRA_PRE}|" \
     -e "s|{{ EXTRA_POST }}|${EXTRA_POST}|" \
     $template