Working MPI hello world scripts.

examples/LUMI/conda_mpi4py_mpich/README.md

@@ -27,6 +27,7 @@ TODO:
 - mpi4py conda packages not compatible with mpich=3.4.3 conda packages (or is it?)
 - Set a better shebang: #!/bin/bash -e
 - Rename cray_python to cray-python in sbatch file names.
+- TL;DR or somehow separate recommended example from tests/benchmarks
 
 
 wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-7.0.1.tar.gz

examples/LUMI/conda_mpi4py_mpich/containers/build_containers.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+module load LUMI/23.03
+module load cotainr
+cotainr build cotainr-mpich3-conda-mpi4py.sif --system=lumi-c --conda-env py310_mpich3_conda_mpi4py.yml --accept-licenses
+cotainr build cotainr-mpich3-pip-mpi4py.sif --system=lumi-c --conda-env py310_mpich3_pip_mpi4py.yml --accept-licenses
+cotainr build cotainr-mpich4-conda-mpi4py.sif --system=lumi-c --conda-env py310_mpich4_conda_mpi4py.yml --accept-licenses
+cotainr build cotainr-mpich4-pip-mpi4py.sif --system=lumi-c --conda-env py310_mpich4_pip_mpi4py.yml --accept-licenses

examples/LUMI/conda_mpi4py_mpich/py310_mpi4py_mpich.yml → examples/LUMI/conda_mpi4py_mpich/containers/py310_mpich3_conda_mpi4py.yml

@@ -1,9 +1,9 @@
-name: py310_mpi4py_mpich
+name: py310_mpich3_conda_mpi4py
 channels:
   - conda-forge
 dependencies:
   - mpi=*=mpich
   - mpich=3.4.3
   - mpi4py=3.1.1
-  - numpy=1.26.0
+  - numpy=1.26.2
   - python=3.10

examples/LUMI/conda_mpi4py_mpich/containers/py310_mpich3_pip_mpi4py.yml

@@ -0,0 +1,12 @@
+name: py310_mpich3_pip_mpi4py
+channels:
+  - conda-forge
+dependencies:
+  - c-compiler
+  - mpi=*=mpich
+  - mpich=3.4.3
+  - numpy=1.26.2
+  - pip
+  - python=3.10
+  - pip:
+    - mpi4py==3.1.1

examples/LUMI/conda_mpi4py_mpich/containers/py310_mpich4_conda_mpi4py.yml

@@ -0,0 +1,9 @@
+name: py310_mpich4_conda_mpi4py
+channels:
+  - conda-forge
+dependencies:
+  - mpi=*=mpich
+  - mpich=4.1.2
+  - mpi4py=3.1.1
+  - numpy=1.26.2
+  - python=3.10

examples/LUMI/conda_mpi4py_mpich/containers/py310_mpich4_pip_mpi4py.yml

@@ -0,0 +1,12 @@
+name: py310_mpich4_pip_mpi4py
+channels:
+  - conda-forge
+dependencies:
+  - c-compiler
+  - mpi=*=mpich
+  - mpich=4.1.2
+  - numpy=1.26.2
+  - pip
+  - python=3.10
+  - pip:
+    - mpi4py==3.1.1

examples/LUMI/conda_mpi4py_mpich/lumi-singularity-bindings.sh

@@ -0,0 +1,36 @@
+## Apparent minimal setup
+export SINGULARITYENV_LD_LIBRARY_PATH=\
+"/opt/cray/pe/mpich/default/ofi/gnu/9.1/lib-abi-mpich:"\
+"/opt/cray/libfabric/1.15.2.0/lib64:"\
+"/opt/cray/pe/gcc-libs:"\
+"/opt/cray/pe/lib64:"\
+"/opt/cray/xpmem/default/lib64:"\
+"/usr/lib64"
+
+export SINGULARITY_BIND=\
+"/var/spool/slurmd,"\
+"/opt/cray,"\
+"/usr/lib64/libbrotlicommon.so.1,"\
+"/usr/lib64/libbrotlidec.so.1,"\
+"/usr/lib64/libcrypto.so.1.1,"\
+"/usr/lib64/libcurl.so.4,"\
+"/usr/lib64/libcxi.so.1,"\
+"/usr/lib64/libgssapi_krb5.so.2,"\
+"/usr/lib64/libidn2.so.0,"\
+"/usr/lib64/libjansson.so.4,"\
+"/usr/lib64/libjitterentropy.so.3,"\
+"/usr/lib64/libjson-c.so.3,"\
+"/usr/lib64/libk5crypto.so.3,"\
+"/usr/lib64/libkeyutils.so.1,"\
+"/usr/lib64/libkrb5.so.3,"\
+"/usr/lib64/libkrb5support.so.0,"\
+"/usr/lib64/liblber-2.4.so.2,"\
+"/usr/lib64/libldap_r-2.4.so.2,"\
+"/usr/lib64/libnghttp2.so.14,"\
+"/usr/lib64/libpcre.so.1,"\
+"/usr/lib64/libpsl.so.5,"\
+"/usr/lib64/libsasl2.so.3,"\
+"/usr/lib64/libssh.so.4,"\
+"/usr/lib64/libssl.so.1.1,"\
+"/usr/lib64/libunistring.so.2,"\
+"/usr/lib64/libzstd.so.1"

examples/LUMI/conda_mpi4py_mpich/run_cotainr_bind_mpi_hello_world.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# A LUMI SLURM batch script for the LUMI mpi4py MPICH example from
+# https://github.com/DeiC-HPC/cotainr
+# This script runs the MPI hello world script
+# using a cotainr container including a generic MPICH, bind mounting the host MPI.
+#
+#SBATCH --job-name=mpi4py_mpi_hello_world_cotainr_bind
+#SBATCH --nodes=2
+#SBATCH --tasks-per-node=1
+#SBATCH --output="output_%x_%j.txt"
+#SBATCH --partition=small
+#SBATCH --exclusive
+#SBATCH --time=00:10:00
+#SBATCH --account=project_<your_project_id>
+
+PROJECT_DIR=
+CONTAINERS=(\
+    "cotainr-mpich3-conda-mpi4py.sif" \
+    "cotainr-mpich3-pip-mpi4py.sif" \
+    "cotainr-mpich4-conda-mpi4py.sif" \
+    "cotainr-mpich4-pip-mpi4py.sif")
+
+export MPIR_CVAR_DEBUG_SUMMARY=1
+export FI_LOG_LEVEL=Info
+
+source lumi-singularity-bindings.sh  # or use the LUMI singularity-bindings module
+
+for container in ${CONTAINERS[@]}; do
+    echo "=============== Run using $container ==============="
+    srun singularity exec \
+        --bind=$PROJECT_DIR \
+        $PROJECT_DIR/containers/$container \
+        python3 $PROJECT_DIR/mpi_hello_world.py
+done

examples/LUMI/conda_mpi4py_mpich/run_cotainr_hybrid_mpi_hello_world.sh

@@ -15,9 +15,19 @@
 #SBATCH --account=project_<your_project_id>
 
 PROJECT_DIR=
-CONTAINER=$PROJECT_DIR/lumi-mpi4py-mpich-demo.sif
+CONTAINERS=(\
+    "cotainr-mpich3-conda-mpi4py.sif" \
+    "cotainr-mpich3-pip-mpi4py.sif" \
+    "cotainr-mpich4-conda-mpi4py.sif" \
+    "cotainr-mpich4-pip-mpi4py.sif")
 
 export MPIR_CVAR_DEBUG_SUMMARY=1
 export FI_LOG_LEVEL=Info
 
-srun --mpi=pmi2 singularity exec --bind=$PROJECT_DIR $CONTAINER python $PROJECT_DIR/mpi_hello_world.py
+for container in ${CONTAINERS[@]}; do
+    echo "=============== Run using $container ==============="
+    srun --mpi=pmi2 singularity exec \
+        --bind=$PROJECT_DIR \
+        $PROJECT_DIR/containers/$container \
+        python3 $PROJECT_DIR/mpi_hello_world.py
+done

examples/LUMI/conda_mpi4py_mpich/run_lumisif_mpi_hello_world.sh

@@ -15,7 +15,7 @@
 #SBATCH --account=project_<your_project_id>
 
 PROJECT_DIR=
-CONTAINER=$PROJECT_DIR/lumi-mpi4py-rocm-5.4.5-python-3.10-mpi4py-3.1.4.sif
+CONTAINER=$PROJECT_DIR/containers/lumi-mpi4py-rocm-5.4.5-python-3.10-mpi4py-3.1.4.sif
 
 rm -rf $PROJECT_DIR/run-script.sh 
 cat > $PROJECT_DIR/run-script.sh << EOF
@@ -33,4 +33,10 @@ chmod +x $PROJECT_DIR/run-script.sh
 export MPIR_CVAR_DEBUG_SUMMARY=1
 export FI_LOG_LEVEL=Info
 
-srun singularity exec --bind=$PROJECT_DIR --bind=/var/spool/slurmd --bind=/opt/cray --bind=/usr/lib64/libcxi.so.1 --bind=/usr/lib64/libjansson.so.4 $CONTAINER $PROJECT_DIR/run-script.sh
+srun singularity exec \
+    --bind=$PROJECT_DIR \
+    --bind=/var/spool/slurmd \
+    --bind=/opt/cray \
+    --bind=/usr/lib64/libcxi.so.1 \
+    --bind=/usr/lib64/libjansson.so.4 \
+    $CONTAINER $PROJECT_DIR/run-script.sh