From 0a3953c94b131c36ecad084af8a4fc063a648d2c Mon Sep 17 00:00:00 2001 From: carlos-encs <110119864+carlos-encs@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:25:31 -0500 Subject: [PATCH 1/2] OpenFoam multinode example --- src/README.md | 14 ++++++++++++++ src/openfoam-multinode.sh | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 src/openfoam-multinode.sh diff --git a/src/README.md b/src/README.md index 012213d..d5198f5 100644 --- a/src/README.md +++ b/src/README.md @@ -207,6 +207,20 @@ pip install git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAP - [Speed-related scripts](https://github.com/d-chante/diviner-tools/tree/development/jobs/speed) + +## OpenFoam-multinode + +This example is taken from OpenFoam tutorials section: $FOAM_TUTORIALS/incompressible/icoFoam/cavity/cavity +1. Go to your speed-scratch directory: `cd /speed-scratch/$USER` +2. open a salloc session +3. Load OpenFoam module: `module load OpenFOAM/v2306/default` +4. Copy the cavity example to your speed-scratch space: `cp -r $FOAM_TUTORIALS/incompressible/icoFoam/cavity/cavity/ .` +5. Modify cavity/system/decomposeParDict: + Remove coeffs section and modify the following: + numberOfSubdomains 10; + method scotch; +6. Exit the salloc session, go to the cavity directory and run the script: `sbatch --mem=10Gb -pps --constraint=el9 openfoam-multinode.sh` + ## OpenISS-yolov3 diff --git a/src/openfoam-multinode.sh b/src/openfoam-multinode.sh new file mode 100644 index 0000000..cb217aa --- /dev/null +++ b/src/openfoam-multinode.sh @@ -0,0 +1,16 @@ +#!/encs/bin/tcsh +#SBATCH --job-name=OpenFOAM-test +#SBATCH --nodes=4 +#SBATCH --ntasks=10 +#SBATCH --time=00:10:00 + +module load OpenFOAM/v2306/default +setenv PATH /encs/pkg/openmpi-4.1.6/root/bin\:$PATH +setenv LD_LIBRARY_PATH $LD_LIBRARY_PATH\:/encs/pkg/openmpi-4.1.6/root/lib64 +setenv CORES $SLURM_NTASKS +setenv OMP_NUM_THREADS $CORES + +blockMesh +mpirun -np $CORES redistributePar -decompose -overwrite -parallel +mpirun -np $CORES icoFoam -parallel +mpirun -np $CORES redistributePar -reconstruct -overwrite -parallel \ No newline at end of file From d93f8e25d69d27fad3f1da1268639664f1ab1868 Mon Sep 17 00:00:00 2001 From: carlos-encs <110119864+carlos-encs@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:31:29 -0500 Subject: [PATCH 2/2] OpenFoam readme add --- src/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/README.md b/src/README.md index d5198f5..8b979ae 100644 --- a/src/README.md +++ b/src/README.md @@ -23,6 +23,7 @@ run certcain things. - Detailed Examples + [efficientdet](#efficientdet) * [Diviner Tools](#diviner-tools) + * [OpenFoam](#openfoam-multinode) * [OpenISS-yolov3](#openiss-yolov3) + [Speed Login Configuration ](#speed-login-configuration) + [Speed Setup and Development Environment Preperation](#speed-setup-and-development-environment-preperation) @@ -59,6 +60,7 @@ These are examples either trivial or some are more elaborate. Some are described - `pytorch-multicpu.txt` -- using Pytorch with Python virtual environment to run on CPUs; with instructions and code ready to paste. - `pytorch-multinode-multigpu.sh` -- using Pytorch with Python virtual environment to run on Multinodes and MultiGpus - `lambdal-singularity.sh` -- an example use of the Singularity container to run LambdaLabs software stack on the GPU node. The container was built from the docker image as a [source](https://github.com/NAG-DevOps/lambda-stack-dockerfiles). + - `openfoam-multinode.sh` -- an example using OpenFoam, icoFoam solver to run on Multinodes-multicpus - `openiss-reid-speed.sh` -- OpenISS computer vision exame for re-edentification, see [more](https://github.com/NAG-DevOps/speed-hpc/tree/master/src#openiss-reid-tfk) in its section - `openiss-yolo-cpu.sh`, `openiss-yolo-gpu.sh`, and `openiss-yolo-interactive.sh` -- OpenISS examples with YOLO, related to `reid`, see [more](https://github.com/NAG-DevOps/speed-hpc/tree/master/src#openiss-yolov3) in the corresponding section