-
Notifications
You must be signed in to change notification settings - Fork 229
[Manual] Devito on CX3 @Imperial College HPC
George Bisbas edited this page Jul 23, 2024
·
4 revisions
Useful: HPC pages:
- https://wiki.imperial.ac.uk/display/HPC/High+Performance+Computing
- https://wiki.imperial.ac.uk/display/HPC/MPI+Jobs
- https://icl-rcs-user-guide.readthedocs.io/en/latest/hpc/queues/classes-of-jobs/
- https://icl-rcs-user-guide.readthedocs.io/en/latest/hpc/queues/mpi-jobs/
Job sizing guidance
# Do `ssh` to your login node (Usually logs to AMD EPYC 7742 64-Core Processor)
ssh [email protected]
# Do `ssh` to your login node (Usually logs to Intel(R) Xeon(R) Platinum 8358 CPU @ 2.60GHz)
ssh -oPubkeyAuthentication=no [email protected]
# Load Anaconda personal
module load anaconda3/personal
# Alternative path with Python3.11
module load tools/prod
# module load Python/3.11.2-GCCcore-12.2.0-bare
module load Python/3.11.5-GCCcore-13.2.0
# Load OpenMPI
module load tools/dev
# module load OpenMPI/4.1.4-GCC-12.2.0
module load OpenMPI/4.1.6-GCC-13.2.0
While these nodes will run software built using the Intel MPI and Compiler, we strongly recommend rebuilding your software using GCC and OpenMPI as we observe greater performance with these.
# Load git module
module load git
# Following instructions from https://github.com/opesci/devito
git clone https://github.com/devitocodes/devito.git
cd devito
conda env create -f environment-dev.yml
source activate devito
pip install -e .
# If everything went fine, you should be able to run a typical operator. i.e.:
source ../../environments/python3.11.5-env/bin/activate
export DEVITO_LOGGING=DEBUG
export DEVITO_LANGUAGE=openmp
python examples/seismic/acoustic/acoustic_example.py
For MPI:
# Instal MPI requirements
python3 -m pip install --force-reinstall --upgrade --no-cache-dir -r requirements-mpi.txt
# Do a test run, it should work
DEVITO_MPI=1 mpirun -n 2 python3 examples/seismic/acoustic/acoustic_example.py
For interactive jobs:
qsub -I -l select=1:ncpus=2:mem=8gb -l walltime=02:00:00
WIP .PBS for some intel machine
#!/bin/bash
#PBS -lselect=1:ncpus=24:mem=120gb:mpiprocs=2:ompthreads=12
#PBS -lwalltime=00:30:00
lscpu
whoami
cd /rdsgpfs/general/user/$(whoami)/home/devitocodes/devito
export DEVITO_HOME=/rdsgpfs/general/user/$(whoami)/home/devitocodes/devito
module load tools/prod
module load Python/3.11.2-GCCcore-12.2.0-bare
source environments/python311-env/bin/activate
module load intel-compilers/2022.2.1
module load mpi
export DEVITO_ARCH=intel
export DEVITO_LANGUAGE=openmp
export DEVITO_LOGGING=DEBUG
export DEVITO_MPI=1
# export TMPDIR=/rds/general/user/$(whoami)/home/devitocodes/cache
mpiexec python3 $DEVITO_HOME/examples/seismic/acoustic/acoustic_example.py d 100 100 100 -so 8 --tn 200 --autotune aggressive
For Rome:
#!/bin/bash
#PBS -lselect=1:ncpus=128:mem=100gb:mpiprocs=8:ompthreads=16
#PBS -lwalltime=02:00:00
lscpu
whoami
cd /rdsgpfs/general/user/$(whoami)/home/devitocodes/devito
export DEVITO_HOME=/rdsgpfs/general/user/$(whoami)/home/devitocodes/devito
module load tools/prod
module load Python/3.11.2-GCCcore-12.2.0-bare
source environments/python311-env/bin/activate
module load intel-compilers/2022.2.1
module load mpi
export DEVITO_ARCH=gcc
export DEVITO_LANGUAGE=openmp
export DEVITO_LOGGING=DEBUG
export DEVITO_MPI=1
# export TMPDIR=/rds/general/user/$(whoami)/home/devitocodes/cache
mpiexec python3 $DEVITO_HOME/examples/seismic/acoustic/acoustic_example.py d 100 100 100 -so 8 --tn 200 --autotune aggressive