From d7d2258db37ba45df47b590301c83616660908f2 Mon Sep 17 00:00:00 2001
From: Gennadi <gle@pml.ac.uk>
Date: Mon, 17 May 2021 11:41:43 +0100
Subject: [PATCH] updated pbs script for gcc

---
 .../EXP2020/Archer2-MultiMonthHindcast.pbs    | 71 ++++++-------------
 1 file changed, 23 insertions(+), 48 deletions(-)

diff --git a/NEMOGCM/CONFIG/AMM7_FABM_ERSEM/EXP2020/Archer2-MultiMonthHindcast.pbs b/NEMOGCM/CONFIG/AMM7_FABM_ERSEM/EXP2020/Archer2-MultiMonthHindcast.pbs
index 2991c0b..a20b7c8 100755
--- a/NEMOGCM/CONFIG/AMM7_FABM_ERSEM/EXP2020/Archer2-MultiMonthHindcast.pbs
+++ b/NEMOGCM/CONFIG/AMM7_FABM_ERSEM/EXP2020/Archer2-MultiMonthHindcast.pbs
@@ -1,15 +1,16 @@
 #!/bin/bash
 #SBATCH --job-name=AMM7_ERSEM_test
-#SBATCH --time=00:20:00
-#SBATCH --nodes=2
-##SBATCH --nodes=1
-#SBATCH --ntasks=124
-##SBATCH --ntasks=2
-#SBATCH --account=n01
+##SBATCH --time=00:20:00
+#SBATCH --time=00:45:00
+##SBATCH --nodes=2
+#SBATCH --nodes=12
+#SBATCH --ntasks=1224
+##SBATCH --ntasks=100
+#SBATCH --account=n01-PML
 #SBATCH --partition=standard
 #SBATCH --qos=standard
-##BATCH --reservation=shortqos
-##BATCH --qos=short
+##SBATCH --reservation=shortqos
+##SBATCH --qos=short
 
 # NOTE:
 # the simulation year and month must be passed to this slurm script at runtime
@@ -18,40 +19,21 @@
 # "sbatch MYH2020slurm.sh 1981 1"
 #
 
-module -s restore /work/n01/shared/acc/n01_modules/ucx_env
-export OMP_NUM_THREADS=1
-
-#cat > myscript_wrapper.sh << EOFB
-##!/bin/ksh
-##
-#set -A map ./xios_server.exe ./nemo.exe
-#exec_map=( 0 1 )
-##
-#exec \${map[\${exec_map[\$SLURM_PROCID]}]}
-###
-#EOFB
-#chmod u+x ./myscript_wrapper.sh
-
-#
-cat > myscript_wrapper2.sh << EOFB
-#!/bin/ksh
-#
-set -A map ./xios_server.exe ./nemo.exe
-exec_map=( 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 )
-#
-exec \${map[\${exec_map[\$SLURM_PROCID]}]}
-##
-EOFB
-chmod u+x ./myscript_wrapper2.sh
-#
+#export OMP_NUM_THREADS=1
 
+module restore /etc/cray-pe.d/PrgEnv-gnu
+module load cray-hdf5-parallel
+module load cray-netcdf-hdf5parallel
+module unload craype-network-ofi
+module unload cray-mpich
+module load craype-network-ucx
+module load cray-mpich-ucx
+module load libfabric
+module load gcc
+module swap gcc/10.1.0 gcc/9.3.0
 
 set -u
 
-#NNODES=64
-#NNODES=7
-#NCORES=$(( NNODES*24 ))
-
 #number of month to launch:
 nmmax=31
 
@@ -185,17 +167,9 @@ do
       exit
    fi
 
-#   echo "Launching $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00 on $NCORES cores"
-
-#   aprun -b -n 1 -N 1 -S 1 ./xios_server.exe : -n $NCORES -N 24 ./nemo.exe
-#   aprun -n 192 -N 24 ./nemo.exe
-
-#srun --mem-bind=local \
-# --cpu-bind=v,mask_cpu:0x1,0x2 ./myscript_wrapper.sh
-
-   srun --mem-bind=local --cpu-bind=v,map_cpu:00,0x4,0x8,0xa,0xc,0xe,0x10,0x12,0x14,0x16,0x18,0x1a,0x1c,0x1e,0x20,0x22,0x24,0x26,0x28,0x2a,0x2c,0x2e,0x30,0x32,0x34,0x36,0x38,0x3a,0x3c,0x3e,0x40,0x42,0x44,0x46,0x48,0x4a,0x4c,0x4e,0x50,0x52,0x54,0x56,0x58,0x5a,0x5c,0x5e,0x60,0x62,0x64,0x66,0x68,0x6a,0x6c,0x6e,0x70,0x72,0x74,0x76,0x78,0x7a,0x7c,0x7e,00,0x4,0x8,0xa,0xc,0xe,0x10,0x12,0x14,0x16,0x18,0x1a,0x1c,0x1e,0x20,0x22,0x24,0x26,0x28,0x2a,0x2c,0x2e,0x30,0x32,0x34,0x36,0x38,0x3a,0x3c,0x3e,0x40,0x42,0x44,0x46,0x48,0x4a,0x4c,0x4e,0x50,0x52,0x54,0x56,0x58,0x5a,0x5c,0x5e,0x60,0x62,0x64,0x66,0x68,0x6a,0x6c,0x6e,0x70,0x72,0x74,0x76,0x78,0x7a,0x7c,0x7e, ./myscript_wrapper2.sh
+   echo "Launching $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00"
 
-#   srun --mem-bind=local --cpu-bind=v,map_cpu:00,0x10,0x20,0x22,0x24,0x26,0x28,0x2a,0x2c,0x2e,0x30,0x32,0x34,0x36,0x38,0x3a,0x3c,0x3e,0x40,0x42,0x44,0x46,0x48,0x4a,0x4c,0x4e,0x50,0x52,0x54,0x56,0x58,0x5a,0x5c,0x5e,0x60,0x62,0x64,0x66,0x68,0x6a,0x6c,0x6e,0x70,0x72,0x74,0x76,0x78,0x7a,0x7c,0x7e,00,0x10,0x20,0x22,0x24,0x26,0x28,0x2a,0x2c,0x2e,0x30,0x32,0x34,0x36,0x38,0x3a,0x3c,0x3e,0x40,0x42,0x44,0x46,0x48,0x4a,0x4c,0x4e,0x50,0x52,0x54,0x56,0x58,0x5a,0x5c,0x5e,0x60,0x62,0x64,0x66,0x68,0x6a,0x6c,0x6e,0x70,0x72,0x74,0x76,0x78,0x7a,0x7c,0x7e, ./myscript_wrapper2.sh
+   ./amm7standard.slurm
 
    echo "Finished $y0 $m0 at $(date +%s) seconds since 1970-01-01 00:00:00"
 
@@ -218,6 +192,7 @@ do
    mv amm7_1m_${d0}_[1-2]???????_ptrc_T.nc $ARCHIVEDIR
    mv amm7_1m_${d0}_[1-2]???????_grid_?.nc $ARCHIVEDIR
    bzip2 ocean.output && mv -f ocean.output.bz2 $ARCHIVEDIR
+   #mv timing.output $ARCHIVEDIR
    mv -f time.step $ARCHIVEDIR
 
    #archive: