Complete mods for data pull scripts. Start mods for chgres

scripts. Fixes ufs-community#559.
GeorgeGayno-NOAA · Sep 21, 2021 · 1939031 · 1939031
1 parent bafbef3
commit 1939031
Showing 1 changed file with 56 additions and 53 deletions.
diff --git a/util/gdas_init/driver.wcoss2.sh b/util/gdas_init/driver.wcoss2.sh
@@ -18,7 +18,6 @@ module list
 module load prod_util/2.0.8
 
 PROJECT_CODE=GFS-DEV
-QUEUE=dev
 
 source config
 
@@ -30,83 +29,83 @@ if [ $EXTRACT_DATA == yes ]; then
   mkdir -p $EXTRACT_DIR
 
   QUEUE=dev_transfer
-  MEM=6000M
+  MEM=500MB
   WALLT="02:00:00"
 
   case $gfs_ver in
     v12 | v13)
-      DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-       -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_pre-v14.data.sh ${CDUMP})
-      DEPEND="-d afterok:$DATAH"
+      DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+              -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh ${CDUMP})
+      DEPEND="-W depend=afterok:$DATAH"
       if [ "$CDUMP" = "gdas" ] ; then
-        DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \
-         -o log.data.enkf -e log.data.enkf ./get_pre-v14.data.sh enkf)
-        DEPEND="-d afterok:$DATAH:$DATA1"
+        DATA1=$(qsub -V -o log.data.enkf -e log.data.enkf -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+              -N get_enkf -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_pre-v14.data.sh enkf)
+        DEPEND="-W depend=afterok:$DATAH:$DATA1"
       fi
       ;;
     v14)
-      DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-       -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v14.data.sh ${CDUMP})
-      DEPEND="-d afterok:$DATAH"
+        DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v14.data.sh ${CDUMP})
+        DEPEND="-W depend=afterok:$DATAH"
       if [ "$CDUMP" = "gdas" ] ; then
-        DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_enkf \
-         -o log.data.enkf -e log.data.enkf ./get_v14.data.sh enkf)
-        DEPEND="-d afterok:$DATAH:$DATA1"
+        DATA1=$(qsub -V -o log.data.enkf -e log.data.enkf -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_enkf -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v14.data.sh enkf)
+        DEPEND="-W depend=afterok:$DATAH:$DATA1"
       fi
       ;;
     v15)
       if [ "$CDUMP" = "gfs" ] ; then
-        DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-         -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP})
-        DEPEND="-d afterok:$DATAH"
+        DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP})
+        DEPEND="-W depend=afterok:$DATAH"
       else
-        DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_${CDUMP} \
-         -o log.data.${CDUMP} -e log.data.${CDUMP} ./get_v15.data.sh ${CDUMP})
-        DATA1=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp1 \
-         -o log.data.grp1 -e log.data.grp1 ./get_v15.data.sh grp1)
-        DATA2=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp2 \
-         -o log.data.grp2 -e log.data.grp2 ./get_v15.data.sh grp2)
-        DATA3=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp3 \
-         -o log.data.grp3 -e log.data.grp3 ./get_v15.data.sh grp3)
-        DATA4=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp4 \
-         -o log.data.grp4 -e log.data.grp4 ./get_v15.data.sh grp4)
-        DATA5=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp5 \
-         -o log.data.grp5 -e log.data.grp5 ./get_v15.data.sh grp5)
-        DATA6=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp6 \
-         -o log.data.grp6 -e log.data.grp6 ./get_v15.data.sh grp6)
-        DATA7=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp7 \
-         -o log.data.grp7 -e log.data.grp7 ./get_v15.data.sh grp7)
-        DATA8=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_grp8 \
-         -o log.data.grp8 -e log.data.grp8 ./get_v15.data.sh grp8)
-        DEPEND="-d afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8"
+        DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh ${CDUMP})
+        DATA1=$(qsub -V -o log.data.grp1 -e log.data.grp1 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp1 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp1)
+        DATA2=$(qsub -V -o log.data.grp2 -e log.data.grp2 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp2 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp2)
+        DATA3=$(qsub -V -o log.data.grp3 -e log.data.grp3 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp3 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp3)
+        DATA4=$(qsub -V -o log.data.grp4 -e log.data.grp4 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp4 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp4)
+        DATA5=$(qsub -V -o log.data.grp5 -e log.data.grp5 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp5)
+        DATA6=$(qsub -V -o log.data.grp6 -e log.data.grp6 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp6 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp6)
+        DATA7=$(qsub -V -o log.data.grp7 -e log.data.grp7 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp7 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp7)
+        DATA8=$(qsub -V -o log.data.grp8 -e log.data.grp8 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+                -N get_grp8 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v15.data.sh grp8)
+        DEPEND="-W depend=afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8"
       fi
       ;;
     v16retro)
-      DATAH=$(sbatch --parsable --partition=service --ntasks=1 --mem=$MEM -t $WALLT -A $PROJECT_CODE -q $QUEUE -J get_v16retro \
-       -o log.data.v16retro -e log.data.v16retro ./get_v16retro.data.sh ${CDUMP})
-      DEPEND="-d afterok:$DATAH"
+      DATAH=$(qsub -V -o log.data.v16retro -e log.data.v16retro -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
+              -N get_v16retro -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16retro.data.sh ${CDUMP})
+      DEPEND="-W depend=afterok:$DATAH"
       ;;
     v16)
       DATAH=$(qsub -V -o log.data.${CDUMP} -e log.data.${CDUMP} -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-              -N get_${CDUMP} -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh ${CDUMP})
+              -N get_${CDUMP} -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh ${CDUMP})
       DEPEND="-W depend=afterok:$DATAH"
       if [ "$CDUMP" = "gdas" ] ; then
         DATA1=$(qsub -V -o log.data.grp1 -e log.data.grp1 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp1 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp1)
+                -N get_grp1 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp1)
         DATA2=$(qsub -V -o log.data.grp2 -e log.data.grp2 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp2 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp2)
+                -N get_grp2 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp2)
         DATA3=$(qsub -V -o log.data.grp3 -e log.data.grp3 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp3 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp3)
+                -N get_grp3 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp3)
         DATA4=$(qsub -V -o log.data.grp4 -e log.data.grp4 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp4 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp4)
+                -N get_grp4 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp4)
         DATA5=$(qsub -V -o log.data.grp5 -e log.data.grp5 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp5 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp5)
+                -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp5)
         DATA6=$(qsub -V -o log.data.grp6 -e log.data.grp6 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp5 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp6)
+                -N get_grp5 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp6)
         DATA7=$(qsub -V -o log.data.grp7 -e log.data.grp7 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp7 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp7)
+                -N get_grp7 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp7)
         DATA8=$(qsub -V -o log.data.grp8 -e log.data.grp8 -q $QUEUE -A $PROJECT_CODE -l walltime=$WALLT \
-                -N get_grp8 -l select=1:ncpus=2:mem=500MB -- ${this_dir}/get_v16.data.sh grp8)
+                -N get_grp8 -l select=1:ncpus=2:mem=$MEM -- ${this_dir}/get_v16.data.sh grp8)
         DEPEND="-W depend=afterok:$DATAH:$DATA1:$DATA2:$DATA3:$DATA4:$DATA5:$DATA6:$DATA7:$DATA8"
       fi
       ;;
@@ -120,16 +119,20 @@ fi  # extract data?
 
 if [ $RUN_CHGRES == yes ]; then
 
-  export APRUN=srun
-  NODES=3
+  QUEUE=dev
+  NODES=1
+  TASKS_PER_NODE=12
   WALLT="0:15:00"
-  export OMP_NUM_THREADS=1
+  MEM=75GB
   if [ $CRES_HIRES == 'C768' ] ; then
     NODES=5
   elif [ $CRES_HIRES == 'C1152' ] ; then
     NODES=8
     WALLT="0:20:00"
   fi
+  (( NCPUS = TASKS_PER_NODE * 2 ))
+  (( TASKS = NODES * TASKS_PER_NODE))
+  export APRUN="mpiexec -n $TASKS -ppn $TASKS_PER_NODE --cpu-bind core"
   case $gfs_ver in
     v12 | v13)
       export OMP_NUM_THREADS=4
@@ -161,8 +164,8 @@ if [ $RUN_CHGRES == yes ]; then
       fi
       ;;
     v16)
-      sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
-      -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP}
+      qsub -V -l select=${NODES}:ncpus=${NCPUS}:ompthreads=1:mem=${MEM} -l walltime=$WALLT -A $PROJECT_CODE -q $QUEUE \
+      -N chgres_${CDUMP} -o log.${CDUMP} -e log.${CDUMP} ${DEPEND} -- ${this_dir}/run_v16.chgres.sh ${CDUMP}
       ;;
   esac