run_OQStandard.sh: Restore "oq engine --run" for Hazard calculation

Also fix many shellcheck warnings
OpenDRR · Oct 31, 2023 · c0df7db · c0df7db
1 parent 36c9be6
commit c0df7db
Showing 1 changed file with 96 additions and 87 deletions.
diff --git a/scripts/run_OQStandard.sh b/scripts/run_OQStandard.sh
@@ -2,9 +2,13 @@
 # ==========================================================================
 # Script for running scenario calculations in the National Canada Risk Model
 # ==========================================================================
+
+set -exo pipefail
+
 usage() {
-echo "Script for running scenario calculations in OpenQuake, using Canadian data. 
-You need to have created the hazard, damage, and risk files already, as well as 
+    cat <<EOF
+Script for running scenario calculations in OpenQuake, using Canadian data.
+You need to have created the hazard, damage, and risk files already, as well as
 functions called therein. Written by Tiegan E. Hobbs, last major update on 10 Dec 2020.
 
 
@@ -18,15 +22,15 @@ USAGE: run_OQStandard.sh NAME [-h -d -r -[b/o] -[s] -[l]]
         -b [b]aseline condition
         -o baseline + [o]ne extra retrofit (level r1)
         -t run retrofit level [t]wo (r2)
-	   -s ini files call a site level exposure file, else default is building level
+        -s ini files call a site level exposure file, else default is building level
     EXAMPLE 1: run_OQStandard.sh AFM7p3_GSM
     EXAMPLE 2: run_OQStandard.sh AFM7p3_GSM -d -b
 
-"
+EOF
 }
 
 #if [[ $LAPTOP != 'True' ]]; then
-### SETUP AWS KILL 
+### SETUP AWS KILL
 #shut_down_ec2_instance() {
 #    echo "Shutting down EC2 instance"
 #    sudo shutdown
@@ -36,7 +40,8 @@ USAGE: run_OQStandard.sh NAME [-h -d -r -[b/o] -[s] -[l]]
 #fi
 
 #    INITIALIZATION
-#Set below flags to 0 to skip those elements of run, and to 1 to include. For retrofits, set flag to 0 for baseline, 1 for baseline plus one level of retrofit, and 2 for baseline plus two levels of retrofit.
+# Set below flags to 0 to skip those elements of run, and to 1 to include.
+# For retrofits, set flag to 0 for baseline, 1 for baseline plus one level of retrofit, and 2 for baseline plus two levels of retrofit.
 HAZFLAG="0"; DMGFLAG="0"; RSKFLAG="0"; RETROFLAG="2"; EXPO='b'
 if [ $# -eq "0" ]; then
     usage
@@ -51,32 +56,35 @@ else
         HAZFLAG="1"; DMGFLAG="1"; RSKFLAG="1"; RETROFLAG="2"
     else
         echo "Reading specifications from flags."
-        while [[ "$1" =~ ^- && ! "$1" == "--" ]]; do case $1 in
-            -d )
+        while [[ "$1" =~ ^- && ! "$1" == "--" ]]; do
+            case $1 in
+            -d)
                 DMGFLAG="1"
                 ;;
-            -h )
+            -h)
                 HAZFLAG="1"
                 ;;
-            -r )
+            -r)
                 RSKFLAG="1"
                 ;;
-            -b )
+            -b)
                 RETROFLAG="0"
                 ;;
-            -o )
+            -o)
                 RETROFLAG="1"
                 ;;
-            -t )
+            -t)
                 RETROFLAG="2"
                 ;;
-            -s ) 
-		        EXPO='s'
-		        ;;
-            -l )
-                LAPTOP='True' #Took laptop flag out of usage statement
+            -s)
+                EXPO='s'
                 ;;
-        esac; shift; done
+            -l)
+                LAPTOP='True'   # Took laptop flag out of usage statement
+                ;;
+            esac
+            shift
+        done
     fi
 fi
 
@@ -89,7 +97,7 @@ echo "Outputs will be placed in '${OUTDIR}'"
 JOBDIR="initializations"
 #JOBDIR="job-files"
 SCRIPTS_LOC="scripts"
-CONSQ_LOC=${SCRIPTS_LOC}"/consequences-v3.10.0.py"
+CONSQ_LOC="${SCRIPTS_LOC}/consequences-v3.10.0.py"
 
 #if [[ $LAPTOP == 'True' ]]; then
 #    SCRIPTS_LOC="/Users/thobbs/Documents/GitHub/earthquake-scenarios/scripts"
@@ -98,41 +106,41 @@ CONSQ_LOC=${SCRIPTS_LOC}"/consequences-v3.10.0.py"
 #    SCRIPTS_LOC="/mnt/storage/earthquake-scenarios/scripts"
 #    CONSQ_LOC=${SCRIPTS_LOC}"/consequences-v3.10.0.py"
 #fi
-AVGHAZLOC=${SCRIPTS_LOC}"/weightHAZ.py"
-AVG_LOC=${SCRIPTS_LOC}"/weightedAverage.py"
+AVGHAZLOC="${SCRIPTS_LOC}/weightHAZ.py"
+AVG_LOC="${SCRIPTS_LOC}/weightedAverage.py"
 
-if [[ $HAZFLAG == "1" ]]; then 
+if [[ $HAZFLAG == "1" ]]; then
     echo "Preparing to run a hazard calculation."
-    if [ ! -f ${JOBDIR}/s_Hazard_${NAME}.ini ]; then
-      echo "Hazard file doesn't exist: ${JOBDIR}/s_Hazard_${NAME}.ini - EXITING"
-      exit 0
+    if [ ! -f "${JOBDIR}/s_Hazard_${NAME}.ini" ]; then
+        echo "Hazard file doesn't exist: ${JOBDIR}/s_Hazard_${NAME}.ini - EXITING"
+        exit 0
     fi
 fi
-if [[ $DMGFLAG == "1" ]]; then 
+if [[ $DMGFLAG == "1" ]]; then
     echo "Preparing to run a damage and consequence calculation."
-    if [ ! -f ${JOBDIR}/s_Damage_${NAME}_b0_${EXPO}.ini ]; then #-o ! -f ${JOBDIR}/s_Damage_${NAME}_r1.ini -o ! -f ${JOBDIR}/s_Damage_${NAME}_r2.ini ]; then
-      echo "Baseline damage file does not exist: ${JOBDIR}/s_Damage_${NAME}_b0_${EXPO}.ini - EXITING"
-      exit 0
+    if [ ! -f "${JOBDIR}/s_Damage_${NAME}_b0_${EXPO}.ini" ]; then #-o ! -f ${JOBDIR}/s_Damage_${NAME}_r1.ini -o ! -f ${JOBDIR}/s_Damage_${NAME}_r2.ini ]; then
+        echo "Baseline damage file does not exist: ${JOBDIR}/s_Damage_${NAME}_b0_${EXPO}.ini - EXITING"
+        exit 0
     fi
     expoSuffix=$EXPO
 fi
-if [[ $RSKFLAG == "1" ]]; then 
+if [[ $RSKFLAG == "1" ]]; then
     echo "Preparing to run a risk calculation."
-    if [ ! -f ${JOBDIR}/s_Risk_${NAME}_b0_${EXPO}.ini ]; then #-o ! -f ${JOBDIR}/s_Risk_${NAME}_r1.ini -o ! -f ${JOBDIR}/s_Risk_${NAME}_r2.ini ]; then
-      echo "Baseline risk file does not exist: ${JOBDIR}/s_Risk_${NAME}_b0_${EXPO}.ini - EXITING"
-      exit 0
+    if [ ! -f "${JOBDIR}/s_Risk_${NAME}_b0_${EXPO}.ini" ]; then #-o ! -f ${JOBDIR}/s_Risk_${NAME}_r1.ini -o ! -f ${JOBDIR}/s_Risk_${NAME}_r2.ini ]; then
+        echo "Baseline risk file does not exist: ${JOBDIR}/s_Risk_${NAME}_b0_${EXPO}.ini - EXITING"
+        exit 0
     fi
-    expoSuffix=$EXPO #`ls ${JOBDIR}/s_Risk_${NAME}_b0_?.ini | awk -F'(.ini|_)' '{print $(NF-1)}'`
+    expoSuffix=$EXPO    # $(ls ${JOBDIR}/s_Risk_${NAME}_b0_?.ini | awk -F'(.ini|_)' '{print $(NF-1)}')
     echo $expoSuffix
 fi
 if [[ $RETROFLAG == "2" ]]; then
-    declare -a arr=("r2") #RETROFITS
+    declare -a arr=("r2")   # RETROFITS
     echo "Running only r2 retrofit."
 elif [[ $RETROFLAG == "1" ]]; then
     declare -a arr=("b0" "r1")
     echo "Running baseline plus r1 retrofit."
 else
-    declare -a arr=("b0") 
+    declare -a arr=("b0")
     echo "Running baseline only."
 fi
 echo "All initialization files exist - You're ready to go."
@@ -147,61 +155,62 @@ echo "All initialization files exist - You're ready to go."
 #exit 0
 #fi
 
+oq dbserver start || true
 
 if [[ $HAZFLAG == "1" ]]; then
     echo "------------------------------------------------"
     echo "RUNNING HAZARD CALCULATION"
     echo "------------------------------------------------"
-    #oq engine --run ${JOBDIR}/s_Hazard_${NAME}.ini &> ./${OUTDIR}/s_Hazard_${NAME}.log;
+    oq engine --run "${JOBDIR}/s_Hazard_${NAME}.ini" 2>&1 | tee "./${OUTDIR}/s_Hazard_${NAME}.log"
     oq export gmf_data -1 -e csv -d temp
-    CALCID=`basename temp/*gmf* .csv | awk -F'_' '{print $NF}'`
-    python3 $AVGHAZLOC $NAME $CALCID
+    CALCID=$(basename temp/*gmf* .csv | awk -F'_' '{print $NF}')
+    python3 "$AVGHAZLOC" "$NAME" "$CALCID"
     SMSFILE="${OUTDIR}/s_sitemesh_${NAME}_${CALCID}.csv"
-    mv temp/*sitemesh* ${SMSFILE}
-    rm -f temp/sigma_epsilon_* temp/*gmf*.csv temp/realizations_${CALCID}.csv
+    mv temp/*sitemesh* "${SMSFILE}"
+    rm -f temp/sigma_epsilon_* temp/*gmf*.csv "temp/realizations_${CALCID}.csv"
 fi
 
 
 if [[ $DMGFLAG == "1" ]]; then
     echo "------------------------------------------------"
     echo "RUNNING DAMAGE & CONSEQUENCE CALCULATIONS"
     echo "------------------------------------------------"
-    if [[ $RETROFLAG == "1" ]]; then 
+    if [[ $RETROFLAG == "1" ]]; then
         # RUN TWO CALCS WITH SHARED HAZ CALC
-        oq engine --run ${JOBDIR}/s_Damage_${NAME}_${arr[0]}_${expoSuffix}.ini ${JOBDIR}/s_Damage_${NAME}_${arr[1]}_${expoSuffix}.ini &> ./${OUTDIR}/s_Damage_${NAME}_b0r1_${expoSuffix}.log;
-        
+        oq engine --run "${JOBDIR}/s_Damage_${NAME}_${arr[0]}_${expoSuffix}.ini" "${JOBDIR}/s_Damage_${NAME}_${arr[1]}_${expoSuffix}.ini" 2>&1 | tee "./${OUTDIR}/s_Damage_${NAME}_b0r1_${expoSuffix}.log"
+
         # EXPORT BASELINE, WHICH IS SECOND TO LAST CALC
         oq export damages-rlzs -2 -e csv -d temp
         oq export realizations -2 -e csv -d temp
-        CALCID=` ls -t temp/avg_damages-rlz-???_* | head -1 | awk -F'[_.]' '{print $(NF-1)}'`
-        python3 $AVG_LOC $NAME ${arr[0]} $CALCID $expoSuffix damage
-        python3 $CONSQ_LOC -2
-        declare -a files=(`ls consequences-rlz-???_-2.csv`) #conseq script doesn't know real calc id, so need to replace the "-2"
-        for file in $files; do mv $file "temp/"`basename $file "-2.csv"`${CALCID}".csv"; done
-        python3 $AVG_LOC $NAME ${arr[0]} $CALCID $expoSuffix consequence
-        rm -f temp/consequences-rlz-???_${CALCID}.csv temp/realizations_${CALCID}.csv temp/avg_damages-rlz-???_${CALCID}.csv #clear temp dir
-        
+        CALCID=$(ls -t temp/avg_damages-rlz-???_* | head -1 | awk -F'[_.]' '{print $(NF-1)}')
+        python3 "$AVG_LOC" "$NAME" "${arr[0]}" "$CALCID" $expoSuffix damage
+        python3 "$CONSQ_LOC" -2
+        declare -a files=($(ls consequences-rlz-???_-2.csv)) #conseq script doesn't know real calc id, so need to replace the "-2"
+        for file in "${files[@]}"; do mv "$file" "temp/$(basename "$file" "-2.csv")${CALCID}.csv"; done
+        python3 "$AVG_LOC" "$NAME" "${arr[0]}" "$CALCID" $expoSuffix consequence
+        rm -f temp/consequences-rlz-???_"${CALCID}".csv temp/realizations_"${CALCID}".csv temp/avg_damages-rlz-???_"${CALCID}".csv #clear temp dir
+
         # EXPORT RETROFIT 1, LAST CALC
         oq export damages-rlzs -1 -e csv -d temp
         oq export realizations -1 -e csv -d temp
-        CALCID=` ls -t temp/avg_damages-rlz-???_* | head -1 | awk -F'[_.]' '{print $(NF-1)}'`
-        python3 $AVG_LOC $NAME ${arr[1]} $CALCID $expoSuffix damage
-        python3 $CONSQ_LOC -1
-        mv consequences-rlz-???_${CALCID}.csv temp/
-        python3 $AVG_LOC $NAME ${arr[1]} $CALCID $expoSuffix consequence
-        rm -f temp/consequences-rlz-???_${CALCID}.csv temp/realizations_${CALCID}.csv temp/avg_damages-rlz-???_${CALCID}.csv #clear temp dir
-        
+        CALCID=$(ls -t temp/avg_damages-rlz-???_* | head -1 | awk -F'[_.]' '{print $(NF-1)}')
+        python3 "$AVG_LOC" "$NAME" "${arr[1]}" "$CALCID" $expoSuffix damage
+        python3 "$CONSQ_LOC" -1
+        mv consequences-rlz-???_"${CALCID}".csv temp/
+        python3 "$AVG_LOC" "$NAME" "${arr[1]}" "$CALCID" $expoSuffix consequence
+        rm -f temp/consequences-rlz-???_"${CALCID}".csv temp/realizations_"${CALCID}".csv temp/avg_damages-rlz-???_"${CALCID}".csv #clear temp dir
+
     else
         # RUN A SINGLE CALC
-        oq engine --run ${JOBDIR}/s_Damage_${NAME}_${arr[0]}_${expoSuffix}.ini > ./${OUTDIR}/s_Damage_${NAME}_${arr[0]}_${expoSuffix}.log
+        oq engine --run "${JOBDIR}/s_Damage_${NAME}_${arr[0]}_${expoSuffix}.ini" 2>&1 | tee "./${OUTDIR}/s_Damage_${NAME}_${arr[0]}_${expoSuffix}.log"
         oq export damages-rlzs -1 -e csv -d temp
         oq export realizations -1 -e csv -d temp
-        CALCID=` ls -t temp/avg_damages-rlz-???_* | head -1 | awk -F'[_.]' '{print $(NF-1)}'`
-        python3 $AVG_LOC $NAME ${arr[0]} $CALCID $expoSuffix damage
+        CALCID=$(ls -t temp/avg_damages-rlz-???_* | head -1 | awk -F'[_.]' '{print $(NF-1)}')
+        python3 $AVG_LOC "$NAME" "${arr[0]}" "$CALCID" $expoSuffix damage
         python3 $CONSQ_LOC -1
-        mv consequences-rlz-???_${CALCID}.csv temp/
-        python3 $AVG_LOC $NAME ${arr[0]} $CALCID $expoSuffix consequence
-        rm -f temp/consequences-rlz-???_${CALCID}.csv temp/realizations_${CALCID}.csv temp/avg_damages-rlz-???_${CALCID}.csv #clear temp dir
+        mv consequences-rlz-???_"${CALCID}".csv temp/
+        python3 $AVG_LOC "$NAME" "${arr[0]}" "$CALCID" $expoSuffix consequence
+        rm -f temp/consequences-rlz-???_"${CALCID}".csv temp/realizations_"${CALCID}".csv temp/avg_damages-rlz-???_"${CALCID}".csv #clear temp dir
     fi
 fi
 
@@ -212,29 +221,29 @@ if [[ $RSKFLAG == "1" ]]; then
     echo "------------------------------------------------"
     if [[ $RETROFLAG == "1" ]]; then
         # RUN TWO CALCS WITH SHARED HAZ CALC
-        oq engine --run ${JOBDIR}/s_Risk_${NAME}_${arr[0]}_${expoSuffix}.ini ${JOBDIR}/s_Risk_${NAME}_${arr[1]}_${expoSuffix}.ini &> ./${OUTDIR}/s_Risk_${NAME}_b0r1_${expoSuffix}.log;
-        
+        oq engine --run "${JOBDIR}/s_Risk_${NAME}_${arr[0]}_${expoSuffix}.ini" "${JOBDIR}/s_Risk_${NAME}_${arr[1]}_${expoSuffix}.ini" 2>&1 | tee "./${OUTDIR}/s_Risk_${NAME}_b0r1_${expoSuffix}.log"
+
         # EXPORT BASELINE, WHICH IS SECOND TO LAST CALC
-        oq export avg_losses-rlzs -2 -e csv -d temp;
-        CALCID=`ls -t temp/avg_losses* | head -1 | awk -F'[_.]' '{print $(NF-1)}'`
+        oq export avg_losses-rlzs -2 -e csv -d temp
+        CALCID=$(ls -t temp/avg_losses* | head -1 | awk -F'[_.]' '{print $(NF-1)}')
         #Run weighted average script, save only that
-        python3 $AVG_LOC $NAME ${arr[0]} $CALCID $expoSuffix loss
-        rm -f temp/avg_losses-rlz-???_${CALCID}.csv temp/realizations_${CALCID}.csv
-        
+        python3 $AVG_LOC "$NAME" "${arr[0]}" "$CALCID" $expoSuffix loss
+        rm -f temp/avg_losses-rlz-???_"${CALCID}".csv temp/realizations_"${CALCID}".csv
+
         # EXPORT RETROFIT 1, LAST CALC
-        oq export avg_losses-rlzs -1 -e csv -d temp;
-        CALCID=`ls -t temp/avg_losses* | head -1 | awk -F'[_.]' '{print $(NF-1)}'`
+        oq export avg_losses-rlzs -1 -e csv -d temp
+        CALCID=$(ls -t temp/avg_losses* | head -1 | awk -F'[_.]' '{print $(NF-1)}')
         #Run weighted average script, save only that
-        python3 $AVG_LOC $NAME ${arr[1]} $CALCID $expoSuffix loss
-        rm -f temp/avg_losses-rlz-???_${CALCID}.csv temp/realizations_${CALCID}.csv
+        python3 $AVG_LOC "$NAME" "${arr[1]}" "$CALCID" $expoSuffix loss
+        rm -f temp/avg_losses-rlz-???_"${CALCID}".csv temp/realizations_"${CALCID}".csv
     else
         # RUN A SINGLE CALC
-        oq engine --run ${JOBDIR}/s_Risk_${NAME}_${arr[0]}_${expoSuffix}.ini &> ./${OUTDIR}/s_Risk_${NAME}_${arr[0]}_${expoSuffix}.log;
-        oq export avg_losses-rlzs -1 -e csv -d temp;
-        CALCID=`ls -t temp/avg_losses* | head -1 | awk -F'[_.]' '{print $(NF-1)}'`
+        oq engine --run "${JOBDIR}/s_Risk_${NAME}_${arr[0]}_${expoSuffix}.ini" 2>&1 | tee "./${OUTDIR}/s_Risk_${NAME}_${arr[0]}_${expoSuffix}.log"
+        oq export avg_losses-rlzs -1 -e csv -d temp
+        CALCID=$(ls -t temp/avg_losses* | head -1 | awk -F'[_.]' '{print $(NF-1)}')
         #Run weighted average script, save only that
-        python3 $AVG_LOC $NAME ${arr[0]} $CALCID $expoSuffix loss
-        rm -f temp/avg_losses-rlz-???_${CALCID}.csv temp/realizations_${CALCID}.csv
+        python3 $AVG_LOC "$NAME" "${arr[0]}" "$CALCID" $expoSuffix loss
+        rm -f temp/avg_losses-rlz-???_"${CALCID}".csv temp/realizations_"${CALCID}".csv
     fi
 fi
 
@@ -249,11 +258,11 @@ exit 0
 # =================================================================
 # Spit out the 4D's
 # =================================================================
-DOLLARS=`awk -F',' 'dollars=$22+$24+$28 {sum += dollars} END {printf "%f",sum}' ${RISKFILE}`
-DEATH=`awk -F',' '{sum += $26} END {printf "%f",sum}' ${RISKFILE}`
-REDTAG=`awk -F',' 'dmg=$28+$30 {sum =+ dmg} END {printf "%f",sum}' ${DMGFILE}`
-DOWNTIME=`awk -F',' '{sum += $12} END {printf "%f",sum/NR}' ${CONSFILE}`
-echo 'Dollars = $'"${DOLLARS}"
+DOLLARS=$(awk -F',' 'dollars=$22+$24+$28 {sum += dollars} END {printf "%f",sum}' "${RISKFILE}")
+DEATH=$(awk -F',' '{sum += $26} END {printf "%f",sum}' "${RISKFILE}")
+REDTAG=$(awk -F',' 'dmg=$28+$30 {sum =+ dmg} END {printf "%f",sum}' "${DMGFILE}")
+DOWNTIME=$(awk -F',' '{sum += $12} END {printf "%f",sum/NR}' "${CONSFILE}")
+echo "Dollars = \$${DOLLARS}"
 echo "Deaths = ${DEATH}"
 echo "Damage = ${REDTAG} red tagged buildings"
 echo "Downtime = ${DOWNTIME} days, on average"