Update utest and ci

.github/workflows/build_test.yml

@@ -111,17 +111,10 @@ jobs:
         sudo docker rmi $(sudo docker image ls | grep -E -m1 '<none>' | awk '{ print $3 }')
         sudo docker rmi $(sudo docker image ls | awk '/ubuntu-hpc/ { print $3 }')
 
-    - name: Prepare artifacts
-      run: |
-        cd tests/ci
-        sudo docker save ${{ needs.setup.outputs.img }} \
-          | gzip >${{ needs.setup.outputs.img }}.tar.gz
-        tar cvjf artifact.tar.bz2 ${{ needs.setup.outputs.img }}.tar.gz ci.sh ci.test
-
     - uses: actions/upload-artifact@v2
       with:
-        name: ${{ matrix.bld_set }}.artifact.tar.bz2
-        path: tests/ci/artifact.tar.bz2
+        name: ${{ matrix.bld_set }}.artifact
+        path: ~/fv3.tar.gz
 
 
   wait:
@@ -162,24 +155,24 @@ jobs:
     steps:
     - name: Clean up in case of left-over files
       run: |
-        rm -f artifact.tar.bz2 ${{ needs.setup.outputs.img }}.tar.gz
+        rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
         docker ps -a --filter "name=my-container" | grep my-container >/dev/null 2>&1 \
           && docker rm my-container >/dev/null 2>&1 && d=$? || d=$?
-        docker image ls | grep ${{ needs.setup.outputs.img }} >/dev/null 2>&1 \
-          && docker rmi ${{ needs.setup.outputs.img }} >/dev/null 2>&1 && d=$? || d=$?
 
-    - uses: actions/download-artifact@v2
+    - uses: actions/checkout@v2
       with:
-        name: ${{ matrix.artifact }}.artifact.tar.bz2
+        submodules: recursive
 
-    - name: Prepare artifacts
-      run: |
-        tar xvjf artifact.tar.bz2 && rm -f artifact.tar.bz2
-        docker load --input ${{ needs.setup.outputs.img }}.tar.gz \
-          && rm -f ${{ needs.setup.outputs.img }}.tar.gz
+    - uses: actions/download-artifact@v2
+      with:
+        name: ${{ matrix.artifact }}.artifact
+        path: ${{ github.workspace}}/tests
 
     - name: Run utest
-      run: ./ci.sh -n ${{ matrix.name }} -r ${{ matrix.case }}
+      run: |
+        cd ${{ github.workspace }}/tests && tar -xvzf fv3.tar.gz && rm -f fv3.tar.gz
+        cd ${{ github.workspace }}/tests/ci
+        ./ci.sh -n ${{ matrix.name }} -r ${{ matrix.case }}
 
     - uses: actions/upload-artifact@v2
       if: ${{ always() }}
@@ -190,8 +183,5 @@ jobs:
     - name: Clean up
       if: ${{ always() }}
       run: |
-        rm -f ci.sh ci.test
-        docker stop my-container && docker rm  my-container \
-          && docker rmi ${{ needs.setup.outputs.img }}:latest
+        docker stop my-container && docker rm my-container
         docker volume rm DataVolume
-        rm -f memory_stat

tests/ci/Dockerfile

@@ -1,4 +1,4 @@
-From noaaemc/ubuntu-hpc:v1.3
+From noaaemc/ubuntu-hpc:v1.3b
 
 ENV HOME=/home/builder
 COPY --chown=builder:builder . $HOME/ufs-weather-model
@@ -15,5 +15,5 @@ ENV RT_COMPILER=gnu
 ENV RT_MACHINE=linux.gnu
 
 WORKDIR $HOME/ufs-weather-model/tests
-RUN ./utest -n $test_name -c $build_case -z
+RUN ./utest -n $test_name -c $build_case -z && tar -cvzf fv3.tar.gz fv3_*.exe modules.fv3_*
 CMD ./utest -n $test_name -c $test_case -x

tests/ci/ci.sh

@@ -62,17 +62,30 @@ fi
 
 if [ $BUILD = "true" ]; then
 
-  sudo docker build --build-arg test_name=$TEST_NAME \
-                    --build-arg build_case=$BUILD_CASE \
-                    --no-cache \
-                    --squash --compress \
-                    -f Dockerfile -t ${IMG_NAME} ../..
-  exit $?
+  docker build --build-arg test_name=$TEST_NAME \
+               --build-arg build_case=$BUILD_CASE \
+               --no-cache \
+               --squash --compress \
+               -f Dockerfile -t ${IMG_NAME} ../..
+
+  docker create --name tmp-container ${IMG_NAME}
+  docker cp -a tmp-container:/home/builder/ufs-weather-model/tests/fv3.tar.gz ~
+  docker rm tmp-container
 
 elif [ $RUN == "true" ]; then
 
-  docker run -d --rm -v DataVolume:/tmp minsukjinoaa/fv3-input-data:input-data-20210115
-  docker run -d -e test_case=${TEST_CASE} --shm-size=512m -v DataVolume:/home/builder/data/NEMSfv3gfs/input-data-20210115 --name my-container ${IMG_NAME}
+  docker run -d --rm -v DataVolume:/tmp minsukjinoaa/input-data:20210428
+
+  docker create -u builder -e "CI_TEST=true" -e "USER=builder" \
+                -e "RT_MACHINE=linux.gnu" -e "RT_COMPILER=gnu" \
+                -w "/home/builder/ufs-weather-model/tests" \
+                -v DataVolume:/home/builder/data/NEMSfv3gfs/input-data-20210428 \
+                --shm-size=512m --name my-container noaaemc/ubuntu-hpc:v1.3b \
+                /bin/bash -c "./utest -n ${TEST_NAME} -c ${TEST_CASE} -x"
+
+  cd $GITHUB_WORKSPACE
+  docker cp . my-container:/home/builder/ufs-weather-model
+  docker start my-container
 
   echo 'cache,rss,shmem' >memory_stat
   sleep 3

tests/ci/ci.test

@@ -1,2 +1,4 @@
 fv3_control
 thr mpi dcp rst bit dbg
+fv3_regional_control
+dcp thr

tests/ci/repo_check.sh

@@ -56,7 +56,7 @@ datm[branch]='develop'
 datm[dir]='DATM'
 
 cdeps[repo]='https://github.com/NOAA-EMC/CDEPS'
-cdeps[branch]='emc/develop'
+cdeps[branch]='develop'
 cdeps[dir]='CDEPS-interface/CDEPS'
 
 cmake[repo]='https://github.com/NOAA-EMC/CMakeModules'

tests/default_vars.sh

@@ -169,15 +169,19 @@ elif [[ $MACHINE_ID = hera.* ]]; then
 
 elif [[ $MACHINE_ID = linux.* ]]; then
 
-  if [[ $CI_TEST = true ]]; then
-  TASKS_dflt=42 ; TPN_dflt=48 ; INPES_dflt=3 ; JNPES_dflt=2
-  else
   TASKS_dflt=150 ; TPN_dflt=40 ; INPES_dflt=3 ; JNPES_dflt=8
-  fi
   TASKS_thrd=84  ; TPN_thrd=20 ; INPES_thrd=3 ; JNPES_thrd=4
   TASKS_stretch=48 ; TPN_stretch=12 ; INPES_stretch=2 ; JNPES_stretch=4
   TASKS_strnest=96 ; TPN_strnest=12 ; INPES_strnest=2 ; JNPES_strnest=4
 
+  TASKS_cpl_dflt=192; TPN_cpl_dflt=40; INPES_cpl_dflt=3; JNPES_cpl_dflt=8
+  THRD_cpl_dflt=1; WPG_cpl_dflt=6;  MPB_cpl_dflt="0 143"; APB_cpl_dflt="0 149"
+  OPB_cpl_dflt="150 179"; IPB_cpl_dflt="180 191"
+
+  TASKS_cpl_thrd=120; TPN_cpl_thrd=40; INPES_cpl_thrd=3; JNPES_cpl_thrd=4
+  THRD_cpl_thrd=2; WPG_cpl_thrd=6;  MPB_cpl_thrd="0 77";  APB_cpl_thrd="0 77"
+  OPB_cpl_thrd="78 107";  IPB_cpl_thrd="108 119"
+
 elif [[ $MACHINE_ID = jet.* ]]; then
 
   TASKS_dflt=150 ; TPN_dflt=24 ; INPES_dflt=3 ; JNPES_dflt=8

tests/fv3_conf/cpld_control_run.IN

@@ -50,6 +50,8 @@ if [[ $CPLWAV == .T. ]]; then
   cp    @[INPUTDATA_ROOT_WW3]/mod_def.* .
 fi
 
+UNIT_TEST=${UNIT_TEST:-false}
+SUFFIX=${RT_SUFFIX}
 # No restart
 if [ $WARM_START = .F. ]; then
   # ICs
@@ -65,14 +67,18 @@ if [ $WARM_START = .F. ]; then
 
 # Restart
 else
+  if [[ ${UNIT_TEST} == true ]]; then
+    SUFFIX=${BL_SUFFIX}
+  fi
+
   # Restart files
-  cp -r ../${DEP_RUN}${RT_SUFFIX}/RESTART/${RESTART_FILE_PREFIX}.* ./INPUT
+  cp -r ../${DEP_RUN}${SUFFIX}/RESTART/${RESTART_FILE_PREFIX}.* ./INPUT
   rm -f INPUT/fv_core.res.*
   rm -f INPUT/fv_srf_wnd.res.*
   rm -f INPUT/fv_tracer.res.*
   rm -f INPUT/phy_data.*
   rm -f INPUT/sfc_data.*
-  for RFILE in ../${DEP_RUN}${RT_SUFFIX}/RESTART/${RESTART_FILE_PREFIX}.*; do
+  for RFILE in ../${DEP_RUN}${SUFFIX}/RESTART/${RESTART_FILE_PREFIX}.*; do
     [ -e $RFILE ] || exit 1
     RFILE_OLD=$(basename $RFILE)
     RFILE_NEW="${RFILE_OLD//${RESTART_FILE_PREFIX}./}"
@@ -81,21 +87,21 @@ else
 
   #if not mx025, then mom6 restart is a single file
   if [[ $OCNRES == '025' ]]; then
-   cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00.nc ./INPUT/MOM.res.nc
-   cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00_1.nc ./INPUT/MOM.res_1.nc
-   cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00_2.nc ./INPUT/MOM.res_2.nc
-   cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00_3.nc ./INPUT/MOM.res_3.nc
+   cp ../${DEP_RUN}${SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00.nc ./INPUT/MOM.res.nc
+   cp ../${DEP_RUN}${SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00_1.nc ./INPUT/MOM.res_1.nc
+   cp ../${DEP_RUN}${SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00_2.nc ./INPUT/MOM.res_2.nc
+   cp ../${DEP_RUN}${SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00_3.nc ./INPUT/MOM.res_3.nc
   else
-   cp ../${DEP_RUN}${RT_SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00.nc ./INPUT/MOM.res.nc
+   cp ../${DEP_RUN}${SUFFIX}/RESTART/MOM.res.${RESTART_FILE_SUFFIX_HRS}-00-00.nc ./INPUT/MOM.res.nc
   fi
 
   # CMEPS restart and pointer files
   RFILE="ufs.cpld.cpl.r.${RESTART_FILE_SUFFIX_SECS}.nc"
-  cp  ../${DEP_RUN}${RT_SUFFIX}/RESTART/${RFILE} .
+  cp  ../${DEP_RUN}${SUFFIX}/RESTART/${RFILE} .
   ls -1 ${RFILE}>rpointer.cpl
 
   # CICE restart and pointer files
   RFILE="iced.${RESTART_FILE_SUFFIX_SECS}.nc"
-  cp  ../${DEP_RUN}${RT_SUFFIX}/RESTART/${RFILE} ./INPUT
+  cp  ../${DEP_RUN}${SUFFIX}/RESTART/${RFILE} ./INPUT
   ls -1 "./INPUT/"${RFILE}>ice.restart_file
 fi

tests/run_test.sh

@@ -156,7 +156,7 @@ if [[ $SCHEDULER = 'none' ]]; then
 
   ulimit -s unlimited
   if [[ $CI_TEST = 'true' ]]; then
-    eval mpiexec -n ${TASKS} ${MPI_PROC_BIND} ./fv3.exe >out 2> >(tee err >&3)
+    eval ${OMP_ENV} mpiexec -n ${TASKS} ${MPI_PROC_BIND} ./fv3.exe >out 2> >(tee err >&3)
   else
     mpiexec -n ${TASKS} ./fv3.exe >out 2> >(tee err >&3)
   fi