Addition of python script to add yaml-based test option + Creation of FV3 diag table documentation #2277 + SKEB fix with d_con = zero #2374

.github/linters/.yaml-lint.yml

@@ -0,0 +1,28 @@
+---
+
+extends: default
+
+rules:
+  braces:
+    level: warning
+    max-spaces-inside: 1
+  brackets:
+    level: warning
+    max-spaces-inside: 1
+  colons:
+    level: warning
+  commas: disable
+  comments: disable
+  comments-indentation: disable
+  document-start: disable
+  empty-lines:
+    level: warning
+  hyphens:
+    level: warning
+  indentation:
+    level: warning
+    indent-sequences: consistent
+  line-length: disable
+  truthy: disable
+  trailing-spaces: disable
+

.github/workflows/aux.yml

@@ -18,43 +18,42 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-    - name: Share helper id
-      run: echo -n ${{ github.run_id }} >~/id_file
+      - name: Share helper id
+        run: echo -n ${{ github.run_id }} >~/id_file
 
-    - uses: actions/cache@v2
-      with:
-        path: ~/id_file
-        key: helperid-${{ github.event.workflow_run.id }}
+      - uses: actions/cache@v2
+        with:
+          path: ~/id_file
+          key: helperid-${{ github.event.workflow_run.id }}
 
   repocheck:
     name: Repo check
     runs-on: ubuntu-20.04
 
     steps:
-    - name: Check up-to-dateness and post comment
-      run: |
-        trap 'echo "exit-code=$?" >> "$GITHUB_OUTPUT"' EXIT
-        head_sha=${{ github.event.pull_request.head.sha }}
-        head_brc=${{ github.event.pull_request.head.ref }}
-        head_url=${{ github.event.pull_request.head.repo.html_url }}
-        git clone -q -b $head_brc $head_url .
-        git checkout -q $head_sha
-        git submodule -q update --init --recursive
-        cd ${{ github.workspace }}/tests/ci
-        url=$GITHUB_API_URL/repos/$GITHUB_REPOSITORY
-        pr_number=$(curl -sS -H $app $url/pulls \
-          | jq -r '.[] | select(.head.sha == "'"$head_sha"'") | .number')
-        echo "pr_number is $pr_number"
-        pr_uid=${{ github.event.pull_request.head.repo.owner.login }}
-        echo "pr_uid is $pr_uid"
-        ./repo_check.sh
-        #comment="$(./repo_check.sh 2>/dev/null)"
-        #echo "comment is $comment"
-        #if [[ -n $comment ]]; then
-        #  curl -sS -X POST -H $app -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-        #    $url/issues/$pr_number/comments -d '{"body": "'"${comment}"'"}'
-        #  echo -n "failure" >~/repocheck_file
-        #else
-        #  echo -n "success" >~/repocheck_file
-        #fi
-
+      - name: Check up-to-dateness and post comment
+        run: |
+          trap 'echo "exit-code=$?" >> "$GITHUB_OUTPUT"' EXIT
+          head_sha=${{ github.event.pull_request.head.sha }}
+          head_brc=${{ github.event.pull_request.head.ref }}
+          head_url=${{ github.event.pull_request.head.repo.html_url }}
+          git clone -q -b $head_brc $head_url .
+          git checkout -q $head_sha
+          git submodule -q update --init --recursive
+          cd ${{ github.workspace }}/tests/ci
+          url=$GITHUB_API_URL/repos/$GITHUB_REPOSITORY
+          pr_number=$(curl -sS -H $app $url/pulls \
+            | jq -r '.[] | select(.head.sha == "'"$head_sha"'") | .number')
+          echo "pr_number is $pr_number"
+          pr_uid=${{ github.event.pull_request.head.repo.owner.login }}
+          echo "pr_uid is $pr_uid"
+          ./repo_check.sh
+          #comment="$(./repo_check.sh 2>/dev/null)"
+          #echo "comment is $comment"
+          #if [[ -n $comment ]]; then
+          #  curl -sS -X POST -H $app -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+          #    $url/issues/$pr_number/comments -d '{"body": "'"${comment}"'"}'
+          #  echo -n "failure" >~/repocheck_file
+          #else
+          #  echo -n "success" >~/repocheck_file
+          #fi

.github/workflows/superlinter.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Super-Linter
         uses: super-linter/[email protected]
         env:
-          LINTER_RULES_PATH: /
+          LINTER_RULES_PATH: '.github/linters/'
           DEFAULT_BRANCH: origin/develop
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           FILTER_REGEX_EXCLUDE: .*(tests/fv3_conf/.*|tests/ci/.*|tests/auto/.*|tests/auto-jenkins/.*|tests/opnReqTests/.*|tests/opnReqTest|tests/atparse.bash).*
@@ -33,5 +33,5 @@ jobs:
           #VALIDATE_GITHUB_ACTIONS: true
           #VALIDATE_LUA: true
           #VALIDATE_MARKDOWN: true
-          #VALIDATE_PYTHON_PYLINT: true
-          #VALIDATE_YAML: true
+          VALIDATE_PYTHON_PYLINT: true
+          VALIDATE_YAML: true

tests-dev/baseline_setup.yaml

@@ -0,0 +1,104 @@
+hera:
+  QUEUE: batch
+  COMPILE_QUEUE: batch
+  PARTITION:
+  dprefix: /scratch1/NCEPDEV
+  DISKNM: /scratch2/NAGAPE/epic/UFS-WM_RT
+  STMP: /scratch1/NCEPDEV/stmp4
+  PTMP: /scratch1/NCEPDEV/stmp2
+  RUNDIR_ROOT:
+  SCHEDULER: slurm
+  INPUTDATA_ROOT: /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /scratch2/NAGAPE/epic/UFS-WM_RT/NEMSfv3gfs/BM_IC-20220207
+orion:
+  QUEUE: batch
+  COMPILE_QUEUE: batch
+  PARTITION: orion
+  dprefix: /work/noaa/stmp/${USER}
+  DISKNM: /work/noaa/epic/UFS-WM_RT
+  STMP: /work/noaa/stmp/${USER}/stmp
+  PTMP: /work/noaa/stmp/${USER}/stmp
+  RUNDIR_ROOT:
+  SCHEDULER: slurm
+  INPUTDATA_ROOT: /work/noaa/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /work/noaa/epic/UFS-WM_RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /work/noaa/epic/UFS-WM_RT/NEMSfv3gfs/BM_IC-20220207  
+hercules:
+  QUEUE: batch
+  COMPILE_QUEUE: batch
+  PARTITION: hercules
+  dprefix: /work2/noaa/stmp/${USER}
+  DISKNM: /work/noaa/epic/hercules/UFS-WM_RT
+  STMP: /work/noaa/epic/hercules/UFS-WM_RT/stmp
+  PTMP: /work/noaa/epic/hercules/UFS-WM_RT/stmp
+  RUNDIR_ROOT:
+  SCHEDULER: slurm
+  INPUTDATA_ROOT: /work/noaa/epic/hercules/UFS-WM_RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /work/noaa/epic/hercules/UFS-WM_RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /work/noaa/epic/hercules/UFS-WM_RT/NEMSfv3gfs/BM_IC-20220207
+gaea:
+  QUEUE: normal
+  COMPILE_QUEUE: normal
+  PARTITION: c5
+  dprefix: /gpfs/f5/${ACCNR}/scratch/${USER}
+  DISKNM: /gpfs/f5/epic/world-shared/UFS-WM_RT
+  STMP: /gpfs/f5/${ACCNR}/scratch/${USER}/RT_BASELINE
+  PTMP: /gpfs/f5/${ACCNR}/scratch/${USER}/RT_RUNDIR
+  RUNDIR_ROOT:
+  SCHEDULER: slurm
+  INPUTDATA_ROOT: /gpfs/f5/epic/world-shared/UFS-WM_RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /gpfs/f5/epic/world-shared/UFS-WM_RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /gpfs/f5/epic/world-shared/UFS-WM_RT/NEMSfv3gfs/BM_IC-20220207
+jet:
+  QUEUE: batch
+  COMPILE_QUEUE: batch
+  PARTITION: xjet
+  dprefix: /mnt/lfs4/HFIP/hfv3gfs/${USER}
+  DISKNM: /mnt/lfs4/HFIP/hfv3gfs/role.epic/RT
+  STMP: /mnt/lfs4/HFIP/hfv3gfs/${USER}/RT_BASELINE
+  PTMP: /mnt/lfs4/HFIP/hfv3gfs/${USER}/RT_RUNDIRS
+  RUNDIR_ROOT:
+  SCHEDULER: slurm
+  INPUTDATA_ROOT: /mnt/lfs4/HFIP/hfv3gfs/role.epic/RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /mnt/lfs4/HFIP/hfv3gfs/role.epic/RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /mnt/lfs4/HFIP/hfv3gfs/role.epic/RT/NEMSfv3gfs/BM_IC-20220207
+derecho:
+  QUEUE: main
+  COMPILE_QUEUE: main
+  PARTITION:
+  dprefix: /glade/derecho/scratch
+  DISKNM: /glade/derecho/scratch/epicufsrt/ufs-weather-model/RT
+  STMP: /glade/derecho/scratch
+  PTMP: /glade/derecho/scratch
+  RUNDIR_ROOT:
+  SCHEDULER: pbs
+  INPUTDATA_ROOT: /glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /glade/derecho/scratch/epicufsrt/ufs-weather-model/RT/NEMSfv3gfs/BM_IC-20220207
+noaacloud:
+  QUEUE: batch
+  COMPILE_QUEUE: batch
+  PARTITION:
+  dprefix: /lustre
+  DISKNM: /contrib/ufs-weather-model/RT
+  STMP: /lustre/stmp4
+  PTMP: /lustre/stmp2
+  RUNDIR_ROOT:
+  SCHEDULER: slurm
+  INPUTDATA_ROOT: /contrib/ufs-weather-model/RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /contrib/ufs-weather-model/RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /contrib/ufs-weather-model/RT/NEMSfv3gfs/BM_IC-20220207
+s4:
+  QUEUE: s4
+  COMPILE_QUEUE: s4
+  PARTITION: s4
+  dprefix: /data/prod
+  DISKNM: /data/prod/emc.nemspara/RT
+  STMP: /scratch/short/users
+  PTMP: /scratch/users
+  RUNDIR_ROOT:
+  SCHEDULER: slurm
+  INPUTDATA_ROOT: /data/prod/emc.nemspara/RT/NEMSfv3gfs/input-data-20240501
+  INPUTDATA_ROOT_WW3: /data/prod/emc.nemspara/RT/NEMSfv3gfs/input-data-20240501/WW3_input_data_20240214
+  INPUTDATA_ROOT_BMIC: /data/prod/emc.nemspara/RT/NEMSfv3gfs/BM_IC-20220207