[develop] Enable UPP 2d decomposition

.cicd/Jenkinsfile

@@ -14,11 +14,10 @@ pipeline {
         // Use the line below to enable the PW AWS cluster
         // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
         // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
-        // Uncomment the following line to re-enable comprehensive tests
-        // booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
+        booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests'
     }
 
     stages {
@@ -89,7 +88,7 @@ pipeline {
                     axis {
                         name 'SRW_PLATFORM'
                         // values 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
-                        values 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        values 'derecho', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
 
                     axis {
@@ -103,7 +102,7 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            values 'gaea', 'gaea-c5', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                            values 'derecho', 'gaea', 'gaea-c5', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                         }
 
                         axis {
@@ -184,11 +183,6 @@ pipeline {
                             dir ("${env.SRW_PLATFORM}") {
                             echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})"
 
-                            // Remove the following line to re-enable comprehensive tests
-                            sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"'
-
-                            // Uncomment the following block to re-enable comprehensive tests
-                            /*
                             // If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set,
                             // override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter
                             script {
@@ -205,7 +199,6 @@ pipeline {
 
                                 sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"'
                             }
-                            */
                             }
                         }
 

Externals.cfg

@@ -3,7 +3,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/UFS_UTILS
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 6a7d534
+hash = dc0e4a6
 local_path = sorc/UFS_UTILS
 required = True
 
@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 788897d
+hash = 020e783
 local_path = sorc/ufs-weather-model
 required = True
 
@@ -21,7 +21,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/UPP
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = baa7751
+hash = fae617b
 local_path = sorc/UPP
 required = True
 

docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst

@@ -1145,6 +1145,10 @@ These parameters set values in the Weather Model's ``model_configure`` file.
 ``WRITE_DOPOST``: (Default: false)
    Flag that determines whether to use the inline post option, which calls the Unified Post Processor (:term:`UPP`) from within the UFS Weather Model. The default ``WRITE_DOPOST: false`` does not use the inline post functionality, and the ``run_post`` tasks are called from outside of the UFS Weather Model. If ``WRITE_DOPOST: true``, the ``WRITE_DOPOST`` flag in the ``model_configure`` file will be set to true, and the post-processing (:term:`UPP`) tasks will be called from within the Weather Model. This means that the post-processed files (in :term:`grib2` format) are output by the Weather Model at the same time that it outputs the ``dynf###.nc`` and ``phyf###.nc`` files. Setting ``WRITE_DOPOST: true`` turns off the separate ``run_post`` task in ``setup.py`` to avoid unnecessary computations. Valid values: ``True`` | ``False``
 
+``ITASKS``: (Default: 1)
+   Variable denoting the number of write tasks in the ``i`` direction in the current group. Used for inline post 2D decomposition. Setting this variable to a value greater than 1 will enable 2D decomposition.
+   Note that 2D decomposition does not yet work with GNU compilers, so this value will be reset to 1 automatically when using GNU compilers (i.e., when ``COMPILER: gnu``).
+
 .. _CompParams:
 
 Computational Parameters
@@ -1286,10 +1290,10 @@ Customized Post Configuration Parameters
 
 Set parameters for customizing the :term:`UPP`.
 
-``USE_CUSTOM_POST_CONFIG_FILE``: (Default: false)
+``USE_CUSTOM_POST_CONFIG_FILE``: (Default: true)
    Flag that determines whether a user-provided custom configuration file should be used for post-processing the model data. If this is set to true, then the workflow will use the custom post-processing (:term:`UPP`) configuration file specified in ``CUSTOM_POST_CONFIG_FP``. Otherwise, a default configuration file provided in the UPP repository will be used. Valid values: ``True`` | ``False``
 
-``CUSTOM_POST_CONFIG_FP``: (Default: "")
+``CUSTOM_POST_CONFIG_FP``: (Default: ``"{{ user.SORCdir }}/ufs-weather-model/tests/parm/postxconfig-NT-fv3lam.txt"``)
    The full path to the custom post flat file, including filename, to be used for post-processing. This is only used if ``CUSTOM_POST_CONFIG_FILE`` is set to true.
 
 ``POST_OUTPUT_DOMAIN_NAME``: (Default: ``'{{ workflow.PREDEF_GRID_NAME }}'``)
@@ -1304,6 +1308,10 @@ Set parameters for customizing the :term:`UPP`.
 ``TESTBED_FIELDS_FN``: (Default: "")
    The file that lists grib2 fields to be extracted for testbed files. An empty string means no need to generate testbed files.
 
+``NUMX``: (Default: 1)
+   The number of ``i`` regions in a 2D decomposition. Each ``i`` row is distributed to ``NUMX`` ranks. Set ``NUMX`` to a value greater than 1 to enable 2D decomposition.
+   Note that 2D decomposition does not yet work with GNU compilers, so this value will be reset to 1 automatically when using GNU compilers (i.e., when ``COMPILER: gnu``).
+
 RUN_PRDGEN Configuration Parameters
 =====================================
 

modulefiles/build_derecho_intel.lua

@@ -3,7 +3,7 @@ This module loads libraries for building the UFS SRW App on
 the CISL machine Derecho (Cray) using Intel-classic-2023.0.0
 ]])
 
-whatis([===[Loads libraries needed for building the UFS SRW App on Cheyenne ]===])
+whatis([===[Loads libraries needed for building the UFS SRW App on Derecho ]===])
 
 load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3"))
 load(pathJoin("ncarenv", os.getenv("ncarenv_ver") or "23.06"))

parm/model_configure

@@ -21,6 +21,7 @@ jchunk2d:                -1
 ichunk3d:                -1
 jchunk3d:                -1
 kchunk3d:                -1
+itasks:                  {{ itasks }}
 quilting:                {{ quilting }}
 {% if quilting  %}
 #

scripts/exregional_run_post.sh

@@ -108,15 +108,15 @@ fi
 cp_vrfy ${post_config_fp} ./postxconfig-NT.txt
 cp_vrfy ${PARMdir}/upp/params_grib2_tbl_new .
 if [ ${USE_CRTM} = "TRUE" ]; then
-  cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Water/Big_Endian/Nalli.IRwater.EmisCoeff.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/MW_Water/Big_Endian/FAST*.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Land/SEcategory/Big_Endian/NPOESS.IRland.EmisCoeff.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Snow/SEcategory/Big_Endian/NPOESS.IRsnow.EmisCoeff.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Ice/SEcategory/Big_Endian/NPOESS.IRice.EmisCoeff.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/AerosolCoeff/Big_Endian/AerosolCoeff.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/CloudCoeff/Big_Endian/CloudCoeff.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/SpcCoeff/Big_Endian/*.bin ./
-  cp_vrfy ${CRTM_DIR}/fix/TauCoeff/ODPS/Big_Endian/*.bin ./
+  cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./
+  cp_vrfy ${CRTM_DIR}/FAST*.bin ./
+  cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./
+  cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./
+  cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./
+  cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./
+  cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./
+  cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./
+  cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./
   print_info_msg "
 ====================================================================
 Copying the external CRTM fix files from CRTM_DIR to the temporary
@@ -201,7 +201,7 @@ fileNameFlux='${phy_file}'
 /
 
  &NAMPGB
- KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,${post_itag_add}
+ KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,${post_itag_add},numx=${NUMX}
  /
 EOF
 #

tests/WE2E/run_WE2E_tests.py

@@ -228,6 +228,15 @@ def run_we2e_tests(homedir, args) -> None:
             test_aqm_input_basedir = machine_defaults['platform']['TEST_AQM_INPUT_BASEDIR']
             test_cfg['cpl_aqm_parm']['DCOMINfire_default'] = f"{test_aqm_input_basedir}/RAVE_fire"
 
+        if args.compiler == "gnu":
+            # 2D decomposition doesn't work with GNU compilers.  Deactivate 2D decomposition for GNU
+            if 'task_run_post' in test_cfg:
+                test_cfg['task_run_post'].update({"NUMX": 1})
+                logging.info(f"NUMX has been reset to 1 due to issues encountered with GNU compilers")
+            if 'task_run_fcst' in test_cfg:
+                test_cfg['task_run_fcst'].update({"ITASKS": 1})
+                logging.info(f"ITASKS has been reset to 1 due to issues encountered with GNU compilers")
+
         logging.debug(f"Writing updated config.yaml for test {test_name}\n"\
                        "based on specified command-line arguments:\n")
         logging.debug(cfg_to_yaml_str(test_cfg))

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml

@@ -4,6 +4,7 @@ metadata:
     completes successfully on the RRFS_CONUS_25km grid using the GFS_v15p2
     physics suite with ICs and LBCs derived from the FV3GFS.
     In addition, this tests the "inline post" option (WRITE_DOPOST: true)
+    and enables UPP inline 2D decomposition.
 user:
   RUN_ENVIR: community
 workflow:
@@ -22,3 +23,4 @@ task_get_extrn_lbcs:
   USE_USER_STAGED_EXTRN_FILES: true
 task_run_fcst:
   WRITE_DOPOST: true
+  ITASKS: 2

tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml

@@ -3,7 +3,7 @@ metadata:
     This test is to ensure that the workflow running in community mode
     completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1beta
     physics suite with ICs derived from the HRRR and LBCs derived from the RAP.
-    Also tests the "DOT_OR_USCORE" option
+    It also tests the "DOT_OR_USCORE" option and enables offline UPP 2D decomposition.
 user:
   RUN_ENVIR: community
 workflow:
@@ -25,3 +25,5 @@ task_get_extrn_lbcs:
   USE_USER_STAGED_EXTRN_FILES: true
   EXTRN_MDL_FILES_LBCS:
     - '{yy}{jjj}{hh}00{fcst_hr:02d}00'
+task_run_post:
+  NUMX: 2

tests/test_python/test_create_model_configure_file.py

@@ -47,6 +47,7 @@ def setUp(self):
         set_env_var("FHROT", 0)
         set_env_var("DT_ATMOS", 1)
         set_env_var("RESTART_INTERVAL", 4)
+        set_env_var("ITASKS", 1)
 
         set_env_var("WRTCMP_write_groups", 1)
         set_env_var("WRTCMP_write_tasks_per_group", 2)

ush/config_defaults.yaml

@@ -1787,12 +1787,21 @@ task_run_fcst:
   # weather model].  If this is set to true, the the run_post task will
   # be deactivated.
   #
+  # ITASKS:
+  # Variable denoting the number of write tasks in the i direction in the
+  # current group.  Used for inline post 2D decomposition.  Setting this
+  # variable to a value greater than 1 will enable 2D decomposition.
+  # Default setting is 1.
+  # Note that 2D decomposition does not yet work with GNU compilers, so this value 
+  # will be reset to 1 automatically when using GNU compilers (i.e., when COMPILER: gnu).
+  #
   #-----------------------------------------------------------------------
   #
   DT_ATMOS: ""
   FHROT: 0
   RESTART_INTERVAL: 0
   WRITE_DOPOST: false
+  ITASKS: 1
   #
   #-----------------------------------------------------------------------
   #
@@ -2067,12 +2076,19 @@ task_run_post:
   # The file that lists grib2 fields to be extracted for testbed files.
   # An empty string means no need to generate testbed files.
   #
+  # NUMX:
+  # The number of i regions in a 2D decomposition.  Each i row is 
+  # distibuted to numx ranks. Default value of numx is 1.
+  # Note that 2D decomposition does not yet work with GNU compilers, so this value 
+  # will be reset to 1 automatically when using GNU compilers (i.e., when COMPILER: gnu).
+  #
   #-----------------------------------------------------------------------
   #
-  USE_CUSTOM_POST_CONFIG_FILE: false
-  CUSTOM_POST_CONFIG_FP: ""
+  USE_CUSTOM_POST_CONFIG_FILE: true
+  CUSTOM_POST_CONFIG_FP: "{{ user.SORCdir }}/ufs-weather-model/tests/parm/postxconfig-NT-fv3lam.txt"
   POST_OUTPUT_DOMAIN_NAME: '{{ workflow.PREDEF_GRID_NAME }}'
   TESTBED_FIELDS_FN: ""
+  NUMX: 1
 
 #----------------------------
 # RUN PRDGEN config parameters

ush/create_model_configure_file.py

@@ -79,6 +79,7 @@ def create_model_configure_file(
         "fhrot": fhrot,
         "dt_atmos": DT_ATMOS,
         "restart_interval": RESTART_INTERVAL,
+        "itasks": ITASKS,
         "write_dopost": f".{lowercase(str(WRITE_DOPOST))}.",
         "quilting": f".{lowercase(str(QUILTING))}.",
         "output_grid": WRTCMP_output_grid,

ush/machine/noaacloud.yaml

@@ -6,6 +6,7 @@ platform:
   TEST_MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc
   TEST_NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc
   TEST_NOHRSC_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/nohrsc/proc
+  DOMAIN_PREGEN_BASEDIR: /contrib/EPIC/UFS_SRW_data/develop/FV3LAM_pregen
   REMOVE_MEMORY: True
   RUN_CMD_FCST: mpiexec -np ${PE_MEMBER01}
   RUN_CMD_POST: mpiexec -np $nprocs