From 4d69a9e0725a80217d5ea7bfefd37126d3cbc153 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 14 Sep 2023 14:30:48 +0000 Subject: [PATCH 01/29] [develop] Update UFS-WM and UPP hashes to enable inline/offline post 2d decomposition. --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index b4a5840568..8101eb46aa 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = 788897d +hash = 1787f26 local_path = sorc/ufs-weather-model required = True @@ -21,7 +21,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = baa7751 +hash = 520cc23 local_path = sorc/UPP required = True From 79fb83393356e0bf981bd1889c00b93ea45c410b Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Mon, 18 Sep 2023 20:02:29 +0000 Subject: [PATCH 02/29] [develop] Update run_fcst, run_post, and plot_allvars scripts to use CRTM coefficients and add CRTM_DIR to Hera machine yaml file. --- scripts/exregional_plot_allvars.py | 2 +- scripts/exregional_plot_allvars_diff.py | 4 ++-- scripts/exregional_run_fcst.sh | 15 ++++++++++++- scripts/exregional_run_post.sh | 30 +++++++++++++------------ ush/config_defaults.yaml | 2 +- ush/machine/hera.yaml | 3 +++ 6 files changed, 37 insertions(+), 19 deletions(-) diff --git a/scripts/exregional_plot_allvars.py b/scripts/exregional_plot_allvars.py index 27eff0f4b0..a8c5611379 100755 --- a/scripts/exregional_plot_allvars.py +++ b/scripts/exregional_plot_allvars.py @@ -429,7 +429,7 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth = ndimage.gaussian_filter(slp, 13.78) # 2-m temperature diff --git a/scripts/exregional_plot_allvars_diff.py b/scripts/exregional_plot_allvars_diff.py index e51a3a6b57..64f6b1b66f 100755 --- a/scripts/exregional_plot_allvars_diff.py +++ b/scripts/exregional_plot_allvars_diff.py @@ -446,9 +446,9 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp_1 = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp_1 = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth_1 = ndimage.gaussian_filter(slp_1, 13.78) - slp_2 = data2.select(name="Pressure reduced to MSL")[0].values * 0.01 + slp_2 = data2.select(name="MSLP (Eta model reduction)")[0].values * 0.01 slpsmooth_2 = ndimage.gaussian_filter(slp_2, 13.78) slp_diff = slp_2 - slp_1 diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 6dfad9673b..931d4b65cf 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -410,7 +410,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then if [ "${CPL_AQM}" = "TRUE" ]; then post_config_fp="${PARMdir}/upp/postxconfig-NT-AQM.txt" else - post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam.txt" + post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam_rrfs.txt" fi print_info_msg " ==================================================================== @@ -420,6 +420,19 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cp_vrfy ${post_config_fp} ./postxconfig-NT_FH00.txt cp_vrfy ${post_config_fp} ./postxconfig-NT.txt cp_vrfy ${PARMdir}/upp/params_grib2_tbl_new . + if [ ${CPL_AQM} != "TRUE" ]; then + if [ ${USE_CRTM} = "TRUE" ]; then + cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/FAST*.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./ + fi + fi # Set itag for inline-post: if [ "${CPL_AQM}" = "TRUE" ]; then post_itag_add="aqf_on=.true.," diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 76a1196329..fb46176f2b 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -95,7 +95,7 @@ else if [ "${CPL_AQM}" = "TRUE" ]; then post_config_fp="${PARMdir}/upp/postxconfig-NT-AQM.txt" else - post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam.txt" + post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam_rrfs.txt" fi print_info_msg " ==================================================================== @@ -107,23 +107,25 @@ temporary work directory (DATA_FHR): fi cp_vrfy ${post_config_fp} ./postxconfig-NT.txt cp_vrfy ${PARMdir}/upp/params_grib2_tbl_new . -if [ ${USE_CRTM} = "TRUE" ]; then - cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Water/Big_Endian/Nalli.IRwater.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/MW_Water/Big_Endian/FAST*.bin ./ - cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Land/SEcategory/Big_Endian/NPOESS.IRland.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Snow/SEcategory/Big_Endian/NPOESS.IRsnow.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/fix/EmisCoeff/IR_Ice/SEcategory/Big_Endian/NPOESS.IRice.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/fix/AerosolCoeff/Big_Endian/AerosolCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/fix/CloudCoeff/Big_Endian/CloudCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/fix/SpcCoeff/Big_Endian/*.bin ./ - cp_vrfy ${CRTM_DIR}/fix/TauCoeff/ODPS/Big_Endian/*.bin ./ - print_info_msg " +if [ ${CPL_AQM} != "TRUE" ]; then + if [ ${USE_CRTM} = "TRUE" ]; then + cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/FAST*.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./ + print_info_msg " ==================================================================== Copying the external CRTM fix files from CRTM_DIR to the temporary work directory (DATA_FHR): - CRTM_DIR = \"${CRTM_DIR}\" - DATA_FHR = \"${DATA_FHR}\" + CRTM_DIR = \"${CRTM_DIR}\" + DATA_FHR = \"${DATA_FHR}\" ====================================================================" + fi fi # #----------------------------------------------------------------------- diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index ac15404277..43681bc391 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1933,7 +1933,7 @@ global: # #----------------------------------------------------------------------- # - USE_CRTM: false + USE_CRTM: true CRTM_DIR: "" # #----------------------------------------------------------------------- diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 545c9320af..ba733c9199 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -35,6 +35,9 @@ platform: FIXshp: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/NaturalEarth EXTRN_MDL_DATA_STORES: hpss aws nomads +global: + CRTM_DIR: /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/impi-2022.1.2/crtm/2.4.0/fix + cpl_aqm_parm: AQM_CONFIG_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/epa/data DCOMINbio_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio From 4e94841837c6dbfb1fe06394487fb5b78028923b Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 19 Sep 2023 20:43:58 +0000 Subject: [PATCH 03/29] [develop] Enable post 2d decomposition for both inline and offline post. Add itasks: 2 to the model_configure file for the grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 test for inline post 2d decomposition and add numx: 2 to the itag namelist option file for the grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta test for offline post 2d decomposition. --- parm/model_configure | 1 + scripts/exregional_run_fcst.sh | 22 +++++++------- scripts/exregional_run_post.sh | 30 +++++++++---------- ...fig.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml | 1 + ...cs_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml | 4 ++- ...m_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml | 4 ++- ush/config.aqm.community.yaml | 1 + ush/config.aqm.nco.realtime.yaml | 1 + ush/config_defaults.yaml | 12 ++++++++ ush/create_model_configure_file.py | 1 + 10 files changed, 47 insertions(+), 30 deletions(-) diff --git a/parm/model_configure b/parm/model_configure index 58ea7378d4..d22adf3f3a 100644 --- a/parm/model_configure +++ b/parm/model_configure @@ -21,6 +21,7 @@ jchunk2d: -1 ichunk3d: -1 jchunk3d: -1 kchunk3d: -1 +itasks: {{ itasks }} quilting: {{ quilting }} {% if quilting %} # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 931d4b65cf..21c4253224 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -420,18 +420,16 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cp_vrfy ${post_config_fp} ./postxconfig-NT_FH00.txt cp_vrfy ${post_config_fp} ./postxconfig-NT.txt cp_vrfy ${PARMdir}/upp/params_grib2_tbl_new . - if [ ${CPL_AQM} != "TRUE" ]; then - if [ ${USE_CRTM} = "TRUE" ]; then - cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/FAST*.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./ - fi + if [ ${USE_CRTM} = "TRUE" ]; then + cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/FAST*.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./ fi # Set itag for inline-post: if [ "${CPL_AQM}" = "TRUE" ]; then diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index fb46176f2b..795efa38f8 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -107,25 +107,23 @@ temporary work directory (DATA_FHR): fi cp_vrfy ${post_config_fp} ./postxconfig-NT.txt cp_vrfy ${PARMdir}/upp/params_grib2_tbl_new . -if [ ${CPL_AQM} != "TRUE" ]; then - if [ ${USE_CRTM} = "TRUE" ]; then - cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/FAST*.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./ - print_info_msg " +if [ ${USE_CRTM} = "TRUE" ]; then + cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/FAST*.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./ + cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./ + print_info_msg " ==================================================================== Copying the external CRTM fix files from CRTM_DIR to the temporary work directory (DATA_FHR): - CRTM_DIR = \"${CRTM_DIR}\" - DATA_FHR = \"${DATA_FHR}\" + CRTM_DIR = \"${CRTM_DIR}\" + DATA_FHR = \"${DATA_FHR}\" ====================================================================" - fi fi # #----------------------------------------------------------------------- @@ -203,7 +201,7 @@ fileNameFlux='${phy_file}' / &NAMPGB - KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,${post_itag_add} + KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,${post_itag_add},numx=${NUMX} / EOF # diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml index 789c5e9674..691c4b5725 100644 --- a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml @@ -49,6 +49,7 @@ task_run_post: POST_OUTPUT_DOMAIN_NAME: 793 USE_CUSTOM_POST_CONFIG_FILE: false global: + USE_CRTM: false DO_ENSEMBLE: false NUM_ENS_MEMBERS: 2 HALO_BLEND: 0 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml index d0d184d288..34d8724a4c 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml @@ -3,7 +3,8 @@ metadata: This test is to ensure that the workflow running in community mode completes successfully on the RRFS_CONUS_25km grid using the GFS_v15p2 physics suite with ICs and LBCs derived from the FV3GFS. - In addition, this tests the "inline post" option (WRITE_DOPOST: true) + In addition, this tests the "inline post" option (WRITE_DOPOST: true). + Enables UPP inline 2d decomposition. user: RUN_ENVIR: community workflow: @@ -22,3 +23,4 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true task_run_fcst: WRITE_DOPOST: true + ITASKS: 2 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml index 2a5124e9ae..0117c41f34 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml @@ -3,7 +3,7 @@ metadata: This test is to ensure that the workflow running in community mode completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1beta physics suite with ICs derived from the HRRR and LBCs derived from the RAP. - Also tests the "DOT_OR_USCORE" option + Also tests the "DOT_OR_USCORE" option. Enables offline UPP 2d decomposition. user: RUN_ENVIR: community workflow: @@ -25,3 +25,5 @@ task_get_extrn_lbcs: USE_USER_STAGED_EXTRN_FILES: true EXTRN_MDL_FILES_LBCS: - '{yy}{jjj}{hh}00{fcst_hr:02d}00' +task_run_post: + NUMX: 2 diff --git a/ush/config.aqm.community.yaml b/ush/config.aqm.community.yaml index 30e391edf2..eabd7c8c20 100644 --- a/ush/config.aqm.community.yaml +++ b/ush/config.aqm.community.yaml @@ -55,6 +55,7 @@ task_run_post: POST_OUTPUT_DOMAIN_NAME: 793 USE_CUSTOM_POST_CONFIG_FILE: false global: + USE_CRTM: false DO_ENSEMBLE: false NUM_ENS_MEMBERS: 2 HALO_BLEND: 0 diff --git a/ush/config.aqm.nco.realtime.yaml b/ush/config.aqm.nco.realtime.yaml index f2299eacc9..d6a35a1229 100644 --- a/ush/config.aqm.nco.realtime.yaml +++ b/ush/config.aqm.nco.realtime.yaml @@ -73,6 +73,7 @@ task_run_fcst: task_run_post: POST_OUTPUT_DOMAIN_NAME: 793 global: + USE_CRTM: false DO_ENSEMBLE: false NUM_ENS_MEMBERS: 2 HALO_BLEND: 0 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 43681bc391..1c74b1668b 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1555,12 +1555,19 @@ task_run_fcst: # weather model]. If this is set to true, the the run_post task will # be deactivated. # + # ITASKS: + # Variable denoting the number of write tasks in the i direction in the + # current group. Used for inline post 2d decomposition. Setting this + # variable to a value greater than 1 will enable 2d decomposition. + # Default setting is 1. + # #----------------------------------------------------------------------- # DT_ATMOS: "" FHROT: 0 RESTART_INTERVAL: 0 WRITE_DOPOST: false + ITASKS: 1 # #----------------------------------------------------------------------- # @@ -1812,12 +1819,17 @@ task_run_post: # to PREDEF_GRID_NAME. Note that this variable is first changed to lower # case before being used to construct the file names. # + # NUMX: + # The number of i regions in a 2D decomposition. Each i row is + # distibuted to numx ranks. Default value of numx is 1. + # #----------------------------------------------------------------------- # USE_CUSTOM_POST_CONFIG_FILE: false CUSTOM_POST_CONFIG_FP: "" POST_OUTPUT_DOMAIN_NAME: '{{ workflow.PREDEF_GRID_NAME }}' TESTBED_FIELDS_FN: "" + NUMX: 1 #---------------------------- # RUN PRDGEN config parameters diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 68d68b724e..0245828ae5 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -81,6 +81,7 @@ def create_model_configure_file( "fhrot": fhrot, "dt_atmos": DT_ATMOS, "restart_interval": RESTART_INTERVAL, + "itasks": ITASKS, "write_dopost": f".{lowercase(str(WRITE_DOPOST))}.", "quilting": f".{lowercase(str(QUILTING))}.", "output_grid": WRTCMP_output_grid, From 136a58f82def431323dc5f9244d42e64338b1d94 Mon Sep 17 00:00:00 2001 From: Michael Lueken Date: Wed, 20 Sep 2023 15:24:22 +0000 Subject: [PATCH 04/29] [develop] Update ush/machine/jet.yaml to include path to CRTM coefficient directory. --- ush/machine/jet.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 92c78aa698..e82f085fe5 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -42,6 +42,8 @@ data: GFS: prepbufr: /public/data/grids/gfs/prepbufr tcvitals: /public/data/grids/gfs/bufr +global: + CRTM_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/impi-2022.1.2/crtm/2.4.0/fix rocoto: tasks: From cdb4cea4b63b46cf928029d800d8b01af58b42c5 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Wed, 20 Sep 2023 13:01:00 -0400 Subject: [PATCH 05/29] [develop] Update ush/machine/gaea.yaml to include path to CRTM coefficient directory. --- ush/machine/gaea.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 8687fd9924..81a5fa0b50 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -38,6 +38,8 @@ data: grib2: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} RAP: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} HRRR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} +global: + CRTM_DIR: /lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/intel-classic-2022.0.2/cray-mpich-7.7.20/crtm/2.4.0/fix rocoto: tasks: metatask_run_ensemble: From d85390e4d7fd16660a70822a877e97bc5ba04678 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Wed, 20 Sep 2023 11:19:05 -0600 Subject: [PATCH 06/29] [develop] Update ush/machine/cheyenne.yaml to include path to CRTM coefficient directory. --- ush/machine/cheyenne.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index 482994235f..10d0d8414a 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -32,3 +32,5 @@ platform: data: ics_lbcs: FV3GFS: /glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh} +global: + CRTM_DIR: /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1_ncdf492/intel-2022.1/mpt-2.25/crtm/2.4.0/fix From 5dc678f26a3d41d37ef58de154f77673157df656 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 21 Sep 2023 08:19:32 -0600 Subject: [PATCH 07/29] [develop] Update ush/machine/derecho.yaml to include path to CRTM coefficient directory and cleaned up comment in build_derecho_intel.lua. --- modulefiles/build_derecho_intel.lua | 2 +- ush/machine/derecho.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modulefiles/build_derecho_intel.lua b/modulefiles/build_derecho_intel.lua index 7c7f3c771e..edc4f92d57 100644 --- a/modulefiles/build_derecho_intel.lua +++ b/modulefiles/build_derecho_intel.lua @@ -3,7 +3,7 @@ This module loads libraries for building the UFS SRW App on the CISL machine Derecho (Cray) using Intel-classic-2023.0.0 ]]) -whatis([===[Loads libraries needed for building the UFS SRW App on Cheyenne ]===]) +whatis([===[Loads libraries needed for building the UFS SRW App on Derecho ]===]) load(pathJoin("cmake", os.getenv("cmake_ver") or "3.26.3")) load(pathJoin("ncarenv", os.getenv("ncarenv_ver") or "23.06")) diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index c7e144b201..98d57fc780 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -41,3 +41,5 @@ data: HRRR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} RAP: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} GSMGFS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} +global: + CRTM_DIR: /glade/work/epicufsrt/contrib/derecho/hpc-stack/intel-classic-2023.0.0/intel-classic-2023.0.0/cray-mpich-8.1.25/crtm/2.4.0/fix From 0b7a1a3c46716220538f205ed264d728d687cf92 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 21 Sep 2023 11:44:56 -0500 Subject: [PATCH 08/29] [develop] Update ush/machine/orion.yaml to include path to CRTM coefficient directory. --- ush/machine/orion.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 9e2ca454ee..212d1431f3 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -32,3 +32,5 @@ platform: FIXsfc: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo FIXshp: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/NaturalEarth EXTRN_MDL_DATA_STORES: aws nomads +global: + CRTM_DIR: /work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2_ncdf492/intel-2022.1.2/impi-2022.1.2/crtm/2.4.0/fix From 91115121089628369abbbcd776eaa0cbdf275606 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 21 Sep 2023 20:00:47 +0000 Subject: [PATCH 09/29] [develop] Updates to allow the test_create_model_configure_file unit test to pass. --- .../test_python/test_create_model_configure_file.py | 2 ++ ush/create_model_configure_file.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_python/test_create_model_configure_file.py b/tests/test_python/test_create_model_configure_file.py index d98f2d56ca..785209ded7 100644 --- a/tests/test_python/test_create_model_configure_file.py +++ b/tests/test_python/test_create_model_configure_file.py @@ -26,6 +26,7 @@ def test_create_model_configure_file(self): sub_hourly_post=True, dt_subhourly_post_mnts=4, dt_atmos=1, + itasks=1, ) ) @@ -47,6 +48,7 @@ def setUp(self): set_env_var("FHROT", 0) set_env_var("DT_ATMOS", 1) set_env_var("RESTART_INTERVAL", 4) + set_env_var("ITASKS", 1) set_env_var("WRTCMP_write_groups", 1) set_env_var("WRTCMP_write_tasks_per_group", 2) diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 0245828ae5..fe189d3574 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -25,7 +25,7 @@ def create_model_configure_file( - cdate, fcst_len_hrs, fhrot, run_dir, sub_hourly_post, dt_subhourly_post_mnts, dt_atmos + cdate, fcst_len_hrs, fhrot, run_dir, sub_hourly_post, dt_subhourly_post_mnts, dt_atmos, itasks ): #pylint: disable=too-many-arguments """Creates a model configuration file in the specified run directory @@ -38,6 +38,7 @@ def create_model_configure_file( sub_hourly_post dt_subhourly_post_mnts dt_atmos + itasks Returns: Boolean """ @@ -300,6 +301,14 @@ def parse_args(argv): help="Forecast model's main time step.", ) + parser.add_argument( + "-it", + "--itasks", + dest="itasks", + required=True, + help="Number of write tasks in the i direction" + ) + parser.add_argument( "-p", "--path-to-defns", @@ -324,4 +333,5 @@ def parse_args(argv): sub_hourly_post=str_to_type(args.sub_hourly_post), dt_subhourly_post_mnts=str_to_type(args.dt_subhourly_post_mnts), dt_atmos=str_to_type(args.dt_atmos), + itasks=str_to_type(args.itasks), ) From fedca0f07170bac097135795adc5dce4f48ca639 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 21 Sep 2023 20:29:18 +0000 Subject: [PATCH 10/29] [develop] Updates to allow the linting and test_create_model_configure_file unit tests to pass. --- .../test_python/test_create_model_configure_file.py | 1 - ush/create_model_configure_file.py | 12 +----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/test_python/test_create_model_configure_file.py b/tests/test_python/test_create_model_configure_file.py index 785209ded7..9475028505 100644 --- a/tests/test_python/test_create_model_configure_file.py +++ b/tests/test_python/test_create_model_configure_file.py @@ -26,7 +26,6 @@ def test_create_model_configure_file(self): sub_hourly_post=True, dt_subhourly_post_mnts=4, dt_atmos=1, - itasks=1, ) ) diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index fe189d3574..0245828ae5 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -25,7 +25,7 @@ def create_model_configure_file( - cdate, fcst_len_hrs, fhrot, run_dir, sub_hourly_post, dt_subhourly_post_mnts, dt_atmos, itasks + cdate, fcst_len_hrs, fhrot, run_dir, sub_hourly_post, dt_subhourly_post_mnts, dt_atmos ): #pylint: disable=too-many-arguments """Creates a model configuration file in the specified run directory @@ -38,7 +38,6 @@ def create_model_configure_file( sub_hourly_post dt_subhourly_post_mnts dt_atmos - itasks Returns: Boolean """ @@ -301,14 +300,6 @@ def parse_args(argv): help="Forecast model's main time step.", ) - parser.add_argument( - "-it", - "--itasks", - dest="itasks", - required=True, - help="Number of write tasks in the i direction" - ) - parser.add_argument( "-p", "--path-to-defns", @@ -333,5 +324,4 @@ def parse_args(argv): sub_hourly_post=str_to_type(args.sub_hourly_post), dt_subhourly_post_mnts=str_to_type(args.dt_subhourly_post_mnts), dt_atmos=str_to_type(args.dt_atmos), - itasks=str_to_type(args.itasks), ) From 28156d4d6b71f11cd8af27c9b0eaeda971248d6a Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 22 Sep 2023 13:35:10 +0000 Subject: [PATCH 11/29] [develop] Update to allow the test_generate_FV3LAM_wflow unit test to pass. --- tests/test_python/test_generate_FV3LAM_wflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py index 9e9e9f5274..9e857d682b 100644 --- a/tests/test_python/test_generate_FV3LAM_wflow.py +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -42,6 +42,11 @@ def run_workflow(USHdir, logfile): logfile = "log.generate_FV3LAM_wflow" sed = get_env_var("SED") + # set USE_CRTM to false in config_defaults.yaml file + run_command( + f"""{sed} -i 's/USE_CRTM: true/USE_CRTM: false/g' {USHdir}/config_defaults.yaml""" + ) + # community test case cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") run_command( From 34523fec107c374f3a90644655a98cac04d3faf3 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 29 Sep 2023 15:47:33 +0000 Subject: [PATCH 12/29] [develop] Update ush/machine/hercules.yaml to include path to CRTM coefficient directory. --- ush/machine/hercules.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 33d5732511..0ceb4d917a 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -43,3 +43,5 @@ data: HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} +global: + CRTM_DIR: /work/noaa/epic/role-epic/contrib/hercules/hpc-stack/intel-2022.2.1/intel-oneapi-compilers-2022.2.1/intel-oneapi-mpi-2021.7.1/crtm/2.4.0/fix From ad5b2f42df6f9b8e9e88c5bf7104c57ce0c12dc7 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 29 Sep 2023 18:09:27 +0000 Subject: [PATCH 13/29] [develop] Update ush/machine/gaea-c5.yaml to include path to CRTM coefficient directory. --- ush/machine/gaea-c5.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/machine/gaea-c5.yaml b/ush/machine/gaea-c5.yaml index 86653be7b5..795ee40fde 100644 --- a/ush/machine/gaea-c5.yaml +++ b/ush/machine/gaea-c5.yaml @@ -52,3 +52,5 @@ rocoto: nnodes: nodesize: ppn: +global: + CRTM_DIR: /lustre/f2/dev/role.epic/contrib/C5/hpc-stack/intel-classic-2023.1.0/intel-classic-2023.1.0/cray-mpich-8.1.25/crtm/2.4.0/fix From 9d9f58bf04dc4ea58416de669d083a0dcbf998d6 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 7 Nov 2023 09:17:08 -0700 Subject: [PATCH 14/29] [develop] Update ufs-weather-model, UPP, and UFS_UTILS hashes to more up-to-date versions. Use configxpost-NT-fv3lam.txt from ufs-weather-model/tests/parm instead of configxpost-NT-fv3lam_rrfs.txt until CRTM issue is addressed on Derecho. Add Derecho label to Jenkinsfile and reactivate comprehensive testing in Jenkins. --- .cicd/Jenkinsfile | 15 ++++----------- Externals.cfg | 6 +++--- scripts/exregional_plot_allvars.py | 2 +- scripts/exregional_plot_allvars_diff.py | 4 ++-- scripts/exregional_run_fcst.sh | 13 +------------ scripts/exregional_run_post.sh | 2 +- .../config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml | 1 - tests/test_python/test_generate_FV3LAM_wflow.py | 5 ----- ush/config.aqm.community.yaml | 1 - ush/config.aqm.nco.realtime.yaml | 1 - ush/config_defaults.yaml | 4 ++-- ush/machine/cheyenne.yaml | 2 -- ush/machine/derecho.yaml | 2 -- ush/machine/gaea-c5.yaml | 2 -- ush/machine/gaea.yaml | 2 -- ush/machine/hera.yaml | 3 --- ush/machine/hercules.yaml | 2 -- ush/machine/jet.yaml | 2 -- ush/machine/orion.yaml | 2 -- 19 files changed, 14 insertions(+), 57 deletions(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 6e453f6f25..4211c37b3f 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -14,11 +14,10 @@ pipeline { // Use the line below to enable the PW AWS cluster // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use') // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') - choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') + choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'derecho', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules'], description: 'Specify the platform(s) to use') // Allow job runner to filter based on compiler choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build') - // Uncomment the following line to re-enable comprehensive tests - // booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests' + booleanParam name: 'SRW_WE2E_COMPREHENSIVE_TESTS', defaultValue: false, description: 'Whether to execute the comprehensive end-to-end tests' } stages { @@ -89,7 +88,7 @@ pipeline { axis { name 'SRW_PLATFORM' // values 'cheyenne', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' - values 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'derecho', 'gaea', 'gaea-c5', 'hera', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -103,7 +102,7 @@ pipeline { exclude { axis { name 'SRW_PLATFORM' - values 'gaea', 'gaea-c5', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' + values 'derecho', 'gaea', 'gaea-c5', 'jet', 'orion', 'hercules' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1' } axis { @@ -184,11 +183,6 @@ pipeline { dir ("${env.SRW_PLATFORM}") { echo "Testing SRW (${env.SRW_COMPILER}) on ${env.SRW_PLATFORM} (using ${env.WORKSPACE}/${env.SRW_PLATFORM})" - // Remove the following line to re-enable comprehensive tests - sh 'SRW_WE2E_COMPREHENSIVE_TESTS=false bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' - - // Uncomment the following block to re-enable comprehensive tests - /* // If executing for a Pull Request, check for the run_we2e_comprehensive_tests. If set, // override the value of the SRW_WE2E_COMPREHENSIVE_TESTS parameter script { @@ -205,7 +199,6 @@ pipeline { sh "SRW_WE2E_COMPREHENSIVE_TESTS=${run_we2e_comprehensive_tests}" + ' bash --login "${WORKSPACE}/${SRW_PLATFORM}/.cicd/scripts/srw_test.sh"' } - */ } } diff --git a/Externals.cfg b/Externals.cfg index 8101eb46aa..7d23e87626 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -3,7 +3,7 @@ protocol = git repo_url = https://github.com/ufs-community/UFS_UTILS # Specify either a branch name or a hash but not both. #branch = develop -hash = 6a7d534 +hash = dc0e4a6 local_path = sorc/UFS_UTILS required = True @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/ufs-weather-model # Specify either a branch name or a hash but not both. #branch = develop -hash = 1787f26 +hash = 020e783 local_path = sorc/ufs-weather-model required = True @@ -21,7 +21,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = 520cc23 +hash = fae617b local_path = sorc/UPP required = True diff --git a/scripts/exregional_plot_allvars.py b/scripts/exregional_plot_allvars.py index a8c5611379..27eff0f4b0 100755 --- a/scripts/exregional_plot_allvars.py +++ b/scripts/exregional_plot_allvars.py @@ -429,7 +429,7 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 + slp = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 slpsmooth = ndimage.gaussian_filter(slp, 13.78) # 2-m temperature diff --git a/scripts/exregional_plot_allvars_diff.py b/scripts/exregional_plot_allvars_diff.py index 64f6b1b66f..e51a3a6b57 100755 --- a/scripts/exregional_plot_allvars_diff.py +++ b/scripts/exregional_plot_allvars_diff.py @@ -446,9 +446,9 @@ def setup_logging(debug=False): t1a = time.perf_counter() # Sea level pressure - slp_1 = data1.select(name="MSLP (Eta model reduction)")[0].values * 0.01 + slp_1 = data1.select(name="Pressure reduced to MSL")[0].values * 0.01 slpsmooth_1 = ndimage.gaussian_filter(slp_1, 13.78) - slp_2 = data2.select(name="MSLP (Eta model reduction)")[0].values * 0.01 + slp_2 = data2.select(name="Pressure reduced to MSL")[0].values * 0.01 slpsmooth_2 = ndimage.gaussian_filter(slp_2, 13.78) slp_diff = slp_2 - slp_1 diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 21c4253224..6dfad9673b 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -410,7 +410,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then if [ "${CPL_AQM}" = "TRUE" ]; then post_config_fp="${PARMdir}/upp/postxconfig-NT-AQM.txt" else - post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam_rrfs.txt" + post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam.txt" fi print_info_msg " ==================================================================== @@ -420,17 +420,6 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cp_vrfy ${post_config_fp} ./postxconfig-NT_FH00.txt cp_vrfy ${post_config_fp} ./postxconfig-NT.txt cp_vrfy ${PARMdir}/upp/params_grib2_tbl_new . - if [ ${USE_CRTM} = "TRUE" ]; then - cp_vrfy ${CRTM_DIR}/Nalli.IRwater.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/FAST*.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRland.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRsnow.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/NPOESS.IRice.EmisCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/AerosolCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/CloudCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/*.SpcCoeff.bin ./ - cp_vrfy ${CRTM_DIR}/*.TauCoeff.bin ./ - fi # Set itag for inline-post: if [ "${CPL_AQM}" = "TRUE" ]; then post_itag_add="aqf_on=.true.," diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 795efa38f8..cfbed2dcfd 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -95,7 +95,7 @@ else if [ "${CPL_AQM}" = "TRUE" ]; then post_config_fp="${PARMdir}/upp/postxconfig-NT-AQM.txt" else - post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam_rrfs.txt" + post_config_fp="${PARMdir}/upp/postxconfig-NT-fv3lam.txt" fi print_info_msg " ==================================================================== diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml index 691c4b5725..789c5e9674 100644 --- a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml @@ -49,7 +49,6 @@ task_run_post: POST_OUTPUT_DOMAIN_NAME: 793 USE_CUSTOM_POST_CONFIG_FILE: false global: - USE_CRTM: false DO_ENSEMBLE: false NUM_ENS_MEMBERS: 2 HALO_BLEND: 0 diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py index 9e857d682b..9e9e9f5274 100644 --- a/tests/test_python/test_generate_FV3LAM_wflow.py +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -42,11 +42,6 @@ def run_workflow(USHdir, logfile): logfile = "log.generate_FV3LAM_wflow" sed = get_env_var("SED") - # set USE_CRTM to false in config_defaults.yaml file - run_command( - f"""{sed} -i 's/USE_CRTM: true/USE_CRTM: false/g' {USHdir}/config_defaults.yaml""" - ) - # community test case cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") run_command( diff --git a/ush/config.aqm.community.yaml b/ush/config.aqm.community.yaml index eabd7c8c20..30e391edf2 100644 --- a/ush/config.aqm.community.yaml +++ b/ush/config.aqm.community.yaml @@ -55,7 +55,6 @@ task_run_post: POST_OUTPUT_DOMAIN_NAME: 793 USE_CUSTOM_POST_CONFIG_FILE: false global: - USE_CRTM: false DO_ENSEMBLE: false NUM_ENS_MEMBERS: 2 HALO_BLEND: 0 diff --git a/ush/config.aqm.nco.realtime.yaml b/ush/config.aqm.nco.realtime.yaml index d6a35a1229..f2299eacc9 100644 --- a/ush/config.aqm.nco.realtime.yaml +++ b/ush/config.aqm.nco.realtime.yaml @@ -73,7 +73,6 @@ task_run_fcst: task_run_post: POST_OUTPUT_DOMAIN_NAME: 793 global: - USE_CRTM: false DO_ENSEMBLE: false NUM_ENS_MEMBERS: 2 HALO_BLEND: 0 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 8f3e4e3675..6abe862d7b 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2080,8 +2080,8 @@ task_run_post: # #----------------------------------------------------------------------- # - USE_CUSTOM_POST_CONFIG_FILE: false - CUSTOM_POST_CONFIG_FP: "" + USE_CUSTOM_POST_CONFIG_FILE: true + CUSTOM_POST_CONFIG_FP: "{{ user.SORCdir }}/ufs-weather-model/tests/parm/postxconfig-NT-fv3lam.txt" POST_OUTPUT_DOMAIN_NAME: '{{ workflow.PREDEF_GRID_NAME }}' TESTBED_FIELDS_FN: "" NUMX: 1 diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index 10d0d8414a..482994235f 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -32,5 +32,3 @@ platform: data: ics_lbcs: FV3GFS: /glade/p/ral/jntp/UFS_CAM/COMGFS/gfs.${yyyymmdd}/${hh} -global: - CRTM_DIR: /glade/work/epicufsrt/contrib/hpc-stack/intel2022.1_ncdf492/intel-2022.1/mpt-2.25/crtm/2.4.0/fix diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index 98d57fc780..c7e144b201 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -41,5 +41,3 @@ data: HRRR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} RAP: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} GSMGFS: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} -global: - CRTM_DIR: /glade/work/epicufsrt/contrib/derecho/hpc-stack/intel-classic-2023.0.0/intel-classic-2023.0.0/cray-mpich-8.1.25/crtm/2.4.0/fix diff --git a/ush/machine/gaea-c5.yaml b/ush/machine/gaea-c5.yaml index 795ee40fde..86653be7b5 100644 --- a/ush/machine/gaea-c5.yaml +++ b/ush/machine/gaea-c5.yaml @@ -52,5 +52,3 @@ rocoto: nnodes: nodesize: ppn: -global: - CRTM_DIR: /lustre/f2/dev/role.epic/contrib/C5/hpc-stack/intel-classic-2023.1.0/intel-classic-2023.1.0/cray-mpich-8.1.25/crtm/2.4.0/fix diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 857b8fbb80..301fa5c5a1 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -38,8 +38,6 @@ data: grib2: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} RAP: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} HRRR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} -global: - CRTM_DIR: /lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/intel-classic-2022.0.2/cray-mpich-7.7.20/crtm/2.4.0/fix rocoto: tasks: metatask_run_ensemble: diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index ba733c9199..545c9320af 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -35,9 +35,6 @@ platform: FIXshp: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/NaturalEarth EXTRN_MDL_DATA_STORES: hpss aws nomads -global: - CRTM_DIR: /scratch1/NCEPDEV/nems/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/impi-2022.1.2/crtm/2.4.0/fix - cpl_aqm_parm: AQM_CONFIG_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/epa/data DCOMINbio_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 0ceb4d917a..33d5732511 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -43,5 +43,3 @@ data: HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} -global: - CRTM_DIR: /work/noaa/epic/role-epic/contrib/hercules/hpc-stack/intel-2022.2.1/intel-oneapi-compilers-2022.2.1/intel-oneapi-mpi-2021.7.1/crtm/2.4.0/fix diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 49ed686e79..2ecd101ae0 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -42,8 +42,6 @@ data: GFS: prepbufr: /public/data/grids/gfs/prepbufr tcvitals: /public/data/grids/gfs/bufr -global: - CRTM_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-2022.1.2/intel-2022.1.2/impi-2022.1.2/crtm/2.4.0/fix rocoto: tasks: diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index c02dfa68a0..38ccd279f9 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -42,5 +42,3 @@ data: HRRR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh} RAP: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh} GSMGFS: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/GSMGFS/${yyyymmdd}${hh} -global: - CRTM_DIR: /work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2_ncdf492/intel-2022.1.2/impi-2022.1.2/crtm/2.4.0/fix From e7205410a2d813b33d3058b47912a30464c80c49 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 7 Nov 2023 10:12:01 -0700 Subject: [PATCH 15/29] [develop] Set USE_CRTM to false in config_defaults.yaml. --- ush/config_defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 6abe862d7b..5ff90eb5f8 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -2243,7 +2243,7 @@ global: # #----------------------------------------------------------------------- # - USE_CRTM: true + USE_CRTM: false CRTM_DIR: "" # #----------------------------------------------------------------------- From da66a8a4eece6fb5d29281211dde32143962004f Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 9 Nov 2023 16:31:33 +0000 Subject: [PATCH 16/29] [develop] Disable 2D decomposition for GNU. --- tests/WE2E/run_WE2E_tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index 3ce6ce39a8..c6b9d2ab37 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -228,6 +228,13 @@ def run_we2e_tests(homedir, args) -> None: test_aqm_input_basedir = machine_defaults['platform']['TEST_AQM_INPUT_BASEDIR'] test_cfg['cpl_aqm_parm']['DCOMINfire_default'] = f"{test_aqm_input_basedir}/RAVE_fire" + if args.compiler == "gnu": + # 2D decomposition doesn't work with GNU compilers. Deactivate 2D decomposition for GNU + if 'task_run_post' in test_cfg: + test_cfg['task_run_post'].update({"NUMX": 1}) + if 'task_run_fcst' in test_cfg: + test_cfg['task_run_fcst'].update({"ITASKS": 1}) + logging.debug(f"Writing updated config.yaml for test {test_name}\n"\ "based on specified command-line arguments:\n") logging.debug(cfg_to_yaml_str(test_cfg)) From b4a5c1aa4195e59347a7093537909d3cdf667e6e Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 9 Nov 2023 14:34:46 -0600 Subject: [PATCH 17/29] [develop] Update ConfigWorkflow.rst to add changes to ush/config_defaults.yaml. --- .../source/CustomizingTheWorkflow/ConfigWorkflow.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index 11d882c4f9..a692eca57c 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1138,6 +1138,9 @@ These parameters set values in the Weather Model's ``model_configure`` file. ``WRITE_DOPOST``: (Default: false) Flag that determines whether to use the inline post option, which calls the Unified Post Processor (:term:`UPP`) from within the UFS Weather Model. The default ``WRITE_DOPOST: false`` does not use the inline post functionality, and the ``run_post`` tasks are called from outside of the UFS Weather Model. If ``WRITE_DOPOST: true``, the ``WRITE_DOPOST`` flag in the ``model_configure`` file will be set to true, and the post-processing (:term:`UPP`) tasks will be called from within the Weather Model. This means that the post-processed files (in :term:`grib2` format) are output by the Weather Model at the same time that it outputs the ``dynf###.nc`` and ``phyf###.nc`` files. Setting ``WRITE_DOPOST: true`` turns off the separate ``run_post`` task in ``setup.py`` to avoid unnecessary computations. Valid values: ``True`` | ``False`` +``ITASKS``: (Default: 1) + Variable denoting the number of write tasks in the i direction in the current group. Used for inline post 2d decomposition. Setting this variable to a value greater than 1 will enable 2d decomposition. + Computational Parameters ---------------------------- @@ -1277,10 +1280,10 @@ Customized Post Configuration Parameters Set parameters for customizing the :term:`UPP`. -``USE_CUSTOM_POST_CONFIG_FILE``: (Default: false) +``USE_CUSTOM_POST_CONFIG_FILE``: (Default: true) Flag that determines whether a user-provided custom configuration file should be used for post-processing the model data. If this is set to true, then the workflow will use the custom post-processing (:term:`UPP`) configuration file specified in ``CUSTOM_POST_CONFIG_FP``. Otherwise, a default configuration file provided in the UPP repository will be used. Valid values: ``True`` | ``False`` -``CUSTOM_POST_CONFIG_FP``: (Default: "") +``CUSTOM_POST_CONFIG_FP``: (Default: ``'{{ user.SORCdir }}/ufs-weather-model/tests/parm/postxconfig-NT-fv3lam.txt'``) The full path to the custom post flat file, including filename, to be used for post-processing. This is only used if ``CUSTOM_POST_CONFIG_FILE`` is set to true. ``POST_OUTPUT_DOMAIN_NAME``: (Default: ``'{{ workflow.PREDEF_GRID_NAME }}'``) @@ -1295,6 +1298,9 @@ Set parameters for customizing the :term:`UPP`. ``TESTBED_FIELDS_FN``: (Default: "") The file that lists grib2 fields to be extracted for testbed files. An empty string means no need to generate testbed files. +``NUMX``: (Default: 1) + The number of i regions in a 2D decomposition. Each i row is distibuted to numx ranks. Set NUMX to a value greater than 1 to enable 2D decomposition. + RUN_PRDGEN Configuration Parameters ===================================== From 161877a129a6988d66d3c914e2c0c3463070cedc Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 10 Nov 2023 14:32:45 +0000 Subject: [PATCH 18/29] [develop] Correct ConfigWorkflow.rst default documentation for CUSTOM_POST_CONFIG_FP. --- .../UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index a692eca57c..cb786f886f 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1283,7 +1283,7 @@ Set parameters for customizing the :term:`UPP`. ``USE_CUSTOM_POST_CONFIG_FILE``: (Default: true) Flag that determines whether a user-provided custom configuration file should be used for post-processing the model data. If this is set to true, then the workflow will use the custom post-processing (:term:`UPP`) configuration file specified in ``CUSTOM_POST_CONFIG_FP``. Otherwise, a default configuration file provided in the UPP repository will be used. Valid values: ``True`` | ``False`` -``CUSTOM_POST_CONFIG_FP``: (Default: ``'{{ user.SORCdir }}/ufs-weather-model/tests/parm/postxconfig-NT-fv3lam.txt'``) +``CUSTOM_POST_CONFIG_FP``: (Default: ``"{{ user.SORCdir }}/ufs-weather-model/tests/parm/postxconfig-NT-fv3lam.txt"``) The full path to the custom post flat file, including filename, to be used for post-processing. This is only used if ``CUSTOM_POST_CONFIG_FILE`` is set to true. ``POST_OUTPUT_DOMAIN_NAME``: (Default: ``'{{ workflow.PREDEF_GRID_NAME }}'``) From 006f2a9cb9fe2f8948cac8984c743e183c67dfa3 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:47:24 -0500 Subject: [PATCH 19/29] [develop] Update docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst with Gillian's suggestions. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- .../UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index cb786f886f..8fec482881 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1139,7 +1139,7 @@ These parameters set values in the Weather Model's ``model_configure`` file. Flag that determines whether to use the inline post option, which calls the Unified Post Processor (:term:`UPP`) from within the UFS Weather Model. The default ``WRITE_DOPOST: false`` does not use the inline post functionality, and the ``run_post`` tasks are called from outside of the UFS Weather Model. If ``WRITE_DOPOST: true``, the ``WRITE_DOPOST`` flag in the ``model_configure`` file will be set to true, and the post-processing (:term:`UPP`) tasks will be called from within the Weather Model. This means that the post-processed files (in :term:`grib2` format) are output by the Weather Model at the same time that it outputs the ``dynf###.nc`` and ``phyf###.nc`` files. Setting ``WRITE_DOPOST: true`` turns off the separate ``run_post`` task in ``setup.py`` to avoid unnecessary computations. Valid values: ``True`` | ``False`` ``ITASKS``: (Default: 1) - Variable denoting the number of write tasks in the i direction in the current group. Used for inline post 2d decomposition. Setting this variable to a value greater than 1 will enable 2d decomposition. + Variable denoting the number of write tasks in the ``i`` direction in the current group. Used for inline post 2D decomposition. Setting this variable to a value greater than 1 will enable 2D decomposition. Computational Parameters ---------------------------- From 835f2c7d49d42f969d079a555da7a10af440ebd6 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:51:20 -0500 Subject: [PATCH 20/29] [develop] Update docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst with Gillian's suggestions. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- .../UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index 8fec482881..b68e579a61 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1299,7 +1299,7 @@ Set parameters for customizing the :term:`UPP`. The file that lists grib2 fields to be extracted for testbed files. An empty string means no need to generate testbed files. ``NUMX``: (Default: 1) - The number of i regions in a 2D decomposition. Each i row is distibuted to numx ranks. Set NUMX to a value greater than 1 to enable 2D decomposition. + The number of ``i`` regions in a 2D decomposition. Each ``i`` row is distributed to ``NUMX`` ranks. Set ``NUMX`` to a value greater than 1 to enable 2D decomposition. RUN_PRDGEN Configuration Parameters ===================================== From 9a3cd6acc41c8b3039e7c613cd8f129e006d7dd6 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:52:07 -0500 Subject: [PATCH 21/29] [develop] Update tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml with Gillian's suggestions. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- ...RFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml index 34d8724a4c..13e7e5f427 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.yaml @@ -3,8 +3,8 @@ metadata: This test is to ensure that the workflow running in community mode completes successfully on the RRFS_CONUS_25km grid using the GFS_v15p2 physics suite with ICs and LBCs derived from the FV3GFS. - In addition, this tests the "inline post" option (WRITE_DOPOST: true). - Enables UPP inline 2d decomposition. + In addition, this tests the "inline post" option (WRITE_DOPOST: true) + and enables UPP inline 2D decomposition. user: RUN_ENVIR: community workflow: From c9613118b286b8163e3f9a2a910178f3dff6d07a Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:52:51 -0500 Subject: [PATCH 22/29] [develop] Update tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml with Gillian's suggestions. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- ...S_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml index 0117c41f34..a65100bf32 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml @@ -3,7 +3,7 @@ metadata: This test is to ensure that the workflow running in community mode completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1beta physics suite with ICs derived from the HRRR and LBCs derived from the RAP. - Also tests the "DOT_OR_USCORE" option. Enables offline UPP 2d decomposition. + It also tests the "DOT_OR_USCORE" option and enables offline UPP 2d decomposition. user: RUN_ENVIR: community workflow: From de0157f73e67e8307219f0bef63376976e1d09d8 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:53:44 -0500 Subject: [PATCH 23/29] [develop] Update docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst with Gillian's suggestions. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index b68e579a61..db9e0efa50 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1300,6 +1300,7 @@ Set parameters for customizing the :term:`UPP`. ``NUMX``: (Default: 1) The number of ``i`` regions in a 2D decomposition. Each ``i`` row is distributed to ``NUMX`` ranks. Set ``NUMX`` to a value greater than 1 to enable 2D decomposition. + Note that 2D decomposition does not yet work with GNU compilers, so this value will be reset to 1 automatically when using GNU compilers (i.e., when ``COMPILER: gnu``). RUN_PRDGEN Configuration Parameters ===================================== From 69c6d08c57690615dd23fabe73ca8aea4662900e Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:54:15 -0500 Subject: [PATCH 24/29] [develop] Update docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst with Gillian's suggestions. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index db9e0efa50..8057252bbd 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1140,6 +1140,7 @@ These parameters set values in the Weather Model's ``model_configure`` file. ``ITASKS``: (Default: 1) Variable denoting the number of write tasks in the ``i`` direction in the current group. Used for inline post 2D decomposition. Setting this variable to a value greater than 1 will enable 2D decomposition. + Note that 2D decomposition does not yet work with GNU compilers, so this value will be reset to 1 automatically when using GNU compilers (i.e., when ``COMPILER: gnu``). Computational Parameters ---------------------------- From e97c1122f61fc5cdb32fd4e661332733ef16fcf0 Mon Sep 17 00:00:00 2001 From: Michael Lueken <63728921+MichaelLueken@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:57:09 -0500 Subject: [PATCH 25/29] [develop] Update ush/config_defaults.yaml with Gillian's suggestions. Co-authored-by: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> --- ush/config_defaults.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 5ff90eb5f8..c8b259fe6f 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1792,6 +1792,8 @@ task_run_fcst: # current group. Used for inline post 2d decomposition. Setting this # variable to a value greater than 1 will enable 2d decomposition. # Default setting is 1. + # Note that 2D decomposition does not yet work with GNU compilers, so this value + # will be reset to 1 automatically when using GNU compilers (i.e., when COMPILER: gnu). # #----------------------------------------------------------------------- # @@ -2077,6 +2079,8 @@ task_run_post: # NUMX: # The number of i regions in a 2D decomposition. Each i row is # distibuted to numx ranks. Default value of numx is 1. + # Note that 2D decomposition does not yet work with GNU compilers, so this value + # will be reset to 1 automatically when using GNU compilers (i.e., when COMPILER: gnu). # #----------------------------------------------------------------------- # From 02f2428438f4ae86fc4da5e238d6e53070a44143 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 14 Nov 2023 15:34:28 +0000 Subject: [PATCH 26/29] [develop] Replace lowercase d with uppercase D for ITASKS 2D decomposition documentation in ush/config_defaults.yaml and added prints regarding resetting NUMX and ITASKS in tests/WE2E/run_WE2E_tests.py. --- tests/WE2E/run_WE2E_tests.py | 2 ++ ush/config_defaults.yaml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index c6b9d2ab37..236720159d 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -232,8 +232,10 @@ def run_we2e_tests(homedir, args) -> None: # 2D decomposition doesn't work with GNU compilers. Deactivate 2D decomposition for GNU if 'task_run_post' in test_cfg: test_cfg['task_run_post'].update({"NUMX": 1}) + logging.debug(f"NUMX has been reset to 1 due to issues encountered with GNU compilers") if 'task_run_fcst' in test_cfg: test_cfg['task_run_fcst'].update({"ITASKS": 1}) + logging.debug(f"ITASKS has been reset to 1 due to issues encountered with GNU compilers") logging.debug(f"Writing updated config.yaml for test {test_name}\n"\ "based on specified command-line arguments:\n") diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index c8b259fe6f..0bd216bb2d 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1789,8 +1789,8 @@ task_run_fcst: # # ITASKS: # Variable denoting the number of write tasks in the i direction in the - # current group. Used for inline post 2d decomposition. Setting this - # variable to a value greater than 1 will enable 2d decomposition. + # current group. Used for inline post 2D decomposition. Setting this + # variable to a value greater than 1 will enable 2D decomposition. # Default setting is 1. # Note that 2D decomposition does not yet work with GNU compilers, so this value # will be reset to 1 automatically when using GNU compilers (i.e., when COMPILER: gnu). From 9f7039be9bf34610a6b39cd20f63b960b35de43c Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 14 Nov 2023 16:26:44 +0000 Subject: [PATCH 27/29] [develop] Replace logging.debug with logging.info for NUMX and ITASKS reset in run_WE2E_tests.py and replace lowercase d with uppercase D in grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta documentation. --- tests/WE2E/run_WE2E_tests.py | 4 ++-- ...CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index 236720159d..63596a7cce 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -232,10 +232,10 @@ def run_we2e_tests(homedir, args) -> None: # 2D decomposition doesn't work with GNU compilers. Deactivate 2D decomposition for GNU if 'task_run_post' in test_cfg: test_cfg['task_run_post'].update({"NUMX": 1}) - logging.debug(f"NUMX has been reset to 1 due to issues encountered with GNU compilers") + logging.info(f"NUMX has been reset to 1 due to issues encountered with GNU compilers") if 'task_run_fcst' in test_cfg: test_cfg['task_run_fcst'].update({"ITASKS": 1}) - logging.debug(f"ITASKS has been reset to 1 due to issues encountered with GNU compilers") + logging.info(f"ITASKS has been reset to 1 due to issues encountered with GNU compilers") logging.debug(f"Writing updated config.yaml for test {test_name}\n"\ "based on specified command-line arguments:\n") diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml index a65100bf32..dd5f5a464a 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.yaml @@ -3,7 +3,7 @@ metadata: This test is to ensure that the workflow running in community mode completes successfully on the RRFS_CONUScompact_25km grid using the RRFS_v1beta physics suite with ICs derived from the HRRR and LBCs derived from the RAP. - It also tests the "DOT_OR_USCORE" option and enables offline UPP 2d decomposition. + It also tests the "DOT_OR_USCORE" option and enables offline UPP 2D decomposition. user: RUN_ENVIR: community workflow: From ec44d97b437fd3011211190bc9d74caaca837c65 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 16 Nov 2023 13:58:10 +0000 Subject: [PATCH 28/29] [develop] Add DOMAIN_PREGEN_BASEDIR entry to ush/machine/noaacloud.yaml to allow for workflow generation on NOAA Cloud platforms. --- ush/machine/noaacloud.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml index a949fd0594..6577667bf6 100644 --- a/ush/machine/noaacloud.yaml +++ b/ush/machine/noaacloud.yaml @@ -5,6 +5,7 @@ platform: TEST_CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc + DOMAIN_PREGEN_BASEDIR: /contrib/EPIC/UFS_SRW_data/develop/FV3LAM_pregen REMOVE_MEMORY: True RUN_CMD_FCST: mpiexec -np ${PE_MEMBER01} RUN_CMD_POST: mpiexec -np $nprocs From 1269cbb873efea63d5a1593788571d3dea874c5e Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Fri, 16 Feb 2024 14:16:05 +0000 Subject: [PATCH 29/29] [feature/upp_2d_decomp] Update numx documentation to note that it is used for offline post 2D decomposition --- .../UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst index db0a1af457..b0b0301973 100644 --- a/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/docs/UsersGuide/source/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -1303,7 +1303,7 @@ Set parameters for customizing the :term:`UPP`. The file that lists grib2 fields to be extracted for testbed files. An empty string means no need to generate testbed files. ``NUMX``: (Default: 1) - The number of ``i`` regions in a 2D decomposition. Each ``i`` row is distributed to ``NUMX`` ranks. Set ``NUMX`` to a value greater than 1 to enable 2D decomposition. + The number of ``i`` regions in a 2D decomposition. Each ``i`` row is distributed to ``NUMX`` ranks. Used for offline post 2D decomposition. Set ``NUMX`` to a value greater than 1 to enable 2D decomposition. Note that 2D decomposition does not yet work with GNU compilers, so this value will be reset to 1 automatically when using GNU compilers (i.e., when ``COMPILER: gnu``). RUN_PRDGEN Configuration Parameters