diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8b6ddf3e51f5..4fb88c2dc67b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,5 @@ updates: reviewers: - "mahf708" - "bartgol" + labels: + - "AT: Integrate Without Testing" diff --git a/.github/workflows/e3sm-gh-ci-cime-tests.yml b/.github/workflows/e3sm-gh-ci-cime-tests.yml index ff9263f9fc7a..d73e96c6a1d1 100644 --- a/.github/workflows/e3sm-gh-ci-cime-tests.yml +++ b/.github/workflows/e3sm-gh-ci-cime-tests.yml @@ -9,6 +9,7 @@ on: jobs: ci: + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/e3sm-gh-pages.yml b/.github/workflows/e3sm-gh-pages.yml index 9c637536ae5e..ccca0c479f26 100644 --- a/.github/workflows/e3sm-gh-pages.yml +++ b/.github/workflows/e3sm-gh-pages.yml @@ -15,7 +15,7 @@ concurrency: jobs: Build-and-Deploy-docs: - if: ${{ github.event.repository.name != 'scream' }} + if: ${{ github.event.repository.name == 'e3sm' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/eamxx_default_files.yml b/.github/workflows/eamxx_default_files.yml new file mode 100644 index 000000000000..5ecdf6dec00c --- /dev/null +++ b/.github/workflows/eamxx_default_files.yml @@ -0,0 +1,62 @@ +name: inputdata + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + schedule: + - cron: '00 00 * * *' + workflow_dispatch: + +jobs: + scream-defaults: + runs-on: ubuntu-latest + outputs: + event_name: ${{ github.event_name }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + with: + show-progress: false + submodules: false + - name: Set up Python 3.11 + uses: actions/setup-python@v5.1.0 + with: + python-version: "3.11" + - name: Run unit tests + working-directory: components/eamxx/cime_config/ + run: | + python -m unittest tests/eamxx_default_files.py -v + + notify-scream-defaults: + needs: scream-defaults + if: ${{ failure() && needs.scream-defaults.outputs.event_name != 'pull_request' }} + runs-on: ubuntu-latest + steps: + - name: Create issue + run: | + previous_issue_number=$(gh issue list \ + --label "$LABELS" \ + --json number \ + --jq '.[0].number') + if [[ -n $previous_issue_number ]]; then + gh issue comment "$previous_issue_number" \ + --body "$BODY" + else + gh issue create \ + --title "$TITLE" \ + --assignee "$ASSIGNEES" \ + --label "$LABELS" \ + --body "$BODY" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TITLE: Inputdata server file missing + ASSIGNEES: mahf708,bartgol + LABELS: bug,input file,notify-file-gh-action + BODY: | + Workflow failed! There's likely a missing file specified in the configs! For more information, please see: + - Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} (number ${{ github.run_number }}, attempt ${{ github.run_attempt }}) + - Workflow SHA: ${{ github.sha }} diff --git a/cime_config/config_grids.xml b/cime_config/config_grids.xml index 5b1a247deedc..57e20d33b746 100755 --- a/cime_config/config_grids.xml +++ b/cime_config/config_grids.xml @@ -4480,6 +4480,11 @@ lnd/clm2/mappingdata/maps/ne240np4/map_0.1x0.1_nomask_to_ne240np4_nomask_aave_da_c120706.nc + + cpl/gridmaps/ne256pg2/map_ne256pg2_to_r0125_mono.200212.nc + cpl/gridmaps/ne256pg2/map_r0125_to_ne256pg2_mono.200212.nc + + cpl/gridmaps/ne1024pg2/map_ne1024pg2_to_r0125_mono.200212.nc cpl/gridmaps/ne1024pg2/map_r0125_to_ne1024pg2_mono.200212.nc diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 14f2d9121eb8..7e713799018a 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2965,10 +2965,10 @@ spectrum-mpi cbronze /usr/workspace/$USER/e3sm_scratch - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/gdata/e3sm/ccsm3data/inputdata + /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 /usr/workspace/$USER/archive/$CASE - /usr/gdata/climdat/baselines/$COMPILER + /usr/gdata/e3sm/baselines/$COMPILER 16 lsf donahue5 -at- llnl.gov @@ -3022,9 +3022,9 @@ y - /usr/gdata/climdat/netcdf/bin:$ENV{PATH} - /usr/gdata/climdat/netcdf/lib:$ENV{LD_LIBRARY_PATH} - /usr/gdata/climdat/netcdf + /usr/gdata/e3sm/netcdf/bin:$ENV{PATH} + /usr/gdata/e3sm/netcdf/lib:$ENV{LD_LIBRARY_PATH} + /usr/gdata/e3sm/netcdf 2 20 2 @@ -3041,11 +3041,11 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/ruby - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/gdata/e3sm/ccsm3data/inputdata + /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/climdat/tools/cprnc + /usr/gdata/e3sm/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov @@ -3073,7 +3073,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/climdat/install/quartz/modulefiles + /usr/gdata/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3084,7 +3084,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/climdat/install/quartz/netcdf-fortran/ + /usr/gdata/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 @@ -3096,11 +3096,11 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/quartz - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/gdata/e3sm/ccsm3data/inputdata + /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/climdat/tools/cprnc + /usr/gdata/e3sm/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov @@ -3128,7 +3128,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/climdat/install/quartz/modulefiles + /usr/gdata/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3139,7 +3139,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/climdat/install/quartz/netcdf-fortran/ + /usr/gdata/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 diff --git a/cime_config/tests.py b/cime_config/tests.py index ab5fa2ce9493..3223ae10804d 100644 --- a/cime_config/tests.py +++ b/cime_config/tests.py @@ -610,12 +610,20 @@ "e3sm_scream_v1_lowres" : { "time" : "01:00:00", + "inherit" : ("e3sm_scream_mam4xx_v1_lowres"), "tests" : ( - "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1", - "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1", - "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero", - "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1", - "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2", + "ERP_D_Lh4.ne4_ne4.F2010-SCREAMv1.scream-output-preset-1", + "ERS_Ln9.ne4_ne4.F2000-SCREAMv1-AQP1.scream-output-preset-2", + "SMS_D_Ln9.ne4_ne4.F2010-SCREAMv1-noAero.scream-output-preset-3", + "ERP_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-4", + "ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-rad_frequency_2--scream-output-preset-5", + ) + }, + + "e3sm_scream_v1_dp-eamxx" : { + "time" : "01:00:00", + "tests" : ( + "ERS_P16_Ln22.ne30_ne30.F2010-SCREAMv1-DP-DYCOMSrf01", # 225 phys cols, roughly size of ne2 ) }, @@ -623,27 +631,26 @@ # should be fast, so we limit it to low res and add some thread tests # specifically for mappy. "e3sm_scream_v1_at" : { - "inherit" : ("e3sm_scream_v1_lowres"), - "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1") + "inherit" : ("e3sm_scream_v1_lowres", "e3sm_scream_v1_dp-eamxx"), + "tests" : ("PET_Ln9_P32x2.ne4pg2_ne4pg2.F2010-SCREAMv1.scream-output-preset-1") }, "e3sm_scream_v1_medres" : { "time" : "02:00:00", "tests" : ( # "SMS_D_Ln2.ne30_ne30.F2000-SCREAMv1-AQP1", # Uncomment once IC file for ne30 is ready - "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.scream-internal_diagnostics_level", - "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1", - "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels", - "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash", + "ERS_Ln22.ne30_ne30.F2010-SCREAMv1.scream-internal_diagnostics_level--scream-output-preset-3", + "PEM_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-spa_remap--scream-output-preset-4", + "ERS_Ln90.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-small_kernels--scream-output-preset-5", + "ERP_Ln22.conusx4v1pg2_r05_oECv3.F2010-SCREAMv1-noAero.scream-bfbhash--scream-output-preset-6", ) }, + # Used to track performance "e3sm_scream_v1_hires" : { - "time" : "03:00:00", + "time" : "01:00:00", "tests" : ( - "SMS_D_Ln12.ne120_r0125_oRRS18to6v3.F2010-SCREAMv1", - "SMS_Ln12.ne120_ne120.F2010-SCREAMv1", -# "SMS_Ln12.ne120_r0125_oRRS18to6v3.F2000-SCREAMv1-AQP1", add when aquap 120 inputs available + "SMS_Ln300.ne30pg2_ne30pg2.F2010-SCREAMv1.scream-perf_test--scream-output-preset-1" ) }, @@ -655,7 +662,7 @@ # Disable the two 111422-commented tests b/c they fail on pm-gpu and # we're not using MPASSI right now. #111422 "ERP_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", - "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", + "ERS_D_Ln22.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off--scream-output-preset-1", # "ERS_Ln22.ne30_oECv3.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", #111422 "PEM_Ln90.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI", # "ERS_Ln22.ne30pg2_EC30to60E2r2.F2010-SCREAMv1-MPASSI.atmlndactive-rtm_off", @@ -679,6 +686,14 @@ ) }, + "e3sm_scream_mam4xx_v1_lowres" : { + "time" : "01:00:00", + "tests" : ( + "SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-optics", + ) + }, + + "e3sm_gpuacc" : { "tests" : ( "SMS_Ld1.T62_oEC60to30v3.CMPASO-NYF", @@ -1000,4 +1015,3 @@ "e3sm_superbfb_atm", "e3sm_superbfb_wcycl"), }, } - diff --git a/components/eam/src/dynamics/se/stepon.F90 b/components/eam/src/dynamics/se/stepon.F90 index 831fd6d87603..38a376935ee6 100644 --- a/components/eam/src/dynamics/se/stepon.F90 +++ b/components/eam/src/dynamics/se/stepon.F90 @@ -585,8 +585,8 @@ subroutine stepon_run3(dtime, cam_out, phys_state, dyn_in, dyn_out) if (dp_crm) then do ie=1,nelemd - out_gridx(:,:) = dyn_in%elem(ie)%spherep(:,:)%lat - out_gridy(:,:) = dyn_in%elem(ie)%spherep(:,:)%lon + out_gridx(:,:) = dyn_in%elem(ie)%spherep(:,:)%lon + out_gridy(:,:) = dyn_in%elem(ie)%spherep(:,:)%lat call outfld('crm_grid_x', out_gridx, npsq, ie) call outfld('crm_grid_y', out_gridy, npsq, ie) enddo diff --git a/components/eam/src/physics/p3/scream/micro_p3.F90 b/components/eam/src/physics/p3/scream/micro_p3.F90 index e59bcc08239c..f345e5abcf70 100644 --- a/components/eam/src/physics/p3/scream/micro_p3.F90 +++ b/components/eam/src/physics/p3/scream/micro_p3.F90 @@ -1846,7 +1846,8 @@ subroutine get_rain_dsd2(qr,nr,mu_r,lamr,cdistr,logn0r) real(rtype), intent(out) :: lamr,mu_r,cdistr,logn0r !local variables: - real(rtype) :: inv_dum,lammax,lammin + real(rtype) :: lammax,lammin + real(rtype) :: mass_to_d3_factor !-------------------------------------------------------------------------- @@ -1858,25 +1859,25 @@ subroutine get_rain_dsd2(qr,nr,mu_r,lamr,cdistr,logn0r) ! find spot in lookup table ! (scaled N/q for lookup table parameter space_ nr = max(nr,nsmall) - inv_dum = bfb_cbrt(qr/(cons1*nr*6._rtype)) ! Apply constant mu_r: Recall the switch to v4 tables means constant mu_r mu_r = mu_r_constant - lamr = bfb_cbrt(cons1*nr*(mu_r+3._rtype)*(mu_r+2._rtype)*(mu_r+1._rtype)/(qr)) ! recalculate slope based on mu_r + mass_to_d3_factor = cons1*(mu_r+3._rtype)*(mu_r+2._rtype)*(mu_r+1._rtype) + lamr = bfb_cbrt(mass_to_d3_factor*nr/qr) ! recalculate slope based on mu_r lammax = (mu_r+1._rtype)*1.e+5_rtype ! check for slope lammin = (mu_r+1._rtype)*500._rtype !500=1/(2mm) is inverse of max allowed number-weighted mean raindrop diameter ! apply lambda limiters for rain if (lamr.lt.lammin) then lamr = lammin - nr = bfb_exp(3._rtype*bfb_log(lamr)+bfb_log(qr)+bfb_log(bfb_gamma(mu_r+1._rtype))-bfb_log(bfb_gamma(mu_r+4._rtype)))/(cons1) + nr = lamr * lamr * lamr * qr / mass_to_d3_factor elseif (lamr.gt.lammax) then lamr = lammax - nr = bfb_exp(3._rtype*bfb_log(lamr)+bfb_log(qr)+bfb_log(bfb_gamma(mu_r+1._rtype))-bfb_log(bfb_gamma(mu_r+4._rtype)))/(cons1) + nr = lamr * lamr * lamr * qr / mass_to_d3_factor endif cdistr = nr/bfb_gamma(mu_r+1._rtype) - logn0r = bfb_log10(nr)+(mu_r+1._rtype)*bfb_log10(lamr)-bfb_log10(bfb_gamma(mu_r+1._rtype)) !note: logn0r is calculated as log10(n0r) + logn0r = bfb_log10(cdistr)+(mu_r+1._rtype)*bfb_log10(lamr) !note: logn0r is calculated as log10(n0r) else diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index e2aae3e97b32..453b2d4be8a3 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -1,7 +1,18 @@ +#################################################################### +# Basic setup: version, cime debug, cmake paths,... # +#################################################################### if (NOT DEFINED PROJECT_NAME) - cmake_minimum_required(VERSION 3.3) + cmake_minimum_required(VERSION 3.14) cmake_policy(SET CMP0057 NEW) set(SCREAM_CIME_BUILD FALSE) + + # Print the sha of the last commit (useful to double check which version was tested on CDash) + execute_process (COMMAND git rev-parse HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE LAST_GIT_COMMIT_SHA + OUTPUT_STRIP_TRAILING_WHITESPACE) + set(EAMXX_GIT_VERSION ${LAST_GIT_COMMIT_SHA} CACHE STRING "The sha of the last git commit." FORCE) + message(STATUS "The sha of the last commit is ${EAMXX_GIT_VERSION}") else() set(SCREAM_CIME_BUILD TRUE) endif() @@ -20,6 +31,13 @@ if ($ENV{SCREAM_FORCE_CONFIG_FAIL}) message(FATAL_ERROR "Failed, as instructed by environment") endif() +string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) +if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") + set (SCREAM_DEBUG TRUE) +else () + set (SCREAM_DEBUG FALSE) +endif() + # Add the ./cmake folder to cmake path. Also add EKAT's cmake folder set (EKAT_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat/cmake) list(APPEND CMAKE_MODULE_PATH @@ -33,6 +51,26 @@ if (SCREAM_CIME_BUILD) ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cime) endif () +# We want to use C++17 in EAMxx +set(CMAKE_CXX_STANDARD 17) + +if (NOT SCREAM_CIME_BUILD) + project(SCREAM CXX C Fortran) + + if (SCREAM_CORI_HACK) + list(APPEND CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifcore") + list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifport") + endif() + +else() + # Ensure our languages are all enabled + enable_language(C CXX Fortran) +endif() + +#################################################################### +# Kokkos/YAKL-related settings # +#################################################################### + if (Kokkos_ENABLE_CUDA) if (Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) if (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK) @@ -56,68 +94,6 @@ endif() # to be on. For now, simply ensure Kokkos Serial is enabled option (Kokkos_ENABLE_SERIAL "" ON) -# We want to use C++17 in EAMxx -set(CMAKE_CXX_STANDARD 17) - -if (NOT SCREAM_CIME_BUILD) - project(SCREAM CXX C Fortran) - - if (SCREAM_CORI_HACK) - list(APPEND CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifcore") - list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifport") - endif() - -else() - # Ensure our languages are all enabled - enable_language(C CXX Fortran) -endif() - -# Print the sha of the last commit (useful to double check which version was tested on CDash) -execute_process (COMMAND git rev-parse HEAD - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE LAST_GIT_COMMIT_SHA - OUTPUT_STRIP_TRAILING_WHITESPACE) -set(EAMXX_GIT_VERSION ${LAST_GIT_COMMIT_SHA} CACHE STRING "The sha of the last git commit.") -message(STATUS "The sha of the last commit is ${EAMXX_GIT_VERSION}") - -set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)") - -# Set the scream base and src directory, to be used across subfolders -set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) -set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) - -# Shortcut function, to print a variable -function (print_var var) - message ("${var}: ${${var}}") -endfunction () - -function (check_pack_size master_pack_size pack_size name) - math (EXPR PACK_MODULO "${master_pack_size} % ${pack_size}") - if ((pack_size GREATER master_pack_size) OR (NOT PACK_MODULO EQUAL 0)) - message (FATAL_ERROR "Invalid '${name}' size of ${pack_size}. Needs to be <= ${master_pack_size} and be a factor of it") - endif() -endfunction () - -# Compute reasonable defaults. This needs to happen before the CACHE variables -# are set. -string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) -if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") - set (SCREAM_DEBUG TRUE) -else () - set (SCREAM_DEBUG FALSE) -endif() - -# Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS -# to turn on the RRTMGP internal checks here as well, via -# option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG}) -# and then adding to scream_config.h: -# #cmakedefine RRTMGP_EXPENSIVE_CHECKS -option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) - -enable_testing() -include(CTest) - set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "") set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? @@ -142,13 +118,16 @@ if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" ) set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp") endif() -### Scream default configuration options +#################################################################### +# EAMxx main configuration options # +#################################################################### + +# First, Compute reasonable defaults. This needs to happen before the CACHE variables are set set(DEFAULT_MAX_RANKS 4) set(DEFAULT_MAX_THREADS 16) set(DEFAULT_MIMIC_GPU FALSE) set(DEFAULT_FPE FALSE) set(DEFAULT_PACK_SIZE 16) -set(DEFAULT_POSSIBLY_NO_PACK FALSE) if (EAMXX_ENABLE_GPU) # On the GPU, the pack size must be 1 set(DEFAULT_PACK_SIZE 1) @@ -203,24 +182,122 @@ if (Kokkos_ENABLE_HIP) set(DEFAULT_SMALL_KERNELS TRUE) endif() -### Set CACHE vars +if (SCREAM_DEBUG) + set(DEFAULT_FPMODEL "strict") + if (SCREAM_PACK_SIZE EQUAL 1 AND NOT EAMXX_ENABLE_GPU) + set(DEFAULT_FPE TRUE) + endif () +endif() + +### Now that reasonable defaults have been computed, set CACHE vars set(SCREAM_MIMIC_GPU ${DEFAULT_MIMIC_GPU} CACHE BOOL "Mimic GPU to correctness-test inter-column parallelism on non-GPU platform") set(SCREAM_PACK_CHECK_BOUNDS FALSE CACHE BOOL "If defined, scream::pack objects check indices against bounds") -set(SCREAM_TEST_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR}/data CACHE PATH "Location of data files generated by tests") set(SCREAM_LIB_ONLY ${DEFAULT_LIB_ONLY} CACHE BOOL "Only build libraries, no exes") set(NetCDF_Fortran_PATH ${DEFAULT_NetCDF_Fortran_PATH} CACHE FILEPATH "Path to netcdf fortran installation") set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C installation") set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) -option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" OFF) +option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" ON) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels") if (NOT SCREAM_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() +# Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS +# to turn on the RRTMGP internal checks here as well, via +# option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG}) +# and then adding to scream_config.h: +# #cmakedefine RRTMGP_EXPENSIVE_CHECKS +option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) + +set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)") + # For now, only used in share/grid/remap/refining_remapper_rma.*pp option (EAMXX_ENABLE_EXPERIMENTAL_CODE "Compile one-sided MPI for refining remappers" OFF) +option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" OFF) + +# Set number of vertical levels +set(SCREAM_NUM_VERTICAL_LEV ${DEFAULT_NUM_VERTICAL_LEV} CACHE STRING + "The number of levels used in the vertical grid." +) +option(SCREAM_HAS_LEAP_YEAR "Whether scream uses leap years or not" ON) + +set(SCREAM_FPMODEL ${DEFAULT_FPMODEL} CACHE STRING "Compiler floating point model") +set(SCREAM_FPE ${DEFAULT_FPE} CACHE BOOL "Enable floating point error exception") + +# Whether to use XYZ as a method to detect memory usage. +option (SCREAM_ENABLE_GETRUSAGE "Whether getrusage can be used to get memory usage." OFF) +option (SCREAM_ENABLE_STATM "Whether /proc/self/statm can be used to get memory usage." OFF) + +# Whether to disable warnings from tpls. +set (SCREAM_DISABLE_TPL_WARNINGS ON CACHE BOOL "") + +# Dycore settings +set(DEFAULT_SCREAM_DYNAMICS_DYCORE "NONE") +if (SCREAM_CIME_BUILD AND SCREAM_DYN_TARGET STREQUAL "theta-l_kokkos") + set (DEFAULT_SCREAM_DYNAMICS_DYCORE "Homme") +endif() + +set(SCREAM_DYNAMICS_DYCORE ${DEFAULT_SCREAM_DYNAMICS_DYCORE} CACHE STRING + "The name of the dycore to be used for dynamics. If NONE, then any code/test requiring dynamics is disabled.") + +string(TOUPPER "${SCREAM_DYNAMICS_DYCORE}" SCREAM_DYNAMICS_DYCORE) +if (NOT ${SCREAM_DOUBLE_PRECISION}) + # Homme cannot handle single precision, for now. This causes tests to fail. + # Fixing this requires adding a config parameter to homme, to switch between + # single and double. That must be done in the upstream repo (E3SM), before + # we can support it here. + # So, for now, if Homme is the requested dyn dycore AND single precision is + # requested, we disable dynamics, printing a warning. + if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") + message("WARNING! Homme dycore cannot be used in a Single Precision build. Turning Homme off.") + set(SCREAM_DYNAMICS_DYCORE "NONE") + endif() +endif() + +# Set the scream base and src directory, to be used across subfolders +set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) +set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +#################################################################### +# Packs-related settings # +#################################################################### + +# Determine the main pack size. +set(SCREAM_PACK_SIZE ${DEFAULT_PACK_SIZE} CACHE STRING + "The number of scalars in a scream::pack::Pack and Mask. Larger packs have good performance on conditional-free loops due to improved caching.") + +# Besides the the main pack size, we have a couple of other pack sizes used across EAMxx +# For some routines, SKX may have better performance with pack_size=1 +set(SCREAM_SMALL_PACK_SIZE ${SCREAM_PACK_SIZE} CACHE STRING + "The number of scalars in a scream::pack::SmallPack and SmallMask. Smaller packs can have better performance in loops with conditionals since more of the packs will have masks with uniform value.") +set(SCREAM_POSSIBLY_NO_PACK "${Kokkos_ARCH_SKX}" CACHE BOOL + "Set possibly-no-pack to this value. You can set it to something else to restore packs on SKX for testing.") + +if (SCREAM_POSSIBLY_NO_PACK) + set (SCREAM_POSSIBLY_NO_PACK_SIZE 1) +else() + set (SCREAM_POSSIBLY_NO_PACK_SIZE ${SCREAM_PACK_SIZE}) +endif () + +function (check_pack_size master_pack_size pack_size name) + math (EXPR PACK_MODULO "${master_pack_size} % ${pack_size}") + if ((pack_size GREATER master_pack_size) OR (NOT PACK_MODULO EQUAL 0)) + message (FATAL_ERROR "Invalid '${name}' size of ${pack_size}. Needs to be <= ${master_pack_size} and be a factor of it") + endif() +endfunction () + +# Checks on pack sizes relative to the master one: +check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_SMALL_PACK_SIZE} "small pack") +# This one is an internal check, as the user cannot set SCREAM_POSSIBLY_NO_PACK_SIZE now. +check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_POSSIBLY_NO_PACK_SIZE} "possibly no pack") + +#################################################################### +# Input-data locations # +#################################################################### + # Handle input root if (SCREAM_MACHINE AND NOT SCREAM_INPUT_ROOT) execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/query-cime ${SCREAM_MACHINE} DIN_LOC_ROOT @@ -274,140 +351,172 @@ set (SCREAM_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/scream CACHE PATH "" FORCE) set (TOPO_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/topo CACHE PATH "" FORCE) set (IOP_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/scam/iop CACHE PATH "" FORCE) -# -# Handle test level -# +#################################################################### +# Tests-related settings # +#################################################################### + +if (NOT SCREAM_LIB_ONLY) + + # Assuming SCREAM_LIB_ONLY is FALSE (else, no exec is built at all), we need to decide + # wether to build baseline-related execs, and wether we are generating baselines or + # comparing against them. These options can help reducing a bit the code that is built + # when generating baselines or when running memory-check tests (no baselines needed there) + option(SCREAM_ONLY_GENERATE_BASELINES "Whether building only baselines-related executables" OFF) + option(SCREAM_ENABLE_BASELINE_TESTS "Whether to run baselines-related tests" ON) + if (SCREAM_ONLY_GENERATE_BASELINES AND NOT SCREAM_ENABLE_BASELINE_TESTS) + message (FATAL_ERROR + "Makes no sense to set SCREAM_ONLY_GENERATE_BASELINES=ON,\n" + "but set SCREAM_ENABLE_BASELINE_TESTS=OFF.") + endif() -# Constants -set(SCREAM_TEST_LEVEL_AT "0") -set(SCREAM_TEST_LEVEL_NIGHTLY "1") -set(SCREAM_TEST_LEVEL_EXPERIMENTAL "2") + set(SCREAM_BASELINES_DIR "UNSET" CACHE PATH "Folder containing baselines data") + if (SCREAM_ENABLE_BASELINE_TESTS) + if (NOT EXISTS ${SCREAM_BASELINES_DIR}/data OR NOT IS_DIRECTORY ${SCREAM_BASELINES_DIR}/data) + string (CONCAT msg + "Error! Baselines tests enabled, but baseline dir is invalid.\n" + " SCREAM_BASELINES_DIR: ${SCREAM_BASELINES_DIR}") + message ("${msg}") + message (FATAL_ERROR "Aborting...") + endif() + endif() -set(SCREAM_TEST_LEVEL "AT" CACHE STRING "The test level to run. Default is AT. NIGHTLY will run additional tests but is not guaranteed to PASS. EXPERIMENTAL will run even more tests with failures being more likely") + # All baselines tests will add a line to the baseline_list file, + # specifyiing the full name of the baseline they generated. + # When test-all-scream has to generate new baselines, it will run + # ctest -L baseline_gen, and then read this file to find out all the + # baseline files to copy into the baseline directory + set (SCREAM_TEST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/data) + file(MAKE_DIRECTORY ${SCREAM_TEST_OUTPUT_DIR}) + file(TOUCH ${SCREAM_TEST_OUTPUT_DIR}/baseline_list) + + set(SCREAM_TEST_LEVEL_AT "0") + set(SCREAM_TEST_LEVEL_NIGHTLY "1") + set(SCREAM_TEST_LEVEL_EXPERIMENTAL "2") + + set(SCREAM_TEST_LEVEL "AT" CACHE STRING "The test level to run. Default is AT. NIGHTLY will run additional tests but is not guaranteed to PASS. EXPERIMENTAL will run even more tests with failures being more likely") + + if (SCREAM_TEST_LEVEL STREQUAL "AT") + set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_AT}) + elseif (SCREAM_TEST_LEVEL STREQUAL "NIGHTLY") + set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_NIGHTLY}) + elseif (SCREAM_TEST_LEVEL STREQUAL "EXPERIMENTAL") + set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_EXPERIMENTAL}) + else() + message(FATAL_ERROR "Unknown SCREAM_TEST_LEVEL '${SCREAM_TEST_LEVEL}'") + endif() -if (SCREAM_TEST_LEVEL STREQUAL "AT") - set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_AT}) -elseif (SCREAM_TEST_LEVEL STREQUAL "NIGHTLY") - set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_NIGHTLY}) -elseif (SCREAM_TEST_LEVEL STREQUAL "EXPERIMENTAL") - set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_EXPERIMENTAL}) -else() - message(FATAL_ERROR "Unknown SCREAM_TEST_LEVEL '${SCREAM_TEST_LEVEL}'") -endif() + set(SCREAM_TEST_MAX_THREADS ${DEFAULT_MAX_THREADS} CACHE STRING "Upper limit on threads per rank for threaded tests") + set(SCREAM_TEST_THREAD_INC 1 CACHE STRING "Thread count increment for threaded tests") + set(SCREAM_TEST_MAX_RANKS ${DEFAULT_MAX_RANKS} CACHE STRING "Upper limit on ranks for mpi tests") + math(EXPR DEFAULT_MAX_TOTAL_THREADS "${SCREAM_TEST_MAX_RANKS}*${SCREAM_TEST_MAX_THREADS}") + set(SCREAM_TEST_MAX_TOTAL_THREADS ${DEFAULT_MAX_TOTAL_THREADS} CACHE STRING "Upper limit on nranks*threads for threaded tests") + + # Make sure SCREAM_TEST_MAX_RANKS and SCREAM_TEST_MAX_THREADS do not individually exceed SCREAM_TEST_MAX_TOTAL_THREADS + if (SCREAM_TEST_MAX_THREADS GREATER ${SCREAM_TEST_MAX_TOTAL_THREADS}) + string(CONCAT msg + "The requested number of max threads/rank (${SCREAM_TEST_MAX_THREADS}) is larger " + "than the max total threads (${SCREAM_TEST_MAX_TOTAL_THREADS}). Setting " + "SCREAM_TEST_MAX_THREADS=${SCREAM_TEST_MAX_TOTAL_THREADS}") + message(STATUS "${msg}") + set (SCREAM_TEST_MAX_THREADS ${SCREAM_TEST_MAX_TOTAL_THREADS}) + endif() + if (SCREAM_TEST_MAX_RANKS GREATER ${SCREAM_TEST_MAX_TOTAL_THREADS}) + string(CONCAT msg + "The requested number of max ranks (${SCREAM_TEST_MAX_RANKS}) is larger " + "than the max total threads (${SCREAM_TEST_MAX_TOTAL_THREADS}). Setting " + "SCREAM_TEST_MAX_RANKS=${SCREAM_TEST_MAX_TOTAL_THREADS}") + message(STATUS "${msg}") + set (SCREAM_TEST_MAX_RANKS ${SCREAM_TEST_MAX_TOTAL_THREADS}) + endif() + # This is a meta-variable, which individual tests can use to set *different* degrees + # of testing, in terms of resolutions. E.g., for SHORT use 3 timesteps, for MEDIUM use 10, + # for LONG use 100. It is *completely* up to the test to decide what short, medium, and long mean. + if (EKAT_ENABLE_COVERAGE OR EKAT_ENABLE_CUDA_MEMCHECK OR EKAT_ENABLE_VALGRIND OR EKAT_ENABLE_COMPUTE_SANITIZER) + set (SCREAM_TEST_SIZE_DEFAULT SHORT) + # also set thread_ing=$max_thread - 1, so we test at most 2 threading configurations + if (SCREAM_TEST_MAX_THREADS GREATER 1) + math (EXPR SCREAM_TEST_THREAD_INC ${SCREAM_TEST_MAX_THREADS}-1) + endif() + else() + set (SCREAM_TEST_SIZE_DEFAULT MEDIUM) + endif() -# Assuming SCREAM_LIB_ONLY is FALSE (else, no exec is built at all), we provide the option -# of building only baseline-related execs. By default, this option is off (menaing "build everything"). -# However, when generating baselines, this can be useful to reduce the amount of stuff compiled. -set(SCREAM_BASELINES_ONLY FALSE CACHE BOOL "Whether building only baselines-related executables") + set(SCREAM_TEST_SIZE ${SCREAM_TEST_SIZE_DEFAULT} CACHE STRING "The kind of testing to perform: SHORT, MEDIUM, LONG. Only applies to certain tests and is generally used to reduce test length when valgrind/cuda-memcheck are on.") + set(SCREAM_TEST_VALID_SIZES "SHORT;MEDIUM;LONG" CACHE INTERNAL "List of valid values for SCREAM_TEST_SIZE") -# Certain builds are not meant to compare against baselines. For instance, a valgrind build -# has the sole purpose of detecting memory errors. For these builds, we can disable baselines tests -set(SCREAM_ENABLE_BASELINE_TESTS TRUE CACHE BOOL "Whether to run baselines-related tests") + if (SCREAM_TEST_SIZE STREQUAL "SHORT") + add_definitions(-DSCREAM_SHORT_TESTS) + endif() -if (SCREAM_BASELINES_ONLY AND NOT SCREAM_ENABLE_BASELINE_TESTS) - message (FATAL_ERROR - "Makes no sense to set SCREAM_BASELINES_ONLY=ON,\n" - "but set SCREAM_ENABLE_BASELINE_TESTS=OFF.") + enable_testing() + include(CTest) endif() -# Set number of vertical levels -set(SCREAM_NUM_VERTICAL_LEV ${DEFAULT_NUM_VERTICAL_LEV} CACHE STRING - "The number of levels used in the vertical grid." -) -option(SCREAM_HAS_LEAP_YEAR "Whether scream uses leap years or not" ON) +#################################################################### +# Configure all tpls and subfolders # +#################################################################### + +if (DEFINED ENV{SCREAM_FAKE_ONLY}) + # We don't really need to build ekat, but we do need to configure the test-launcher + + # Declare some vars that Ekat would have declared, and may be used later + option (EKAT_ENABLE_MPI "Whether EKAT requires MPI." ON) + option (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES "Whether test-launcher should try to manage thread distribution. Requires a ctest resource file to be effective." OFF) + option (EKAT_ENABLE_VALGRIND "Whether to run tests with valgrind" OFF) + set(EKAT_VALGRIND_SUPPRESSION_FILE "" CACHE FILEPATH "Use this valgrind suppression file if valgrind is enabled.") + set (EKAT_ENABLE_GPU False) + if (Kokkos_ENABLE_CUDA OR Kokkos_ENABLE_HIP OR Kokkos_ENABLE_SYCL) + set (EKAT_ENABLE_GPU True) + endif () -## Work out pack sizes. -# Determine the master pack size. -set(SCREAM_PACK_SIZE ${DEFAULT_PACK_SIZE} CACHE STRING - "The number of scalars in a scream::pack::Pack and Mask. Larger packs have good performance on conditional-free loops due to improved caching.") -# With the master pack size determined, we have constraints on the others. -set(DEFAULT_SMALL_PACK_SIZE ${SCREAM_PACK_SIZE}) -# For some routines, SKX may have better performance with pksize=1 -if (Kokkos_ARCH_SKX) - set(DEFAULT_POSSIBLY_NO_PACK TRUE) -endif () -set(SCREAM_SMALL_PACK_SIZE ${DEFAULT_SMALL_PACK_SIZE} CACHE STRING - "The number of scalars in a scream::pack::SmallPack and SmallMask. Smaller packs can have better performance in loops with conditionals since more of the packs will have masks with uniform value.") -set(SCREAM_POSSIBLY_NO_PACK ${DEFAULT_POSSIBLY_NO_PACK} CACHE BOOL - "Set possibly-no-pack to this value. You can set it to something else to restore packs on SKX for testing.") -set (DEFAULT_POSSIBLY_NO_PACK_SIZE ${SCREAM_PACK_SIZE}) + if (EKAT_TEST_LAUNCHER_MANAGE_RESOURCES) + set (TEST_LAUNCHER_MANAGE_RESOURCES True) + else() + set (TEST_LAUNCHER_MANAGE_RESOURCES False) + endif() + if (EKAT_ENABLE_GPU) + set (TEST_LAUNCHER_ON_GPU True) + else() + set (TEST_LAUNCHER_ON_GPU False) + endif() -if (SCREAM_POSSIBLY_NO_PACK) - set (DEFAULT_POSSIBLY_NO_PACK_SIZE 1) -endif () -set (SCREAM_POSSIBLY_NO_PACK_SIZE ${DEFAULT_POSSIBLY_NO_PACK_SIZE}) -# Checks on pack sizes relative to the master one: -check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_SMALL_PACK_SIZE} "small pack") -# This one is an internal check, as the user cannot set SCREAM_POSSIBLY_NO_PACK_SIZE now. -check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_POSSIBLY_NO_PACK_SIZE} "possibly no pack") + set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat) -## Now we have pack sizes. Proceed with other config options that depend on -## these. + if (EKAT_ENABLE_MPI) + find_package(MPI REQUIRED COMPONENTS C) -if (SCREAM_DEBUG) - set(DEFAULT_FPMODEL "strict") - if (${SCREAM_PACK_SIZE} EQUAL 1 AND NOT ${EAMXX_ENABLE_GPU}) - set(DEFAULT_FPE TRUE) - endif () -endif() -set(SCREAM_FPMODEL ${DEFAULT_FPMODEL} CACHE STRING "Compiler floating point model") -set(SCREAM_FPE ${DEFAULT_FPE} CACHE BOOL "Enable floating point error exception") -### Experimental, under development + + F20TR-SCREAMv1 + 20TR_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_MOSART_SGLC_SWAV + + + + F2010-SCREAMv1-DP-DYCOMSrf01 + 2010_SCREAM_ELM%SPBC_CICE%PRES_DOCN%DOM_SROF_SGLC_SWAV_SIAC_SESP%DP-EAMxx%DYCOMSrf01 + Experimental, under development + + 2016-08-01 2020-01-20 + 1999-07-10 + + + + + + + 864 + + + + + + TRUE + + + + + + TRUE + + + + + + 31.5 + + + + + + 238.5 + + + + + + 225 + + + + + + 1 + + + + + + 225 + + + + + + 1 diff --git a/components/eamxx/cime_config/eamxx_buildnml.py b/components/eamxx/cime_config/eamxx_buildnml.py index db05bd2e20e1..0c7ed8e52aac 100644 --- a/components/eamxx/cime_config/eamxx_buildnml.py +++ b/components/eamxx/cime_config/eamxx_buildnml.py @@ -21,7 +21,7 @@ from utils import ensure_yaml # pylint: disable=no-name-in-module ensure_yaml() import yaml -from yaml_utils import Bools,Ints,Floats,Strings,array_representer +from yaml_utils import Bools,Ints,Floats,Strings,array_representer,array_constructor _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","cime") sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) @@ -465,7 +465,18 @@ def expand_cime_vars(element, case): child.text = do_cime_vars(child.text, case) ############################################################################### -def _create_raw_xml_file_impl(case, xml): +def write_pretty_xml(filepath, xml): +############################################################################### + with open(filepath, "w") as fd: + # dom has better pretty printing than ET in older python versions < 3.9 + dom = md.parseString(ET.tostring(xml, encoding="unicode")) + pretty_xml = dom.toprettyxml(indent=" ") + pretty_xml = os.linesep.join([s for s in pretty_xml.splitlines() + if s.strip()]) + fd.write(pretty_xml) + +############################################################################### +def _create_raw_xml_file_impl(case, xml, filepath=None): ############################################################################### """ On input, xml contains the parsed content of namelist_defaults_scream.xml. @@ -610,32 +621,40 @@ def _create_raw_xml_file_impl(case, xml): selectors = get_valid_selectors(xml) # 1. Evaluate all selectors - evaluate_selectors(xml, case, selectors) + try: + evaluate_selectors(xml, case, selectors) - # 2. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that may alter - # which atm processes are used - apply_atm_procs_list_changes_from_buffer (case,xml) + # 2. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that may alter + # which atm processes are used + apply_atm_procs_list_changes_from_buffer (case,xml) - # 3. Resolve all inheritances - resolve_all_inheritances(xml) + # 3. Resolve all inheritances + resolve_all_inheritances(xml) - # 4. Expand any CIME var that appears inside XML nodes text - expand_cime_vars(xml,case) + # 4. Expand any CIME var that appears inside XML nodes text + expand_cime_vars(xml,case) - # 5. Grab the atmosphere_processes macro list, with all the defaults - atm_procs_defaults = get_child(xml,"atmosphere_processes_defaults",remove=True) + # 5. Grab the atmosphere_processes macro list, with all the defaults + atm_procs_defaults = get_child(xml,"atmosphere_processes_defaults",remove=True) - # 6. Get atm procs list - atm_procs_list = get_child(atm_procs_defaults,"atm_procs_list",remove=True) + # 6. Get atm procs list + atm_procs_list = get_child(atm_procs_defaults,"atm_procs_list",remove=True) - # 7. Form the nested list of atm procs needed, append to atmosphere_driver section - atm_procs = gen_atm_proc_group(atm_procs_list.text, atm_procs_defaults) - atm_procs.tag = "atmosphere_processes" - xml.append(atm_procs) + # 7. Form the nested list of atm procs needed, append to atmosphere_driver section + atm_procs = gen_atm_proc_group(atm_procs_list.text, atm_procs_defaults) + atm_procs.tag = "atmosphere_processes" + xml.append(atm_procs) - # 8. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that do not alter - # which atm processes are used - apply_non_atm_procs_list_changes_from_buffer (case,xml) + # 8. Apply all changes in the SCREAM_ATMCHANGE_BUFFER that do not alter + # which atm processes are used + apply_non_atm_procs_list_changes_from_buffer (case,xml) + except BaseException as e: + if filepath is not None: + dbg_xml_path = filepath.replace(".xml", ".dbg.xml") + write_pretty_xml(dbg_xml_path, xml) + print(f"Error during XML creation, writing {dbg_xml_path}") + + raise e perform_consistency_checks (case, xml) @@ -666,17 +685,11 @@ def create_raw_xml_file(case, caseroot): # be processed early by treating them as if they were made to the defaults file. with open(src, "r") as fd: defaults = ET.parse(fd).getroot() - raw_xml = _create_raw_xml_file_impl(case, defaults) + raw_xml = _create_raw_xml_file_impl(case, defaults, filepath=raw_xml_file) check_all_values(raw_xml) - with open(raw_xml_file, "w") as fd: - # dom has better pretty printing than ET in older python versions < 3.9 - dom = md.parseString(ET.tostring(raw_xml, encoding="unicode")) - pretty_xml = dom.toprettyxml(indent=" ") - pretty_xml = os.linesep.join([s for s in pretty_xml.splitlines() - if s.strip()]) - fd.write(pretty_xml) + write_pretty_xml(raw_xml_file, raw_xml) ############################################################################### def convert_to_dict(element): @@ -877,6 +890,11 @@ def get_file_parameters(caseroot): result = [] for item in raw_xml.findall('.//*[@type="file"]'): + # Certain configurations may not need a file (e.g., a remap + # file for SPA may not be needed if the model resolution + # matches the data file resolution + if item.text is None or item.text=="": + continue result.append(item.text.strip()) for item in raw_xml.findall('.//*[@type="array(file)"]'): @@ -886,7 +904,7 @@ def get_file_parameters(caseroot): return list(OrderedDict.fromkeys(result)) ############################################################################### -def create_input_data_list_file(caseroot): +def create_input_data_list_file(case,caseroot): ############################################################################### """ Create the scream.input_data_list file for this case. This will tell CIME @@ -894,18 +912,51 @@ def create_input_data_list_file(caseroot): """ files_to_download = get_file_parameters(caseroot) + # Add array parsing knowledge to yaml loader + loader = yaml.SafeLoader + loader.add_constructor("!bools",array_constructor) + loader.add_constructor("!ints",array_constructor) + loader.add_constructor("!floats",array_constructor) + loader.add_constructor("!strings",array_constructor) + + # Grab all the output yaml files, open them, and check if horiz_remap_file or vertical_remap_file is used + rundir = case.get_value("RUNDIR") + eamxx_xml_file = os.path.join(caseroot, "namelist_scream.xml") + with open(eamxx_xml_file, "r") as fd: + eamxx_xml = ET.parse(fd).getroot() + + scorpio = get_child(eamxx_xml,'Scorpio') + out_files_xml = get_child(scorpio,"output_yaml_files",must_exist=False) + # out_files = out_files_xml.text.split(",") if (out_files_xml is not None and out_files_xml.text is not None) else [] + # for fn in out_files: + if (out_files_xml is not None and out_files_xml.text is not None): + for fn in out_files_xml.text.split(","): + # Get full name + src_yaml = os.path.expanduser(os.path.join(fn.strip())) + dst_yaml = os.path.expanduser(os.path.join(rundir,'data',os.path.basename(src_yaml))) + + # Load file, and look for the remap file entries + content = yaml.load(open(dst_yaml,"r"),Loader=loader) + if 'horiz_remap_file' in content.keys(): + files_to_download += [content['horiz_remap_file']] + if 'vertical_remap_file' in content.keys(): + files_to_download += [content['vertical_remap_file']] + input_data_list_file = "{}/Buildconf/scream.input_data_list".format(caseroot) if os.path.exists(input_data_list_file): os.remove(input_data_list_file) + din_loc_root = case.get_value("DIN_LOC_ROOT") with open(input_data_list_file, "w") as fd: - for idx, file_path in enumerate(files_to_download): - fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) + for idx, file_path in enumerate(list(set(files_to_download))): + # Only add files whose full path starts with the CIME's input data location + if file_path.startswith(din_loc_root): + fd.write("scream_dl_input_{} = {}\n".format(idx, file_path)) + ############################################################################### def do_cime_vars_on_yaml_output_files(case, caseroot): ############################################################################### - from yaml_utils import array_constructor rundir = case.get_value("RUNDIR") eamxx_xml_file = os.path.join(caseroot, "namelist_scream.xml") @@ -956,14 +1007,15 @@ def do_cime_vars_on_yaml_output_files(case, caseroot): # produces an output at t=0, which is not present in the restarted run, and # which also causes different timestamp in the file name. # Hence, change default output settings to perform a single AVERAGE step at the end of the run - if case.get_value("TESTCASE") in ["ERP", "ERS"]: - test_env = case.get_env('test') - stop_n = int(test_env.get_value("STOP_N")) - stop_opt = test_env.get_value("STOP_OPTION") - content['output_control']['Frequency'] = stop_n - content['output_control']['frequency_units'] = stop_opt - content['Averaging Type'] = 'AVERAGE' - print ("WARNING: ERS/ERP tests hard code output to consist of a single AVERAGE output step at the end of the run.") + if case.get_value("TESTCASE") in ["ERP", "ERS"] and content['Averaging Type'].upper()=="INSTANT": + hist_n = int(case.get_value("HIST_N",resolved=True)) + hist_opt = case.get_value("HIST_OPTION",resolved=True) + content['output_control']['Frequency'] = hist_n + content['output_control']['frequency_units'] = hist_opt + content['output_control']['skip_t0_output'] = True + print ("ERS/ERP test with INSTANT output detected. Adjusting output control specs:\n") + print (" - setting skip_t0_output=true\n") + print (" - setting freq and freq_units to HIST_N and HIST_OPTION respectively\n") ordered_dump(content, open(dst_yaml, "w")) diff --git a/components/eamxx/cime_config/eamxx_buildnml_impl.py b/components/eamxx/cime_config/eamxx_buildnml_impl.py index 2fa00b7f4a4b..e0ecab146246 100644 --- a/components/eamxx/cime_config/eamxx_buildnml_impl.py +++ b/components/eamxx/cime_config/eamxx_buildnml_impl.py @@ -162,11 +162,12 @@ def refine_type(entry, force_type=None): >>> e = '1.0' >>> refine_type(e,force_type='my_type') Traceback (most recent call last): - NameError: ERROR: Invalid/unsupported force type 'my_type' + CIME.utils.CIMEError: ERROR: Invalid/unsupported force type 'my_type' >>> e = 'true,falsE' >>> refine_type(e,'logical') Traceback (most recent call last): - ValueError: Could not refine 'true,falsE' as type 'logical' + CIME.utils.CIMEError: ERROR: Could not refine 'true,falsE' as type 'logical': + ERROR: For entry of type 'logical', expected 'true' or 'false', got 'true,falsE' >>> refine_type(e,'array(logical)') [True, False] >>> refine_type('', 'array(string)') @@ -176,15 +177,14 @@ def refine_type(entry, force_type=None): >>> refine_type(None, 'array(real)') [] """ - - # If force type is unspecified, try to deduce it + # If force type is unspecified, try to deduce it if force_type is None: expect (entry is not None, "If an entry is None, you must specify the force_type") else: elem_valid = ["logical","integer","real","string","file"] valid = elem_valid + ["array("+e+")" for e in elem_valid] - expect (force_type in valid, exc_type=NameError, + expect (force_type in valid, error_msg=f"Invalid/unsupported force type '{force_type}'") if is_array_type(force_type): @@ -208,7 +208,8 @@ def refine_type(entry, force_type=None): elif entry.upper() == "FALSE": return False else: - return bool(int(entry)) + expect(False, f"For entry of type 'logical', expected 'true' or 'false', got '{entry}'", + exc_type=ValueError) elif elem_type == "integer": tmp = float(entry) @@ -220,7 +221,7 @@ def refine_type(entry, force_type=None): return str(entry) except ValueError as e: - raise ValueError (f"Could not refine '{entry}' as type '{force_type}'") from e + expect(False, f"Could not refine '{entry}' as type '{force_type}':\n{e}") # No force type provided. Try to infer from value if entry.upper() == "TRUE": @@ -273,7 +274,7 @@ def derive_type(entry): elif isinstance(elem_value, str): elem_type = "string" else: - raise RuntimeError("Couldn't derive type of '{}'".format(entry)) + expect(False, "Couldn't derive type of '{}'".format(entry)) if isinstance(refined_value,list): return "array(" + elem_type + ")" @@ -293,7 +294,8 @@ def check_value(elem, value): >>> root = ET.fromstring(xml) >>> check_value(root,'1.5') Traceback (most recent call last): - ValueError: Could not refine '1.5' as type 'integer' + CIME.utils.CIMEError: ERROR: Could not refine '1.5' as type 'integer': + ERROR: Cannot interpret 1.5 as int >>> check_value(root,'3') Traceback (most recent call last): CIME.utils.CIMEError: ERROR: Invalid value '3' for element 'a'. Value not in the valid list ('[1, 2]') diff --git a/components/eamxx/cime_config/namelist_defaults_scream.xml b/components/eamxx/cime_config/namelist_defaults_scream.xml index 71c1bfd4f828..07ef2793f38c 100644 --- a/components/eamxx/cime_config/namelist_defaults_scream.xml +++ b/components/eamxx/cime_config/namelist_defaults_scream.xml @@ -48,7 +48,7 @@ be lost if SCREAM_HACK_XML is not enabled. ctl_nl - driver_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters + driver_options,iop_options,atmosphere_processes,grids_manager,initial_conditions,Scorpio,e3sm_parameters @@ -119,9 +119,8 @@ be lost if SCREAM_HACK_XML is not enabled. for the atmosphere processes section(s). 11) The attribute 'locked="true"' is to be used for entries that cannot be changed - via atmchange (see scripts/atmchange). For instance, the overall list of atm procs cannot be changed, - since it would require to re-parse the defaults, to re-generate the correct - defaults for the (possibly) new atm procs. + via atmchange (see scripts/atmchange). If an element is locked, then all children + will be locked as well. 12) The attribute 'constraints' allows to specify constraints on values. Valid constraints are lt, le, ne, gt, ge, and mod. Except the latter (which has slightly different syntax, @@ -203,6 +202,18 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/tables/vn_table_vals.dat8, ${DIN_LOC_ROOT}/atm/scream/tables/vm_table_vals.dat8 + 1350.0 + 1.0 + 1.0 + 67.0 + 0.5 + 1.0 + 50.0 + 900.0 + 0.65 + 0.304 + 1.0 + 0.00028 @@ -226,20 +237,38 @@ be lost if SCREAM_HACK_XML is not enabled. + + + + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + ${DIN_LOC_ROOT}/atm/scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + + - + 0 false - false + TIME_DEPENDENT_3D_PROFILE - + - UNSET - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30_to_ne4_mono_20220502.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc - none - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne30pg2_mono.20220714.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120np4_mono_20220502.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne512np4_mono_20220506.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne120pg2_intbilin_20221012.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne256pg2_intbilin_20221011.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne512pg2_intbilin_20221012.nc - ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30np4_to_ne1024pg2_intbilin_20221012.nc - - ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne30_20220428.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne120pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne256pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne512pg2_20231201.nc + ${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne1024pg2_20231201.nc + + + + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne30_20220428.nc + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne30pg2_20240111.nc + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne4_20220428.nc + ${DIN_LOC_ROOT}/atm/scream/init/spa_file_unified_and_complete_ne4pg2_20231222.nc @@ -327,6 +356,7 @@ be lost if SCREAM_HACK_XML is not enabled. 3 3 3 + 3 4 true false @@ -343,6 +373,9 @@ be lost if SCREAM_HACK_XML is not enabled. > false + + true + @@ -356,11 +389,15 @@ be lost if SCREAM_HACK_XML is not enabled. 6 2 1 + 1 5 - 10 + 10 + 1 + 1 + 1 1 hours @@ -397,10 +434,8 @@ be lost if SCREAM_HACK_XML is not enabled. ${DIN_LOC_ROOT}/atm/scream/init/screami_ne256np4L128_ifs-20200120_20220914.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne512np4L128_20220823.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne1024np4L128_era5-20131001-topoadj-16x_20220914.nc - ${DIN_LOC_ROOT}/atm/scream/init/screami_ne1024np4L128_ifs-20160801-topoadjx6t_20221011.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_ne1024np4L128_ifs-20200120-topoadjx6t_20221011.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_aquaplanet_ne4np4L72_20220823.nc - ${DIN_LOC_ROOT}/atm/scream/init/screami_aquaplanet_ne30np4L128_20220823.nc ${DIN_LOC_ROOT}/atm/scream/init/screami_conusx4v1np4L72-topo12x_013023.nc @@ -453,11 +488,19 @@ be lost if SCREAM_HACK_XML is not enabled. 0.0 0.0 0.0 + + + + T_mid + false + 0 + 0.001 + 900.0 - ${SRCROOT}/components/eamxx/data/scream_default_output.yaml + ./${CASE}.scream @@ -475,12 +518,19 @@ be lost if SCREAM_HACK_XML is not enabled. doc="Verbosity level for the atm logger"> info + + warn + false 1e-10 1e-14 Warning true phis,landfrac + false + true @@ -488,11 +538,28 @@ be lost if SCREAM_HACK_XML is not enabled. ${CASE} + + + true + UNSET + ${DIN_LOC_ROOT}/atm/cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc + -999 + 31.5 + -999 + 238.5 + false + true + false + true + + 0 + 2 False 2 + 1 1 6 0 @@ -506,6 +573,7 @@ be lost if SCREAM_HACK_XML is not enabled. 1 1 3.4e-08 + 0.216784 UNSET 250000.0 250000.0 @@ -513,6 +581,7 @@ be lost if SCREAM_HACK_XML is not enabled. 4.0e4 2.0e4 1.0e4 + 1.0e4 100000.0 1 0 @@ -522,6 +591,7 @@ be lost if SCREAM_HACK_XML is not enabled. 0 0 sphere + plane 9 UNSET 4 @@ -530,19 +600,28 @@ be lost if SCREAM_HACK_XML is not enabled. 256 512 1024 + 1024 0 0 + 5 0 + 5 + 0 + 50000 + 0 + 50000 -1 4 cube - UNSET + plane + UNSET 600 300 75 33.33333333333 16.6666666666666 8.3333333333333 + 8.3333333333333 75 9999 1 diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands index d3a4a39b668a..cf3ca97f6dd3 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/internal_diagnostics_level/shell_commands @@ -1,2 +1,2 @@ $CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b -./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/tests/postrun/check_hashes_ers.py" +./xmlchange POSTRUN_SCRIPT="$CIMEROOT/../components/eamxx/scripts/check-hashes-ers" diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands new file mode 100644 index 000000000000..1c22bd9ee454 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/mam4xx/optics/shell_commands @@ -0,0 +1,9 @@ + +#Default scream has 10 tracers, MAM4xx adds another 31 making a total of 41 tracer +#Set total number of tracers to 41. We are using append here as last entry wins while parsing xml options +./xmlchange --append SCREAM_CMAKE_OPTIONS="SCREAM_NUM_TRACERS 41" + +$CIMEROOT/../components/eamxx/scripts/atmchange initial_conditions::Filename='$DIN_LOC_ROOT/atm/scream/init/screami_mam4xx_ne4np4L72_c20240208.nc' -b +$CIMEROOT/../components/eamxx/scripts/atmchange physics::atm_procs_list="mac_aero_mic,mam4_optics,rrtmgp" -b + + diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README new file mode 100644 index 000000000000..5d33c71fed37 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/README @@ -0,0 +1,9 @@ +The testmods in this folder contain different configuration +of output to request to eamxx. All the presets create a single +output file, all with a different name, which means one can +use 2+ presets at the same time. + + +Each preset does basically two things: + 1. create a yaml file in the case folder + 2. add the yaml file to the yaml_output_files xml setting of eamxx diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands new file mode 100644 index 000000000000..904f46b580f2 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/diags/shell_commands @@ -0,0 +1,37 @@ +# This script generates a (single) yaml file for EAMxx output. +# The output will be INSTANT, with only some diags fields as ouput + +CASEROOT=$(./xmlquery --value CASEROOT) +CASE=$(./xmlquery --value CASE) + +# Scripts location +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange +YAML_FILE=$CASEROOT/eamxx_diags_output.yaml + +# Figure out the suffix for the physics grid +ATM_GRID=$(./xmlquery --value ATM_GRID) +if [[ $ATM_GRID == *"pg2"* ]]; then + PGTYPE="PG2" +else + PGTYPE="GLL" +fi + +# List of output fields +FIELDS='Exner LiqWaterPath dz geopotential_int PotentialTemperature' +FIELDS+=' precip_liq_surf_mass_flux wind_speed ShortwaveCloudForcing' +FIELDS+=' T_mid_at_model_bot T_mid_at_900hPa' +FIELDS+=' horiz_winds_at_100m_above_surface horiz_winds_at_100m_above_sealevel' + +# Generate the file +$YAML_EDIT_SCRIPT -g \ + -f $YAML_FILE \ + --avg-type INSTANT \ + --freq HIST_N \ + --freq-units HIST_OPTION \ + --prefix ${CASE}.scream.diags.hi \ + --grid "Physics ${PGTYPE}" \ + --fields ${FIELDS} + +# Add this output yaml file to the list of eamxx output streams +$ATMCHANGE output_yaml_files+=$YAML_FILE -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands new file mode 100644 index 000000000000..99289d5e411b --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/hremap_to_ne4/shell_commands @@ -0,0 +1,15 @@ +# Look for all the eamxx_***_output.yaml files in the case folder and +# sets horiz remap if atm grid is ne30pg2 + +CASEROOT=$(./xmlquery --value CASEROOT) +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATM_GRID=$(./xmlquery --value ATM_GRID) + +if [[ $ATM_GRID = "ne30np4.pg2" ]];then + YAML_FILES=$(ls -1 | grep 'eamxx_.*_output.yaml') + for fname in ${YAML_FILES}; do + $YAML_EDIT_SCRIPT -f $fname --horiz-remap-file \${DIN_LOC_ROOT}/atm/scream/maps/map_ne30pg2_to_ne4pg2_20231201.nc + done +else + echo "Note: testmod 'hremap_to_ne4' only works for ne30pg2 atm grid" +fi diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands new file mode 100644 index 000000000000..116cdf111b43 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys/shell_commands @@ -0,0 +1,34 @@ +# This script generates a (single) yaml file for EAMxx output. +# The output will be INSTANT, with only a few state vars + +CASEROOT=$(./xmlquery --value CASEROOT) +CASE=$(./xmlquery --value CASE) + +# Scripts location +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange +YAML_FILE=$CASEROOT/eamxx_phys_output.yaml + +# Figure out the suffix for the physics grid +ATM_GRID=$(./xmlquery --value ATM_GRID) +if [[ $ATM_GRID == *"pg2"* ]]; then + PGTYPE="PG2" +else + PGTYPE="GLL" +fi + +# List of output fields +FIELDS='horiz_winds T_mid tracers pseudo_density p_mid p_int' + +# Generate the file +$YAML_EDIT_SCRIPT -g \ + -f $YAML_FILE \ + --avg-type INSTANT \ + --freq HIST_N \ + --freq-units HIST_OPTION \ + --prefix ${CASE}.scream.phys.hi \ + --grid "Physics ${PGTYPE}" \ + --fields ${FIELDS} + +# Add this output yaml file to the list of eamxx output streams +$ATMCHANGE output_yaml_files+=$YAML_FILE -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands new file mode 100644 index 000000000000..0952b4afe3f9 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/phys_dyn/shell_commands @@ -0,0 +1,41 @@ +# This script generates a (single) yaml file for EAMxx output. +# The output will be INSTANT, with only dyn state vars + +CASEROOT=$(./xmlquery --value CASEROOT) +CASE=$(./xmlquery --value CASE) + +# Scripts location +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream +ATMCHANGE=$CIMEROOT/../components/eamxx/scripts/atmchange +YAML_FILE=$CASEROOT/eamxx_dyn_output.yaml + +# Figure out the suffix for the physics grid +ATM_GRID=$(./xmlquery --value ATM_GRID) +if [[ $ATM_GRID == *"pg2"* ]]; then + PGTYPE="PG2" +else + PGTYPE="GLL" +fi + +# List of output fields +FIELDS='v_dyn vtheta_dp_dyn dp3d_dyn w_int_dyn phis_dyn phi_int_dyn ps_dyn omega_dyn Qdp_dyn' + +# Generate the file +$YAML_EDIT_SCRIPT -g \ + -f $YAML_FILE \ + --avg-type INSTANT \ + --freq HIST_N \ + --freq-units HIST_OPTION \ + --prefix ${CASE}.scream.phys_dyn.hi \ + --grid Dynamics \ + --io-grid 'Physics GLL' \ + --fields ${FIELDS} + +# Add also a couple of fields on the phys grid, to trigger 2-grid in same stream +$YAML_EDIT_SCRIPT \ + -f $YAML_FILE \ + --grid "Physics ${PGTYPE}" \ + --fields T_mid horiz_winds + +# Add this output yaml file to the list of eamxx output streams +$ATMCHANGE output_yaml_files+=$YAML_FILE -b diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands new file mode 100644 index 000000000000..74bd2a4d0213 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/1/shell_commands @@ -0,0 +1,9 @@ +# This preset uses the three output streams (phys_dyn, phys, and diags) +# It does not add remap, and uses INSTANT output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the three streams +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/phys_dyn/shell_commands +. $SCRIPTS_DIR/diags/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands new file mode 100644 index 000000000000..aea7feb5bbd6 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/2/shell_commands @@ -0,0 +1,12 @@ +# This preset uses the three output streams (phys_dyn, phys, and diags) +# It does not add remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the three streams +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/phys_dyn/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Change avg-type to AVERAGE for all streams +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands new file mode 100644 index 000000000000..da1d02dc5de1 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/3/shell_commands @@ -0,0 +1,11 @@ +# This preset uses the three output streams (phys, and diags) +# It adds horiz remap, and uses INSTANT output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add horiz remap +. $SCRIPTS_DIR/hremap_to_ne4/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands new file mode 100644 index 000000000000..521ada7f5082 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/4/shell_commands @@ -0,0 +1,14 @@ +# This preset uses the three output streams (phys, and diags) +# It adds horizontal remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add horiz remap +. $SCRIPTS_DIR/hremap_to_ne4/shell_commands + +# Use AVERAGE +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands new file mode 100644 index 000000000000..bd60a0472a87 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/5/shell_commands @@ -0,0 +1,15 @@ +# This preset uses the three output streams (phys_dyn, phys, and diags) +# It adds vertical remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/dyn/diags streams +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/phys_dyn/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add vertical remap +. $SCRIPTS_DIR/vremap/shell_commands + +# Use AVERAGE +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands new file mode 100644 index 000000000000..937d16468516 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/preset/6/shell_commands @@ -0,0 +1,17 @@ +# This preset uses the three output streams (phys, and diags) +# It adds horizontal and remap, and uses AVERAGE output + +SCRIPTS_DIR=$CIMEROOT/../components/eamxx/cime_config/testdefs/testmods_dirs/scream/output + +# Add the phys/diags streams (cannot add phys_dyn, b/c we use horiz remap) +. $SCRIPTS_DIR/phys/shell_commands +. $SCRIPTS_DIR/diags/shell_commands + +# Add horiz remap +. $SCRIPTS_DIR/hremap_to_ne4/shell_commands + +# Add vert remap +. $SCRIPTS_DIR/vremap/shell_commands + +# Use AVERAGE +. $SCRIPTS_DIR/set_avg/shell_commands diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands new file mode 100644 index 000000000000..dc58bc6301a7 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/set_avg/shell_commands @@ -0,0 +1,10 @@ +# Look for all the eamxx_***_output.yaml files in the case folder and +# change the avg type to average. + +CASEROOT=$(./xmlquery --value CASEROOT) +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream + +YAML_FILES=$(ls -1 | grep 'eamxx_.*_output.yaml') +for fname in ${YAML_FILES}; do + $YAML_EDIT_SCRIPT -f $fname --avg-type average +done diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands new file mode 100644 index 000000000000..151928669d1a --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/output/vremap/shell_commands @@ -0,0 +1,10 @@ +# Look for all the eamxx_***_output.yaml files in the case folder and +# sets vertical remap + +CASEROOT=$(./xmlquery --value CASEROOT) +YAML_EDIT_SCRIPT=$CIMEROOT/../components/eamxx/scripts/edit-output-stream + +YAML_FILES=$(ls -1 | grep 'eamxx_.*_output.yaml') +for fname in ${YAML_FILES}; do + $YAML_EDIT_SCRIPT -f $fname --vertical-remap-file \${DIN_LOC_ROOT}/atm/scream/maps/vrt_remapping_p_levs_20230926.nc +done diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands new file mode 100644 index 000000000000..d4e7c2a95377 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/perf_test/shell_commands @@ -0,0 +1,5 @@ + +# Force us to use 1 node to eliminate network noise +if [ `./xmlquery --value MACH` == frontier-scream-gpu ]; then + ./xmlchange NTASKS=8 +fi diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands index 04989a22796a..e6773dce4199 100644 --- a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/small_kernels/shell_commands @@ -1,7 +1,2 @@ ./xmlchange --append SCREAM_CMAKE_OPTIONS='SCREAM_SMALL_KERNELS On' $CIMEROOT/../components/eamxx/scripts/atmchange --all internal_diagnostics_level=1 atmosphere_processes::internal_diagnostics_level=0 -b - -f=$(./xmlquery --value MACH) -if [ $f == chrysalis ]; then - ./xmlchange BATCH_COMMAND_FLAGS="--time 00:30:00 -p debug --account e3sm --exclude=chr-0512" -fi diff --git a/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands new file mode 100644 index 000000000000..5a07fb378464 --- /dev/null +++ b/components/eamxx/cime_config/testdefs/testmods_dirs/scream/spa_remap/shell_commands @@ -0,0 +1,2 @@ +$CIMEROOT/../components/eamxx/scripts/atmchange -b spa_data_file='${DIN_LOC_ROOT}'/atm/scream/init/spa_file_unified_and_complete_ne4pg2_20231222.nc +$CIMEROOT/../components/eamxx/scripts/atmchange -b spa_remap_file='${DIN_LOC_ROOT}'/atm/scream/maps/map_ne4pg2_to_ne30pg2_20231201.nc diff --git a/components/eamxx/cime_config/tests/eamxx_default_files.py b/components/eamxx/cime_config/tests/eamxx_default_files.py new file mode 100644 index 000000000000..b39d2d5c1553 --- /dev/null +++ b/components/eamxx/cime_config/tests/eamxx_default_files.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import os +import http +import pathlib +import unittest +import urllib.request +import xml.etree.ElementTree as ET + + +class testNamelistDefaultsScream(unittest.TestCase): + def setUp(self): + """ + Set up the environment for the test by setting the DIN_LOC_ROOT + environment variable. Parse the 'namelist_defaults_scream.xml' + file and extract the files of interest based on the DIN_LOC_ROOT + variable or the array(file) type. Assign the extracted files + to the 'my_files' attribute of the test instance. + """ + + os.environ["DIN_LOC_ROOT"] = "https://web.lcrc.anl.gov/public/e3sm/inputdata/" + + scream_defaults_path = pathlib.Path(__file__) + tree = ET.parse(f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml") + root = tree.getroot() + + files_of_interest = [ + child.text for child in root.findall(".//") + if child.text and child.text.startswith("${DIN_LOC_ROOT}") + ] + + more_files_of_interest = [ + child.text for child in root.findall(".//") + if child.text and "type" in child.attrib.keys() and child.attrib["type"]=="array(file)" + ] + + files_of_interest.extend( + text.strip() for text_list in more_files_of_interest for text in text_list.split(",") + if text.strip().startswith("${DIN_LOC_ROOT}") + ) + + self.my_files = [ + file.replace("${DIN_LOC_ROOT}/", "") + for file in files_of_interest + ] + + self.my_lines = [] + with open( + f"{scream_defaults_path.parent.parent}/namelist_defaults_scream.xml", + "r" + ) as the_file: + for a_line in the_file: + self.my_lines.append(a_line) + + def test_ascii_lines(self): + """ + Test that all lines are ASCII + """ + + for i_line, a_line in enumerate(self.my_lines): + with self.subTest(i_line=i_line): + self.assertTrue( + a_line.isascii(), + msg=f"\nERROR! This line is not ASCII!\n{a_line}" + ) + + def test_opening_files(self): + """ + Test the opening of files from the inputdata server. + """ + + for i_file in range(len(self.my_files)): + with self.subTest(i_file=i_file): + try: + request_return = urllib.request.urlopen( + f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" + ) + self.assertIsInstance(request_return, http.client.HTTPResponse) + except urllib.error.HTTPError: + file_name = f"{os.environ['DIN_LOC_ROOT']}{self.my_files[i_file]}" + self.assertTrue( + False, + msg=f"\nERROR! This file doesn't exist!\n{file_name}" + ) + + def test_expected_fail(self): + """ + Test an expected failure by manipulating the file name. + """ + + with self.assertRaises(urllib.error.HTTPError): + some_phony_file = f"{self.my_files[5][:-5]}some_phony_file.nc" + urllib.request.urlopen( + f"{os.environ['DIN_LOC_ROOT']}{some_phony_file}" + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/components/eamxx/cmake/machine-files/lassen.cmake b/components/eamxx/cmake/machine-files/lassen.cmake deleted file mode 100644 index 36b69c7f0253..000000000000 --- a/components/eamxx/cmake/machine-files/lassen.cmake +++ /dev/null @@ -1,9 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -set(NetCDF_PATH /usr/gdata/climdat/netcdf CACHE STRING "") -set(NetCDF_Fortran_PATH /usr/gdata/climdat/netcdf CACHE STRING "") -set(LAPACK_LIBRARIES /usr/lib64/liblapack.so CACHE STRING "") -set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) - -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata/" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/mappy.cmake b/components/eamxx/cmake/machine-files/mappy.cmake index 7c1fc8cf25ea..29fb2e74b8a1 100644 --- a/components/eamxx/cmake/machine-files/mappy.cmake +++ b/components/eamxx/cmake/machine-files/mappy.cmake @@ -1,3 +1,5 @@ include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) common_setup() -set(PYTHON_EXECUTABLE "/ascldap/users/jgfouca/packages/Python-3.8.5/bin/python3.8" CACHE STRING "" FORCE) \ No newline at end of file +set(PYTHON_EXECUTABLE "/ascldap/users/jgfouca/packages/Python-3.8.5/bin/python3.8" CACHE STRING "" FORCE) + +set(CMAKE_Fortran_FLAGS "-fallow-argument-mismatch" CACHE STRING "" FORCE) diff --git a/components/eamxx/cmake/machine-files/quartz-intel.cmake b/components/eamxx/cmake/machine-files/quartz-intel.cmake index 753c782702db..defd8cbb2d33 100644 --- a/components/eamxx/cmake/machine-files/quartz-intel.cmake +++ b/components/eamxx/cmake/machine-files/quartz-intel.cmake @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) -execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) +execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/quartz.cmake b/components/eamxx/cmake/machine-files/quartz.cmake index ee9a3dcbffd3..e4b4fcbd8a57 100644 --- a/components/eamxx/cmake/machine-files/quartz.cmake +++ b/components/eamxx/cmake/machine-files/quartz.cmake @@ -16,4 +16,4 @@ elseif ("${COMPILER}" STREQUAL "gnu") set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/" CACHE STRING "" FORCE) endif() -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "") +set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/ruby-intel.cmake b/components/eamxx/cmake/machine-files/ruby-intel.cmake index 63fff478fdaf..9c6318da4952 100644 --- a/components/eamxx/cmake/machine-files/ruby-intel.cmake +++ b/components/eamxx/cmake/machine-files/ruby-intel.cmake @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) -execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) +execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/ruby.cmake b/components/eamxx/cmake/machine-files/ruby.cmake index d0a9de4baf4b..77d6e6618c71 100644 --- a/components/eamxx/cmake/machine-files/ruby.cmake +++ b/components/eamxx/cmake/machine-files/ruby.cmake @@ -12,4 +12,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "") +set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/syrah.cmake b/components/eamxx/cmake/machine-files/syrah.cmake deleted file mode 100644 index f03fc1e9d469..000000000000 --- a/components/eamxx/cmake/machine-files/syrah.cmake +++ /dev/null @@ -1,13 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) - -# Enable Sandy Bridge arch in Kokkos -option(Kokkos_ARCH_SNB "" ON) - -set(CMAKE_CXX_FLAGS "-w -cxxlib=/usr/tce/packages/gcc/gcc-8.3.1/rh" CACHE STRING "" FORCE) -set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/ -mkl" CACHE STRING "" FORCE) - -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata/" CACHE STRING "") diff --git a/components/eamxx/data/SCREAM_YAML_README b/components/eamxx/data/SCREAM_YAML_README deleted file mode 100644 index daf425845e31..000000000000 --- a/components/eamxx/data/SCREAM_YAML_README +++ /dev/null @@ -1,52 +0,0 @@ - -INTRO: - -The scream_input.yaml is the key file for configuring a SCREAM run. This file will be -processed and copied to $case/run/scream_input.yaml by scream's buidnml script, which -is called during case.setup. Note, this is for runtime coniguration -only. Cmake/build-time configuration should be done through SCREAM_CMAKE_OPTIONS. - -For inline comments, see the version of scream_input.yaml that lives in the repo -(components/eamxx/data/scream_input.yaml) - -Note, the $case/run/scream_input.yaml will NEVER be overwritten by subsequent -calls to case.setup/buildnml in order to avoid blowing away potential local -modifications. To force a regeneration of this file, it should be removed from the -case and `./case.setup --reset` should be called. - -SECTIONS: - - Atmosphere Driver: Contains settings for the AD. Can turn off processes by editing "Number of Entries" and - changing the Process N list. - - SCREAM: For general SCREAM settings - - HOMME: For HOMME settings. These settings will be translated into data/namelist.nl - -SYNTAX: - -This file supports some special syntax in addition to basic YAML: -'${VAR}' will be used to refer to env variables in the CIME case - -' val1 : key2 => val2 : elseval>' will be used to express conditional -statements. If switch_val matches key1, then the expression evaluates to val1; if switch_val -matches key2, then the expression evaluates to val2; if it matches neither, then -the expression evaluates to elseval. The elseval component of this expression is optional. -You can have any number (N>=1) of key => val sections. - -Example, if you wanted tstep to depend on atm grid resolution: - - tstep: "<${ATM_GRID} : ne4np4 => 300 : 30>" - -This would give all ne4 cases a timestep of 300, otherwise it would be 30. - -You could specify multiple grid->timestep relationships this way: - - tstep: "<${ATM_GRID} : ne4np4 => 300 : ne30np4 => 100 : 30>" - -Regex matching is supported: - - tstep: "<${ATM_GRID} : .*ne4.* => 300 : .*ne30.* => 100 : 30>" - -Note: none of this special syntax will be automatically reprocessed if the case XML values -are changed. Regenerating this file is necessary if relevant case XML values are modified. diff --git a/components/eamxx/data/scream_default_output.yaml b/components/eamxx/data/scream_default_output.yaml deleted file mode 100644 index 7e0a45f12d6a..000000000000 --- a/components/eamxx/data/scream_default_output.yaml +++ /dev/null @@ -1,75 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: ${CASE}.scream.hi -# WARNING: ERS/ERP tets will override this with AVERAGE -Averaging Type: Instant -# One output every 31 days if output frequency is set to once per hour -Max Snapshots Per File: 744 -Fields: - Physics ${PHYSICS_GRID_TYPE}: - Field Names: - # HOMME - - ps - - pseudo_density - - omega - - p_int - - p_mid - # SHOC + HOMME - - horiz_winds - # SHOC - - cldfrac_liq - - eddy_diff_mom - - sgs_buoy_flux - - tke - - pbl_height - # CLD - - cldfrac_ice_for_analysis - - cldfrac_tot_for_analysis - # P3 - - bm - - nc - - ni - - nr - - qi - - qm - - qr - - eff_radius_qc - - eff_radius_qi - - eff_radius_qr - - precip_ice_surf_mass - - precip_liq_surf_mass - - rainfrac - # SHOC + P3 - - qc - - qv - # SHOC + P3 + RRTMGP + HOMME - - T_mid - # RRTMGP - - sfc_alb_dir_vis - - LW_flux_dn - - LW_flux_up - - SW_flux_dn - - SW_flux_up - - sfc_flux_lw_dn - - sfc_flux_sw_net - - cldtot - - cldlow - - cldmed - - cldhgh - # Surface Fluxes - - surf_evap - - surf_sens_flux - # Diagnostics - - PotentialTemperature - # GLL output for homme states. - Dynamics: - Field Names: - - ps_dyn - - dp3d_dyn - - omega_dyn - IO Grid Name: Physics GLL -output_control: -# WARNING: ERS/ERP tets will override this with STOP_N/STOP_OPTION - Frequency: ${HIST_N} - frequency_units: ${HIST_OPTION} -... diff --git a/components/eamxx/data/scream_default_remap.yaml b/components/eamxx/data/scream_default_remap.yaml deleted file mode 100644 index 8bf47386c76d..000000000000 --- a/components/eamxx/data/scream_default_remap.yaml +++ /dev/null @@ -1,67 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: ${CASE}.scream.arm_sites.hi -Averaging Type: Instant -Max Snapshots Per File: 744 # One output every 31 days -#remap_file: /g/g17/donahue5/Code/e3sm/scream-docs/regional_output_sites/20221123_ARM_sites_map.nc -remap_file: /usr/gdata/climdat/ccsm3data/inputdata/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc -Fields: - Physics ${PHYSICS_GRID_TYPE}: - Field Names: - # HOMME - - ps - - pseudo_density - - omega - - p_int - - p_mid - # SHOC + HOMME - - horiz_winds - # SHOC - - cldfrac_liq - - eddy_diff_mom - - sgs_buoy_flux - - tke - - pbl_height - # CLD - - cldfrac_ice - - cldfrac_tot - # P3 - - bm - - nc - - ni - - nr - - qi - - qm - - qr - - eff_radius_qc - - eff_radius_qi - - eff_radius_qr - - precip_ice_surf_mass - - precip_liq_surf_mass - - rainfrac - # SHOC + P3 - - qc - - qv - # SHOC + P3 + RRTMGP + HOMME - - T_mid - # RRTMGP - - sfc_alb_dir_vis - - LW_flux_dn - - LW_flux_up - - SW_flux_dn - - SW_flux_up - - sfc_flux_lw_dn - - sfc_flux_sw_net - - cldtot - - cldlow - - cldmed - - cldhgh - # Surface Fluxes - - surf_evap - - surf_sens_flux - # Diagnostics -# - PotentialTemperature -output_control: - Frequency: ${HIST_N} - frequency_units: ${HIST_OPTION} -... diff --git a/components/eamxx/docs/developer/standalone_testing.md b/components/eamxx/docs/developer/standalone_testing.md index aedb7b2cbaad..63ea01fc612e 100644 --- a/components/eamxx/docs/developer/standalone_testing.md +++ b/components/eamxx/docs/developer/standalone_testing.md @@ -33,9 +33,9 @@ make baseline ``` The tests will run, automatically using the baseline file, which is located in -the CMake-configurable path `${SCREAM_TEST_DATA_DIR}`. By default, this path is -set to `data/` within your build directory (which is `$RUN_ROOT_DIR`, in -our case). +the CMake-configurable path `${SCREAM_BASELINES_DIR}`. By default, this path is +set to an invalid string. If baselines tests are enabled, we check that a valid +path has been provided. To run all of SCREAM's tests, make sure you're in `$RUN_ROOT_DIR` and type diff --git a/components/eamxx/docs/user/coarse_nudging.md b/components/eamxx/docs/user/coarse_nudging.md index c52ce9a0eb24..3d7309c42aaa 100644 --- a/components/eamxx/docs/user/coarse_nudging.md +++ b/components/eamxx/docs/user/coarse_nudging.md @@ -20,6 +20,6 @@ In other words, the following options are needed: ```shell ./atmchange atm_procs_list=(sc_import,nudging,homme,physics,sc_export) ./atmchange nudging_fields=U,V -./atmchange nudging_filename=/path/to/nudging_data_ne4pg2_L72.nc +./atmchange nudging_filenames_patterns=/path/to/nudging_data_ne4pg2_L72.nc ./atmchange nudging_refine_remap_mapfile=/another/path/to/mapping_file_ne4pg2_to_ne120pg2.nc ``` diff --git a/components/eamxx/scripts/atm_manip.py b/components/eamxx/scripts/atm_manip.py index 2a763bc1990d..fbd8381e1727 100755 --- a/components/eamxx/scripts/atm_manip.py +++ b/components/eamxx/scripts/atm_manip.py @@ -170,7 +170,7 @@ def modify_ap_list(xml_root, group, ap_list_str, append_this): 'p1,p2,p1' >>> modify_ap_list(tree,node,"p1,p3",False) Traceback (most recent call last): - ValueError: ERROR: Unrecognized atm proc name 'p3'. To declare a new group, prepend and append '_' to the name. + SystemExit: ERROR: Unrecognized atm proc name 'p3'. To declare a new group, prepend and append '_' to the name. >>> modify_ap_list(tree,node,"p1,_my_group_",False) True >>> get_child(node,"atm_procs_list").text @@ -203,11 +203,11 @@ def modify_ap_list(xml_root, group, ap_list_str, append_this): new_aps = [n for n in add_aps if find_node(ap_defaults,n) is None] for ap in new_aps: - expect (ap[0]=="_" and ap[-1]=="_" and len(ap)>2, exc_type=ValueError, - error_msg=f"Unrecognized atm proc name '{ap}'. To declare a new group, prepend and append '_' to the name.") + expect (ap[0]=="_" and ap[-1]=="_" and len(ap)>2, + f"Unrecognized atm proc name '{ap}'. To declare a new group, prepend and append '_' to the name.") group = gen_atm_proc_group("", ap_defaults) group.tag = ap - + ap_defaults.append(group) # Update the 'atm_procs_list' in this node @@ -217,6 +217,25 @@ def modify_ap_list(xml_root, group, ap_list_str, append_this): curr_apl.text = ','.join(ap_list) return True +############################################################################### +def is_locked_impl(node): +############################################################################### + return "locked" in node.attrib.keys() and str(node.attrib["locked"]).upper() == "TRUE" + +############################################################################### +def is_locked(xml_root, node): +############################################################################### + if is_locked_impl(node): + return True + else: + parent_map = create_parent_map(xml_root) + parents = get_parents(node, parent_map) + for parent in parents: + if is_locked_impl(parent): + return True + + return False + ############################################################################### def apply_change(xml_root, node, new_value, append_this): ############################################################################### @@ -231,6 +250,7 @@ def apply_change(xml_root, node, new_value, append_this): if append_this: + expect (not is_locked(xml_root, node), f"Cannot change {node.tag}, it is locked") expect ("type" in node.attrib.keys(), f"Error! Missing type information for {node.tag}") type_ = node.attrib["type"] @@ -238,7 +258,11 @@ def apply_change(xml_root, node, new_value, append_this): "Error! Can only append with array and string types.\n" f" - name: {node.tag}\n" f" - type: {type_}") - if is_array_type(type_): + + if node.text is None: + node.text = "" + + if is_array_type(type_) and node.text!="": node.text += ", " + new_value else: node.text += new_value @@ -246,6 +270,7 @@ def apply_change(xml_root, node, new_value, append_this): any_change = True elif node.text != new_value: + expect (not is_locked(xml_root, node), f"Cannot change {node.tag}, it is locked") check_value(node,new_value) node.text = new_value any_change = True @@ -283,16 +308,37 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): """ >>> xml = ''' ... - ... 1,2,3 - ... 1 - ... 1 - ... one - ... one - ... one - ... - ... two - ... 2 - ... + ... 1,2,3 + ... 1 + ... 1 + ... one + ... one + ... one + ... + ... two + ... 2 + ... + ... + ... + ... + ... hi + ... + ... + ... + ... + ... + ... + ... hi + ... + ... + ... + ... + ... + ... + ... hi + ... + ... + ... ... ... ''' >>> import xml.etree.ElementTree as ET @@ -306,7 +352,8 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): >>> ################ INVALID TYPE ####################### >>> atm_config_chg_impl(tree,'prop2=two') Traceback (most recent call last): - ValueError: Could not refine 'two' as type 'integer' + CIME.utils.CIMEError: ERROR: Could not refine 'two' as type 'integer': + could not convert string to float: 'two' >>> ################ INVALID VALUE ####################### >>> atm_config_chg_impl(tree,'prop2=3') Traceback (most recent call last): @@ -350,6 +397,16 @@ def atm_config_chg_impl(xml_root, change, all_matches=False): True >>> get_xml_nodes(tree,'e')[0].text 'one, two' + >>> ################ Test locked ################## + >>> atm_config_chg_impl(tree, 'lprop2=yo') + Traceback (most recent call last): + SystemExit: ERROR: Cannot change lprop2, it is locked + >>> atm_config_chg_impl(tree, 'lprop3=yo') + Traceback (most recent call last): + SystemExit: ERROR: Cannot change lprop3, it is locked + >>> atm_config_chg_impl(tree, 'lprop4=yo') + Traceback (most recent call last): + SystemExit: ERROR: Cannot change lprop4, it is locked """ node_name, new_value, append_this = parse_change(change) matches = get_xml_nodes(xml_root, node_name) diff --git a/components/eamxx/scripts/change-param-pattern b/components/eamxx/scripts/change-param-pattern index 5ab26a64df3b..a54c7292d246 100755 --- a/components/eamxx/scripts/change-param-pattern +++ b/components/eamxx/scripts/change-param-pattern @@ -35,4 +35,4 @@ for bad_name_under in bad_name_unders: run_cmd_no_fail(f"sed -i -e 's/{bad_name_ws}/{good_name}/g' $(git grep -l '{bad_name_ws}')") run_cmd_no_fail(f"git commit -a -m '{bad_name_ws} -> {good_name}'") print(" Testing") - run_cmd_no_fail("./create_test ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1 --compiler=gnu9", from_dir="../../cime/scripts") + run_cmd_no_fail("./create_test ERS_D_Ln22.ne4pg2_ne4pg2.F2010-SCREAMv1", from_dir="../../cime/scripts") diff --git a/components/eamxx/scripts/check-hashes-ers b/components/eamxx/scripts/check-hashes-ers new file mode 100755 index 000000000000..6d9da4b2f98e --- /dev/null +++ b/components/eamxx/scripts/check-hashes-ers @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 + +""" +See https://acme-climate.atlassian.net/wiki/spaces/NGDNA/pages/3831923056/EAMxx+BFB+hashing +for full explanation. + +This script is used by the scream-internal_diagnostics_level testmod to check +hash output after a test has run. +""" + +import sys, re, glob, pathlib, argparse + +from utils import run_cmd_no_fail, expect + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n{0} [=] ... +OR +{0} --help + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Run hash checker on /my/case/dir \033[0m + > {0} /my/case/dir +""".format(pathlib.Path(args[0]).name), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "case_dir", + help="The test case you want to check" + ) + + return parser.parse_args(args[1:]) + +############################################################################### +def readall(fn): +############################################################################### + with open(fn,'r') as f: + txt = f.read() + return txt + +############################################################################### +def greptxt(pattern, txt): +############################################################################### + return re.findall('(?:' + pattern + ').*', txt, flags=re.MULTILINE) + +############################################################################### +def grep(pattern, fn): +############################################################################### + txt = readall(fn) + return greptxt(pattern, txt) + +############################################################################### +def get_log_glob_from_atm_modelio(case_dir): +############################################################################### + filename = case_dir / 'CaseDocs' / 'atm_modelio.nml' + ln = grep('diro = ', filename)[0] + run_dir = pathlib.Path(ln.split()[2].split('"')[1]) + ln = grep('logfile = ', filename)[0] + atm_log_fn = ln.split()[2].split('"')[1] + id_ = atm_log_fn.split('.')[2] + return str(run_dir / '**' / f'e3sm.log.{id_}*') + +############################################################################### +def get_hash_lines(fn): +############################################################################### + rlns = run_cmd_no_fail(f'zgrep exxhash {fn}').splitlines() + lns = [] + if len(rlns) == 0: return lns + for rln in rlns: + pos = rln.find('exxhash') + lns.append(rln[pos:]) + return lns + +############################################################################### +def parse_time(hash_ln): +############################################################################### + return hash_ln.split()[1:3] + +############################################################################### +def all_equal(t1, t2): +############################################################################### + if len(t1) != len(t2): return False + for i in range(len(t1)): + if t1[i] != t2[i]: return False + return True + +############################################################################### +def find_first_index_at_time(lns, time): +############################################################################### + for i, ln in enumerate(lns): + t = parse_time(ln) + if all_equal(time, t): return i + return None + +############################################################################### +def diff(l1, l2): +############################################################################### + diffs = [] + for i in range(len(l1)): + if l1[i] != l2[i]: + diffs.append((l1[i], l2[i])) + return diffs + +############################################################################### +def check_hashes_ers(case_dir): +############################################################################### + case_dir_p = pathlib.Path(case_dir) + expect(case_dir_p.is_dir(), f"{case_dir} is not a dir") + + # Look for the two e3sm.log files. + glob_pat = get_log_glob_from_atm_modelio(case_dir_p) + e3sm_fns = glob.glob(glob_pat, recursive=True) + if len(e3sm_fns) == 0: + print('Could not find e3sm.log files with glob string {}'.format(glob_pat)) + return False + e3sm_fns.sort() + if len(e3sm_fns) == 1: + # This is the first run. Exit and wait for the second + # run. (POSTRUN_SCRIPT is called after each of the two runs.) + print('Exiting on first run.') + return True + print('Diffing base {} and restart {}'.format(e3sm_fns[0], e3sm_fns[1])) + + # Because of the prefixed 1: and 2: on some systems, we can't just use + # zdiff. + lns = [] + for f in e3sm_fns: + lns.append(get_hash_lines(f)) + time = parse_time(lns[1][0]) + time_idx = find_first_index_at_time(lns[0], time) + if time_idx is None: + print('Could not find a start time.') + return False + lns[0] = lns[0][time_idx:] + if len(lns[0]) != len(lns[1]): + print('Number of hash lines starting at restart time do not agree.') + return False + diffs = diff(lns[0], lns[1]) + + # Flushed prints to e3sm.log can sometimes conflict with other + # output. Permit up to 'thr' diffs so we don't fail due to badly printed + # lines. This isn't a big loss in checking because an ERS_Ln22 second run + # writes > 1000 hash lines, and a true loss of BFBness is nearly certain to + # propagate to a large number of subsequent hashes. + thr = 5 + if len(lns[0]) < 100: thr = 0 + + ok = True + if len(diffs) > thr: + print('DIFF') + print(diffs[-10:]) + ok = False + else: + print('OK') + + return ok + +############################################################################### +def _main_func(description): +############################################################################### + success = check_hashes_ers(**vars(parse_command_line(sys.argv, description))) + sys.exit(0 if success else 1) + +############################################################################### + +if (__name__ == "__main__"): + _main_func(__doc__) diff --git a/components/eamxx/scripts/cime-nml-tests b/components/eamxx/scripts/cime-nml-tests index 0a60376bd45f..b128121cf95f 100755 --- a/components/eamxx/scripts/cime-nml-tests +++ b/components/eamxx/scripts/cime-nml-tests @@ -222,9 +222,10 @@ class TestBuildnml(unittest.TestCase): # Append to an existing entry name = 'output_yaml_files' out = run_cmd_no_fail(f"./atmchange {name}+=a.yaml", from_dir=case) + out = run_cmd_no_fail(f"./atmchange {name}+=b.yaml", from_dir=case) # Get the yaml files - expected =f'{EAMXX_DIR / "data/scream_default_output.yaml"}, a.yaml' + expected =f'a.yaml, b.yaml' self._get_values(case, name, value=expected, expect_equal=True) ########################################################################### diff --git a/components/eamxx/scripts/eamxx-params-docs-autogen b/components/eamxx/scripts/eamxx-params-docs-autogen index 3024677d96b0..9bea5242d5a9 100755 --- a/components/eamxx/scripts/eamxx-params-docs-autogen +++ b/components/eamxx/scripts/eamxx-params-docs-autogen @@ -20,6 +20,7 @@ from mdutils import Html sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "cime_config")) from eamxx_buildnml_impl import resolve_all_inheritances, get_valid_selectors +from atm_manip import is_locked_impl ############################################################################### def parse_command_line(args, description): @@ -34,14 +35,15 @@ def parse_command_line(args, description): return parser.parse_args(args[1:]) ########################################################################### -def add_param(docs,scope,item): +def add_param(docs, scope, item): ########################################################################### # Locked parameters are not to be configured at runtime, so don't even bother # E.g, a locked param is something we need to get in the input file, like # the restart write frequency, but we don't want the user to modify it # via atmchange - if "locked" in item.attrib.keys(): + if is_locked_impl(item): return + docs.new_line(f"* {scope}{item.tag}:") pdoc = item.attrib['doc'] if 'doc' in item.attrib.keys() else "**MISSING**" @@ -53,21 +55,23 @@ def add_param(docs,scope,item): pvalid = item.attrib['valid_values'] if 'valid_values' in item.attrib.keys() else None if pvalid is not None: docs.new_line(f" - valid values: {pvalid}") + pconstr = item.attrib['constraints'] if 'constraints' in item.attrib.keys() else None if pconstr is not None: docs.new_line(f" - constraints: {pconstr}") ########################################################################### -def add_children(docs,xml,scope=""): +def add_children(docs, elem, scope=""): ########################################################################### done = [] # Locked parameters are not to be configured at runtime, so don't even bother # E.g, a locked param is something we need to get in the input file, like # the restart write frequency, but we don't want the user to modify it # via atmchange - if "locked" in xml.attrib.keys(): + if is_locked_impl(elem): return - for item in xml: + + for item in elem: # The same entry may appear multiple times in the XML defaults file, # each time with different selectors. We don't want to generate the # same documentation twice. @@ -75,9 +79,10 @@ def add_children(docs,xml,scope=""): continue done.append(item.tag) if len(item)>0: - add_children (docs,item,f"{scope}{xml.tag}::") + add_children (docs,item,f"{scope}{elem.tag}::") else: - add_param(docs,f"{scope}{xml.tag}::",item) + add_param(docs,f"{scope}{elem.tag}::",item) + docs.new_line() ########################################################################### @@ -107,7 +112,7 @@ def generate_params_docs(): continue docs.new_header(level=2,title=ap.tag) add_children(docs,ap) - + ic = xml_defaults.find('initial_conditions') docs.new_header(level=1,title="Initial Conditions Parameters") add_children(docs,ic) @@ -123,6 +128,7 @@ def generate_params_docs(): homme = xml_defaults.find('ctl_nl') docs.new_header(level=1,title='Homme namelist') add_children(docs,homme) + docs.create_md_file() print("Generating eamxx params documentation ... SUCCESS!") diff --git a/components/eamxx/scripts/edit-output-stream b/components/eamxx/scripts/edit-output-stream new file mode 100755 index 000000000000..76c88e69ca39 --- /dev/null +++ b/components/eamxx/scripts/edit-output-stream @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +""" +Edit (or create) an output stream yaml file +""" + +import argparse, sys, pathlib + +from edit_output_stream import edit_output_stream_impl + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n{0} = [=] ... +OR +{0} --help + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Generate empty file (with invalid options)'\033[0m + > {0} -g -f my_output.yaml + + \033[1;32m# Generate empty file (with invalid options), overwrite if existing'\033[0m + > {0} -g -O -f my_output.yaml + + \033[1;32m# Change avg type to Instant, output frequency to 1 day\033[0m + > {0} -f my_output.yaml --avg-type instant --freq 1 --freq-units ndays + + \033[1;32m# Set horiz remaping\033[0m + > {0} -f my_output.yaml --horiz-map-file /path/to/map.nc +""".format(pathlib.Path(args[0]).name), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "-f","--filename", + required=True, + type=str, + help="The name of the yaml file to be configured", + ) + + parser.add_argument( + "--prefix", + type=str, + help="The prefix of the output files", + ) + + parser.add_argument( + "-g", "--generate", + default=False, + dest="generate", + action="store_true", + help="Generate a new file", + ) + parser.add_argument( + "-O", "--overwrite", + default=False, + action="store_true", + help="When generating a new file, overwrite existing file (if any)", + ) + + parser.add_argument( + "-r","--reset", + default=None, + nargs="+", + type=str, + help="List of options to remove (or reset to default values) BEFORE doing any other edit" + ) + + parser.add_argument( + "--avg-type", + default=None, + type=str.lower, + help="Set the averaging type", + choices=['instant','average','max','min'], + ) + + parser.add_argument( + "--skip-t0-output", + action="store_true", + help="Skip t=case_t0 output (only relevant for INSTANT avg)" + ) + + parser.add_argument( + "--freq-units", + default=None, + type=str.lower, + help="Set the output frequency units", + ) + + parser.add_argument( + "--freq", + type=str.lower, + default=None, + help="Set the output frequency", + ) + + parser.add_argument( + "--grid", + default=None, + type=str, + help="Specify grid for which --fields/--io-grid options apply to", + ) + + parser.add_argument( + "--fields", + default=[], + nargs="+", + help="Fields to add to output", + ) + + parser.add_argument( + "--io-grid", + default=None, + type=str, + help="Name of grid onto which to remap fields before outputing them", + ) + + parser.add_argument( + "--horiz-remap-file", + default=None, + help="Map file to use for horizontal remap", + ) + + parser.add_argument( + "--vertical-remap-file", + default=None, + help="Map file to use for horizontal remap", + ) + + return parser.parse_args(args[1:]) + +############################################################################### +def _main_func(description): +############################################################################### + if "--test" in sys.argv: + from doctest import testmod + import edit_output_stream + testmod() + testmod(m=edit_output_stream) + else: + edit_output_stream_impl(**vars(parse_command_line(sys.argv, description))) + sys.exit(0) + +############################################################################### + +if (__name__ == "__main__"): + _main_func(__doc__) diff --git a/components/eamxx/scripts/edit_output_stream.py b/components/eamxx/scripts/edit_output_stream.py new file mode 100644 index 000000000000..8ca194a47f98 --- /dev/null +++ b/components/eamxx/scripts/edit_output_stream.py @@ -0,0 +1,223 @@ +import pathlib + +from utils import expect, ensure_yaml + +ensure_yaml() +import yaml + +############################################################################### +def generate_empty_yaml(filename,overwrite): +############################################################################### + """ + Generate a yaml file with basic fields set, but containing empty or invalid values + >>> fname = "__test__.yaml" + >>> generate_empty_yaml(fname,False) + >>> generate_empty_yaml(fname,False) + Traceback (most recent call last): + SystemExit: ERROR: YAML file already exist. Re-run with -O/--overwrite to overwrite existing file + >>> generate_empty_yaml(fname,True) + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> len(data) + 4 + >>> data["filename_prefix"] + 'UNSET' + >>> data["Averaging Type"] + 'INVALID' + >>> len(data["Fields"]) + 0 + >>> oc = data["output_control"] + >>> len(oc) + 2 + >>> oc["Frequency"] + -1 + >>> oc["frequency_units"] + 'never' + >>> # Clean up the file + >>> pathlib.Path(fname).unlink() + """ + file = pathlib.Path(filename).resolve() + + expect (overwrite or not file.exists(), + "YAML file already exist. Re-run with -O/--overwrite to overwrite existing file") + + if file.exists(): + file.unlink() + + data = {} + data["filename_prefix"] = "UNSET" + data["Averaging Type"] = "INVALID" + data["Fields"] = {} + data["output_control"] = {} + data["output_control"]["skip_t0_output"] = "false" + data["output_control"]["Frequency"] = -1 + data["output_control"]["frequency_units"] = "never" + + with open(file,'w') as fd: + yaml.dump(data,fd,Dumper=yaml.SafeDumper,explicit_start=True,explicit_end=True,version=(1,2)) + +############################################################################### +def edit_output_stream_impl(filename,prefix=None,generate=False,overwrite=False, + avg_type=None,skip_t0_output=False,freq_units=None,freq=None, + grid=None,fields=[],reset=None,io_grid=None, + horiz_remap_file=None,vertical_remap_file=None): +############################################################################### + """ + Apply the requested changes to the output stream yaml file + >>> fname = '__test__.yaml' + >>> # Create the file + >>> edit_output_stream_impl(fname,generate=True,prefix='foo') + >>> # Set some basic options, and then check + >>> edit_output_stream_impl(fname,avg_type='max',freq_units='ndays',freq=10) + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> data['filename_prefix'] + 'foo' + >>> data['Averaging Type'] + 'max' + >>> data['output_control']['Frequency'] + 10 + >>> data['output_control']['frequency_units'] + 'ndays' + >>> # Set fields options, and then check + >>> edit_output_stream_impl(fname,fields=['a','b'],grid='my_grid',io_grid='other_grid') + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> f = data['Fields']['my_grid']['Field Names'] + >>> f.sort() + >>> f + ['a', 'b'] + >>> data['Fields']['my_grid']['IO Grid Name'] + 'other_grid' + >>> # No remap if online remap (IO Grid Name) is set + >>> edit_output_stream_impl(fname,horiz_remap_file='blah') + Traceback (most recent call last): + SystemExit: ERROR: Cannot use online remap and horiz/vert remap at the same time. + >>> edit_output_stream_impl(fname,vertical_remap_file='blah') + Traceback (most recent call last): + SystemExit: ERROR: Cannot use online remap and horiz/vert remap at the same time. + >>> # Remove io grid and fields + >>> edit_output_stream_impl(fname,reset=['fields','io-grid']) + Traceback (most recent call last): + SystemExit: ERROR: Fields reset requested, but no grid name provided. Re-run with --grid GRID_NAME + >>> edit_output_stream_impl(fname,reset=['fields','io-grid'],grid='my_grid') + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> 'my_grid' in data['Fields'].keys() + False + >>> # Set remap options, and then check + >>> edit_output_stream_impl(fname,horiz_remap_file='blah1',vertical_remap_file='blah2') + >>> data = yaml.load(open(fname,'r'),Loader=yaml.SafeLoader) + >>> data['horiz_remap_file'] + 'blah1' + >>> data['vertical_remap_file'] + 'blah2' + >>> # Clean up the file + >>> pathlib.Path(fname).unlink() + """ + + if generate: + generate_empty_yaml(filename,overwrite) + + file = pathlib.Path(filename).resolve() + + expect (file.exists(), + "YAML file does not exist. Re-run with -g/--generate to create") + + data = yaml.load(open(file,"r"),Loader=yaml.SafeLoader) + + # Before adding new options, process all the reset requests + if reset is not None: + for s in reset: + if s=="avg-type": + data["Averaging Type"] = "INVALID" + elif s=="skip_t0_output": + data["skip_t0_output"] = "false" + elif s=="preifx": + data["filename_prefix"] = "UNSET" + elif s=="freq": + data["output_control"]["Frequency"] = -1 + elif s=="freq_units": + data["output_control"]["frequency_units"] = "never" + elif s=="horiz_remap_file": + del data["horiz_remap_file"] + elif s=="vert_remap_file": + del data["vert_remap_file"] + elif s=="fields": + expect (grid is not None, + "Fields reset requested, but no grid name provided. Re-run with --grid GRID_NAME") + if grid in data["Fields"].keys(): + data["Fields"][grid]["Field Names"] = [] + + # Remove this grid if there are no other options set + if len(data["Fields"][grid])==1: + del data["Fields"][grid] + elif s=="io-grid": + expect (grid is not None, + "IO grid reset requested, but no grid name provided. Re-run with --grid GRID_NAME") + if grid in data["Fields"].keys(): + del data["Fields"][grid]["IO Grid Name"] + + # Remove this grid if there's not other options set other than + # fields names, and field names is an empty list + if len(data["Fields"][grid])==1 and len(data["Fields"][grid]["Field Names"])==0: + del data["Fields"][grid] + + if prefix is not None: + data["filename_prefix"] = prefix + + if avg_type is not None: + data["Averaging Type"] = avg_type + + if skip_t0_output is not None: + data["skip_t0_output"] = skip_t0_output + + if freq is not None: + expect (freq.lstrip('-+').isnumeric() or freq=='hist_n', + f"Invalid value '{freq}' for --freq. Valid options are\n" + " - an integer\n" + " - HIST_N\n") + data["output_control"]["Frequency"] = int(freq) if freq.lstrip('-+').isnumeric() else f"${{{freq.upper()}}}" + + if freq_units is not None: + explicit = ['nsteps','nsecs','nmins','nhours','ndays','nmonths','nyears'] + expect (freq_units in explicit or freq_units=='hist_option', + f"Invalid value '{freq_units}' for --freq-units. Valid options are (case insensitive)\n" + " - explicit values: 'nsteps','nsecs','nmins','nhours','ndays','nmonths','nyears'\n" + " - CIME variables : 'HIST_OPTION'\n") + + data["output_control"]["frequency_units"] = freq_units if freq_units in explicit else f"${{{freq_units.upper()}}}" + + if horiz_remap_file is not None: + data["horiz_remap_file"] = horiz_remap_file + + if vertical_remap_file is not None: + data["vertical_remap_file"] = vertical_remap_file + + if len(fields)>0 or io_grid is not None: + expect (grid is not None, + "Fields list specified, but no grid name provided. Re-run with --grid GRID_NAME") + + section = data["Fields"].setdefault(grid,{}) + if "Field Names" not in section.keys(): + section["Field Names"] = [] + + fnames = section["Field Names"] + fnames += fields + fnames = list(set(fnames)) + section["Field Names"] = fnames + + if io_grid is not None: + section["IO Grid Name"] = io_grid + # If not already present, add an empty list of field names + section.setdefault("Field Names",[]) + + data["Fields"][grid] = section + + # We cannot do online remap (typically dyn->physGLL) if horiz or vert remap is used + has_online_remap = False + for k,v in data["Fields"].items(): + has_online_remap = has_online_remap or "IO Grid Name" in v.keys(); + has_vert_remap = "vertical_remap_file" in data.keys() + has_horiz_remap = "horiz_remap_file" in data.keys() + expect (not has_online_remap or (not has_vert_remap and not has_horiz_remap), + "Cannot use online remap and horiz/vert remap at the same time.") + + with open(file,'w') as fd: + yaml.dump(dict(data),fd,Dumper=yaml.SafeDumper,explicit_start=True,explicit_end=True,version=(1,2)) diff --git a/components/eamxx/scripts/git_utils.py b/components/eamxx/scripts/git_utils.py index e3aa70025398..a7202bfbfaba 100644 --- a/components/eamxx/scripts/git_utils.py +++ b/components/eamxx/scripts/git_utils.py @@ -55,7 +55,7 @@ def get_current_head(repo=None): return branch ############################################################################### -def git_refs_difference (cmp_ref, head="HEAD", repo=None): +def git_refs_difference(cmp_ref, head="HEAD", repo=None): ############################################################################### """ Return the difference in commits between cmp_ref and head. @@ -136,6 +136,17 @@ def merge_git_ref(git_ref, repo=None, verbose=False, dry_run=False): print ("git ref {} successfully merged.".format(git_ref)) print_last_commit() +############################################################################### +def create_backup_commit (repo=None, dry_run=False): +############################################################################### + + bkp_cmd = "git add -A && git commit -m 'WARNING: test-all-scream backup commit'" + if dry_run: + print (f"Would run: {bkp_cmd}") + else: + run_cmd_no_fail(bkp_cmd, from_dir=repo) + expect(is_repo_clean(repo=repo), "Something went wrong while performing the backup commit") + ############################################################################### def print_last_commit(git_ref=None, repo=None, dry_run=False): ############################################################################### @@ -182,7 +193,7 @@ def get_git_toplevel_dir(repo=None): return output if stat == 0 else None ############################################################################### -def cleanup_repo(orig_branch, orig_commit, repo=None, dry_run=False): +def cleanup_repo(orig_branch, orig_commit, has_backup_commit=False, repo=None, dry_run=False): ############################################################################### """ Discards all unstaged changes, as well as untracked files @@ -206,4 +217,10 @@ def cleanup_repo(orig_branch, orig_commit, repo=None, dry_run=False): # NOTE: if you reset the branch, don't forget to re-update the modules!! if curr_commit != orig_commit and not dry_run: run_cmd_no_fail("git reset --hard {}".format(orig_commit), from_dir=repo) + if has_backup_commit: + # This can happen if we ran an integration test with a dirty repo. + # test_all_scream will create a temporary backup commit, which we + # need to undo, but leaving the changed files in the workspace. + # So DON'T add --hard to this call! + run_cmd_no_fail("git reset {HEAD~1}", from_dir=repo) update_submodules(repo=repo) diff --git a/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh b/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh index 942be64d8df6..3282f86e7158 100755 --- a/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_cleanup_impl.sh @@ -1,11 +1,10 @@ #!/bin/bash -xe -# Adjust this number to keep more/less builds -echo "WORKSPACE: ${WORKSPACE}, BUILD_ID: ${BUILD_ID}" +echo "RUNNING CLEANUP FOR WORKSPACE: ${WORKSPACE}, BUILD_ID: ${BUILD_ID}" cd ${WORKSPACE} -NUM_KEEP=30 +NUM_KEEP=12 # Adjust this number to keep more/fewer builds KEEP_LAST=${BUILD_ID} KEEP_FIRST=$((${BUILD_ID}-${NUM_KEEP})) KEEP="$(seq ${KEEP_FIRST} 1 ${KEEP_LAST})" @@ -15,3 +14,11 @@ REMOVE_THESE="$(ls -1 | grep -vF "${KEEP}")" echo "Purging old builds: ${REMOVE_THESE}." /bin/rm -rf $REMOVE_THESE + +# Now clean up the scratch area +if [[ "$NODE_NAME" == "mappy" ]]; then + # Ensure we have a newer python + source $JENKINS_SCRIPT_DIR/${NODE_NAME}_setup + + $JENKINS_SCRIPT_DIR/scratch_cleanup.py +fi diff --git a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh index fe293debe446..664c911585e2 100755 --- a/components/eamxx/scripts/jenkins/jenkins_common_impl.sh +++ b/components/eamxx/scripts/jenkins/jenkins_common_impl.sh @@ -72,7 +72,12 @@ if [ $skip_testing -eq 0 ]; then # IF such dir is not found, then the default (ctest-build/baselines) is used BASELINES_DIR=AUTO - TAS_ARGS="--baseline-dir $BASELINES_DIR \$compiler -c EKAT_DISABLE_TPL_WARNINGS=ON -p -i -m \$machine" + TAS_ARGS="--baseline-dir $BASELINES_DIR \$compiler -p -c EKAT_DISABLE_TPL_WARNINGS=ON -i -m \$machine" + # pm-gpu needs to do work in scratch area in order not to fill home quota + if [[ "$SCREAM_MACHINE" == "pm-gpu" ]]; then + TAS_ARGS="${TAS_ARGS} -w /pscratch/sd/e/e3smtest/e3sm_scratch/pm-gpu/ctest-build" + fi + # Now that we are starting to run things that we expect could fail, we # do not want the script to exit on any fail since this will prevent # later tests from running. @@ -120,7 +125,9 @@ if [ $skip_testing -eq 0 ]; then fi fi - if [[ "$SCREAM_MACHINE" == "weaver" ]]; then + if [[ -z "$SCREAM_FAKE_ONLY" && "$SCREAM_MACHINE" == "weaver" ]]; then + # The fake-only tests don't launch any kernels which will cause all + # the compute-sanitizer runs to fail. ./scripts/gather-all-data "./scripts/test-all-scream -t csm -t csr -t csi -t css ${TAS_ARGS}" -l -m $SCREAM_MACHINE if [[ $? != 0 ]]; then fails=$fails+1; @@ -138,20 +145,6 @@ if [ $skip_testing -eq 0 ]; then # Run scripts-tests if [[ $test_scripts == 1 ]]; then - # JGF: I'm not sure there's much value in these dry-run comparisons - # since we aren't changing HEADs - ./scripts/scripts-tests -g -m $SCREAM_MACHINE - if [[ $? != 0 ]]; then - fails=$fails+1; - scripts_fail=1 - fi - - ./scripts/scripts-tests -c -m $SCREAM_MACHINE - if [[ $? != 0 ]]; then - fails=$fails+1; - scripts_fail=1 - fi - ./scripts/scripts-tests -f -m $SCREAM_MACHINE if [[ $? != 0 ]]; then fails=$fails+1; @@ -195,7 +188,7 @@ if [ $skip_testing -eq 0 ]; then if [[ $test_v1 == 1 ]]; then # AT runs should be fast. => run only low resolution - this_output=$(../../cime/scripts/create_test e3sm_scream_v1_at --compiler=gnu9 -c -b master --wait) + this_output=$(../../cime/scripts/create_test e3sm_scream_v1_at -c -b master --wait) if [[ $? != 0 ]]; then fails=$fails+1; v1_fail=1 diff --git a/components/eamxx/scripts/jenkins/pm-gpu_setup b/components/eamxx/scripts/jenkins/pm-gpu_setup new file mode 100644 index 000000000000..7bc04f72f9da --- /dev/null +++ b/components/eamxx/scripts/jenkins/pm-gpu_setup @@ -0,0 +1,2 @@ +source /global/common/software/e3sm/anaconda_envs/load_latest_cime_env.sh +SCREAM_MACHINE=pm-gpu diff --git a/components/eamxx/scripts/jenkins/scratch_cleanup.py b/components/eamxx/scripts/jenkins/scratch_cleanup.py new file mode 100755 index 000000000000..0195195a157e --- /dev/null +++ b/components/eamxx/scripts/jenkins/scratch_cleanup.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 + +""" +Clean up old files in the scratch area for mappy. +""" + +from pathlib import Path +import re, time, shutil, sys, argparse + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n{0} [-c HOURS] +OR +{0} --help + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Purge files older than 20 hours \033[0m + > {0} -c 20 +""".format(Path(args[0]).name), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument("-c", "--cutoff", type=int, default=30, help="The cutoff age for purging in hours") + + parser.add_argument("-d", "--dry-run", action="store_true", help="Do a dry run, don't actually remove files") + + args = parser.parse_args(args[1:]) + + return args + +############################################################################### +def scratch_cleanup(cutoff, dry_run): +############################################################################### + scratch = Path('/ascldap/users/e3sm-jenkins/acme/scratch') + + timestamp_re = re.compile(r'.*(20[0-9]{6}_[0-9]{6}).*') + timestamps = set() + for item in scratch.iterdir(): + basename = item.name + re_match = timestamp_re.match(basename) + if re_match: + timestamps.add(re_match.groups()[0]) + + tformat = "%Y%m%d_%H%M%S" + curr_time = time.time() + + for timestamp in timestamps: + timestamp_time = time.mktime(time.strptime(timestamp, tformat)) + age_in_hours = (curr_time - timestamp_time) / 3600 + if age_in_hours > cutoff: + print(f"Timestamp {timestamp} is {age_in_hours} hours old and corresponding files will be removed") + files_to_remove = scratch.glob(f"*{timestamp}*") + for file_to_remove in files_to_remove: + print(f" Removing {file_to_remove}") + if not dry_run: + if file_to_remove.is_dir(): + shutil.rmtree(file_to_remove) + else: + file_to_remove.unlink() + + return True + +############################################################################### +def _main_func(description): +############################################################################### + success = scratch_cleanup(**vars(parse_command_line(sys.argv, description))) + + sys.exit(0 if success else 1) + +############################################################################### + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index cd717cba6b97..df89ae7b5add 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -4,6 +4,8 @@ ensure_psutil() import psutil +CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","cime") + # MACHINE -> (env_setup, # list of shell commands to set up scream-approved env # compilers, # list of compilers [CXX, F90, C] # batch submit prefix, # string shell commmand prefix @@ -26,22 +28,22 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q rhel8 -n 4 -gpu num=4", "/home/projects/e3sm/scream/pr-autotester/master-baselines/weaver/"), - "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 sems-archive-gcc/9.2.0 sems-archive-git/2.10.1 acme-openmpi/4.0.7 acme-netcdf/4.7.4/acme"], + "mappy" : (["module purge", "module load sems-archive-env acme-env acme-cmake/3.26.3 acme-gcc/11.2.0 sems-archive-git/2.10.1 acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme", "export GATOR_INITIAL_MB=4000MB"], ["mpicxx","mpifort","mpicc"], "", "/sems-data-store/ACME/baselines/scream/master-baselines"), "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", - "export PATH=/usr/gdata/climdat/netcdf/bin:$PATH", - "export LD_LIBRARY_PATH=/usr/gdata/climdat/netcdf/lib:$LD_LIBRARY_PATH", + "export PATH=/usr/gdata/e3sm/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/gdata/e3sm/netcdf/lib:$LD_LIBRARY_PATH", ], ["mpicxx","mpifort","mpicc"], "bsub -Ip -qpdebug", ""), - "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), - "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), @@ -57,15 +59,19 @@ ["mpicxx","mpifort","mpicc"], "bsub -I -q batch -W 0:30 -P cli115 -nnodes 1", "/gpfs/alpine/cli115/proj-shared/scream/master-baselines"), - "pm-gpu" : (["module load PrgEnv-gnu gcc/10.3.0 cudatoolkit craype-accel-nvidia80 cray-libsci craype cray-mpich cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf cmake evp-patch","module unload craype-accel-host perftools-base perftools darshan", "export NVCC_WRAPPER_DEFAULT_COMPILER=CC", "export NVCC_WRAPPER_DEFAULT_ARCH=sm_80"], + "pm-cpu" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.pm-cpu_gnu)"], + ["CC","ftn","cc"], + "salloc --time 00:30:00 --nodes=1 --constraint=cpu -q debug --account e3sm_g", + "/global/cfs/cdirs/e3sm/baselines/gnu/scream/pm-cpu"), + "pm-gpu" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env -c SMS.ne4pg2_ne4pg2.F2010-SCREAMv1.pm-gpu_gnugpu)", "echo cuda=true"], ["CC","ftn","cc"], - "srun --time 00:30:00 --nodes=1 --constraint=gpu --exclusive -q regular --account e3sm_g", - ""), + "salloc --time 02:00:00 --nodes=4 --constraint=gpu --gpus-per-node=4 --gpu-bind=none --exclusive -q regular --account e3sm_g", + "/global/cfs/cdirs/e3sm/baselines/gnugpu/scream/pm-gpu"), "compy" : (["module purge", "module load cmake/3.19.6 gcc/8.1.0 mvapich2/2.3.1 python/3.7.3"], ["mpicxx","mpifort","mpicc"], "srun --time 02:00:00 --nodes=1 -p short --exclusive --account e3sm", ""), - "chrysalis" : (["eval $(../../cime/CIME/Tools/get_case_env)", "export OMP_NUM_THREADS=1"], + "chrysalis" : ([f"eval $({CIMEROOT}/CIME/Tools/get_case_env)", "export OMP_NUM_THREADS=1"], ["mpic++","mpif90","mpicc"], "srun --mpi=pmi2 -l -N 1 --kill-on-bad-exit --cpu_bind=cores", "/lcrc/group/e3sm/baselines/chrys/intel/scream"), @@ -193,10 +199,20 @@ def get_mach_testing_resources(machine): of jobs across cores. """ if is_cuda_machine(machine): - return int(run_cmd_no_fail("nvidia-smi -L | wc -l")) + prefix = "srun " if is_salloc(machine) else "" + return int(run_cmd_no_fail(f"{prefix}nvidia-smi -L | wc -l")) else: return get_available_cpu_count() +############################################################################### +def is_salloc(machine): +############################################################################### + """ + Return true if we are running on an salloc'd job. + """ + bcmd = get_mach_batch_command(machine) + return "salloc" in bcmd and "srun" not in bcmd + ############################################################################### def is_cuda_machine(machine): ############################################################################### diff --git a/components/eamxx/scripts/scripts-tests b/components/eamxx/scripts/scripts-tests index b70a697a7d55..b5af87320243 100755 --- a/components/eamxx/scripts/scripts-tests +++ b/components/eamxx/scripts/scripts-tests @@ -3,14 +3,7 @@ """ Script containing python test suite for SCREAM test infrastructure. This suite should be run to confirm overall -correctness. You should run this test once in generation mode to -generate baseline results using your reference commit (common -ancestor) and once in comparison mode to compare against these -baselines using your development commit. Baseline and compare runs -will use dry-run modes so we are only comparing hypothetical shell -commands, not actually running them. - -You can also do a full run which will actually execute the commands. +correctness. If you are on a batch machine, it is expected that you are on a compute node. @@ -26,43 +19,17 @@ from machines_specs import is_machine_supported, is_cuda_machine from git_utils import get_current_branch, get_current_commit, get_current_head, git_refs_difference, \ is_repo_clean, get_common_ancestor, checkout_git_ref, get_git_toplevel_dir -import unittest, argparse, sys, difflib, shutil, os +import unittest, argparse, sys, shutil, os from pathlib import Path # Globals TEST_DIR = Path(__file__).resolve().parent CONFIG = { "machine" : None, - "compare" : False, - "generate" : False, "full" : False, "jenkins" : False } -############################################################################### -def run_cmd_store_output(test_obj, cmd, output_file): -############################################################################### - output_file.parent.mkdir(parents=True, exist_ok=True) - output = run_cmd_assert_result(test_obj, cmd, from_dir=TEST_DIR) - head = get_current_head() - output = output.replace(head, "CURRENT_HEAD_NORMALIZED") - output_file.write_text(output) - -############################################################################### -def run_cmd_check_baseline(test_obj, cmd, baseline_path): -############################################################################### - test_obj.assertTrue(baseline_path.is_file(), msg="Missing baseline {}".format(baseline_path)) - output = run_cmd_assert_result(test_obj, cmd, from_dir=TEST_DIR) - head = get_current_head() - output = output.replace(head, "CURRENT_HEAD_NORMALIZED") - diff = difflib.unified_diff( - baseline_path.read_text().splitlines(), - output.splitlines(), - fromfile=str(baseline_path), - tofile=cmd) - diff_output = "\n".join(diff) - test_obj.assertEqual("", diff_output, msg=diff_output) - ############################################################################### def test_cmake_cache_contents(test_obj, build_name, cache_var, expected_value): ############################################################################### @@ -150,8 +117,6 @@ class TestBaseOuter: # Hides the TestBase class from test scanner self._source_file = source_file self._cmds = list(cmds) self._machine = CONFIG["machine"] - self._compare = CONFIG["compare"] - self._generate = CONFIG["generate"] self._full = CONFIG["full"] self._jenkins = CONFIG["jenkins"] @@ -160,42 +125,20 @@ class TestBaseOuter: # Hides the TestBase class from test scanner self._results = TEST_DIR.joinpath("results") self._results.mkdir(parents=True, exist_ok=True) # pylint: disable=no-member - def get_baseline(self, cmd, machine): - return self._results.joinpath(self._source_file).with_suffix("").\ - joinpath(machine, self.get_cmd(cmd, machine, dry_run=False).translate(str.maketrans(" /='", "____"))) - - def get_cmd(self, cmd, machine, dry_run=True): - return "{}{}".format(cmd.replace("$machine", machine).replace("$results", str(self._results)), - " --dry-run" if (dry_run and "--dry-run" not in cmd) else "") + def get_cmd(self, cmd, machine): + return cmd.replace("$machine", machine).replace("$results", str(self._results)) def test_doctests(self): run_cmd_assert_result(self, "python3 -m doctest {}".format(self._source_file), from_dir=TEST_DIR) def test_pylint(self): ensure_pylint() - run_cmd_assert_result(self, "python3 -m pylint --disable C --disable R {}".format(self._source_file), from_dir=TEST_DIR, verbose=True) - - def test_gen_baseline(self): - if self._generate: - for cmd in self._cmds: - run_cmd_store_output(self, self.get_cmd(cmd, self._machine), self.get_baseline(cmd, self._machine)) - else: - self.skipTest("Skipping dry run baseline generation") - - def test_cmp_baseline(self): - if self._compare: - for cmd in self._cmds: - if "-p" in cmd: - # Parallel builds can generate scrambled output. Skip them. - continue - run_cmd_check_baseline(self, self.get_cmd(cmd, self._machine), self.get_baseline(cmd, self._machine)) - else: - self.skipTest("Skipping dry run baseline comparison") + run_cmd_assert_result(self, "python3 -m pylint --disable C --disable R {}".format(self._source_file), from_dir=TEST_DIR) def test_full(self): if self._full: for cmd in self._cmds: - run_cmd_assert_result(self, self.get_cmd(cmd, self._machine, dry_run=False), from_dir=TEST_DIR) + run_cmd_assert_result(self, self.get_cmd(cmd, self._machine), from_dir=TEST_DIR) else: self.skipTest("Skipping full run") @@ -271,9 +214,8 @@ class TestTestAllScream(TestBaseOuter.TestBase): ############################################################################### CMDS_TO_TEST = [ - "./test-all-scream -m $machine -b HEAD -k -p", - "./test-all-scream -m $machine -b HEAD -k -t dbg", - "./test-all-scream --baseline-dir $results -p -c EKAT_DISABLE_TPL_WARNINGS=ON -i -m $machine --submit --dry-run", # always dry run + "./test-all-scream -m $machine -p -i -c EKAT_DISABLE_TPL_WARNINGS=ON", + "./test-all-scream -m $machine -t dbg", ] def __init__(self, *internal_args): @@ -288,9 +230,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the 'dbg' test in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t dbg --config-only" + options = "-t dbg --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "full_debug", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "full_debug", "SCREAM_DOUBLE_PRECISION", "TRUE") @@ -307,9 +249,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the 'sp' test in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t sp --config-only" + options = "-t sp --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "full_sp_debug", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "full_sp_debug", "SCREAM_DOUBLE_PRECISION", "FALSE") @@ -329,9 +271,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): if is_cuda_machine(self._machine): self.skipTest("Skipping FPE check on cuda") else: - options = "-b HEAD -k -t fpe --config-only" + options = "-t fpe --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "debug_nopack_fpe", "CMAKE_BUILD_TYPE", "Debug") test_cmake_cache_contents(self, "debug_nopack_fpe", "SCREAM_DOUBLE_PRECISION", "TRUE") @@ -345,9 +287,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the mem (default mem-check build) in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t mem --config-only" + options = "-t mem --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) builddir = "compute_sanitizer_memcheck" if is_cuda_machine(self._machine) else "valgrind" test_cmake_cache_contents(self, builddir, "CMAKE_BUILD_TYPE", "Debug") @@ -365,9 +307,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test the 'opt' test in test-all-scream. It should set certain CMake values """ if not self._jenkins: - options = "-b HEAD -k -t opt --config-only" + options = "-t opt --config-only" cmd = self.get_cmd("./test-all-scream -m $machine {}".format(options), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_cmake_cache_contents(self, "release", "CMAKE_BUILD_TYPE", "Release") else: @@ -379,7 +321,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the 'dbg' test in test-all-scream detects and returns non-zero if there's a cmake configure error """ if self._full: - cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_CONFIG_FAIL=True -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_CONFIG_FAIL=True -m $machine -t dbg", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: self.skipTest("Skipping full run") @@ -389,7 +331,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the 'dbg' test in test-all-scream detects and returns non-zero if there's a build error """ if self._full: - cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_BUILD_FAIL=True -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_BUILD_FAIL=True -m $machine -t dbg", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: self.skipTest("Skipping full run") @@ -399,7 +341,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that a failure from ctest in the testing phase is caught by test-all-scream """ if self._full: - cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_RUN_FAIL=True -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -e SCREAM_FORCE_RUN_FAIL=True -m $machine -t dbg", self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: self.skipTest("Skipping full run") @@ -409,7 +351,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the at test level works """ if self._full: - cmd = self.get_cmd("./test-all-scream -x -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -x -m $machine -t dbg", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_test_levels_were_run(self, output, ["AT"], ["NIGHTLY", "EXPERIMENTAL"]) else: @@ -420,7 +362,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the nightly test level works """ if self._full: - cmd = self.get_cmd("./test-all-scream -x --test-level=nightly -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -x --test-level=nightly -m $machine -t dbg", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_test_levels_were_run(self, output, ["AT", "NIGHTLY"], ["EXPERIMENTAL"]) else: @@ -431,7 +373,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): Test that the experimental test level works """ if self._full: - cmd = self.get_cmd("./test-all-scream -x --test-level=experimental -m $machine -b HEAD -k -t dbg", self._machine, dry_run=False) + cmd = self.get_cmd("./test-all-scream -x --test-level=experimental -m $machine -t dbg", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_test_levels_were_run(self, output, ["AT", "NIGHTLY", "EXPERIMENTAL"], []) else: @@ -445,13 +387,12 @@ class TestTestAllScream(TestBaseOuter.TestBase): So set SCREAM_FAKE_ONLY=ON, to lower the build time """ if self._full: - baseline_dir = TEST_DIR.parent/"ctest-build"/"baselines" - # Start a couple new tests, baselines will be generated + # Start a couple new tests, baselines will be generated in ctest-build/baselines env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE1" - opts = "-b HEAD -k -t dbg -t sp --no-tests" - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = " -g -t dbg -t sp --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") @@ -459,39 +400,42 @@ class TestTestAllScream(TestBaseOuter.TestBase): # Re-run reusing baselines from above env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -t sp --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -t sp --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") # Re-run dbg reusing baselines from above with a fake commit that's not ahead + # The flag -u implies -g, but nothing should happen, since SCREAM_FAKE_AHEAD=0 env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=0 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -u --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -u --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE1") test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") # Re-run dbg reusing baselines from above but expire them + # The flag -u implies -g, and since SCREAM_FAKE_AHEAD=1, baseline should be regenerated env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=1 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -u --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -u --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE2") test_baseline_has_sha(self, TEST_DIR, "full_sp_debug", "FAKE1") # Re-run reusing some baselines and expiring others + # The dbg baselines were generated in the prev step, so this should only gen the sp baselines env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_AHEAD=1 SCREAM_FAKE_GIT_HEAD=FAKE2" - opts = "--baseline-dir={} -b HEAD -k -t dbg -t sp -u --no-tests".format(baseline_dir) - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "--baseline-dir LOCAL -t dbg -t sp -u --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE2") @@ -499,9 +443,9 @@ class TestTestAllScream(TestBaseOuter.TestBase): # Re-run without reusing baselines, should force regeneration env = "SCREAM_FAKE_ONLY=ON SCREAM_FAKE_GIT_HEAD=FAKE3" - opts = "-b HEAD -k -t dbg -t sp --no-tests" - cmd = self.get_cmd("{} ./test-all-scream -m $machine {}".format(env,opts), - self._machine, dry_run=False) + opts = "-g -t dbg -t sp --no-tests" + cmd = self.get_cmd(f"{env} ./test-all-scream -m $machine {opts}", + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_baseline_has_sha(self, TEST_DIR, "full_debug", "FAKE3") @@ -518,21 +462,13 @@ class TestTestAllScream(TestBaseOuter.TestBase): can manage resources correctly. """ if self._full and self._machine == "mappy": - spread_test_opts = "-x -e SCREAM_TEST_THREAD_SPREAD=True -c EKAT_TEST_LAUNCHER_MANAGE_RESOURCES=True -c EKAT_MPIRUN_EXE=mpiexec -c EKAT_MPI_EXTRA_ARGS='-bind-to core' -c EKAT_MPI_NP_FLAG='--map-by' -c EKAT_MPI_THREAD_FLAG='' -m $machine -b HEAD -k -t dbg" + spread_test_opts = "-x -e SCREAM_TEST_THREAD_SPREAD=True -e SCREAM_TEST_RANK_SPREAD=True -c EKAT_TEST_LAUNCHER_MANAGE_RESOURCES=True -c EKAT_MPIRUN_EXE=mpiexec -c EKAT_MPI_EXTRA_ARGS='-bind-to core' -c EKAT_MPI_NP_FLAG='--map-by' -c EKAT_MPI_THREAD_FLAG='' -m $machine -t dbg" - cmd = self.get_cmd(f"./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) + cmd = self.get_cmd(f"./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_omp_spread(self, output, 40) - cmd = self.get_cmd(f"./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) - output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - test_omp_spread(self, output, 40) - - cmd = self.get_cmd(f"taskset -c 8-47 ./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) - output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) - test_omp_spread(self, output, 48, begin=8) - - cmd = self.get_cmd(f"taskset -c 8-47 ./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine, dry_run=False) + cmd = self.get_cmd(f"taskset -c 8-47 ./test-all-scream {spread_test_opts} --ctest-parallel-level=40 ", self._machine) output = run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) test_omp_spread(self, output, 48, begin=8) @@ -547,7 +483,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): # We set PULLREQUESTNUM to block dashboard submission # We set SCREAM_FAKE_AUTO to not interere with real baselines cmd = self.get_cmd("PR_LABELS= NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) else: @@ -561,7 +497,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): # We set PULLREQUESTNUM to block dashboard submission # We set SCREAM_FAKE_AUTO to not interere with real baselines cmd = self.get_cmd("PR_LABELS= NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR) else: @@ -574,7 +510,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): if self._jenkins: # Any fail will do, we already checked test-all-scream captures all the fail types cmd = self.get_cmd("PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM=42 ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: @@ -587,7 +523,7 @@ class TestTestAllScream(TestBaseOuter.TestBase): if self._jenkins: # Any fail will do, we already checked test-all-scream captures all the fail types cmd = self.get_cmd("PR_LABELS= SCREAM_FORCE_CONFIG_FAIL=True NODE_NAME={} SCREAM_FAKE_AUTO=TRUE PULLREQUESTNUM= ./jenkins/jenkins_common.sh".format(self._machine), - self._machine, dry_run=False) + self._machine) run_cmd_assert_result(self, cmd, from_dir=TEST_DIR, expect_works=False) else: @@ -598,7 +534,7 @@ class TestGatherAllData(TestBaseOuter.TestBase): ############################################################################### CMDS_TO_TEST = [ - "./gather-all-data './scripts/test-all-scream -m $machine -b HEAD -k' -l -m $machine", + "./gather-all-data 'echo -m $machine' -l -m $machine", ] def __init__(self, *internal_args): @@ -637,27 +573,10 @@ OR \033[1;32m# Run pylint tests for test_all_scream \033[0m > {0} TestTestAllScream.test_pylint - \033[1;32m# Do a dry-run generation for test_all_scream \033[0m - > {0} -g TestTestAllScream -m $machine - - \033[1;32m# Do a dry-run comparison for test_all_scream \033[0m - > {0} -c TestTestAllScream -m $machine - \033[1;32m# Do a full test run of test_all_scream \033[0m > {0} -f -m $machine TestTestAllScream - \033[1;32m# Do a full test run of everything \033[0m - > {0} -f -m $machine - - \033[1;32m# Do a dry-run generation for everything \033[0m - > {0} -g -m $machine - - \033[1;32m# Do a dry-run comparison for comparison \033[0m - > {0} -c -m $machine - \033[1;32m# Run every possible test. This should be done before a PR is issued \033[0m - > {0} -g -m $machine # You likely want to do this for a reference commit - > {0} -c -m $machine > {0} -f -m $machine \033[1;32m# Test Jenkins script \033[0m @@ -673,12 +592,6 @@ OR parser.add_argument("-m", "--machine", help="Provide machine name. This is required for full (not dry) runs") - parser.add_argument("-g", "--generate", action="store_true", - help="Do a dry run with baseline generation") - - parser.add_argument("-c", "--compare", action="store_true", - help="Do a dry run with baseline comparison") - parser.add_argument("-f", "--full", action="store_true", help="Do a full (not dry) run") @@ -691,7 +604,7 @@ OR return args ############################################################################### -def scripts_tests(machine=None, generate=False, compare=False, full=False, jenkins=False): +def scripts_tests(machine=None, full=False, jenkins=False): ############################################################################### os.environ["SCREAM_FAKE_ONLY"] = "True" @@ -700,9 +613,6 @@ def scripts_tests(machine=None, generate=False, compare=False, full=False, jenki expect(is_machine_supported(machine), "Machine {} is not supported".format(machine)) CONFIG["machine"] = machine - expect(not (generate and compare), "Cannot do generate and compare in the same run") - CONFIG["compare"] = compare - CONFIG["generate"] = generate CONFIG["jenkins"] = jenkins if full: diff --git a/components/eamxx/scripts/test-all-scream b/components/eamxx/scripts/test-all-scream index 4a6d0710b4e3..bb55ad23c216 100755 --- a/components/eamxx/scripts/test-all-scream +++ b/components/eamxx/scripts/test-all-scream @@ -16,6 +16,7 @@ check_minimum_python_version(3, 4) import argparse, sys, pathlib +from test_factory import get_test_name_dict from test_all_scream import TestAllScream ############################################################################### @@ -35,10 +36,6 @@ OR > cd $scream_repo/components/eamxx > ./scripts/{0} --preserve-env -m melvin - \033[1;32m# Run all tests on current machine with default behavior except using a custom ref for baseline generation\033[0m - > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin -b BASELINE_REF - \033[1;32m# Run all tests on current machine with default behavior except using pre-existing baselines (skips baseline generation) \033[0m > cd $scream_repo/components/eamxx > ./scripts/{0} -m melvin --baseline-dir=PATH_TO_BASELINES @@ -49,7 +46,7 @@ OR \033[1;32m# Run all tests on current machine with default behavior on a repo with uncommitted changes\033[0m > cd $scream_repo/components/eamxx - > ./scripts/{0} -m melvin -k -b HEAD + > ./scripts/{0} -m melvin -k -b AUTO """.format(pathlib.Path(args[0]).name), description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter @@ -63,17 +60,14 @@ OR parser.add_argument("-p", "--parallel", action="store_true", help="Launch the different build types stacks in parallel") - parser.add_argument("-f", "--fast-fail", action="store_true", - help="Stop testing when the first failure is detected") - - parser.add_argument("-b", "--baseline-ref", default=None, # default will be computed later - help="What commit to use to generate baselines. Default is merge-base of current commit and origin/master (or HEAD if --keep-tree)") + parser.add_argument("-g", "--generate", action="store_true", + help="Instruct test-all-scream to generate baselines from current commit. Skips tests") - parser.add_argument("--baseline-dir", default=None, - help="Use baselines from the given directory, skip baseline creation.") + parser.add_argument("-b", "--baseline-dir", default=None, + help="Directory where baselines should be read from (or written to, if -g/-i is used)") parser.add_argument("-u", "--update-expired-baselines", action="store_true", - help="Update baselines that appear to be expired.") + help="Update baselines that appear to be expired (only used with -g)") parser.add_argument("-m", "--machine", help="Provide machine name. This is *always* required. It can, but does not" @@ -86,9 +80,6 @@ OR parser.add_argument("--config-only", action="store_true", help="In the testing phase, only run config step, skip build and tests") - parser.add_argument("-k", "--keep-tree", action="store_true", - help="Allow to keep the current work tree when testing against HEAD (only valid with `-b HEAD`)") - parser.add_argument("-c", "--custom-cmake-opts", action="append", default=[], help="Extra custom options to pass to cmake. Can use multiple times for multiple cmake options. The -D is added for you") @@ -100,12 +91,12 @@ OR parser.add_argument("--preserve-env", action="store_true", help="Whether to skip machine env setup, and preserve the current user env (useful to manually test new modules)") - choices_doc = ", ".join(["'{}' ({})".format(k, v) for k, v in TestAllScream.get_test_name_desc().items()]) + choices_doc = ", ".join(["'{}' ({})".format(k, v) for k, v in get_test_name_dict().items()]) parser.add_argument("-t", "--test", dest="tests", action="append", default=[], help=f"Only run specific test configurations, choices={choices_doc}") parser.add_argument("-i", "--integration-test", action="store_true", - help="Merge origin/master into this branch before testing.") + help="Merge origin/master into this branch before testing (implies -u).") parser.add_argument("-l", "--local", action="store_true", help="Allow to not specify a machine name, and have test-all-scream to look" @@ -123,9 +114,6 @@ OR parser.add_argument("--quick-rerun-failed", action="store_true", help="Do not clean the build dir, and do not reconfigure. Just (incremental) build and retest failed tests only.") - parser.add_argument("-d", "--dry-run", action="store_true", - help="Do a dry run, commands will be printed but not executed") - parser.add_argument("--make-parallel-level", action="store", type=int, default=0, help="Max number of jobs to be created during compilation. If not provided, use default for given machine.") diff --git a/components/eamxx/scripts/test_all_scream.py b/components/eamxx/scripts/test_all_scream.py index dfade8beda0d..ffbe70e94f68 100644 --- a/components/eamxx/scripts/test_all_scream.py +++ b/components/eamxx/scripts/test_all_scream.py @@ -1,6 +1,10 @@ -from utils import run_cmd, run_cmd_no_fail, expect, check_minimum_python_version, ensure_psutil +from utils import run_cmd, run_cmd_no_fail, expect, check_minimum_python_version, ensure_psutil, \ + SharedArea, safe_copy from git_utils import get_current_head, get_current_commit, get_current_branch, is_repo_clean, \ - cleanup_repo, merge_git_ref, checkout_git_ref, git_refs_difference, print_last_commit + cleanup_repo, merge_git_ref, git_refs_difference, print_last_commit, \ + create_backup_commit, checkout_git_ref + +from test_factory import create_tests, COV from machines_specs import get_mach_compilation_resources, get_mach_testing_resources, \ get_mach_baseline_root_dir, setup_mach_env, is_cuda_machine, \ @@ -18,270 +22,19 @@ import psutil import re -from collections import OrderedDict from pathlib import Path -############################################################################### -class TestProperty(object): -############################################################################### - - """ - Parent class of predefined test types for SCREAM standalone. test-all-scream - offers a number of customization points, but you may need to just use - cmake if you need maximal customization. You can run test-all-scream --dry-run - to get the corresponding cmake command which can then be used as a starting - point for making your own cmake command. - """ - - def __init__(self, longname, description, cmake_args, - uses_baselines=True, on_by_default=True, default_test_len=None): - # What the user uses to select tests via test-all-scream CLI. - # Should also match the class name when converted to caps - self.shortname = type(self).__name__.lower() - - # A longer name used to name baseline and test directories for a test. - # Also used in output/error messages to refer to the test - self.longname = longname - - # A longer decription of the test - self.description = description - - # Cmake config args for this test. Check that quoting is done with - # single quotes. - self.cmake_args = cmake_args - for name, arg in self.cmake_args: - expect('"' not in arg, - f"In test definition for {longname}, found cmake args with double quotes {name}='{arg}'" - "Please use single quotes if quotes are needed.") - - # Does the test do baseline testing - self.uses_baselines = uses_baselines - - # Should this test be run if the user did not specify tests at all? - self.on_by_default = on_by_default - - # Should this test have a default test size - self.default_test_len = default_test_len - - # - # Properties not set by constructor (Set by the main TestAllScream object) - # - - # Resources used by this test. - self.compile_res_count = None - self.testing_res_count = None - - # Does this test need baselines - self.missing_baselines = False - - # - # Common - # - - if not self.uses_baselines: - self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] - - def disable_baselines(self): - if self.uses_baselines: - self.uses_baselines = False - self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] - - # Tests will generally be referred to via their longname - def __str__(self): - return self.longname - -############################################################################### -class DBG(TestProperty): -############################################################################### - - CMAKE_ARGS = [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True")] - - def __init__(self, _): - TestProperty.__init__( - self, - "full_debug", - "debug", - self.CMAKE_ARGS, - ) - -############################################################################### -class SP(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "full_sp_debug", - "debug single precision", - DBG.CMAKE_ARGS + [("SCREAM_DOUBLE_PRECISION", "False")], - ) - -############################################################################### -class FPE(TestProperty): -############################################################################### - - def __init__(self, tas): - TestProperty.__init__( - self, - "debug_nopack_fpe", - "debug pksize=1 floating point exceptions on", - DBG.CMAKE_ARGS + [("SCREAM_PACK_SIZE", "1"), ("SCREAM_FPE","True")], - uses_baselines=False, - on_by_default=(tas is not None and not tas.on_cuda()) - ) - -############################################################################### -class OPT(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "release", - "release", - [("CMAKE_BUILD_TYPE", "Release")], - ) - -############################################################################### -class COV(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "coverage", - "debug coverage", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COVERAGE", "True")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class VALG(TestProperty): -############################################################################### - - def __init__(self, tas): - TestProperty.__init__( - self, - "valgrind", - "debug with valgrind", - [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_VALGRIND", "True")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - if tas is not None: - # If a stored suppression file exists for this machine, use it - persistent_supp_file = tas.get_root_dir() / "scripts" / "jenkins" / "valgrind" / f"{tas.get_machine()}.supp" - if persistent_supp_file.exists(): - self.cmake_args.append( ("EKAT_VALGRIND_SUPPRESSION_FILE", str(persistent_supp_file)) ) - -############################################################################### -class CSM(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_memcheck", - "debug with compute sanitizer memcheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class CSR(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_racecheck", - "debug with compute sanitizer racecheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=racecheck --racecheck-detect-level=error'")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class CSI(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_initcheck", - "debug with compute sanitizer initcheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -class CSS(TestProperty): -############################################################################### - - def __init__(self, _): - TestProperty.__init__( - self, - "compute_sanitizer_synccheck", - "debug with compute sanitizer synccheck", - [("CMAKE_BUILD_TYPE", "Debug"), - ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), - ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], - uses_baselines=False, - on_by_default=False, - default_test_len="short" - ) - -############################################################################### -def test_factory(user_req_tests, tas): -############################################################################### - testclasses = TestProperty.__subclasses__() - if not user_req_tests: - result = [testclass(tas) for testclass in testclasses - if testclass(tas).on_by_default] - else: - valid_names = [testclass(tas).shortname for testclass in testclasses] - for user_req_test in user_req_tests: - expect(user_req_test in valid_names, f"'{user_req_test}' is not a known test") - - result = [testclass(tas) for testclass in testclasses if testclass(tas).shortname in user_req_tests] - - return result - ############################################################################### class TestAllScream(object): ############################################################################### - ########################################################################### - @classmethod - def get_test_name_desc(cls): - ########################################################################### - """ - Returns a dict mapping short test names to full names - """ - testclasses = TestProperty.__subclasses__() - return OrderedDict([(testc(None).shortname, testc(None).description) for testc in testclasses]) - ########################################################################### def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, - submit=False, parallel=False, fast_fail=False, - baseline_ref=None, baseline_dir=None, machine=None, no_tests=False, config_only=False, keep_tree=False, + submit=False, parallel=False, generate=False, no_tests=False, + baseline_dir=None, machine=None, config_only=False, custom_cmake_opts=(), custom_env_vars=(), preserve_env=False, tests=(), integration_test=False, local=False, root_dir=None, work_dir=None, - quick_rerun=False,quick_rerun_failed=False,dry_run=False, + quick_rerun=False,quick_rerun_failed=False, make_parallel_level=0, ctest_parallel_level=0, update_expired_baselines=False, extra_verbose=False, limit_test_regex=None, test_level="at", test_size=None, force_baseline_regen=False): @@ -299,13 +52,10 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._c_compiler = c_compiler self._submit = submit self._parallel = parallel - self._fast_fail = fast_fail - self._baseline_ref = baseline_ref self._machine = machine self._local = local - self._perform_tests = not no_tests + self._run_tests = not no_tests self._config_only = config_only - self._keep_tree = keep_tree self._baseline_dir = baseline_dir self._custom_cmake_opts = custom_cmake_opts self._custom_env_vars = custom_env_vars @@ -315,15 +65,16 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._integration_test = integration_test self._quick_rerun = quick_rerun self._quick_rerun_failed = quick_rerun_failed - self._dry_run = dry_run - self._update_expired_baselines= update_expired_baselines self._extra_verbose = extra_verbose self._limit_test_regex = limit_test_regex self._test_level = test_level self._test_size = test_size self._force_baseline_regen = force_baseline_regen - - # Not all builds are ment to perform comparisons against pre-built baselines + # Integration test always updates expired baselines + self._update_expired_baselines= update_expired_baselines or self._integration_test or self._force_baseline_regen + # If we are to update expired baselines, then we must run the generate phase + # NOTE: the gen phase will do nothing if baselines are present and not expired + self._generate = generate or self._update_expired_baselines if self._quick_rerun_failed: self._quick_rerun = True @@ -355,30 +106,22 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, self._root_dir = Path(__file__).resolve().parent.parent else: self._root_dir = Path(self._root_dir).resolve() - expect(self._root_dir.is_dir() and self._root_dir.parts()[-2:] == ("scream", "components"), - f"Bad root-dir '{self._root_dir}', should be: $scream_repo/components/eamxx") + expect(self._root_dir.is_dir() and list(self._root_dir.parts)[-2:] == ["components","eamxx"], + f"Bad root-dir '{self._root_dir}', should end with: /components/eamxx") # Make our test objects! Change mem to default mem-check test for current platform if "mem" in tests: tests[tests.index("mem")] = "csm" if self.on_cuda() else "valg" - self._tests = test_factory(tests, self) + self._tests = create_tests(tests, self) if self._work_dir is not None: self._work_dir = Path(self._work_dir).absolute() - expect(self._work_dir.is_dir(), - f"Error! Work directory '{self._work_dir}' does not exist.") else: self._work_dir = self._root_dir.absolute().joinpath("ctest-build") - self._work_dir.mkdir(exist_ok=True) - os.chdir(str(self._root_dir)) # needed, or else every git command will need repo=root_dir - expect(get_current_commit(), f"Root dir: {self._root_dir}, does not appear to be a git repo") - - # Print some info on the branch - self._original_branch = get_current_branch() - self._original_commit = get_current_commit() + self._work_dir.mkdir(parents=True, exist_ok=True) - print_last_commit(git_ref=self._original_branch, dry_run=self._dry_run) + os.chdir(str(self._root_dir)) # needed, or else every git command will need repo=root_dir ################################### # Compilation/testing resources # @@ -445,80 +188,84 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, # Setup the env on this machine setup_mach_env(self._machine, ctest_j=ctest_max_jobs) + ############################################ + # Check repo status # + ############################################ + + expect(get_current_commit(), f"Root dir: {self._root_dir}, does not appear to be a git repo") + + # Get git status info. Besides printing this info, we will need it to restore the repo initial + # configuration if we are running an integration test (where baselines need to be created + # from the origin/master commit) + self._original_branch = get_current_branch() + self._original_commit = get_current_commit() + + print_last_commit(git_ref=self._original_branch) + + # If we have an integration test, we need to merge master. Hence, do two things: + # 1) create bkp commit for all uncommitted/unstaged changes + # 2) save commit, so we can undo the merge after testing + self._has_backup_commit = False + if self._integration_test: + if not is_repo_clean(): + # Back up work in a temporary commit + create_backup_commit() + self._has_backup_commit = True + + self._original_commit = get_current_commit() + ################################### # Compute baseline info # ################################### expect (not self._baseline_dir or self._work_dir != self._baseline_dir, - f"Error! For your safety, do NOT use '{self._work_dir}' to store baselines. Move them to a different directory (even a subdirectory if that works).") - - # If no baseline ref/dir was provided, use default master baseline dir for this machine - # NOTE: if user specifies baseline ref, baseline dir will be set later to a path within work dir - if self._baseline_dir is None and self._baseline_ref is None: - self._baseline_dir = "AUTO" - print ("No '--baseline-dir XYZ' nor '-b XYZ' provided. Testing against default baselines dir for this machine.") - - # If -k was used, make sure it's allowed - if self._keep_tree: - expect(not self._integration_test, "Should not be doing keep-tree with integration testing") - print("WARNING! You have uncommitted changes in your repo.", - " The PASS/FAIL status may depend on these changes", - " so if you want to keep them, don't forget to create a commit.",sep="\n") - if self._baseline_dir is None: - # Make sure the baseline ref is HEAD - expect(self._baseline_ref == "HEAD", - "The option --keep-tree is only available when testing against pre-built baselines " - "(--baseline-dir) or HEAD (-b HEAD)") + f"Error! For your safety, do NOT use '{self._work_dir}' (the work_dir) to store baselines. Move them to a different directory (even a subdirectory if that works).") + + # These two dir are special dir for "on-the-fly baselines" and "machine's official baselines" + local_baseline_dir = self._work_dir/"baselines" + auto_dir = Path(get_mach_baseline_root_dir(self._machine)).absolute() + # Handle the "fake" auto case, used in scripts tests + if "SCREAM_FAKE_AUTO" in os.environ: + auto_dir = auto_dir / "fake" + + if self._baseline_dir == "LOCAL": + self._baseline_dir = local_baseline_dir + elif self._baseline_dir == "AUTO": + self._baseline_dir = auto_dir + elif self._baseline_dir is None: + if self._generate and not self._integration_test: + print ("No '--baseline-dir XYZ' provided. Baselines will be generated in {local_baseline_dir}.") + print ("NOTE: test-all-scream will proceed as if --force-baseline-regen was passed") + self._baseline_dir = local_baseline_dir + self._force_baseline_regen = True + self._update_expired_baselines = True else: - # Make sure the baseline ref is unset (or HEAD) - expect(self._baseline_ref is None or self._baseline_ref == "HEAD", - "The option --keep-tree is only available when testing against pre-built baselines " - "(--baseline-dir) or HEAD (-b HEAD)") - else: - expect(self._dry_run or is_repo_clean(), - "Repo must be clean before running. If testing against HEAD or pre-built baselines, " - "you can pass `--keep-tree` to allow non-clean repo.") + print ("No '--baseline-dir XYZ' provided. Testing against default baselines dir for this machine.") + self._baseline_dir = auto_dir - # For integration test, enforce baseline_ref==origin/master, and proceed to merge origin/master - if self._integration_test: - expect (self._baseline_ref is None or self._baseline_ref=="origin/master", - "Error! Integration tests cannot be done against an arbitrary baseline ref.") + self._baseline_dir = Path(self._baseline_dir).absolute() - # Set baseline ref and merge it - self._baseline_ref = "origin/master" - merge_git_ref(git_ref=self._baseline_ref, verbose=True, dry_run=self._dry_run) + # Only integration tests can overwrite the mach-specific baselines + if self._baseline_dir==auto_dir: + expect (not self._generate or self._integration_test or self._force_baseline_regen, + "You are not allowed to overwrite baselines in AUTO dir folder. Only -i and --force-baseline-regen can do that\n" + f" AUTO dir: {auto_dir}") - # Always update expired baselines if this is an integration test - self._update_expired_baselines = True + # Make the baseline dir, if not already existing. + if self._generate: + self.create_tests_dirs(self._baseline_dir, clean=False) - # By now, we should have at least one between baseline_dir and baseline_ref set (possibly both) - default_baselines_root_dir = self._work_dir/"baselines" - if self._baseline_dir is None: - # Use default baseline dir, and create it if necessary - self._baseline_dir = Path(default_baselines_root_dir).absolute() - self.create_tests_dirs(self._baseline_dir, True) # Wipe out previous baselines + # For now, assume baselines are generated from HEAD. If -i was used, we'll change this + self._baseline_ref = "origin/master" if self._integration_test else self._original_commit - else: - if self._baseline_dir == "AUTO": - expect (self._baseline_ref is None or self._baseline_ref == "origin/master", - "Do not specify `-b XYZ` when using `--baseline-dir AUTO`. The AUTO baseline dir should be used for the master baselines only.\n" - " `-b XYZ` needs to probably build baselines for ref XYZ. However, no baselines will be built if the dir already contains baselines.\n") - # We treat the "AUTO" string as a request for automatic baseline dir. - auto_dir = get_mach_baseline_root_dir(self._machine) - self._baseline_dir = Path(auto_dir) if auto_dir else default_baselines_root_dir - if "SCREAM_FAKE_AUTO" in os.environ: - self._baseline_dir = self._baseline_dir/"fake" - else: - self._baseline_dir = Path(self._baseline_dir).absolute() - - # Make sure the baseline folders exist (but do not purge content if they exist) - self.create_tests_dirs(self._baseline_dir, False) - - # Do not do baseline operations if mem checking is on + # Check baselines status print (f"Checking baselines directory: {self._baseline_dir}") - self.baselines_are_present() + missing_baselines = self.check_baselines_are_present() + expect (len(missing_baselines)==0 or self._generate, + f"Missing baselines for builds {missing_baselines}. Re-run with -g to generate them") + if self._update_expired_baselines: - self.baselines_are_expired() + self.check_baselines_are_expired() ############################################ # Deduce compilers if needed/possible # @@ -531,16 +278,11 @@ def __init__(self, cxx_compiler=None, f90_compiler=None, c_compiler=None, if self._c_compiler is None: self._c_compiler = get_mach_c_compiler(self._machine) - if not self._dry_run: - self._f90_compiler = run_cmd_no_fail(f"which {self._f90_compiler}") - self._cxx_compiler = run_cmd_no_fail(f"which {self._cxx_compiler}") - self._c_compiler = run_cmd_no_fail(f"which {self._c_compiler}") - ############################################################################### def create_tests_dirs(self, root, clean): ############################################################################### - # Make sure the baseline root directory exists + # Make sure the tests root directory exists root.mkdir(parents=True,exist_ok=True) # Create build directories (one per test) @@ -553,9 +295,11 @@ def create_tests_dirs(self, root, clean): # TypeError: lstat: illegal type for path parameter shutil.rmtree(str(test_dir)) - # Create this baseline's build dir - if not test_dir.exists(): - test_dir.mkdir(parents=True) + # Create this built type's build dir (if not already existing) + test_dir.mkdir(parents=True,exist_ok=True) + + # Create the 'data' subdir (if not already existing) + (test_dir / "data").mkdir(parents=False,exist_ok=True) ############################################################################### def get_baseline_file_sha(self, test): @@ -568,8 +312,9 @@ def get_baseline_file_sha(self, test): return None ############################################################################### - def set_baseline_file_sha(self, test, sha): + def set_baseline_file_sha(self, test): ############################################################################### + sha = get_current_commit() baseline_file = (self.get_preexisting_baseline(test).parent)/"baseline_git_sha" with baseline_file.open("w", encoding="utf-8") as fd: return fd.write(sha) @@ -601,7 +346,7 @@ def get_preexisting_baseline(self, test): return self._baseline_dir/str(test)/"data" ############################################################################### - def baselines_are_present(self): + def check_baselines_are_present(self): ############################################################################### """ Check that all baselines are present (one subdir for all values of self._tests) @@ -611,19 +356,23 @@ def baselines_are_present(self): expect(self._baseline_dir is not None, "Error! Baseline directory not correctly set.") + missing = [] for test in self._tests: if test.uses_baselines: data_dir = self.get_preexisting_baseline(test) if not data_dir.is_dir(): - test.missing_baselines = True + test.baselines_missing = True + missing += [test.longname] print(f" -> Test {test} is missing baselines") else: print(f" -> Test {test} appears to have baselines") else: print(f" -> Test {test} does not use baselines") + return missing + ############################################################################### - def baselines_are_expired(self): + def check_baselines_are_expired(self): ############################################################################### """ Baselines are expired if either: @@ -632,40 +381,50 @@ def baselines_are_expired(self): """ baseline_ref_sha = get_current_commit(commit=self._baseline_ref) - # Sanity check - expect(self._baseline_dir is not None, "Error! This routine should only be called when testing against pre-existing baselines.") - for test in self._tests: - if test.uses_baselines and not test.missing_baselines: - # this test is not missing a baseline, but it may be expired. - - baseline_file_sha = self.get_baseline_file_sha(test) - if baseline_file_sha is None: - test.missing_baselines = True - print(f" -> Test {test} has no stored sha so must be considered expired") - else: - num_ref_is_behind_file, num_ref_is_ahead_file = git_refs_difference(baseline_file_sha, baseline_ref_sha) - - # If the copy in our repo is behind, then we need to update the repo - expect (num_ref_is_behind_file==0 or not self._integration_test, + if not test.uses_baselines or test.baselines_missing: + continue + + if self._force_baseline_regen: + test.baselines_expired = True + print(f" -> Test {test} baselines are expired because self._force_baseline_regen=True") + continue + + # this test is not missing a baseline, but it may be expired. + baseline_file_sha = self.get_baseline_file_sha(test) + if baseline_file_sha is None: + test.baselines_missing = True + print(f" -> Test {test} has no stored sha so must be considered expired") + continue + + # There is a sha file, so check how it compares with self._baseline_ref + try: + num_ref_is_behind_file, num_ref_is_ahead_file = git_refs_difference(baseline_file_sha, baseline_ref_sha) + except SystemExit as e: + test.baselines_expired = True + reason = f"Failed to get refs difference between {baseline_file_sha} and {baseline_ref_sha} because: {e}" + print(f" -> Test {test} baselines are expired because {reason}") + continue + + # If the copy in our repo is behind, then we need to update the repo + expect (num_ref_is_behind_file==0 or not self._integration_test, f"""Error! Your repo seems stale, since the baseline sha in your repo is behind the one last used to generated them. We do *not* allow an integration test to replace baselines with older ones, for security reasons. If this is a legitimate case where baselines need to be 'rewound', e.g. b/c of a (hopefully VERY RARE) force push to master, then remove existing baselines first. Otherwise, please run 'git fetch $remote'. - - baseline_ref: {self._baseline_ref} - - repo baseline sha: {baseline_ref_sha} - - last used baseline sha: {baseline_file_sha}""") - - # If the copy in our repo is not ahead, then baselines are not expired - if num_ref_is_ahead_file > 0 or self._force_baseline_regen: - test.missing_baselines = True - reason = "forcing baseline regen" if self._force_baseline_regen \ - else f"{self._baseline_ref} is ahead of the baseline commit by {num_ref_is_ahead_file}" - print(f" -> Test {test} baselines are expired because {reason}") - else: - print(f" -> Test {test} baselines are valid and do not need to be regenerated") +- baseline_ref: {self._baseline_ref} +- repo baseline sha: {baseline_ref_sha} +- last used baseline sha: {baseline_file_sha}""") + + # If the copy in our repo is ahead, then baselines are expired + if num_ref_is_ahead_file > 0: + test.baselines_expired = True + reason = f"{self._baseline_ref} is ahead of the existing baseline commit {baseline_file_sha} by {num_ref_is_ahead_file}" + print(f" -> Test {test} baselines are expired because {reason}") + else: + print(f" -> Test {test} baselines are valid and do not need to be regenerated") ############################################################################### def get_machine_file(self): @@ -691,6 +450,9 @@ def generate_cmake_config(self, test, for_ctest=False): stat, c_path, _ = run_cmd("nc-config --prefix") if stat == 0: result += f" -DNetCDF_C_PATH={c_path}" + stat, pc_path, _ = run_cmd("pnetcdf-config --prefix") + if stat == 0: + result += f" -DPnetCDF_C_PATH={pc_path}" # Test-specific cmake options for key, value in test.cmake_args: @@ -815,7 +577,7 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): result += f"--resource-spec-file {test_dir}/ctest_resource_file.json " if self._baseline_dir is not None and test.uses_baselines: - cmake_config += f" -DSCREAM_TEST_DATA_DIR={self.get_preexisting_baseline(test)}" + cmake_config += f" -DSCREAM_BASELINES_DIR={self.get_preexisting_baseline(test).parent}" if not self._submit: result += "-DNO_SUBMIT=True " @@ -847,91 +609,111 @@ def generate_ctest_config(self, cmake_config, extra_configs, test): return result ############################################################################### - def generate_baselines(self, test, commit): + def generate_baselines(self, test): ############################################################################### expect(test.uses_baselines, f"Something is off. generate_baseline should have not be called for test {test}") - test_dir = self.get_test_dir(self._baseline_dir, test) + baseline_dir = self.get_test_dir(self._baseline_dir, test) + test_dir = self.get_test_dir(self._work_dir / "tas_baseline_build", test) + if test_dir.exists(): + shutil.rmtree(test_dir) + test_dir.mkdir() + num_test_res = self.create_ctest_resource_file(test,test_dir) cmake_config = self.generate_cmake_config(test) - cmake_config += " -DSCREAM_BASELINES_ONLY=ON" - cmake_config += f" -DSCREAM_TEST_DATA_DIR={test_dir}/data" + cmake_config += " -DSCREAM_ONLY_GENERATE_BASELINES=ON" + cmake_config += f" -DSCREAM_BASELINES_DIR={baseline_dir}" + cmake_config += f" -DSCREAM_TEST_MAX_TOTAL_THREADS={num_test_res}" print("===============================================================================") print(f"Generating baseline for test {test} with config '{cmake_config}'") print("===============================================================================") - success = True + # We cannot just crash if we fail to generate baselines, since we would + # not get a dashboard report if we did that. Instead, just ensure there is + # no baseline file to compare against if there's a problem. + stat, _, err = run_cmd(f"{cmake_config} {self._root_dir}", + from_dir=test_dir, verbose=True) + if stat != 0: + print (f"WARNING: Failed to create baselines (config phase):\n{err}") + return False - try: - # We cannot just crash if we fail to generate baselines, since we would - # not get a dashboard report if we did that. Instead, just ensure there is - # no baseline file to compare against if there's a problem. - stat, _, err = run_cmd(f"{cmake_config} {self._root_dir}", - from_dir=test_dir, verbose=True, dry_run=self._dry_run) - if stat != 0: - print (f"WARNING: Failed to configure baselines:\n{err}") - success = False + cmd = f"make -j{test.compile_res_count}" + if self._parallel: + start, end = self.get_taskset_range(test) + cmd = f"taskset -c {start}-{end} sh -c '{cmd}'" - else: - cmd = f"make -j{test.compile_res_count} && make -j{test.testing_res_count} baseline" - if self._parallel: - start, end = self.get_taskset_range(test) - cmd = f"taskset -c {start}-{end} sh -c '{cmd}'" + stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True) - stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True, dry_run=self._dry_run) + if stat != 0: + print (f"WARNING: Failed to create baselines (build phase):\n{err}") + return False - if stat != 0: - print(f"WARNING: Failed to create baselines:\n{err}") - success = False + cmd = f"ctest -j{test.testing_res_count}" + cmd += " -L baseline_gen" + cmd += f" --resource-spec-file {test_dir}/ctest_resource_file.json" + stat, _, err = run_cmd(cmd, from_dir=test_dir, verbose=True) - finally: - # Clean up the directory, by removing everything but the 'data' subfolder. This must - # happen unconditionally or else subsequent runs could be corrupted - run_cmd_no_fail(r"find -maxdepth 1 -not -name data ! -path . -exec rm -rf {} \;", - from_dir=test_dir, verbose=True, dry_run=self._dry_run) + if stat != 0: + print (f"WARNING: Failed to create baselines (run phase):\n{err}") + return False - if success: - # Store the sha used for baselines generation - self.set_baseline_file_sha(test, commit) - test.missing_baselines = False + # Read list of nc files to copy to baseline dir + with open(test_dir/"data/baseline_list","r",encoding="utf-8") as fd: + files = fd.read().splitlines() - return success + with SharedArea(): + for fn in files: + # In case appending to the file leaves an empty line at the end + if fn != "": + src = Path(fn) + dst = baseline_dir / "data" / src.name + safe_copy(src, dst) + + # Store the sha used for baselines generation + self.set_baseline_file_sha(test) + test.baselines_missing = False + + # Clean up the directory by removing everything + shutil.rmtree(test_dir) + + return True ############################################################################### def generate_all_baselines(self): ############################################################################### - git_head_ref = get_current_head() + + tests_needing_baselines = self.baselines_to_be_generated() + if len(tests_needing_baselines)==0: + return True + + # Switch to baseline ref + checkout_git_ref (self._baseline_ref) print("###############################################################################") - print(f"Generating baselines for ref {self._baseline_ref}") + print(f"Generating baselines from git ref {self._baseline_ref}") print("###############################################################################") - commit = get_current_commit(commit=self._baseline_ref) - - # Switch to the baseline commit - checkout_git_ref(self._baseline_ref, verbose=True, dry_run=self._dry_run) + tas_baseline_bld = self._work_dir / "tas_baseline_build" + if tas_baseline_bld.exists(): + shutil.rmtree(tas_baseline_bld) + tas_baseline_bld.mkdir() success = True - tests_needing_baselines = [test for test in self._tests if test.missing_baselines] num_workers = len(tests_needing_baselines) if self._parallel else 1 with threading3.ProcessPoolExecutor(max_workers=num_workers) as executor: future_to_test = { - executor.submit(self.generate_baselines, test, commit) : test + executor.submit(self.generate_baselines, test) : test for test in tests_needing_baselines} for future in threading3.as_completed(future_to_test): test = future_to_test[future] success &= future.result() - if not success and self._fast_fail: - print(f"Generation of baselines for test {test} failed") - return False - - # Switch back to the branch commit - checkout_git_ref(git_head_ref, verbose=True, dry_run=self._dry_run) + # Restore original commit + checkout_git_ref (self._original_commit) return success @@ -957,12 +739,13 @@ def run_test(self, test): if self._quick_rerun_failed: ctest_config += "--rerun-failed " else: - # This directory might have been used also to build the model to generate baselines. + # This directory might have been used before during another test-all-scream run. # Although it's ok to build in the same dir, we MUST make sure to erase cmake's cache - # and internal files from the previous build (CMakeCache.txt and CMakeFiles folder) - run_cmd_no_fail("rm -rf CMake*", from_dir=test_dir, dry_run=self._dry_run) + # and internal files from the previous build (CMakeCache.txt and CMakeFiles folder), + # Otherwise, we may not pick up changes in certain cmake vars that are already cached. + run_cmd_no_fail("rm -rf CMake*", from_dir=test_dir) - success = run_cmd(ctest_config, from_dir=test_dir, arg_stdout=None, arg_stderr=None, verbose=True, dry_run=self._dry_run)[0] == 0 + success = run_cmd(ctest_config, from_dir=test_dir, arg_stdout=None, arg_stderr=None, verbose=True)[0] == 0 return success @@ -973,9 +756,6 @@ def run_all_tests(self): print("Running tests!") print("###############################################################################") - # First, create build directories (one per test). If existing, nuke the content - self.create_tests_dirs(self._work_dir, not self._quick_rerun) - success = True tests_success = { test : False @@ -991,10 +771,6 @@ def run_all_tests(self): test = future_to_test[future] tests_success[test] = future.result() success &= tests_success[test] - # If failed, and fast fail is requested, return immediately - # Note: this is effective only if num_worksers=1 - if not success and self._fast_fail: - break for t,s in tests_success.items(): if not s: @@ -1049,6 +825,23 @@ def get_last_ctest_file(self,test,phase): else: return None + ############################################################################### + def baselines_to_be_generated(self): + ############################################################################### + """ + Return list of baselines to generate. Baselines need to be generated if + - they are missing + - they are expired and we asked to update expired baselines + """ + ret = [] + for test in self._tests: + if test.baselines_missing: + ret.append(test) + elif self._update_expired_baselines and test.baselines_expired: + ret.append(test) + + return ret + ############################################################################### def test_all_scream(self): ############################################################################### @@ -1060,25 +853,30 @@ def test_all_scream(self): success = True try: - # If needed, generate baselines first - tests_needing_baselines = [test for test in self._tests if test.missing_baselines] - if tests_needing_baselines: - expect(self._baseline_ref is not None, "Missing baseline ref") + if self._integration_test: + # Merge origin/master + merge_git_ref(git_ref=self._baseline_ref, verbose=True) + + if self._generate: success = self.generate_all_baselines() if not success: print ("Error(s) occurred during baselines generation phase") + + # Do not continue testing, as you may be testing against old/invalid baselines return False - # If requested, run tests - if self._perform_tests: + if self._run_tests: + # First, create build directories (one per test). If existing, nuke the content + self.create_tests_dirs(self._work_dir, not self._quick_rerun) + success &= self.run_all_tests() if not success: print ("Error(s) occurred during test phase") finally: - if not self._keep_tree: - # Cleanup the repo if needed - cleanup_repo(self._original_branch, self._original_commit, dry_run=self._dry_run) + # Cleanup the repo if needed + if self._original_commit!=get_current_commit(): + cleanup_repo(self._original_branch, self._original_commit, self._has_backup_commit) return success diff --git a/components/eamxx/scripts/test_factory.py b/components/eamxx/scripts/test_factory.py new file mode 100644 index 000000000000..3c480e3c6d8b --- /dev/null +++ b/components/eamxx/scripts/test_factory.py @@ -0,0 +1,258 @@ +# This module contains classes that describe a test type for test-all-scream +# Each test type (here represented by a TestProperty object) can have different +# flags, build type, profiling, cmake options, etc. +# The function "test_factory" can be used to get a list of test types from +# their string representation. + +from collections import OrderedDict +from utils import expect + +############################################################################### +class TestProperty(object): +############################################################################### + + """ + Parent class of predefined test types for SCREAM standalone. test-all-scream + offers a number of customization points, but you may need to just use + cmake if you need maximal customization. You can run test-all-scream --dry-run + to get the corresponding cmake command which can then be used as a starting + point for making your own cmake command. + """ + + def __init__(self, longname, description, cmake_args, + uses_baselines=True, on_by_default=True, default_test_len=None): + # What the user uses to select tests via test-all-scream CLI. + # Should also match the class name when converted to caps + self.shortname = type(self).__name__.lower() + + # A longer name used to name baseline and test directories for a test. + # Also used in output/error messages to refer to the test + self.longname = longname + + # A longer decription of the test + self.description = description + + # Cmake config args for this test. Check that quoting is done with + # single quotes. + self.cmake_args = cmake_args + for name, arg in self.cmake_args: + expect('"' not in arg, + f"In test definition for {longname}, found cmake args with double quotes {name}='{arg}'" + "Please use single quotes if quotes are needed.") + + # Does the test do baseline testing + self.uses_baselines = uses_baselines + + # Should this test be run if the user did not specify tests at all? + self.on_by_default = on_by_default + + # Should this test have a default test size + self.default_test_len = default_test_len + + # + # Properties not set by constructor (Set by the main TestAllScream object) + # + + # Resources used by this test. + self.compile_res_count = None + self.testing_res_count = None + + # Does this test need baselines + self.baselines_missing = False + self.baselines_expired = False + + # + # Common + # + + if not self.uses_baselines: + self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] + + def disable_baselines(self): + if self.uses_baselines: + self.uses_baselines = False + self.cmake_args += [("SCREAM_ENABLE_BASELINE_TESTS", "False")] + + # Tests will generally be referred to via their longname + def __str__(self): + return self.longname + +############################################################################### +class DBG(TestProperty): +############################################################################### + + CMAKE_ARGS = [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_DEFAULT_BFB", "True")] + + def __init__(self, _): + TestProperty.__init__( + self, + "full_debug", + "debug", + self.CMAKE_ARGS, + ) + +############################################################################### +class SP(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "full_sp_debug", + "debug single precision", + DBG.CMAKE_ARGS + [("SCREAM_DOUBLE_PRECISION", "False")], + ) + +############################################################################### +class FPE(TestProperty): +############################################################################### + + def __init__(self, tas): + TestProperty.__init__( + self, + "debug_nopack_fpe", + "debug pksize=1 floating point exceptions on", + DBG.CMAKE_ARGS + [("SCREAM_PACK_SIZE", "1"), ("SCREAM_FPE","True")], + uses_baselines=False, + on_by_default=(tas is not None and not tas.on_cuda()) + ) + +############################################################################### +class OPT(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "release", + "release", + [("CMAKE_BUILD_TYPE", "Release")], + ) + +############################################################################### +class COV(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "coverage", + "debug coverage", + [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_COVERAGE", "True")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class VALG(TestProperty): +############################################################################### + + def __init__(self, tas): + TestProperty.__init__( + self, + "valgrind", + "debug with valgrind", + [("CMAKE_BUILD_TYPE", "Debug"), ("EKAT_ENABLE_VALGRIND", "True")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + if tas is not None: + # If a stored suppression file exists for this machine, use it + persistent_supp_file = tas.get_root_dir() / "scripts" / "jenkins" / "valgrind" / f"{tas.get_machine()}.supp" + if persistent_supp_file.exists(): + self.cmake_args.append( ("EKAT_VALGRIND_SUPPRESSION_FILE", str(persistent_supp_file)) ) + +############################################################################### +class CSM(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_memcheck", + "debug with compute sanitizer memcheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=memcheck")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class CSR(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_racecheck", + "debug with compute sanitizer racecheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "'--tool=racecheck --racecheck-detect-level=error'")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class CSI(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_initcheck", + "debug with compute sanitizer initcheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=initcheck")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +class CSS(TestProperty): +############################################################################### + + def __init__(self, _): + TestProperty.__init__( + self, + "compute_sanitizer_synccheck", + "debug with compute sanitizer synccheck", + [("CMAKE_BUILD_TYPE", "Debug"), + ("EKAT_ENABLE_COMPUTE_SANITIZER", "True"), + ("EKAT_COMPUTE_SANITIZER_OPTIONS", "--tool=synccheck")], + uses_baselines=False, + on_by_default=False, + default_test_len="short" + ) + +############################################################################### +def create_tests(user_req_tests, tas): +############################################################################### + testclasses = TestProperty.__subclasses__() + if not user_req_tests: + result = [testclass(tas) for testclass in testclasses + if testclass(tas).on_by_default] + else: + valid_names = [testclass(tas).shortname for testclass in testclasses] + for user_req_test in user_req_tests: + expect(user_req_test in valid_names, f"'{user_req_test}' is not a known test") + + result = [testclass(tas) for testclass in testclasses if testclass(tas).shortname in user_req_tests] + + return result + +########################################################################### +def get_test_name_dict(): +########################################################################### + """ + Returns a dict mapping short test names to full names + """ + testclasses = TestProperty.__subclasses__() + return OrderedDict([(testc(None).shortname, testc(None).description) for testc in testclasses]) diff --git a/components/eamxx/scripts/utils.py b/components/eamxx/scripts/utils.py index d08075a5e34b..9aafd09ae8ae 100644 --- a/components/eamxx/scripts/utils.py +++ b/components/eamxx/scripts/utils.py @@ -2,9 +2,11 @@ Utilities """ -import os, sys, re, signal, subprocess, site, time +import os, sys, re, signal, subprocess, site, time, shutil from importlib import import_module +import stat as statlib from pathlib import Path +from distutils import file_util # pylint: disable=deprecated-module ############################################################################### def expect(condition, error_msg, exc_type=SystemExit, error_prefix="ERROR:"): @@ -415,3 +417,81 @@ def ensure_yaml(): _ensure_pylib_impl("yaml", pip_libname="pyyaml",min_version def ensure_pylint(): _ensure_pylib_impl("pylint") def ensure_psutil(): _ensure_pylib_impl("psutil") def ensure_netcdf4(): _ensure_pylib_impl("netCDF4") + +############################################################################### +def safe_copy(src_path, tgt_path, preserve_meta=True): +############################################################################### + """ + A flexbile and safe copy routine. Will try to copy file and metadata, but this + can fail if the current user doesn't own the tgt file. A fallback data-only copy is + attempted in this case. Works even if overwriting a read-only file. + + tgt_path can be a directory, src_path must be a file + + most of the complexity here is handling the case where the tgt_path file already + exists. This problem does not exist for the tree operations so we don't need to wrap those. + + preserve_meta toggles if file meta-data, like permissions, should be preserved. If you are + copying baseline files, you should be within a SharedArea context manager and preserve_meta + should be false so that the umask set up by SharedArea can take affect regardless of the + permissions of the src files. + """ + + # Only works for str paths for now + src_path = str(src_path) + tgt_path = str(tgt_path) + + tgt_path = ( + os.path.join(tgt_path, os.path.basename(src_path)) + if os.path.isdir(tgt_path) + else tgt_path + ) + + # Handle pre-existing file + if os.path.isfile(tgt_path): + st = os.stat(tgt_path) + owner_uid = st.st_uid + + # Handle read-only files if possible + if not os.access(tgt_path, os.W_OK): + if owner_uid == os.getuid(): + # I am the owner, make writeable + os.chmod(tgt_path, st.st_mode | statlib.S_IWRITE) + else: + # I won't be able to copy this file + raise OSError( + "Cannot copy over file {}, it is readonly and you are not the owner".format( + tgt_path + ) + ) + + if owner_uid == os.getuid(): + # I am the owner, copy file contents, permissions, and metadata + file_util.copy_file( + src_path, + tgt_path, + preserve_mode=preserve_meta, + preserve_times=preserve_meta, + verbose=0, + ) + else: + # I am not the owner, just copy file contents + shutil.copyfile(src_path, tgt_path) + + else: + # We are making a new file, copy file contents, permissions, and metadata. + # This can fail if the underlying directory is not writable by current user. + file_util.copy_file( + src_path, + tgt_path, + preserve_mode=preserve_meta, + preserve_times=preserve_meta, + verbose=0, + ) + + # If src file was executable, then the tgt file should be too + st = os.stat(tgt_path) + if os.access(src_path, os.X_OK) and st.st_uid == os.getuid(): + os.chmod( + tgt_path, st.st_mode | statlib.S_IXUSR | statlib.S_IXGRP | statlib.S_IXOTH + ) diff --git a/components/eamxx/src/CMakeLists.txt b/components/eamxx/src/CMakeLists.txt index 9568d3cf85ce..59a5d6646443 100644 --- a/components/eamxx/src/CMakeLists.txt +++ b/components/eamxx/src/CMakeLists.txt @@ -4,9 +4,6 @@ add_subdirectory(dynamics) add_subdirectory(physics) add_subdirectory(diagnostics) add_subdirectory(control) -if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") - add_subdirectory(doubly-periodic) -endif() if (PROJECT_NAME STREQUAL "E3SM") add_subdirectory(mct_coupling) endif() diff --git a/components/eamxx/src/control/atmosphere_driver.cpp b/components/eamxx/src/control/atmosphere_driver.cpp index 96adc56127d4..a5dacd856d75 100644 --- a/components/eamxx/src/control/atmosphere_driver.cpp +++ b/components/eamxx/src/control/atmosphere_driver.cpp @@ -167,44 +167,44 @@ init_time_stamps (const util::TimeStamp& run_t0, const util::TimeStamp& case_t0) void AtmosphereDriver:: -setup_intensive_observation_period () +setup_iop () { - // At this point, must have comm, params, initialized timestamps, and grids created. - check_ad_status(s_comm_set | s_params_set | s_ts_inited | s_grids_created); + // At this point, must have comm, params, initialized timestamps created. + check_ad_status(s_comm_set | s_params_set | s_ts_inited); // Check to make sure iop is not already initialized - EKAT_REQUIRE_MSG(not m_intensive_observation_period, "Error! setup_intensive_observation_period() is " - "called, but IOP already set up.\n"); + EKAT_REQUIRE_MSG(not m_iop, "Error! setup_iop() is called, but IOP already set up.\n"); // This function should only be called if we are enabling IOP const bool enable_iop = - m_atm_params.sublist("driver_options").get("enable_intensive_observation_period", false); - EKAT_REQUIRE_MSG(enable_iop, "Error! setup_intensive_observation_period() is called, but " - "enable_intensive_observation_period=false " + m_atm_params.sublist("driver_options").get("enable_iop", false); + EKAT_REQUIRE_MSG(enable_iop, "Error! setup_iop() is called, but enable_iop=false " "in driver_options parameters.\n"); - // Params must include intensive_observation_period_options sublist. - const auto iop_sublist_exists = m_atm_params.isSublist("intensive_observation_period_options"); + // Params must include iop_options sublist. + const auto iop_sublist_exists = m_atm_params.isSublist("iop_options"); EKAT_REQUIRE_MSG(iop_sublist_exists, - "Error! setup_intensive_observation_period() is called, but no intensive_observation_period_options " + "Error! setup_iop() is called, but no iop_options " "defined in parameters.\n"); - const auto iop_params = m_atm_params.sublist("intensive_observation_period_options"); + const auto iop_params = m_atm_params.sublist("iop_options"); const auto phys_grid = m_grids_manager->get_grid("Physics"); const auto nlevs = phys_grid->get_num_vertical_levels(); const auto hyam = phys_grid->get_geometry_data("hyam"); const auto hybm = phys_grid->get_geometry_data("hybm"); - m_intensive_observation_period = - std::make_shared(m_atm_comm, - iop_params, - m_run_t0, - nlevs, - hyam, - hybm); + m_iop = std::make_shared(m_atm_comm, + iop_params, + m_run_t0, + nlevs, + hyam, + hybm); auto dx_short_f = phys_grid->get_geometry_data("dx_short"); - m_intensive_observation_period->set_grid_spacing(dx_short_f.get_view()()); + m_iop->set_grid_spacing(dx_short_f.get_view()()); + + // Set IOP object in atm processes + m_atm_process_group->set_iop(m_iop); } void AtmosphereDriver::create_atm_processes() @@ -279,6 +279,14 @@ void AtmosphereDriver::create_grids() setup_shoc_tms_links(); } + // IOP object needs the grids_manager to have been created, but is then needed in set_grids() + // implementation of some processes, so setup here. + const bool enable_iop = + m_atm_params.sublist("driver_options").get("enable_iop", false); + if (enable_iop) { + setup_iop (); + } + // Set the grids in the processes. Do this by passing the grids manager. // Each process will grab what they need m_atm_process_group->set_grids(m_grids_manager); @@ -338,10 +346,6 @@ void AtmosphereDriver::setup_surface_coupling_processes () const std::shared_ptr importer = std::dynamic_pointer_cast(atm_proc); importer->setup_surface_coupling_data(*m_surface_coupling_import_data_manager); - - if (m_intensive_observation_period) { - importer->set_intensive_observation_period(m_intensive_observation_period); - } } if (atm_proc->type() == AtmosphereProcessType::SurfaceCouplingExporter) { exporter_found = true; @@ -691,9 +695,9 @@ void AtmosphereDriver::initialize_output_managers () { checkpoint_params.set("Frequency",-1); if (io_params.isSublist("model_restart")) { auto restart_pl = io_params.sublist("model_restart"); - m_output_managers.emplace_back(); + restart_pl.set("Averaging Type","Instant"); restart_pl.sublist("provenance") = m_atm_params.sublist("provenance"); - auto& om = m_output_managers.back(); + auto& om = m_output_managers.emplace_back(); if (fvphyshack) { // Don't save CGLL fields from ICs to the restart file. std::map fms; @@ -839,7 +843,7 @@ initialize_fields () auto hw = fm->get_field("horiz_winds"); const auto& fid = hw.get_header().get_identifier(); const auto& layout = fid.get_layout(); - const int vec_dim = layout.get_vector_dim(); + const int vec_dim = layout.get_vector_component_idx(); const auto& units = fid.get_units(); auto U = hw.subfield("U",units,vec_dim,0); auto V = hw.subfield("V",units,vec_dim,1); @@ -855,7 +859,7 @@ initialize_fields () auto hw = fm->get_field("surf_mom_flux"); const auto& fid = hw.get_header().get_identifier(); const auto& layout = fid.get_layout(); - const int vec_dim = layout.get_vector_dim(); + const int vec_dim = layout.get_vector_component_idx(); const auto& units = fid.get_units(); auto surf_mom_flux_U = hw.subfield("surf_mom_flux_U",units,vec_dim,0); auto surf_mom_flux_V = hw.subfield("surf_mom_flux_V",units,vec_dim,1); @@ -937,29 +941,34 @@ void AtmosphereDriver::create_logger () { auto& driver_options_pl = m_atm_params.sublist("driver_options"); ci_string log_fname = driver_options_pl.get("Atm Log File","atm.log"); - ci_string log_level_str = driver_options_pl.get("atm_log_level","info"); + ci_string log_level = driver_options_pl.get("atm_log_level","info"); + ci_string flush_level = driver_options_pl.get("atm_flush_level","warn"); EKAT_REQUIRE_MSG (log_fname!="", "Invalid string for 'Atm Log File': '" + log_fname + "'.\n"); - LogLevel log_level; - if (log_level_str=="trace") { - log_level = LogLevel::trace; - } else if (log_level_str=="debug") { - log_level = LogLevel::debug; - } else if (log_level_str=="info") { - log_level = LogLevel::info; - } else if (log_level_str=="warn") { - log_level = LogLevel::warn; - } else if (log_level_str=="err") { - log_level = LogLevel::err; - } else if (log_level_str=="off") { - log_level = LogLevel::off; - } else { - EKAT_ERROR_MSG ("Invalid choice for 'atm_log_level': " + log_level_str + "\n"); - } + auto str2lev = [](const std::string& s, const std::string& name) { + LogLevel lev; + if (s=="trace") { + lev = LogLevel::trace; + } else if (s=="debug") { + lev = LogLevel::debug; + } else if (s=="info") { + lev = LogLevel::info; + } else if (s=="warn") { + lev = LogLevel::warn; + } else if (s=="err") { + lev = LogLevel::err; + } else if (s=="off") { + lev = LogLevel::off; + } else { + EKAT_ERROR_MSG ("Invalid choice for '" + name + "': " + s + "\n"); + } + return lev; + }; using logger_t = Logger; - m_atm_logger = std::make_shared(log_fname,log_level,m_atm_comm,""); + m_atm_logger = std::make_shared(log_fname,str2lev(log_level,"atm_log_level"),m_atm_comm,""); + m_atm_logger->flush_on(str2lev(flush_level,"atm_flush_level")); m_atm_logger->set_no_format(); // In CIME runs, this is already set to false, so atm log does not pollute e3sm.loc. @@ -1129,7 +1138,7 @@ void AtmosphereDriver::set_initial_conditions () } } - if (m_intensive_observation_period) { + if (m_iop) { // For runs with IOP, call to setup io grids and lat // lon information needed for reading from file for (const auto& it : m_field_mgrs) { @@ -1140,7 +1149,7 @@ void AtmosphereDriver::set_initial_conditions () ic_pl.get("Filename") : ic_pl.get("topography_filename"); - m_intensive_observation_period->setup_io_info(file_name, it.second->get_grid()); + m_iop->setup_io_info(file_name, it.second->get_grid()); } } } @@ -1152,15 +1161,15 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" [EAMxx] IC filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_intensive_observation_period) { + if (not m_iop) { read_fields_from_file (ic_fields_names[grid_name],it.second->get_grid(),file_name,m_current_ts); } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_intensive_observation_period->read_fields_from_file_for_iop(file_name, - ic_fields_names[grid_name], - m_current_ts, - it.second); + m_iop->read_fields_from_file_for_iop(file_name, + ic_fields_names[grid_name], + m_current_ts, + it.second); } } } @@ -1227,7 +1236,7 @@ void AtmosphereDriver::set_initial_conditions () m_atm_logger->info(" filename: " + file_name); for (const auto& it : m_field_mgrs) { const auto& grid_name = it.first; - if (not m_intensive_observation_period) { + if (not m_iop) { // Topography files always use "ncol_d" for the GLL grid value of ncol. // To ensure we read in the correct value, we must change the name for that dimension auto io_grid = it.second->get_grid(); @@ -1243,11 +1252,11 @@ void AtmosphereDriver::set_initial_conditions () } else { // For IOP enabled, we load from file and copy data from the closest // lat/lon column to every other column - m_intensive_observation_period->read_fields_from_file_for_iop(file_name, - topography_file_fields_names[grid_name], - topography_eamxx_fields_names[grid_name], - m_current_ts, - it.second); + m_iop->read_fields_from_file_for_iop(file_name, + topography_file_fields_names[grid_name], + topography_eamxx_fields_names[grid_name], + m_current_ts, + it.second); } } // Store in provenance list, for later usage in output file metadata @@ -1268,16 +1277,16 @@ void AtmosphereDriver::set_initial_conditions () m_atm_params.sublist("provenance").set("topography_file","NONE"); } - if (m_intensive_observation_period) { + if (m_iop) { // Load IOP data file data for initial time stamp - m_intensive_observation_period->read_iop_file_data(m_current_ts); + m_iop->read_iop_file_data(m_current_ts); // Now that ICs are processed, set appropriate fields using IOP file data. // Since ICs are loaded on GLL grid, we set those fields only and dynamics // will take care of the rest (for PG2 case). if (m_field_mgrs.count("Physics GLL") > 0) { const auto& fm = m_field_mgrs.at("Physics GLL"); - m_intensive_observation_period->set_fields_from_iop_data(fm); + m_iop->set_fields_from_iop_data(fm); } } @@ -1441,12 +1450,15 @@ initialize_constant_field(const FieldIdentifier& fid, // The user provided a constant value for this field. Simply use that. const auto& layout = f.get_header().get_identifier().get_layout(); - // For vector fields, we expect something like "fname: [val0,...,valN], - // where the field dim is N+1. For scalars, "fname: val". So check the - // field layout first, so we know what to get from the parameter list. - if (layout.is_vector_layout()) { - const auto idim = layout.get_vector_dim(); - const auto vec_dim = layout.dim(idim); + // For vector fields, we allow either single value init or vector value init. + // That is, both these are ok + // fname: val + // fname: [val1,...,valN] + // In the first case, all entries of the field are inited to val, while in the latter, + // each component is inited to the corresponding entry of the array. + if (layout.is_vector_layout() and ic_pl.isType>(name)) { + const auto idim = layout.get_vector_component_idx(); + const auto vec_dim = layout.get_vector_dim(); const auto& values = ic_pl.get>(name); EKAT_REQUIRE_MSG (values.size()==static_cast(vec_dim), "Error! Initial condition values array for '" + name + "' has the wrong dimension.\n" @@ -1554,12 +1566,6 @@ initialize (const ekat::Comm& atm_comm, create_grids (); - const bool enable_iop = - m_atm_params.sublist("driver_options").get("enable_intensive_observation_period", false); - if (enable_iop) { - setup_intensive_observation_period (); - } - create_fields (); initialize_fields (); diff --git a/components/eamxx/src/control/atmosphere_driver.hpp b/components/eamxx/src/control/atmosphere_driver.hpp index bf694da3fb3e..1cd8d4db08e7 100644 --- a/components/eamxx/src/control/atmosphere_driver.hpp +++ b/components/eamxx/src/control/atmosphere_driver.hpp @@ -72,7 +72,7 @@ class AtmosphereDriver void init_scorpio (const int atm_id = 0); // Setup IntensiveObservationPeriod - void setup_intensive_observation_period (); + void setup_iop (); // Create atm processes, without initializing them void create_atm_processes (); @@ -207,7 +207,7 @@ class AtmosphereDriver std::shared_ptr m_surface_coupling_import_data_manager; std::shared_ptr m_surface_coupling_export_data_manager; - std::shared_ptr m_intensive_observation_period; + std::shared_ptr m_iop; // This is the time stamp at the beginning of the time step. util::TimeStamp m_current_ts; diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp index 1bf32b924215..e816801fa618 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.cpp @@ -170,9 +170,11 @@ void SurfaceCouplingImporter::do_import(const bool called_during_initialization) } }); - // If IOP is defined, potentially overwrite imports with data from IOP file - if (m_intensive_observation_period) { - overwrite_iop_imports(called_during_initialization); + if (m_iop) { + if (m_iop->get_params().get("iop_srf_prop")) { + // Overwrite imports with data from IOP file + overwrite_iop_imports(called_during_initialization); + } } } // ========================================================================================= @@ -181,11 +183,9 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in using policy_type = KokkosTypes::RangePolicy; using C = physics::Constants; - const auto& iop = m_intensive_observation_period; - - const auto has_lhflx = iop->has_iop_field("lhflx"); - const auto has_shflx = iop->has_iop_field("shflx"); - const auto has_Tg = iop->has_iop_field("Tg"); + const auto has_lhflx = m_iop->has_iop_field("lhflx"); + const auto has_shflx = m_iop->has_iop_field("shflx"); + const auto has_Tg = m_iop->has_iop_field("Tg"); static constexpr Real latvap = C::LatVap; static constexpr Real stebol = C::stebol; @@ -205,19 +205,19 @@ void SurfaceCouplingImporter::overwrite_iop_imports (const bool called_during_in // Store IOP surf data into col_val Real col_val(std::nan("")); if (fname == "surf_evap" && has_lhflx) { - const auto f = iop->get_iop_field("lhflx"); + const auto f = m_iop->get_iop_field("lhflx"); f.sync_to_host(); col_val = f.get_view()()/latvap; } else if (fname == "surf_sens_flux" && has_shflx) { - const auto f = iop->get_iop_field("shflx"); + const auto f = m_iop->get_iop_field("shflx"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_radiative_T" && has_Tg) { - const auto f = iop->get_iop_field("Tg"); + const auto f = m_iop->get_iop_field("Tg"); f.sync_to_host(); col_val = f.get_view()(); } else if (fname == "surf_lw_flux_up" && has_Tg) { - const auto f = iop->get_iop_field("Tg"); + const auto f = m_iop->get_iop_field("Tg"); f.sync_to_host(); col_val = stebol*std::pow(f.get_view()(), 4); } else { diff --git a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp index 5884c9d40af2..3a34a8b2951e 100644 --- a/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp +++ b/components/eamxx/src/control/atmosphere_surface_coupling_importer.hpp @@ -5,8 +5,6 @@ #include "ekat/ekat_parameter_list.hpp" #include "share/atm_process/SCDataManager.hpp" -#include "control/intensive_observation_period.hpp" - #include "surface_coupling_utils.hpp" #include @@ -36,7 +34,6 @@ class SurfaceCouplingImporter : public AtmosphereProcess using uview_2d = Unmanaged>; using name_t = char[32]; - using iop_ptr = std::shared_ptr; // Constructors SurfaceCouplingImporter (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -64,9 +61,6 @@ class SurfaceCouplingImporter : public AtmosphereProcess // Overwrite imports for IOP cases with IOP file surface data void overwrite_iop_imports (const bool called_during_initialization); - void set_intensive_observation_period (const iop_ptr& iop) { - m_intensive_observation_period = iop; - } protected: // The three main overrides for the subcomponent @@ -102,9 +96,6 @@ class SurfaceCouplingImporter : public AtmosphereProcess view_1d m_column_info_d; decltype(m_column_info_d)::HostMirror m_column_info_h; - // Intensive observation period object. - iop_ptr m_intensive_observation_period; - // The grid is needed for property checks std::shared_ptr m_grid; }; // class SurfaceCouplingImporter diff --git a/components/eamxx/src/control/intensive_observation_period.cpp b/components/eamxx/src/control/intensive_observation_period.cpp index f4ab466c17ba..d925b21bfaa3 100644 --- a/components/eamxx/src/control/intensive_observation_period.cpp +++ b/components/eamxx/src/control/intensive_observation_period.cpp @@ -116,7 +116,7 @@ IntensiveObservationPeriod(const ekat::Comm& comm, EKAT_REQUIRE_MSG(m_params.isParameter("target_latitude") && m_params.isParameter("target_longitude"), "Error! Using intensive observation period files requires " "target_latitude and target_longitude be gives as parameters in " - "\"intensive_observation_period_options\" in the input yaml file.\n"); + "\"iop_options\" in the input yaml file.\n"); const auto target_lat = m_params.get("target_latitude"); const auto target_lon = m_params.get("target_longitude"); EKAT_REQUIRE_MSG(-90 <= target_lat and target_lat <= 90, @@ -135,16 +135,18 @@ IntensiveObservationPeriod(const ekat::Comm& comm, if (not m_params.isParameter("iop_nudge_tscale")) m_params.set("iop_nudge_tscale", 10800); if (not m_params.isParameter("zero_non_iop_tracers")) m_params.set("zero_non_iop_tracers", false); + // Store hybrid coords in helper fields + m_helper_fields.insert({"hyam", hyam}); + m_helper_fields.insert({"hybm", hybm}); + // Use IOP file to initialize parameters // and timestepping information - initialize_iop_file(run_t0, model_nlevs, hyam, hybm); + initialize_iop_file(run_t0, model_nlevs); } void IntensiveObservationPeriod:: initialize_iop_file(const util::TimeStamp& run_t0, - int model_nlevs, - const Field& hyam, - const Field& hybm) + int model_nlevs) { EKAT_REQUIRE_MSG(m_params.isParameter("iop_file"), "Error! Using IOP requires defining an iop_file parameter.\n"); @@ -184,6 +186,8 @@ initialize_iop_file(const util::TimeStamp& run_t0, if (scorpio::has_variable(iop_file, srf_varname)) { m_iop_field_surface_varnames.insert({iop_varname, srf_varname}); } + // Store that the IOP variable is found in the IOP file + m_iop_field_type.insert({iop_varname, IOPFieldType::FromFile}); // Allocate field for variable FieldIdentifier fid(iop_varname, fl, ekat::units::Units::nondimensional(), ""); @@ -191,6 +195,10 @@ initialize_iop_file(const util::TimeStamp& run_t0, EKAT_REQUIRE_MSG(field_rank <= 1, "Error! Unexpected field rank "+std::to_string(field_rank)+" for iop file fields.\n"); Field field(fid); + if (fl.has_tag(FieldTag::LevelMidPoint) or fl.has_tag(FieldTag::LevelInterface)) { + // Request packsize allocation for level layout + field.get_header().get_alloc_properties().request_allocation(Pack::n); + } field.allocate_view(); m_iop_fields.insert({iop_varname, field}); } @@ -230,13 +238,42 @@ initialize_iop_file(const util::TimeStamp& run_t0, setup_iop_field({"Q2"}, fl_vector); setup_iop_field({"omega"}, fl_vector, "Ptend"); - // Make sure Ps, T, and q are defined in the iop file + // Require Ps, T, q, divT, divq are all defined in the iop file EKAT_REQUIRE_MSG(has_iop_field("Ps"), - "Error! Using IOP file requires variable \"Ps\".\n"); + "Error! IOP file required to contain variable \"Ps\".\n"); EKAT_REQUIRE_MSG(has_iop_field("T"), - "Error! Using IOP file requires variable \"T\".\n"); + "Error! IOP file required to contain variable \"T\".\n"); EKAT_REQUIRE_MSG(has_iop_field("q"), - "Error! Using IOP file requires variable \"q\".\n"); + "Error! IOP file required to contain variable \"q\".\n"); + EKAT_REQUIRE_MSG(has_iop_field("divT"), + "Error! IOP file required to contain variable \"divT\".\n"); + EKAT_REQUIRE_MSG(has_iop_field("divq"), + "Error! IOP file required to contain variable \"divq\".\n"); + + // If we have the vertical component of T/Q forcing, define 3d forcing as a computed field. + if (has_iop_field("vertdivT")) { + FieldIdentifier fid("divT3d", fl_vector, ekat::units::Units::nondimensional(), ""); + Field field(fid); + field.get_header().get_alloc_properties().request_allocation(Pack::n); + field.allocate_view(); + m_iop_fields.insert({"divT3d", field}); + m_iop_field_type.insert({"divT3d", IOPFieldType::Computed}); + } + if (has_iop_field("vertdivq")) { + FieldIdentifier fid("divq3d", fl_vector, ekat::units::Units::nondimensional(), ""); + Field field(fid); + field.get_header().get_alloc_properties().request_allocation(Pack::n); + field.allocate_view(); + m_iop_fields.insert({"divq3d", field}); + m_iop_field_type.insert({"divq3d", IOPFieldType::Computed}); + } + + // Enforce that 3D forcing is all-or-nothing. + const bool both = (has_iop_field("divT3d") and has_iop_field("divq3d")); + const bool neither = (not (has_iop_field("divT3d") or has_iop_field("divq3d"))); + EKAT_REQUIRE_MSG(both or neither, + "Error! Either T and q both have 3d forcing, or neither have 3d forcing.\n"); + m_params.set("use_3d_forcing", both); // Initialize time information int bdate; @@ -286,6 +323,7 @@ initialize_iop_file(const util::TimeStamp& run_t0, ekat::units::Units::nondimensional(), ""); Field iop_file_pressure(fid); + iop_file_pressure.get_header().get_alloc_properties().request_allocation(Pack::n); iop_file_pressure.allocate_view(); auto data = iop_file_pressure.get_view().data(); read_variable_from_file(iop_file, "lev", "real", {"lev"}, -1, data); @@ -300,12 +338,9 @@ initialize_iop_file(const util::TimeStamp& run_t0, fl_vector, ekat::units::Units::nondimensional(), ""); Field model_pressure(model_pres_fid); + model_pressure.get_header().get_alloc_properties().request_allocation(Pack::n); model_pressure.allocate_view(); m_helper_fields.insert({"model_pressure", model_pressure}); - - // Store hyam and hybm in helper fields - m_helper_fields.insert({"hyam", hyam}); - m_helper_fields.insert({"hybm", hybm}); } void IntensiveObservationPeriod:: @@ -400,7 +435,8 @@ read_fields_from_file_for_iop (const std::string& file_name, const vos& field_names_nc, const vos& field_names_eamxx, const util::TimeStamp& initial_ts, - const field_mgr_ptr field_mgr) + const field_mgr_ptr field_mgr, + const int time_index) { const auto dummy_units = ekat::units::Units::nondimensional(); @@ -456,7 +492,7 @@ read_fields_from_file_for_iop (const std::string& file_name, // Read data from file AtmosphereInput file_reader(file_name,io_grid,io_fields); - file_reader.read_variables(); + file_reader.read_variables(time_index); file_reader.finalize(); // For each field, broadcast data from closest lat/lon column to all processors @@ -502,27 +538,27 @@ read_fields_from_file_for_iop (const std::string& file_name, void IntensiveObservationPeriod:: read_iop_file_data (const util::TimeStamp& current_ts) { - const auto iop_file = m_params.get("iop_file"); + // Query to see if we need to load data from IOP file. + // If we are still in the time interval as the previous + // read from iop file, there is no need to reload data. const auto iop_file_time_idx = m_time_info.get_iop_file_time_idx(current_ts); - - // Sanity check EKAT_REQUIRE_MSG(iop_file_time_idx >= m_time_info.time_idx_of_current_data, "Error! Attempting to read previous iop file data time index.\n"); - - // If we are still in the time interval as the previous read from iop file, - // there is no need to reload data. Return early if (iop_file_time_idx == m_time_info.time_idx_of_current_data) return; + const auto iop_file = m_params.get("iop_file"); const auto file_levs = scorpio::get_dimlen(iop_file, "lev"); const auto iop_file_pressure = m_helper_fields["iop_file_pressure"]; const auto model_pressure = m_helper_fields["model_pressure"]; const auto surface_pressure = m_iop_fields["Ps"]; - // Loop through iop fields, if rank 1 fields exist we need to - // gather information for vertically interpolating views + // Loop through iop fields, if any rank 1 fields are loaded from file, + // we need to gather information for vertical interpolation bool has_level_data = false; for (auto& it : m_iop_fields) { - if (it.second.rank() == 1) { + if (it.second.rank() == 1 + and + m_iop_field_type.at(it.first)==IOPFieldType::FromFile) { has_level_data = true; break; } @@ -589,6 +625,10 @@ read_iop_file_data (const util::TimeStamp& current_ts) Kokkos::Max(iop_file_start), Kokkos::Min(iop_file_end)); + // If no file pressures are found outide the reference pressure range, set to file level endpoints + if (iop_file_start == Kokkos::reduction_identity::max()) iop_file_start = 0; + if (iop_file_end == Kokkos::reduction_identity::min()) iop_file_end = adjusted_file_levs; + // Find model pressure levels just inside range of file pressure levels Kokkos::parallel_reduce(model_nlevs, KOKKOS_LAMBDA (const int& ilev, int& lmin, int& lmax) { if (model_pres_v(ilev) >= iop_file_pres_v(iop_file_start) && ilev < lmin) { @@ -600,6 +640,10 @@ read_iop_file_data (const util::TimeStamp& current_ts) }, Kokkos::Min(model_start), Kokkos::Max(model_end)); + + // If not reference pressures are found inside file pressures, set to model level endpoints + if (model_start == Kokkos::reduction_identity::min()) model_start = model_nlevs-1; + if (model_end == Kokkos::reduction_identity::max()) model_end = 1; } // Loop through fields and store data from file @@ -607,6 +651,9 @@ read_iop_file_data (const util::TimeStamp& current_ts) auto fname = it.first; auto field = it.second; + // If this is a computed field, do not attempt to load from file + if (m_iop_field_type.at(fname)==IOPFieldType::Computed) continue; + // File may use different varname than IOP class auto file_varname = (m_iop_file_varnames.count(fname) > 0) ? m_iop_file_varnames[fname] : fname; @@ -625,6 +672,7 @@ read_iop_file_data (const util::TimeStamp& current_ts) ekat::units::Units::nondimensional(), ""); Field iop_file_field(fid); + iop_file_field.get_header().get_alloc_properties().request_allocation(Pack::n); iop_file_field.allocate_view(); // Read data from iop file. @@ -657,8 +705,8 @@ read_iop_file_data (const util::TimeStamp& current_ts) iop_file_field.sync_to_dev(); // Vertically interpolate iop file data to iop fields. - // Note: ekat lininterp requires packs. Use 1d packs here. - // TODO: allow for nontrivial packsize. + // Note: ekat lininterp requires packs. Use 1d packs here + // to easily mask out levels which we do not want to interpolate. const auto iop_file_pres_v = iop_file_pressure.get_view(); const auto model_pres_v = model_pressure.get_view(); const auto iop_file_v = iop_file_field.get_view(); @@ -685,22 +733,42 @@ read_iop_file_data (const util::TimeStamp& current_ts) // the interpolated region with the value at model_start/model_end if (fname == "T" || fname == "q" || fname == "u" || fname == "u_ls" || fname == "v" || fname == "v_ls") { - if (model_start > 0) { - Kokkos::parallel_for(Kokkos::RangePolicy<>(0, model_start), - KOKKOS_LAMBDA (const int ilev) { - iop_field_v(ilev) = iop_field_v(model_start); - }); - } - if (model_end < total_nlevs) { - Kokkos::parallel_for(Kokkos::RangePolicy<>(model_end, total_nlevs), - KOKKOS_LAMBDA (const int ilev) { - iop_field_v(ilev) = iop_field_v(model_end-1); - }); - } + Kokkos::parallel_for(Kokkos::RangePolicy<>(0, model_start+1), + KOKKOS_LAMBDA (const int ilev) { + iop_field_v(ilev) = iop_file_v(0); + }); + Kokkos::parallel_for(Kokkos::RangePolicy<>(model_end-1, total_nlevs), + KOKKOS_LAMBDA (const int ilev) { + iop_field_v(ilev) = iop_file_v(adjusted_file_levs-1); + }); } } } + // Calculate 3d forcing (if applicable). + if (has_iop_field("divT3d")) { + if (m_iop_field_type.at("divT3d")==IOPFieldType::Computed) { + const auto divT = get_iop_field("divT").get_view(); + const auto vertdivT = get_iop_field("vertdivT").get_view(); + const auto divT3d = get_iop_field("divT3d").get_view(); + const auto nlevs = get_iop_field("divT3d").get_header().get_identifier().get_layout().dim(0); + Kokkos::parallel_for(nlevs, KOKKOS_LAMBDA (const int ilev) { + divT3d(ilev) = divT(ilev) + vertdivT(ilev); + }); + } + } + if (has_iop_field("divq3d")) { + if (m_iop_field_type.at("divq3d")==IOPFieldType::Computed) { + const auto divq = get_iop_field("divq").get_view(); + const auto vertdivq = get_iop_field("vertdivq").get_view(); + const auto divq3d = get_iop_field("divq3d").get_view(); + const auto nlevs = get_iop_field("divq3d").get_header().get_identifier().get_layout().dim(0); + Kokkos::parallel_for(nlevs, KOKKOS_LAMBDA (const int ilev) { + divq3d(ilev) = divq(ilev) + vertdivq(ilev); + }); + } + } + // Now that data is loaded, reset the index of the currently loaded data. m_time_info.time_idx_of_current_data = iop_file_time_idx; } diff --git a/components/eamxx/src/control/intensive_observation_period.hpp b/components/eamxx/src/control/intensive_observation_period.hpp index 6e70f44a0f56..6c0b3640ffb2 100644 --- a/components/eamxx/src/control/intensive_observation_period.hpp +++ b/components/eamxx/src/control/intensive_observation_period.hpp @@ -26,6 +26,8 @@ class IntensiveObservationPeriod using KT = ekat::KokkosTypes; using ESU = ekat::ExeSpaceUtils; + using Pack = ekat::Pack; + using Pack1d = ekat::Pack; template using view_1d = KT::template view_1d; @@ -35,14 +37,13 @@ class IntensiveObservationPeriod using view_3d = KT::template view_3d; template using view_1d_host = typename view_1d::HostMirror; - using Pack1d = ekat::Pack; public: // Constructor // Input: // - comm: MPI communicator - // - params: Input yaml file needs intensive_observation_period_options sublist + // - params: Input yaml file needs iop_options sublist // - run_t0: Initial timestamp for the simulation // - model_nlevs: Number of vertical levels in the simulation. Needed since // the iop file contains a (potentially) different number of levels @@ -69,31 +70,35 @@ class IntensiveObservationPeriod void setup_io_info (const std::string& file_name, const grid_ptr& grid); - // Read ICs from file for IOP cases. We set all columns in the - // given fields to the values of the column in the file with the - // closest lat,lon pair to the target lat,lon in the parameters. - // The setup_io_info must be called for the correct grids before - // this function can be called. - // Input: + // Read ICs and SPA data from file and remap to fields in field_mgr. + // The remap is defined by setting all columns in the given fields to the + // values of the column in the file with the closest lat,lon pair to + // the target lat,lon in the parameters. + // The function setup_io_info() must be called for the grids corresponding + // to the file data before this function can be called. + // Fields in the field_mgr must have the same number of levels as the file. + // Inputs and outputs: // - file_name: Name of the file used to load field data (IC or topo file) // - field_names_nc: Field names used by the input file // - field_names_eamxx: Field names used by eamxx - // - initial_ts: Inital timestamp - // Input/output + // - initial_ts: Inital timestamp. // - field_mgr: Field manager containing fields that need data read from files + // - time_index: Time index of read. time_index=-1 will read the latest time in file. void read_fields_from_file_for_iop(const std::string& file_name, const vos& field_names_nc, const vos& field_names_eamxx, const util::TimeStamp& initial_ts, - const field_mgr_ptr field_mgr); + const field_mgr_ptr field_mgr, + const int time_index = -1); // Version of above, but where nc and eamxx field names are identical void read_fields_from_file_for_iop(const std::string& file_name, const vos& field_names, const util::TimeStamp& initial_ts, - const field_mgr_ptr field_mgr) + const field_mgr_ptr field_mgr, + const int time_index = -1) { - read_fields_from_file_for_iop(file_name, field_names, field_names, initial_ts, field_mgr); + read_fields_from_file_for_iop(file_name, field_names, field_names, initial_ts, field_mgr, time_index); } // Set fields using data loaded from the iop file @@ -175,10 +180,13 @@ class IntensiveObservationPeriod } }; + enum IOPFieldType { + FromFile, + Computed + }; + void initialize_iop_file(const util::TimeStamp& run_t0, - int model_nlevs, - const Field& hyam, - const Field& hybm); + int model_nlevs); ekat::Comm m_comm; ekat::ParameterList m_params; @@ -195,6 +203,7 @@ class IntensiveObservationPeriod std::map m_iop_file_varnames; std::map m_iop_field_surface_varnames; + std::map m_iop_field_type; }; // class IntensiveObservationPeriod } // namespace control diff --git a/components/eamxx/src/control/tests/CMakeLists.txt b/components/eamxx/src/control/tests/CMakeLists.txt index 43aa5cebab1b..401c0f576268 100644 --- a/components/eamxx/src/control/tests/CMakeLists.txt +++ b/components/eamxx/src/control/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT ${SCREAM_BASELINES_ONLY}) +if (NOT ${SCREAM_ONLY_GENERATE_BASELINES}) include (ScreamUtils) # Unit test the ad diff --git a/components/eamxx/src/diagnostics/CMakeLists.txt b/components/eamxx/src/diagnostics/CMakeLists.txt index 5818c4d836a8..f34d5b99638f 100644 --- a/components/eamxx/src/diagnostics/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/CMakeLists.txt @@ -16,6 +16,7 @@ set(DIAGNOSTIC_SRCS vertical_layer.cpp virtual_temperature.cpp water_path.cpp + wind_speed.cpp ) add_library(diagnostics ${DIAGNOSTIC_SRCS}) diff --git a/components/eamxx/src/diagnostics/exner.hpp b/components/eamxx/src/diagnostics/exner.hpp index 5e029b716bf0..dead0a5cbf64 100644 --- a/components/eamxx/src/diagnostics/exner.hpp +++ b/components/eamxx/src/diagnostics/exner.hpp @@ -16,9 +16,6 @@ class ExnerDiagnostic : public AtmosphereDiagnostic // Constructors ExnerDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "Exner"; } diff --git a/components/eamxx/src/diagnostics/field_at_height.cpp b/components/eamxx/src/diagnostics/field_at_height.cpp index 7759dfe5811d..38ae5e3e5503 100644 --- a/components/eamxx/src/diagnostics/field_at_height.cpp +++ b/components/eamxx/src/diagnostics/field_at_height.cpp @@ -38,6 +38,13 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereDiagnostic(comm,params) { m_field_name = m_params.get("field_name"); + auto surf_ref = m_params.get("surface_reference"); + EKAT_REQUIRE_MSG(surf_ref == "sealevel" or surf_ref == "surface", + "Error! Invalid surface reference for FieldAtHeight.\n" + " - field name: " + m_field_name + "\n" + " - surface reference: " + surf_ref + "\n" + " - valid options: sealevel, surface\n"); + m_z_name = (surf_ref == "sealevel") ? "z" : "geopotential"; const auto& location = m_params.get("vertical_location"); auto chars_start = location.find_first_not_of("0123456789."); EKAT_REQUIRE_MSG (chars_start!=0 && chars_start!=std::string::npos, @@ -52,7 +59,7 @@ FieldAtHeight (const ekat::Comm& comm, const ekat::ParameterList& params) "Error! Invalid string for height value for FieldAtHeight.\n" " - input string : " + location + "\n" " - expected format: Nm, with N integer\n"); - m_diag_name = m_field_name + "_at_" + m_params.get("vertical_location"); + m_diag_name = m_field_name + "_at_" + m_params.get("vertical_location") + "_above_" + surf_ref; } void FieldAtHeight:: @@ -62,8 +69,8 @@ set_grids (const std::shared_ptr grids_manager) add_field(m_field_name,gname); // We don't know yet which one we need - add_field("z_mid",gname); - add_field("z_int",gname); + add_field(m_z_name+"_mid",gname); + add_field(m_z_name+"_int",gname); } void FieldAtHeight:: @@ -90,7 +97,7 @@ initialize_impl (const RunType /*run_type*/) " - field layout: " + to_string(layout) + "\n"); // Figure out the z value - m_z_name = tag==LEV ? "z_mid" : "z_int"; + m_z_suffix = tag==LEV ? "_mid" : "_int"; // All good, create the diag output FieldIdentifier d_fid (m_diag_name,layout.strip_dim(tag),fid.get_units(),fid.get_grid_name()); @@ -111,7 +118,7 @@ initialize_impl (const RunType /*run_type*/) // ========================================================================================= void FieldAtHeight::compute_diagnostic_impl() { - const auto z_view = get_field_in(m_z_name).get_view(); + const auto z_view = get_field_in(m_z_name + m_z_suffix).get_view(); const Field& f = get_field_in(m_field_name); const auto& fl = f.get_header().get_identifier().get_layout(); diff --git a/components/eamxx/src/diagnostics/field_at_height.hpp b/components/eamxx/src/diagnostics/field_at_height.hpp index 54843b4f1a0e..e6198153f940 100644 --- a/components/eamxx/src/diagnostics/field_at_height.hpp +++ b/components/eamxx/src/diagnostics/field_at_height.hpp @@ -33,6 +33,7 @@ class FieldAtHeight : public AtmosphereDiagnostic std::string m_diag_name; std::string m_z_name; + std::string m_z_suffix; std::string m_field_name; Real m_z; diff --git a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp index 23ef59891e9f..4e44dff9dc76 100644 --- a/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/longwave_cloud_forcing.cpp @@ -19,6 +19,8 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -33,7 +35,7 @@ void LongwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("LW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); + FieldIdentifier fid (name(), scalar2d_layout_col, radflux_units, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(); diff --git a/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp b/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp index 9703330ce0cb..efe0e7c04760 100644 --- a/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp +++ b/components/eamxx/src/diagnostics/longwave_cloud_forcing.hpp @@ -16,9 +16,6 @@ class LongwaveCloudForcingDiagnostic : public AtmosphereDiagnostic // Constructors LongwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "LongwaveCloudForcing"; } diff --git a/components/eamxx/src/diagnostics/potential_temperature.hpp b/components/eamxx/src/diagnostics/potential_temperature.hpp index 1e0af49fb597..933a6935005d 100644 --- a/components/eamxx/src/diagnostics/potential_temperature.hpp +++ b/components/eamxx/src/diagnostics/potential_temperature.hpp @@ -20,9 +20,6 @@ class PotentialTemperatureDiagnostic : public AtmosphereDiagnostic // Constructors PotentialTemperatureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "PotentialTemperature"; } diff --git a/components/eamxx/src/diagnostics/register_diagnostics.hpp b/components/eamxx/src/diagnostics/register_diagnostics.hpp index 181a531a7c87..1f0c3bd63e3f 100644 --- a/components/eamxx/src/diagnostics/register_diagnostics.hpp +++ b/components/eamxx/src/diagnostics/register_diagnostics.hpp @@ -19,6 +19,7 @@ #include "diagnostics/field_at_pressure_level.hpp" #include "diagnostics/precip_surf_mass_flux.hpp" #include "diagnostics/surf_upward_latent_heat_flux.hpp" +#include "diagnostics/wind_speed.hpp" namespace scream { @@ -45,6 +46,7 @@ inline void register_diagnostics () { diag_factory.register_product("VaporFlux",&create_atmosphere_diagnostic); diag_factory.register_product("precip_surf_mass_flux",&create_atmosphere_diagnostic); diag_factory.register_product("surface_upward_latent_heat_flux",&create_atmosphere_diagnostic); + diag_factory.register_product("wind_speed",&create_atmosphere_diagnostic); } } // namespace scream diff --git a/components/eamxx/src/diagnostics/sea_level_pressure.hpp b/components/eamxx/src/diagnostics/sea_level_pressure.hpp index 8c522a2e75fe..0ae793658cd1 100644 --- a/components/eamxx/src/diagnostics/sea_level_pressure.hpp +++ b/components/eamxx/src/diagnostics/sea_level_pressure.hpp @@ -25,9 +25,6 @@ class SeaLevelPressureDiagnostic : public AtmosphereDiagnostic // Constructors SeaLevelPressureDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "SeaLevelPressure"; } diff --git a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp index e0e815549c8a..9ca5ea4a8a14 100644 --- a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp +++ b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.cpp @@ -17,7 +17,8 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptrget_grid("Physics"); const auto& grid_name = grid->name(); @@ -28,13 +29,13 @@ void ShortwaveCloudForcingDiagnostic::set_grids(const std::shared_ptr("SW_flux_dn", scalar3d_layout_mid, W/m2, grid_name); - add_field("SW_flux_up", scalar3d_layout_mid, W/m2, grid_name); - add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, W/m2, grid_name); - add_field("SW_clrsky_flux_up", scalar3d_layout_mid, W/m2, grid_name); + add_field("SW_flux_dn", scalar3d_layout_mid, radflux_units, grid_name); + add_field("SW_flux_up", scalar3d_layout_mid, radflux_units, grid_name); + add_field("SW_clrsky_flux_dn", scalar3d_layout_mid, radflux_units, grid_name); + add_field("SW_clrsky_flux_up", scalar3d_layout_mid, radflux_units, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid (name(), scalar2d_layout_col, W/m2, grid_name); + FieldIdentifier fid (name(), scalar2d_layout_col, radflux_units, grid_name); m_diagnostic_output = Field(fid); auto& C_ap = m_diagnostic_output.get_header().get_alloc_properties(); C_ap.request_allocation(); diff --git a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp index 9d676338a765..421d06d3fe07 100644 --- a/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp +++ b/components/eamxx/src/diagnostics/shortwave_cloud_forcing.hpp @@ -16,9 +16,6 @@ class ShortwaveCloudForcingDiagnostic : public AtmosphereDiagnostic // Constructors ShortwaveCloudForcingDiagnostic (const ekat::Comm& comm, const ekat::ParameterList& params); - // Set type to diagnostic - AtmosphereProcessType type () const { return AtmosphereProcessType::Diagnostic; } - // The name of the diagnostic std::string name () const { return "ShortwaveCloudForcing"; } diff --git a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp index 19a49ad595c5..009f8e6dd77f 100644 --- a/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp +++ b/components/eamxx/src/diagnostics/surf_upward_latent_heat_flux.cpp @@ -22,6 +22,9 @@ set_grids (const std::shared_ptr grids_manager) { const auto m2 = ekat::units::m * ekat::units::m; const auto W = ekat::units::W; + auto radflux_units = W/(m2); + radflux_units.set_string("W/m2"); + const auto surf_evap_units = ekat::units::kg / m2 / ekat::units::s; auto grid = grids_manager->get_grid("Physics"); @@ -35,7 +38,7 @@ set_grids (const std::shared_ptr grids_manager) add_field("surf_evap", scalar2d_layout_mid, surf_evap_units, grid_name); // Construct and allocate the diagnostic field - FieldIdentifier fid(name(), scalar2d_layout_mid, W/m2, grid_name); + FieldIdentifier fid(name(), scalar2d_layout_mid, radflux_units, grid_name); // handle parent class member variables m_diagnostic_output = Field(fid); m_diagnostic_output.get_header().get_alloc_properties().request_allocation(); diff --git a/components/eamxx/src/diagnostics/tests/CMakeLists.txt b/components/eamxx/src/diagnostics/tests/CMakeLists.txt index 0882d3f6119d..d413b013f444 100644 --- a/components/eamxx/src/diagnostics/tests/CMakeLists.txt +++ b/components/eamxx/src/diagnostics/tests/CMakeLists.txt @@ -6,7 +6,7 @@ function (createDiagTest test_name test_srcs) LABELS diagnostics) endfunction () -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include(ScreamUtils) # Test extracting a single level of a field @@ -59,4 +59,6 @@ if (NOT SCREAM_BASELINES_ONLY) # Test surface latent heat flux CreateDiagTest(surface_upward_latent_heat_flux "surf_upward_latent_heat_flux_tests.cpp") + # Test wind speed diagnostic + CreateDiagTest(wind_speed "wind_speed_tests.cpp") endif() diff --git a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp index 0d45eb62e879..57a057116102 100644 --- a/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/field_at_height_tests.cpp @@ -9,6 +9,10 @@ namespace scream { +void f_z_src(const Real y0, const Real m, const Field& z_data, Field& out_data); +void f_z_tgt(const Real y0, const Real m, const Real z_target, const Field& z_data, Field& out_data); +bool views_are_approx_equal(const Field& f0, const Field& f1, const Real tol, const bool msg = true); + TEST_CASE("field_at_height") { using namespace ShortFieldTagsNames; @@ -18,11 +22,12 @@ TEST_CASE("field_at_height") // Get an MPI comm group for test ekat::Comm comm(MPI_COMM_WORLD); - constexpr int nruns = 10; + constexpr int nruns = 100; util::TimeStamp t0 ({2022,1,1},{0,0,0}); // Create a grids manager w/ a point grid + constexpr Real tol = std::numeric_limits::epsilon()*1e5; int ncols = 3; int ndims = 4; int nlevs = 10; @@ -31,35 +36,57 @@ TEST_CASE("field_at_height") gm->build_grids(); auto grid = gm->get_grid("Point Grid"); - // Create input test fields, as well as z_mid/int fields const auto m = ekat::units::m; + // Create input data test fields FieldIdentifier s_mid_fid ("s_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); FieldIdentifier s_int_fid ("s_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); FieldIdentifier v_mid_fid ("v_mid",FieldLayout({COL,CMP, LEV},{ncols,ndims,nlevs }),m,grid->name()); FieldIdentifier v_int_fid ("v_int",FieldLayout({COL,CMP,ILEV},{ncols,ndims,nlevs+1}),m,grid->name()); - FieldIdentifier z_mid_fid ("z_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); - FieldIdentifier z_int_fid ("z_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + // Create vertical fields z and geo on both midpoints and interfaces + FieldIdentifier z_surf_fid ("z_surf", FieldLayout({COL },{ncols }),m,grid->name()); + FieldIdentifier z_mid_fid ("z_mid", FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); + FieldIdentifier z_int_fid ("z_int", FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + FieldIdentifier geo_mid_fid ("geopotential_mid",FieldLayout({COL, LEV},{ncols, nlevs }),m,grid->name()); + FieldIdentifier geo_int_fid ("geopotential_int",FieldLayout({COL, ILEV},{ncols, nlevs+1}),m,grid->name()); + // Keep track of reference fields for comparison + FieldIdentifier s_tgt_fid ("scalar_target",FieldLayout({COL },{ncols }),m,grid->name()); + FieldIdentifier v_tgt_fid ("vector_target",FieldLayout({COL,CMP},{ncols,ndims}),m,grid->name()); - Field s_mid (s_mid_fid); - Field s_int (s_int_fid); - Field v_mid (v_mid_fid); - Field v_int (v_int_fid); - Field z_mid (z_mid_fid); - Field z_int (z_int_fid); + Field s_mid (s_mid_fid); + Field s_int (s_int_fid); + Field v_mid (v_mid_fid); + Field v_int (v_int_fid); + Field z_surf (z_surf_fid); + Field z_mid (z_mid_fid); + Field z_int (z_int_fid); + Field geo_mid (geo_mid_fid); + Field geo_int (geo_int_fid); + Field s_tgt (s_tgt_fid); + Field v_tgt (v_tgt_fid); s_mid.allocate_view(); s_int.allocate_view(); v_mid.allocate_view(); v_int.allocate_view(); + z_surf.allocate_view(); z_mid.allocate_view(); z_int.allocate_view(); + geo_mid.allocate_view(); + geo_int.allocate_view(); + s_tgt.allocate_view(); + v_tgt.allocate_view(); s_mid.get_header().get_tracking().update_time_stamp(t0); s_int.get_header().get_tracking().update_time_stamp(t0); v_mid.get_header().get_tracking().update_time_stamp(t0); v_int.get_header().get_tracking().update_time_stamp(t0); + z_surf.get_header().get_tracking().update_time_stamp(t0); z_mid.get_header().get_tracking().update_time_stamp(t0); z_int.get_header().get_tracking().update_time_stamp(t0); + geo_mid.get_header().get_tracking().update_time_stamp(t0); + geo_int.get_header().get_tracking().update_time_stamp(t0); + s_tgt.get_header().get_tracking().update_time_stamp(t0); + v_tgt.get_header().get_tracking().update_time_stamp(t0); auto print = [&](const std::string& msg) { if (comm.am_i_root()) { @@ -69,16 +96,19 @@ TEST_CASE("field_at_height") auto engine = scream::setup_random_test(&comm); using IPDF = std::uniform_int_distribution; + using RPDF = std::uniform_real_distribution; IPDF pdf_fields (0,1000); - IPDF pdf_levs (1,nlevs-1); + RPDF pdf_m (1,10); + RPDF pdf_y0 (0,5); // Lambda to create and run a diag, and return output auto run_diag = [&](const Field& f, const Field& z, - const std::string& loc) { + const std::string& loc, const std::string& surf_ref) { util::TimeStamp t0 ({2022,1,1},{0,0,0}); auto& factory = AtmosphereDiagnosticFactory::instance(); ekat::ParameterList pl; + pl.set("surface_reference",surf_ref); pl.set("vertical_location",loc); pl.set("field_name",f.name()); pl.set("grid_name",grid->name()); @@ -92,135 +122,231 @@ TEST_CASE("field_at_height") return diag->get_diagnostic(); }; - // Create z(i,j)=nlevs-j, which makes testing easier - for (auto f : {z_mid, z_int}) { - auto v = f.get_view(); - const auto& dims = f.get_header().get_identifier().get_layout().dims(); - for (int i=0; i(); + const auto& zmid_v = z_mid.get_view(); + const auto& zsurf_v = z_surf.get_view(); + const auto& geoint_v = geo_int.get_view(); + const auto& geomid_v = geo_mid.get_view(); + int min_col_thickness = z_top; + int max_surf = 0; + for (int ii=0; ii max_surf ? zsurf_v(ii) : max_surf; + const Real col_thickness = z_top - zsurf_v(ii); + min_col_thickness = min_col_thickness < col_thickness ? col_thickness : min_col_thickness; + const Real dz = (z_top - zsurf_v(ii))/nlevs; + zint_v(ii,0) = z_top; + geoint_v(ii,0) = z_top - zsurf_v(ii); // Note, the distance above surface needs to consider the surface height. + for (int jj=0; jj Testing throws error with unsupported reference height...\n"); + { + REQUIRE_THROWS(run_diag (s_mid,geo_mid,"1m","foobar")); } + print(" -> Testing throws error with unsupported reference height... OK\n"); // Run many times - Real z_tgt,lev_tgt; + int z_tgt; std::string loc; - for (int irun=0; irun(); - const auto& size = f.get_header().get_identifier().get_layout().size(); - for (int i=0; i Testing for a reference height above %s...\n",surf_ref.c_str()); + const auto mid_src = surf_ref == "sealevel" ? z_mid : geo_mid; + const auto int_src = surf_ref == "sealevel" ? z_int : geo_int; + const int max_surf_4test = surf_ref == "sealevel" ? max_surf : 0; + for (int irun=0; irun Testing with z_tgt coinciding with a z level\n"); - { - print(" -> scalar midpoint field...............\n"); - auto d = run_diag (s_mid,z_mid,loc); - auto tgt = s_mid.subfield(1,static_cast(lev_tgt)); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar midpoint field............... OK!\n"); - } - { - print(" -> scalar interface field...............\n"); - auto d = run_diag (s_int,z_int,loc); - // z_mid = nlevs+1-ilev, so the tgt slice is nlevs+1-z_tgt - auto tgt = s_int.subfield(1,static_cast(lev_tgt)); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar interface field............... OK!\n"); - } - { - print(" -> vector midpoint field...............\n"); - auto d = run_diag (v_mid,z_mid,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(lev_tgt)); - REQUIRE (views_are_equal(di,tgt,&comm)); + // Set target z-slice for testing to a random value. + z_tgt = pdf_levs(engine)+max_surf_4test; + loc = std::to_string(z_tgt) + "m"; + printf(" -> test at height of %s.............\n",loc.c_str()); + { + print(" -> scalar midpoint field...............\n"); + auto d = run_diag(s_mid,mid_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,mid_src,s_tgt); + REQUIRE (views_are_approx_equal(d,s_tgt,tol)); + print(" -> scalar midpoint field............... OK!\n"); + } + { + print(" -> scalar interface field...............\n"); + auto d = run_diag (s_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (views_are_approx_equal(d,s_tgt,tol)); + print(" -> scalar interface field............... OK!\n"); + } + { + print(" -> vector midpoint field...............\n"); + auto d = run_diag (v_mid,mid_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,mid_src,v_tgt); + REQUIRE (views_are_approx_equal(d,v_tgt,tol)); + print(" -> vector midpoint field............... OK!\n"); + } + { + print(" -> vector interface field...............\n"); + auto d = run_diag (v_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,v_tgt); + REQUIRE (views_are_approx_equal(d,v_tgt,tol)); + print(" -> vector interface field............... OK!\n"); + } + { + print(" -> Forced fail, give incorrect location...............\n"); + const int z_tgt_adj = (z_tgt+max_surf_4test)/2; + std::string loc_err = std::to_string(z_tgt_adj) + "m"; + auto d = run_diag(s_int,int_src,loc_err,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (!views_are_approx_equal(d,s_tgt,tol,false)); + print(" -> Forced fail, give incorrect location............... OK!\n"); } - print(" -> vector midpoint field............... OK!\n"); } { - print(" -> vector interface field...............\n"); - auto d = run_diag (v_int,z_int,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(lev_tgt)); - REQUIRE (views_are_equal(di,tgt,&comm)); - } - print(" -> vector interface field............... OK!\n"); + print(" -> Forced extrapolation ...............\n"); + auto slope = pdf_m(engine); + auto inter = pdf_y0(engine); + f_z_src(inter, slope, int_src, s_int); + print(" -> at top...............\n"); + z_tgt = 2*z_top; + std::string loc = std::to_string(z_tgt) + "m"; + auto dtop = run_diag(s_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (views_are_approx_equal(dtop,s_tgt,tol)); + print(" -> at bot...............\n"); + z_tgt = 0; + loc = std::to_string(z_tgt) + "m"; + auto dbot = run_diag(s_int,int_src,loc,surf_ref); + f_z_tgt(inter,slope,z_tgt,int_src,s_tgt); + REQUIRE (views_are_approx_equal(dbot,s_tgt,tol)); + print(" -> Forced extrapolation............... OK!\n"); } + printf(" -> Testing for a reference height above %s... OK!\n",surf_ref.c_str()); + } +} - z_tgt = pdf_levs(engine) + 0.5; - lev_tgt = nlevs-z_tgt; - loc = std::to_string(z_tgt) + "m"; - - auto zp1 = static_cast(std::round(lev_tgt+0.5)); - auto zm1 = static_cast(std::round(lev_tgt-0.5)); - - print(" -> Testing with z_tgt between levels\n"); - { - print(" -> scalar midpoint field...............\n"); - auto d = run_diag (s_mid,z_mid,loc); - auto tgt = s_mid.subfield(1,zp1).clone(); - tgt.update(s_mid.subfield(1,zm1),0.5,0.5); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar midpoint field............... OK!\n"); +//------------------------------- +// Set up the inpute data. To make the test simple we assume a linear distribution of the data +// with height. That way we can exactly calculate what a linear interpolation to a random +// height would be. +void f_z_src(const Real y0, const Real m, const Field& z_data, Field& out_data) { + using namespace ShortFieldTagsNames; + const auto layout = out_data.get_header().get_identifier().get_layout(); + if (layout.has_tag(CMP)) { // Is a vector layout, meaning different dims than z_data. + const auto& dims = layout.dims(); + const auto& z_view = z_data.get_view(); + const auto& out_view = out_data.get_view(); + for (int ii=0; ii scalar interface field...............\n"); - auto d = run_diag (s_int,z_int,loc); - auto tgt = s_int.subfield(1,zp1).clone(); - tgt.update(s_int.subfield(1,zm1),0.5,0.5); - REQUIRE (views_are_equal(d,tgt,&comm)); - print(" -> scalar interface field............... OK!\n"); + } else { // Not a vector output, easier to deal with + const auto z_view = z_data.get_internal_view_data(); + const auto& size = z_data.get_header().get_identifier().get_layout().size(); + auto out_view = out_data.get_internal_view_data(); + for (int ii=0; ii vector midpoint field...............\n"); - auto d = run_diag (v_mid,z_mid,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(); + const auto& zdims = z_data.get_header().get_identifier().get_layout().dims(); + if (layout.has_tag(CMP)) { // Is a vector layout, meaning different dims than z_target. + const auto& dims = layout.dims(); + const auto& out_view = out_data.get_view(); + for (int ii=0; ii z_view(ii,0)) { + out_view(ii,nd) = y0 + m*(nd+1)*z_view(ii,0); + } else if ( z_target < z_view(ii,zdims[1]-1)) { + out_view(ii,nd) = y0 + m*(nd+1)*z_view(ii,zdims[1]-1); + } else { + out_view(ii,nd) = y0 + m*(nd+1)*z_target; + } } - print(" -> vector midpoint field............... OK!\n"); } - { - print(" -> vector interface field...............\n"); - auto d = run_diag (v_int,z_int,loc); - // We can't subview over 3rd index and keep layout right, - // so do all cols separately - for (int i=0; i(); + for (int ii=0; ii z_view(ii,0)) { + out_view(ii) = y0 + m*z_view(ii,0); + } else if ( z_target < z_view(ii,zdims[1]-1)) { + out_view(ii) = y0 + m*z_view(ii,zdims[1]-1); + } else { + out_view(ii) = y0 + m*z_target; } - print(" -> vector interface field............... OK!\n"); } } + out_data.sync_to_dev(); +} +/*-----------------------------------------------------------------------------------------------*/ +bool views_are_approx_equal(const Field& f0, const Field& f1, const Real tol, const bool msg) +{ + const auto& l0 = f0.get_header().get_identifier().get_layout(); + const auto& l1 = f1.get_header().get_identifier().get_layout(); + EKAT_REQUIRE_MSG(l0==l1,"Error! views_are_approx_equal - the two fields don't have matching layouts."); + // Take advantage of field utils update, min and max to assess the max difference between the two fields + // simply. + auto ft = f0.clone(); + ft.update(f1,1.0,-1.0); + auto d_min = field_min(ft); + auto d_max = field_max(ft); + if (std::abs(d_min) > tol or std::abs(d_max) > tol) { + if (msg) { + printf("The two copies of (%16s) are NOT approx equal within a tolerance of %e.\n The min and max errors are %e and %e respectively.\n",f0.name().c_str(),tol,d_min,d_max); + } + return false; + } else { + return true; + } + } } // namespace scream diff --git a/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp b/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp index aceb705a61c7..bcd7efc8bb87 100644 --- a/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp +++ b/components/eamxx/src/diagnostics/tests/vapor_flux_tests.cpp @@ -83,7 +83,7 @@ void run(std::mt19937_64& engine) REQUIRE_THROWS (diag_factory.create("VaporFlux",comm,params)); // No 'Wind Component' params.set("Wind Component","foo"); REQUIRE_THROWS (diag_factory.create("VaporFlux",comm,params)); // Invalid 'Wind Component' - for (const std::string& which_comp : {"Zonal", "Meridional"}) { + for (const std::string which_comp : {"Zonal", "Meridional"}) { // Construct the Diagnostic params.set("Wind Component",which_comp); auto diag = diag_factory.create("VaporFlux",comm,params); diff --git a/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp new file mode 100644 index 000000000000..7e3affedacb2 --- /dev/null +++ b/components/eamxx/src/diagnostics/tests/wind_speed_tests.cpp @@ -0,0 +1,96 @@ +#include "catch2/catch.hpp" + +#include "diagnostics/register_diagnostics.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/field/field_utils.hpp" + +namespace scream { + +std::shared_ptr +create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { + + const int num_global_cols = ncols*comm.size(); + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", num_global_cols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + + return gm; +} + +TEST_CASE("wind_speed") +{ + using namespace ShortFieldTagsNames; + using namespace ekat::units; + + // A world comm + ekat::Comm comm(MPI_COMM_WORLD); + + // A time stamp + util::TimeStamp t0 ({2022,1,1},{0,0,0}); + + // Create a grids manager - single column for these tests + constexpr int nlevs = 33; + const int ngcols = 2*comm.size();; + auto gm = create_gm(comm,ngcols,nlevs); + auto grid = gm->get_grid("Physics"); + + // Input (randomized) velocity + auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + FieldIdentifier uv_fid ("horiz_winds",vector3d,m/s,grid->name()); + Field uv(uv_fid); + uv.allocate_view(); + uv.get_header().get_tracking().update_time_stamp(t0); + + // Construct random number generator stuff + using RPDF = std::uniform_real_distribution; + RPDF pdf(-1,1); + auto engine = scream::setup_random_test(); + + // Construct the Diagnostics + std::map> diags; + auto& diag_factory = AtmosphereDiagnosticFactory::instance(); + register_diagnostics(); + + constexpr int ntests = 5; + for (int itest=0; itestset_grids(gm); + diag->set_required_field(uv); + diag->initialize(t0,RunType::Initial); + + // Run diag + diag->compute_diagnostic(); + + // Check result + uv.sync_to_host(); + diag->get_diagnostic().sync_to_host(); + + auto uv_h = uv.get_view(); + auto ws_h = diag->get_diagnostic().get_view(); + + for (int icol=0; icolget_num_local_dofs(); ++icol) { + for (int ilev=0; ilev + +namespace scream +{ + +WindSpeed:: +WindSpeed (const ekat::Comm& comm, const ekat::ParameterList& params) + : AtmosphereDiagnostic(comm,params) +{ + // Nothing to do here +} + +void WindSpeed:: +set_grids(const std::shared_ptr grids_manager) +{ + using namespace ekat::units; + using namespace ShortFieldTagsNames; + + auto grid = grids_manager->get_grid("Physics"); + const auto& grid_name = grid->name(); + + m_ncols = grid->get_num_local_dofs(); + m_nlevs = grid->get_num_vertical_levels(); + + auto scalar3d = grid->get_3d_scalar_layout(true); + auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + + // The fields required for this diagnostic to be computed + add_field("horiz_winds", vector3d, Pa, grid_name); + + // Construct and allocate the 3d wind_speed field + FieldIdentifier fid ("wind_speed", scalar3d, m/s, grid_name); + m_diagnostic_output = Field(fid); + m_diagnostic_output.allocate_view(); +} + +void WindSpeed::compute_diagnostic_impl() +{ + using KT = KokkosTypes; + using RP = typename KT::RangePolicy; + + const auto uv = get_field_in("horiz_winds").get_view(); + const auto ws = m_diagnostic_output.get_view(); + + const int nlevs = m_nlevs; + Kokkos::parallel_for("Compute " + name(), RP(0,m_nlevs*m_ncols), + KOKKOS_LAMBDA(const int& idx) { + const int icol = idx / nlevs; + const int ilev = idx % nlevs; + const auto& u = uv(icol,0,ilev); + const auto& v = uv(icol,1,ilev); + ws (icol,ilev) = sqrt(u*u + v*v); + }); +} + +} //namespace scream diff --git a/components/eamxx/src/diagnostics/wind_speed.hpp b/components/eamxx/src/diagnostics/wind_speed.hpp new file mode 100644 index 000000000000..91ac551a1b76 --- /dev/null +++ b/components/eamxx/src/diagnostics/wind_speed.hpp @@ -0,0 +1,37 @@ +#ifndef EAMXX_WIND_SPEED_HPP +#define EAMXX_WIND_SPEED_HPP + +#include "share/atm_process/atmosphere_diagnostic.hpp" + +namespace scream +{ + +/* + * This diagnostic will compute the magnitute of the horiz_winds vector + */ + +class WindSpeed : public AtmosphereDiagnostic +{ +public: + // Constructors + WindSpeed (const ekat::Comm& comm, const ekat::ParameterList& params); + + // The name of the diagnostic + std::string name () const override { return "wind_speed"; } + + // Set the grid + void set_grids (const std::shared_ptr grids_manager) override; + +protected: +#ifdef KOKKOS_ENABLE_CUDA +public: +#endif + void compute_diagnostic_impl () override; + + int m_ncols; + int m_nlevs; +}; + +} //namespace scream + +#endif // EAMXX_WIND_SPEED_HPP diff --git a/components/eamxx/src/doubly-periodic/CMakeLists.txt b/components/eamxx/src/doubly-periodic/CMakeLists.txt deleted file mode 100644 index 09dab68a8e28..000000000000 --- a/components/eamxx/src/doubly-periodic/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ -include (ScreamUtils) - -set(DP_SRCS - dp_f90.cpp - dp_iso_c.f90 - #${SCREAM_BASE_DIR}/../eam/src/control/apply_iop_forcing.F90 - #${SCREAM_BASE_DIR}/../eam/src/dynamics/se/se_iop_intr_mod.F90", - #${SCREAM_BASE_DIR}/../eam/src/control/iop_data_mod.F90", - #${SCREAM_BASE_DIR}/../eam/src/control/history_iop.F90" -) - -# Set cmake config options for Homme -if (NOT "${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") - message(FATAL_ERROR "Requires homme") -endif() - -# Get or create the dynamics lib -# HOMME_TARGET NP PLEV QSIZE_D -CreateDynamicsLib("theta-l_kokkos" 4 72 10) - -if (NOT SCREAM_LIB_ONLY) - list(APPEND DP_SRCS - dp_functions_f90.cpp - ) # Add f90 bridges needed for testing -endif() - -# Add ETI source files if not on CUDA/HIP -if (NOT EAMXX_ENABLE_GPU OR Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR Kokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) - list(APPEND DP_SRCS - eti/dp_advance_iop_forcing.cpp - eti/dp_advance_iop_nudging.cpp - eti/dp_advance_iop_subsidence.cpp - eti/dp_iop_setinitial.cpp - eti/dp_iop_broadcast.cpp - eti/dp_apply_iop_forcing.cpp - eti/dp_iop_domain_relaxation.cpp - eti/dp_crm_resolved_turb.cpp - eti/dp_iop_default_opts.cpp - eti/dp_iop_setopts.cpp - eti/dp_setiopupdate_init.cpp - eti/dp_setiopupdate.cpp - eti/dp_readiopdata.cpp - eti/dp_iop_intht.cpp - ) # DP ETI SRCS -endif() - -add_library(dp ${DP_SRCS}) -set_target_properties(dp PROPERTIES - Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules -) -target_include_directories(dp PUBLIC - ${CMAKE_CURRENT_BINARY_DIR}/modules - ${CMAKE_CURRENT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/impl -) -target_link_libraries(dp PUBLIC physics_share scream_share ${dynLibName}) - -#if (NOT SCREAM_LIB_ONLY) -# add_subdirectory(tests) -#endif() diff --git a/components/eamxx/src/doubly-periodic/dp_constants.hpp b/components/eamxx/src/doubly-periodic/dp_constants.hpp deleted file mode 100644 index c974106f83ec..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_constants.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef DP_CONSTANTS_HPP -#define DP_CONSTANTS_HPP - -namespace scream { -namespace dp { - -/* - * Mathematical constants used by dp. - */ - -template -struct Constants -{ - static constexpr Scalar iop_nudge_tq_low = 1050; - static constexpr Scalar iop_nudge_tq_high = 0; - static constexpr Scalar iop_nudge_tscale = 10800; -}; - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/dp_f90.cpp b/components/eamxx/src/doubly-periodic/dp_f90.cpp deleted file mode 100644 index 2802e5266fd5..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_f90.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "dp_f90.hpp" -#include "physics_constants.hpp" - -#include "ekat/ekat_assert.hpp" - -using scream::Int; - -extern "C" { - -void init_time_level_c (const int& nm1, const int& n0, const int& np1, - const int& nstep, const int& nstep0); - -} - -namespace scream { -namespace dp { - -void dp_init(const bool force_reinit) { - static bool is_init = false; - if (!is_init || force_reinit) { - init_time_level_c(10, 3, 11, 5, 4); - is_init = true; - } -} - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/dp_f90.hpp b/components/eamxx/src/doubly-periodic/dp_f90.hpp deleted file mode 100644 index 338a583f777a..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_f90.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef SCREAM_DP_F90_HPP -#define SCREAM_DP_F90_HPP - -#include "share/scream_types.hpp" - -#include -#include - -namespace scream { -namespace dp { - -// Initialize DP. This is only for standalone DP testing. -void dp_init(const bool force_reinit=false); - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/dp_functions.hpp b/components/eamxx/src/doubly-periodic/dp_functions.hpp deleted file mode 100644 index 18e6ec58ba9e..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_functions.hpp +++ /dev/null @@ -1,325 +0,0 @@ -#ifndef DP_FUNCTIONS_HPP -#define DP_FUNCTIONS_HPP - -#include "physics/share/physics_constants.hpp" -#include "dp_constants.hpp" - -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack_kokkos.hpp" -#include "ekat/ekat_workspace.hpp" - -#include "Elements.hpp" -#include "Tracers.hpp" - -namespace scream { -namespace dp { - -/* - * Functions is a stateless struct used to encapsulate a - * number of functions for DP. We use the ETI pattern for - * these functions. - * - * DP assumptions: - * - Kokkos team policies have a vector length of 1 - */ - -using element_t = Homme::Elements; -using tracer_t = Homme::Tracers; -struct hvcoord_t{}; -struct timelevel_t{}; -struct hybrid_t{}; - -template -struct Functions -{ - // - // ------- Types -------- - // - - using Scalar = ScalarT; - using Device = DeviceT; - - template - using BigPack = ekat::Pack; - template - using SmallPack = ekat::Pack; - - using IntSmallPack = SmallPack; - using Pack = BigPack; - using Spack = SmallPack; - - using Mask = ekat::Mask; - using Smask = ekat::Mask; - - using KT = ekat::KokkosTypes; - using ExeSpace = typename KT::ExeSpace; - - using C = physics::Constants; - using DPC = dp::Constants; - - template - using view_1d = typename KT::template view_1d; - template - using view_2d = typename KT::template view_2d; - template - using view_3d = typename KT::template view_3d; - - template - using view_1d_ptr_array = typename KT::template view_1d_ptr_carray; - - template - using uview_1d = typename ekat::template Unmanaged >; - - template - using uview_2d = typename ekat::template Unmanaged >; - - using MemberType = typename KT::MemberType; - - using WorkspaceMgr = typename ekat::WorkspaceManager; - using Workspace = typename WorkspaceMgr::Workspace; - - // - // --------- Functions --------- - // - - // --------------------------------------------------------------------- - // Define the pressures of the interfaces and midpoints from the - // coordinate definitions and the surface pressure. - // --------------------------------------------------------------------- - KOKKOS_FUNCTION - static void plevs0( - // Input arguments - const Int& nver, // vertical dimension - const Scalar& ps, // Surface pressure (pascals) - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - // Kokkos stuff - const MemberType& team, - // Output arguments - const uview_1d& pint, // Pressure at model interfaces - const uview_1d& pmid, // Pressure at model levels - const uview_1d& pdel); // Layer thickness (pint(k+1) - pint(k)) - - //----------------------------------------------------------------------- - // advance_iop_forcing - // Purpose: - // Apply large scale forcing for t, q, u, and v as provided by the - // case IOP forcing file. - // - // Author: - // Original version: Adopted from CAM3.5/CAM5 - // Updated version for E3SM: Peter Bogenschutz (bogenschutz1@llnl.gov) - // and replaces the forecast.F90 routine in CAM3.5/CAM5/CAM6/E3SMv1/E3SMv2 - // CXX version: James Foucar (jgfouca@sandia.gov) - // - //----------------------------------------------------------------------- - KOKKOS_FUNCTION - static void advance_iop_forcing( - // Input arguments - const Int& plev, // number of vertical levels - const Int& pcnst, // number of advected constituents including cloud water - const bool& have_u, // dataset contains u - const bool& have_v, // dataset contains v - const bool& dp_crm, // use 3d forcing - const bool& use_3dfrc, // use 3d forcing - const Scalar& scm_dt, // model time step [s] - const Scalar& ps_in, // surface pressure [Pa] - const uview_1d& u_in, // zonal wind [m/s] - const uview_1d& v_in, // meridional wind [m/s] - const uview_1d& t_in, // temperature [K] - const uview_2d& q_in, // q tracer array [units vary] - const uview_1d& t_phys_frc, // temperature forcing from physics [K/s] - const uview_1d& divt3d, // 3D T advection - const uview_2d& divq3d, // 3D q advection - const uview_1d& divt, // Divergence of temperature - const uview_2d& divq, // Divergence of moisture - const uview_1d& wfld, // Vertical motion (slt) - const uview_1d& uobs, // actual u wind - const uview_1d& vobs, // actual v wind - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& u_update, // updated temperature [K] - const uview_1d& v_update, // updated q tracer array [units vary] - const uview_1d& t_update, // updated zonal wind [m/s] - const uview_2d& q_update); // updated meridional wind [m/s] - - //----------------------------------------------------------------------- - // advance_iop_nudging - // Purpose: - // Option to nudge t and q to observations as specified by the IOP file - // - // Author: - // Original version: Adopted from CAM3.5/CAM5 - // Updated version for E3SM: Peter Bogenschutz (bogenschutz1@llnl.gov) - // CXX version: Conrad Clevenger (tccleve@sandia.gov) - // - //----------------------------------------------------------------------- - KOKKOS_FUNCTION - static void advance_iop_nudging( - // Input arguments - const Int& plev, // number of vertical levels - const Scalar& scm_dt, // model time step [s] - const Scalar& ps_in, // surface pressure [Pa] - const uview_1d& t_in, // temperature [K] - const uview_1d& q_in, // water vapor mixing ratio [kg/kg] - const uview_1d& tobs, // observed temperature [K] - const uview_1d& qobs, // observed vapor mixing ratio [kg/kg] - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& t_update, // updated temperature [K] - const uview_1d& q_update, // updated water vapor [kg/kg] - const uview_1d& relaxt, // relaxation of temperature [K/s] - const uview_1d& relaxq); // relaxation of vapor [kg/kg/s] - - KOKKOS_INLINE_FUNCTION - static void do_advance_iop_subsidence_update( - const Int& k, - const Int& plev, - const Spack& fac, - const Spack& swfldint, - const Spack& swfldint_p1, - const uview_1d& in, - const uview_1d& in_s, - const uview_1d& update); - - //----------------------------------------------------------------------- - // - // Purpose: - // Option to compute effects of large scale subsidence on T, q, u, and v. - // Code originated from CAM3.5/CAM5 Eulerian subsidence computation for SCM - // in the old forecast.f90 routine. - //----------------------------------------------------------------------- - KOKKOS_FUNCTION - static void advance_iop_subsidence( - // Input arguments - const Int& plev, // number of vertical levels - const Int& pcnst, // number of advected constituents including cloud water - const Scalar& scm_dt, // model time step [s] - const Scalar& ps_in, // surface pressure [Pa] - const uview_1d& u_in, // zonal wind [m/s] - const uview_1d& v_in, // meridional wind [m/s] - const uview_1d& t_in, // temperature [K] - const uview_2d& q_in, // tracer [vary] - const uview_1d& hyai, // ps0 component of hybrid coordinate - interfaces - const uview_1d& hyam, // ps0 component of hybrid coordinate - midpoints - const uview_1d& hybi, // ps component of hybrid coordinate - interfaces - const uview_1d& hybm, // ps component of hybrid coordinate - midpoints - const uview_1d& wfld, // Vertical motion (slt) - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& u_update, // zonal wind [m/s] - const uview_1d& v_update, // meridional wind [m/s] - const uview_1d& t_update, // temperature [m/s] - const uview_2d& q_update); // tracer [vary] - - //--------------------------------------------------------- - // Purpose: Set initial values from IOP files (where available) - // when running SCM or DP-CRM. - //---------------------------------------------------------- - static void iop_setinitial( - // Input arguments - const Int& plev, // number of vertical levels - const Int& pcnst, // number of advected constituents including cloud water - const Int& nelemd, // number of elements per MPI task - const Int& np, // NP - const Int& nstep, // the timestep number - const bool& use_replay, // use e3sm generated forcing - const bool& dynproc, // Designation of a dynamics processor - AaronDonahue - const bool& have_t, // dataset contains t - const bool& have_q, // dataset contains q - const bool& have_ps, // dataset contains ps - const bool& have_u, // dataset contains u - const bool& have_v, // dataset contains v - const bool& have_numliq, // dataset contains numliq - const bool& have_cldliq, // dataset contains cldliq - const bool& have_numice, // dataset contains numice - const bool& have_cldice, // dataset contains cldice - const bool& scm_zero_non_iop_tracers, // Ignore all tracers from initial conditions file, and default all tracers not specified in IOP to minimum value (usually zero) - const bool& is_first_restart_step, // is first restart step - const uview_1d& qmin, // minimum permitted constituent concentration (kg/kg) (pcnst) - const uview_1d& uobs, // actual u wind - const uview_1d& vobs, // actual v wind - const uview_1d& numliqobs, // actual ??? - const uview_1d& numiceobs, // actual ??? - const uview_1d& cldliqobs, // actual ??? - const uview_1d& cldiceobs, // actual ??? - const Scalar& psobs, // ??? - const uview_1d& dx_short, // short length scale in km (nelemd) - // Input/Output arguments - Scalar& dyn_dx_size, // for use in doubly periodic CRM mode - tracer_t& tracers, // tracers - element_t& elem, // elements - const uview_1d& tobs, // actual temperature, dims=(plev) - const uview_1d& qobs); // actual W.V. Mixing ratio - - KOKKOS_FUNCTION - static void iop_broadcast(); - - KOKKOS_FUNCTION - static void apply_iop_forcing(const Int& nelemd, const uview_1d& elem, hvcoord_t& hvcoord, const hybrid_t& hybrid, const timelevel_t& tl, const Int& n, const bool& t_before_advance, const Int& nets, const Int& nete); - - KOKKOS_FUNCTION - static void iop_domain_relaxation(const Int& nelemd, const Int& np, const Int& nlev, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const uview_1d& dp, const Int& nelemd_todo, const Int& np_todo, const Spack& dt); - - KOKKOS_FUNCTION - static void crm_resolved_turb(const Int& nelemd, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const Int& nelemd_todo, const Int& np_todo); - - static void iop_default_opts(Spack& scmlat_out, Spack& scmlon_out, std::string& iopfile_out, bool& single_column_out, bool& scm_iop_srf_prop_out, bool& iop_nudge_tq_out, bool& iop_nudge_uv_out, Spack& iop_nudge_tq_low_out, Spack& iop_nudge_tq_high_out, Spack& iop_nudge_tscale_out, bool& scm_observed_aero_out, bool& iop_dosubsidence_out, bool& scm_multcols_out, bool& dp_crm_out, Spack& iop_perturb_high_out, bool& precip_off_out, bool& scm_zero_non_iop_tracers_out); - - static void iop_setopts(const Spack& scmlat_in, const Spack& scmlon_in, const std::string& iopfile_in, const bool& single_column_in, const bool& scm_iop_srf_prop_in, const bool& iop_nudge_tq_in, const bool& iop_nudge_uv_in, const Spack& iop_nudge_tq_low_in, const Spack& iop_nudge_tq_high_in, const Spack& iop_nudge_tscale_in, const bool& scm_observed_aero_in, const bool& iop_dosubsidence_in, const bool& scm_multcols_in, const bool& dp_crm_in, const Spack& iop_perturb_high_in, const bool& precip_off_in, const bool& scm_zero_non_iop_tracers_in); - - KOKKOS_FUNCTION - static void setiopupdate_init(); - - KOKKOS_FUNCTION - static void setiopupdate(); - - KOKKOS_FUNCTION - static void readiopdata(const Int& plev, const bool& iop_update_phase1, const uview_1d& hyam, const uview_1d& hybm); - - KOKKOS_FUNCTION - static void iop_intht(); -}; // struct Functions - -} // namespace dp -} // namespace scream - -// If a GPU build, without relocatable device code enabled, make all code available -// to the translation unit; otherwise, ETI is used. -#if defined(EAMXX_ENABLE_GPU) && !defined(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) \ - && !defined(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) - -# include "impl/dp_advance_iop_forcing_impl.hpp" -# include "impl/dp_advance_iop_nudging_impl.hpp" -# include "impl/dp_advance_iop_subsidence_impl.hpp" -# include "impl/dp_iop_setinitial_impl.hpp" -# include "impl/dp_iop_broadcast_impl.hpp" -# include "impl/dp_apply_iop_forcing_impl.hpp" -# include "impl/dp_iop_domain_relaxation_impl.hpp" -# include "impl/dp_crm_resolved_turb_impl.hpp" -# include "impl/dp_iop_default_opts_impl.hpp" -# include "impl/dp_iop_setopts_impl.hpp" -# include "impl/dp_setiopupdate_init_impl.hpp" -# include "impl/dp_setiopupdate_impl.hpp" -# include "impl/dp_readiopdata_impl.hpp" -# include "impl/dp_iop_intht_impl.hpp" -#endif // GPU && !KOKKOS_ENABLE_*_RELOCATABLE_DEVICE_CODE - -#endif // DP_FUNCTIONS_HPP diff --git a/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp b/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp deleted file mode 100644 index bad3c68dcb2d..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_functions_f90.cpp +++ /dev/null @@ -1,526 +0,0 @@ -#include "dp_functions_f90.hpp" - -#include "dp_f90.hpp" - -#include "ekat/ekat_assert.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/ekat_pack_kokkos.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" - -#include "share/util/scream_deep_copy.hpp" - -#include - -using scream::Real; -using scream::Int; - -using scream::dp::element_t; -using scream::dp::hvcoord_t; -using scream::dp::hybrid_t; -using scream::dp::timelevel_t; - -// -// A C interface to DP fortran calls. The stubs below will link to fortran definitions in dp_iso_c.f90 -// - -extern "C" { - -void advance_iop_forcing_c(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* u_update, Real* v_update, Real* t_update, Real* q_update); -void advance_iop_nudging_c(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* t_update, Real* q_update, Real* relaxt, Real* relaxq); -void advance_iop_subsidence_c(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* u_update, Real* v_update, Real* t_update, Real* q_update); -void iop_setinitial_c(Int nelemd, element_t* elem); -void iop_broadcast_c(); -void apply_iop_forcing_c(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t* hybrid, timelevel_t* tl, Int n, bool t_before_advance, Int nets, Int nete); -void iop_domain_relaxation_c(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt); -void crm_resolved_turb_c(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo); -void iop_default_opts_c(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out); -void iop_setopts_c(Real scmlat_in, Real scmlon_in, const char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in); -void setiopupdate_init_c(); -void setiopupdate_c(); -void readiopdata_c(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm); -void iop_intht_c(); -} // extern "C" : end _c decls - -namespace scream { -namespace dp { - -// -// Glue functions to call fortran from from C++ with the Data struct -// - -void advance_iop_forcing(AdvanceIopForcingData& d) -{ - dp_init(); - d.transpose(); - advance_iop_forcing_c(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.t_phys_frc, d.u_update, d.v_update, d.t_update, d.q_update); - d.transpose(); -} - - -void advance_iop_nudging(AdvanceIopNudgingData& d) -{ - dp_init(); - advance_iop_nudging_c(d.plev, d.scm_dt, d.ps_in, d.t_in, d.q_in, d.t_update, d.q_update, d.relaxt, d.relaxq); -} - -void advance_iop_subsidence(AdvanceIopSubsidenceData& d) -{ - dp_init(); - d.transpose(); - advance_iop_subsidence_c(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.u_update, d.v_update, d.t_update, d.q_update); - d.transpose(); -} - -void iop_setinitial(IopSetinitialData& d) -{ - dp_init(); - //iop_setinitial_c(d.nelemd, d.elem); -} - -void iop_broadcast(IopBroadcastData& d) -{ - dp_init(); - iop_broadcast_c(); -} - -void apply_iop_forcing(ApplyIopForcingData& d) -{ - dp_init(); - apply_iop_forcing_c(d.nelemd, d.elem, &d.hvcoord, &d.hybrid, &d.tl, d.n, d.t_before_advance, d.nets, d.nete); -} - -void iop_domain_relaxation(IopDomainRelaxationData& d) -{ - dp_init(); - d.transpose(); - iop_domain_relaxation_c(d.nelemd, d.np, d.nlev, d.elem, d.hvcoord, d.hybrid, d.t1, d.dp, d.nelemd_todo, d.np_todo, d.dt); - d.transpose(); -} - -void crm_resolved_turb(CrmResolvedTurbData& d) -{ - dp_init(); - crm_resolved_turb_c(d.nelemd, d.elem, d.hvcoord, d.hybrid, d.t1, d.nelemd_todo, d.np_todo); -} - -void iop_default_opts(IopDefaultOptsData& d) -{ - dp_init(); - char cbuff[512] = ""; - char* buffptr = cbuff; - iop_default_opts_c(&d.scmlat_out, &d.scmlon_out, &buffptr, &d.single_column_out, &d.scm_iop_srf_prop_out, &d.iop_nudge_tq_out, &d.iop_nudge_uv_out, &d.iop_nudge_tq_low_out, &d.iop_nudge_tq_high_out, &d.iop_nudge_tscale_out, &d.scm_observed_aero_out, &d.iop_dosubsidence_out, &d.scm_multcols_out, &d.dp_crm_out, &d.iop_perturb_high_out, &d.precip_off_out, &d.scm_zero_non_iop_tracers_out); - d.iopfile_out = std::string(buffptr); -} - -void iop_setopts(IopSetoptsData& d) -{ - dp_init(); - const char* cptr = d.iopfile_in.c_str(); - iop_setopts_c(d.scmlat_in, d.scmlon_in, &cptr, d.single_column_in, d.scm_iop_srf_prop_in, d.iop_nudge_tq_in, d.iop_nudge_uv_in, d.iop_nudge_tq_low_in, d.iop_nudge_tq_high_in, d.iop_nudge_tscale_in, d.scm_observed_aero_in, d.iop_dosubsidence_in, d.scm_multcols_in, d.dp_crm_in, d.iop_perturb_high_in, d.precip_off_in, d.scm_zero_non_iop_tracers_in); -} - -void setiopupdate_init(SetiopupdateInitData& d) -{ - dp_init(); - setiopupdate_init_c(); -} - -void setiopupdate(SetiopupdateData& d) -{ - dp_init(); - setiopupdate_c(); -} - -void readiopdata(ReadiopdataData& d) -{ - dp_init(); - readiopdata_c(d.plev, d.iop_update_phase1, d.hyam, d.hybm); -} - -void iop_intht(IopInthtData& d) -{ - dp_init(); - iop_intht_c(); -} - -// end _c impls - -// -// _f function definitions. These expect data in C layout -// - -void advance_iop_forcing_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, bool have_u, bool have_v, bool dp_crm, bool use_3dfrc, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* divt3d, Real* divq3d, Real* divt, Real* divq, Real* wfld, Real* uobs, Real* vobs, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* u_update, Real* v_update, Real* t_update, Real* q_update) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using view_1d = typename DPF::view_1d; - using view_2d = typename DPF::view_2d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - // Some of the workspaces need plev+1 items - const Int plev_pack = ekat::npack(plev); - const Int plevp_pack = ekat::npack(plev+1); - - // Set up views - std::vector temp_d(AdvanceIopForcingData::NUM_ARRAYS-4); - std::vector temp_2d_d(4); - - ekat::host_to_device({u_in, v_in, t_in, t_phys_frc, divt3d, divt, wfld, uobs, vobs, hyai, hyam, hybi, hybm, u_update, v_update, t_update}, - plev, temp_d); - - ekat::host_to_device({ q_in, divq3d, divq, q_update }, - pcnst, plev, temp_2d_d, true); - - view_1d - u_in_d (temp_d[0]), - v_in_d (temp_d[1]), - t_in_d (temp_d[2]), - t_phys_frc_d (temp_d[3]), - divt3d_d (temp_d[4]), - divt_d (temp_d[5]), - wfld_d (temp_d[6]), - uobs_d (temp_d[7]), - vobs_d (temp_d[8]), - hyai_d (temp_d[9]), - hyam_d (temp_d[10]), - hybi_d (temp_d[11]), - hybm_d (temp_d[12]), - u_update_d (temp_d[13]), - v_update_d (temp_d[14]), - t_update_d (temp_d[15]); - - view_2d - q_in_d (temp_2d_d[0]), - divq3d_d (temp_2d_d[1]), - divq_d (temp_2d_d[2]), - q_update_d(temp_2d_d[3]); - - // Call core function from kernel - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); - ekat::WorkspaceManager wsm(plevp_pack, 3, policy); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - - DPF::advance_iop_forcing( - plev, pcnst, have_u, have_v, dp_crm, use_3dfrc, scm_dt, ps_in, - u_in_d, v_in_d, t_in_d, q_in_d, t_phys_frc_d, divt3d_d, divq3d_d, divt_d, divq_d, wfld_d, uobs_d, vobs_d, hyai_d, hyam_d, hybi_d, hybm_d, - team, wsm.get_workspace(team), - u_update_d, v_update_d, t_update_d, q_update_d); - }); - - // Sync back to host - std::vector inout_views = {t_update_d, u_update_d, v_update_d}; - std::vector inout_views_2d = {q_update_d}; - - ekat::device_to_host({t_update, u_update, v_update}, plev, inout_views); - ekat::device_to_host({q_update}, pcnst, plev, inout_views_2d, true); -} - -void advance_iop_nudging_f(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* tobs, Real* qobs, - Real* hyai, Real* hyam, Real* hybi, Real* hybm, - Real* t_update, Real* q_update, Real* relaxt, Real* relaxq) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using view_1d = typename DPF::view_1d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - const Int plev_pack = ekat::npack(plev); - const Int plevp_pack = ekat::npack(plev+1); - - // Set up views - std::vector temp_d(AdvanceIopNudgingData::NUM_ARRAYS); - - ekat::host_to_device({t_in, q_in, tobs, qobs, hyai, hyam, hybi, hybm, t_update, q_update, relaxt, relaxq}, - plev, temp_d); - - int counter=0; - view_1d - t_in_d (temp_d[counter++]), - q_in_d (temp_d[counter++]), - tobs_d (temp_d[counter++]), - qobs_d (temp_d[counter++]), - hyai_d (temp_d[counter++]), - hyam_d (temp_d[counter++]), - hybi_d (temp_d[counter++]), - hybm_d (temp_d[counter++]), - t_update_d(temp_d[counter++]), - q_update_d(temp_d[counter++]), - relaxt_d (temp_d[counter++]), - relaxq_d (temp_d[counter++]); - - // Call core function from kernel - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); - ekat::WorkspaceManager wsm(plevp_pack, 4, policy); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - - DPF::advance_iop_nudging( - plev, scm_dt, ps_in, t_in_d, q_in_d, tobs_d, qobs_d, - hyai_d, hyam_d, hybi_d, hybm_d, - team, wsm.get_workspace(team), - t_update_d, q_update_d, relaxt_d, relaxq_d); - }); - - // Sync back to host - std::vector out_views = {t_update_d, q_update_d, relaxt_d, relaxq_d}; - - ekat::device_to_host({t_update, q_update, relaxt, relaxq}, plev, out_views); -} - -void advance_iop_subsidence_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* wfld, Real* u_update, Real* v_update, Real* t_update, Real* q_update) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using view_1d = typename DPF::view_1d; - using view_2d = typename DPF::view_2d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - // Some of the workspaces need plev+1 items - const Int plev_pack = ekat::npack(plev); - const Int plevp_pack = ekat::npack(plev+1); - - // Set up views - std::vector temp_d(AdvanceIopSubsidenceData::NUM_ARRAYS-2); - std::vector temp_2d_d(2); - - ekat::host_to_device({u_in, v_in, t_in, hyai, hyam, hybi, hybm, wfld, u_update, v_update, t_update}, - plev, temp_d); - - ekat::host_to_device({ q_in, q_update }, - pcnst, plev, temp_2d_d, true); - - view_1d - u_in_d (temp_d[0]), - v_in_d (temp_d[1]), - t_in_d (temp_d[2]), - hyai_d (temp_d[3]), - hyam_d (temp_d[4]), - hybi_d (temp_d[5]), - hybm_d (temp_d[6]), - wfld_d (temp_d[7]), - u_update_d (temp_d[8]), - v_update_d (temp_d[9]), - t_update_d (temp_d[10]); - - view_2d - q_in_d (temp_2d_d[0]), - q_update_d(temp_2d_d[1]); - - // Call core function from kernel - auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_pack); - ekat::WorkspaceManager wsm(plevp_pack, 4, policy); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const MemberType& team) { - - DPF::advance_iop_subsidence( - plev, pcnst, scm_dt, ps_in, - u_in_d, v_in_d, t_in_d, q_in_d, hyai_d, hyam_d, hybi_d, hybm_d, wfld_d, - team, wsm.get_workspace(team), - u_update_d, v_update_d, t_update_d, q_update_d); - }); - - // Sync back to host - std::vector inout_views = {t_update_d, u_update_d, v_update_d}; - std::vector inout_views_2d = {q_update_d}; - - ekat::device_to_host({t_update, u_update, v_update}, plev, inout_views); - ekat::device_to_host({q_update}, pcnst, plev, inout_views_2d, true); -} - -void iop_setinitial_f(Int plev, Int pcnst, Int nelemd, Int np, Int nstep, Real psobs, bool use_replay, bool dynproc, bool have_t, bool have_q, bool have_ps, bool have_u, bool have_v, bool have_numliq, bool have_cldliq, bool have_numice, bool have_cldice, bool scm_zero_non_iop_tracers, bool is_first_restart_step, Real* qmin, Real* uobs, Real* vobs, Real* numliqobs, Real* numiceobs, Real* cldliqobs, Real* cldiceobs, Real* dx_short, tracer_t* tracers, element_t* elem, Real* dyn_dx_size, Real* tobs, Real* qobs) -{ - using DPF = Functions; - - using Spack = typename DPF::Spack; - using Scalarp = ekat::Pack; - using view_1d = typename DPF::view_1d; - using view_1ds = typename DPF::view_1d; - using sview_1ds = typename DPF::view_1d; - using KT = typename DPF::KT; - using ExeSpace = typename KT::ExeSpace; - using MemberType = typename DPF::MemberType; - - // Some of the workspaces need plev+1 items - const Int plev_pack = ekat::npack(plev); - - // Set up views - std::vector temp_d(8); - std::vector temp2_d(2); - - ekat::host_to_device({uobs, vobs, numliqobs, numiceobs, cldliqobs, cldiceobs, tobs, qobs}, - plev, temp_d); - - std::vector scalar_sizes = {nelemd, pcnst}; - ekat::host_to_device({dx_short, qmin}, scalar_sizes, temp2_d); - - view_1d - uobs_d (temp_d[0]), - vobs_d (temp_d[1]), - numliqobs_d (temp_d[2]), - numiceobs_d (temp_d[3]), - cldliqobs_d (temp_d[4]), - cldiceobs_d (temp_d[5]), - tobs_d (temp_d[6]), - qobs_d (temp_d[7]); - - sview_1ds - dx_short_d(reinterpret_cast(temp2_d[0].data()), scalar_sizes[0]), - qmin_d (reinterpret_cast(temp2_d[1].data()), scalar_sizes[1]); - - // Call core function - DPF::iop_setinitial(plev, pcnst, nelemd, np, nstep, use_replay, dynproc, have_t, have_ps, have_q, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step, qmin_d, uobs_d, vobs_d, numliqobs_d, numiceobs_d, cldliqobs_d, cldiceobs_d, psobs, dx_short_d, *dyn_dx_size, *tracers, *elem, tobs_d, qobs_d); - - // Sync back to host - std::vector inout_views = {tobs_d, qobs_d}; - - ekat::device_to_host({tobs, qobs}, plev, inout_views); -} - -void iop_broadcast_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::iop_broadcast(); - }); -#endif - -} -void apply_iop_forcing_f(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t hybrid, timelevel_t tl, Int n, bool t_before_advance, Int nets, Int nete) -{ - // TODO -} -void iop_domain_relaxation_f(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt) -{ - // TODO -} -void crm_resolved_turb_f(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo) -{ - // TODO -} -void iop_default_opts_f(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out) -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - using view_1d = typename PF::view_1d; - using bview_1d = typename PF::view_1d; - - view_1d t_d("t_d", 6); - const auto t_h = Kokkos::create_mirror_view(t_d); - - bview_1d bt_d("bt_d", 10); - const auto bt_h = Kokkos::create_mirror_view(bt_d); - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - Spack iop_nudge_tq_high_out_(), iop_nudge_tq_low_out_(), iop_nudge_tscale_out_(), iop_perturb_high_out_(), scmlat_out_(), scmlon_out_(); - bool dp_crm_out_(), iop_dosubsidence_out_(), iop_nudge_tq_out_(), iop_nudge_uv_out_(), precip_off_out_(), scm_iop_srf_prop_out_(), scm_multcols_out_(), scm_observed_aero_out_(), scm_zero_non_iop_tracers_out_(), single_column_out_(); - PF::iop_default_opts(scmlat_out_, scmlon_out_, iopfile_out_, single_column_out_, scm_iop_srf_prop_out_, iop_nudge_tq_out_, iop_nudge_uv_out_, iop_nudge_tq_low_out_, iop_nudge_tq_high_out_, iop_nudge_tscale_out_, scm_observed_aero_out_, iop_dosubsidence_out_, scm_multcols_out_, dp_crm_out_, iop_perturb_high_out_, precip_off_out_, scm_zero_non_iop_tracers_out_); - t_d(0) = iop_nudge_tq_high_out_[0]; - t_d(1) = iop_nudge_tq_low_out_[0]; - t_d(2) = iop_nudge_tscale_out_[0]; - t_d(3) = iop_perturb_high_out_[0]; - t_d(4) = scmlat_out_[0]; - t_d(5) = scmlon_out_[0]; - bt_d(0) = dp_crm_out_; - bt_d(1) = iop_dosubsidence_out_; - bt_d(2) = iop_nudge_tq_out_; - bt_d(3) = iop_nudge_uv_out_; - bt_d(4) = precip_off_out_; - bt_d(5) = scm_iop_srf_prop_out_; - bt_d(6) = scm_multcols_out_; - bt_d(7) = scm_observed_aero_out_; - bt_d(8) = scm_zero_non_iop_tracers_out_; - bt_d(9) = single_column_out_; - }); - Kokkos::deep_copy(t_h, t_d); - Kokkos::deep_copy(bt_h, bt_d); - *iop_nudge_tq_high_out = t_h(0); - *iop_nudge_tq_low_out = t_h(1); - *iop_nudge_tscale_out = t_h(2); - *iop_perturb_high_out = t_h(3); - *scmlat_out = t_h(4); - *scmlon_out = t_h(5); - *dp_crm_out = bt_h(0); - *iop_dosubsidence_out = bt_h(1); - *iop_nudge_tq_out = bt_h(2); - *iop_nudge_uv_out = bt_h(3); - *precip_off_out = bt_h(4); - *scm_iop_srf_prop_out = bt_h(5); - *scm_multcols_out = bt_h(6); - *scm_observed_aero_out = bt_h(7); - *scm_zero_non_iop_tracers_out = bt_h(8); - *single_column_out = bt_h(9); -#endif - -} -void iop_setopts_f(Real scmlat_in, Real scmlon_in, char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in) -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - Spack iop_nudge_tq_high_in_(iop_nudge_tq_high_in), iop_nudge_tq_low_in_(iop_nudge_tq_low_in), iop_nudge_tscale_in_(iop_nudge_tscale_in), iop_perturb_high_in_(iop_perturb_high_in), scmlat_in_(scmlat_in), scmlon_in_(scmlon_in); - PF::iop_setopts(scmlat_in_, scmlon_in_, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in_, iop_nudge_tq_high_in_, iop_nudge_tscale_in_, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in_, precip_off_in, scm_zero_non_iop_tracers_in); - }); -#endif - -} -void setiopupdate_init_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::setiopupdate_init(); - }); -#endif - -} -void setiopupdate_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::setiopupdate(); - }); -#endif - -} -void readiopdata_f(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm) -{ - // TODO -} -void iop_intht_f() -{ -#if 0 - using PF = Functions; - - using Spack = typename PF::Spack; - - Kokkos::parallel_for(1, KOKKOS_LAMBDA(const Int&) { - PF::iop_intht(); - }); -#endif - -} -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp b/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp deleted file mode 100644 index 055bfe4ad683..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_functions_f90.hpp +++ /dev/null @@ -1,306 +0,0 @@ -#ifndef SCREAM_DP_FUNCTIONS_F90_HPP -#define SCREAM_DP_FUNCTIONS_F90_HPP - -#include "share/scream_types.hpp" -#include "physics/share/physics_test_data.hpp" - -#include "dp_functions.hpp" -#include "physics_constants.hpp" - -#include -#include -#include - -// -// Bridge functions to call fortran version of dp functions from C++ -// - -namespace scream { -namespace dp { - -struct AdvanceIopForcingData : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 20; - - // Inputs - Int plev, pcnst; - Real scm_dt, ps_in; - bool have_u, have_v, dp_crm, use_3dfrc; - Real *u_in, *v_in, *t_in, *q_in, *t_phys_frc, *divt3d, *divq3d, *divt, - *divq, *wfld, *uobs, *vobs, *hyai, *hyam, *hybi, *hybm; - - // Outputs - Real *u_update, *v_update, *t_update, *q_update; - - AdvanceIopForcingData(Int plev_, Int pcnst_, Real scm_dt_, Real ps_in_, bool have_u_, bool have_v_, bool dp_crm_, bool use_3dfrc_) : - PhysicsTestData( - {{ plev_ }, { pcnst_, plev_ }}, - { - { &u_in, &v_in, &t_in, &t_phys_frc, &divt3d, &divt, &divq, &wfld, &uobs, &vobs, &hyai, &hyam, &hybi, &hybm, &u_update, &v_update, &t_update }, - { &q_in, &divq3d, &divq, &q_update } - }), - plev(plev_), pcnst(pcnst_), scm_dt(scm_dt_), ps_in(ps_in_), have_u(have_u_), have_v(have_v_), dp_crm(dp_crm_), use_3dfrc(use_3dfrc_) {} - - PTD_STD_DEF(AdvanceIopForcingData, 8, plev, pcnst, scm_dt, ps_in, have_u, have_v, dp_crm, use_3dfrc); -}; - - -struct AdvanceIopNudgingData : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 12; - - // Inputs - Int plev; - Real scm_dt, ps_in; - Real *t_in, *q_in, *tobs, *qobs, *hyai, *hyam, *hybi, *hybm; - - // Outputs - Real *t_update, *q_update, *relaxt, *relaxq; - - AdvanceIopNudgingData(Int plev_, Real scm_dt_, Real ps_in_) : - PhysicsTestData({{ plev_ }}, {{ &t_in, &q_in, &tobs, &qobs, &hyai, &hyam, &hybi, &hybm, &t_update, &q_update, &relaxt, &relaxq }}), - plev(plev_), scm_dt(scm_dt_), ps_in(ps_in_) {} - - PTD_STD_DEF(AdvanceIopNudgingData, 3, plev, scm_dt, ps_in); -}; - -struct AdvanceIopSubsidenceData : public PhysicsTestData { - static constexpr size_t NUM_ARRAYS = 13; - - // Inputs - Int plev, pcnst; - Real scm_dt, ps_in; - Real *u_in, *v_in, *t_in, *q_in, *hyai, *hyam, *hybi, *hybm, *wfld; - - // Outputs - Real *u_update, *v_update, *t_update, *q_update; - - AdvanceIopSubsidenceData(Int plev_, Int pcnst_, Real scm_dt_, Real ps_in_) : - PhysicsTestData( - {{ plev_ }, { plev_, pcnst_ }}, - { - { &u_in, &v_in, &t_in, &hyai, &hyam, &hybi, &hybm, &wfld, &u_update, &v_update, &t_update }, - { &q_in, &q_update } - }), - plev(plev_), pcnst(pcnst_), scm_dt(scm_dt_), ps_in(ps_in_) {} - - PTD_STD_DEF(AdvanceIopSubsidenceData, 4, plev, pcnst, scm_dt, ps_in); -}; - -struct IopSetinitialData : public PhysicsTestData { - // Inputs - Int plev, pcnst, nelemd, np, nstep; - - bool use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step; - - Real psobs; - - Real* qmin, *uobs, *vobs, *numliqobs, *numiceobs, *cldliqobs, *cldiceobs, *dx_short; - - // Inputs/Outputs - tracer_t tracers; - element_t elem; - - Real dyn_dx_size; - - Real* tobs, *qobs; - - IopSetinitialData( - Int plev_, Int pcnst_, Int nelemd_, Int np_, Int nstep_, Real psobs_, - bool use_replay_, bool dynproc_, bool have_t_, bool have_q_, bool have_ps_, bool have_u_, bool have_v_, bool have_numliq_, bool have_cldliq_, bool have_numice_, bool have_cldice_, bool scm_zero_non_iop_tracers_, bool is_first_restart_step_) : - PhysicsTestData( - {{nelemd_}, {pcnst_}, {plev_}}, - { - {&dx_short}, - {&qmin}, - {&uobs, &vobs, &numliqobs, &numiceobs, &cldliqobs, &cldiceobs, &tobs, &qobs} - }), - plev(plev_), pcnst(pcnst_), nelemd(nelemd_), np(np_), nstep(nstep_), psobs(psobs_), - use_replay(use_replay_), dynproc(dynproc_), have_t(have_t_), have_q(have_q_), have_ps(have_ps_), have_u(have_u_), have_v(have_v_), have_numliq(have_numliq_), have_cldliq(have_cldliq_), have_numice(have_numice_), have_cldice(have_cldice_), scm_zero_non_iop_tracers(scm_zero_non_iop_tracers_), is_first_restart_step(is_first_restart_step_) { } - - PTD_STD_DEF(IopSetinitialData, 19, plev, pcnst, nelemd, np, nstep, psobs, use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step); - - void init() - { - tracers.init(nelemd, QSIZE_D); - elem.init(nelemd, true, true, 2); - } - - void randomize(std::mt19937_64& engine) - { - PhysicsTestData::randomize(engine); - init(); - - tracers.randomize(engine(), 0, 1); - elem.randomize(engine()); - - // Where tobs starts being non-zero is important to the algorithm - std::uniform_int_distribution default_int_dist(0, plev); - Int start_nz = default_int_dist(engine); - - for (Int k = 0; k < start_nz; ++k) { - tobs[k] = 0; - } - } -}; - -struct IopBroadcastData : public PhysicsTestData { - // Inputs - Int plev; - - IopBroadcastData(Int plev_=0) : - PhysicsTestData({}, {}), plev(plev_) {} - - PTD_STD_DEF(IopBroadcastData, 1, plev); -}; - -struct ApplyIopForcingData : public PhysicsTestData { - // Inputs - Int plev, nelemd, n, nets, nete; - hybrid_t hybrid; - timelevel_t tl; - bool t_before_advance; - - // Inputs/Outputs - element_t *elem; - hvcoord_t hvcoord; - - ApplyIopForcingData(Int plev_, Int nelemd_, Int n_, Int nets_, Int nete_, bool t_before_advance_) : - PhysicsTestData({}, {}), plev(plev_), nelemd(nelemd_), n(n_), nets(nets_), nete(nete_), t_before_advance(t_before_advance_) {} - - PTD_STD_DEF(ApplyIopForcingData, 6, plev, nelemd, n, nets, nete, t_before_advance); -}; - -struct IopDomainRelaxationData : public PhysicsTestData { - // Inputs - Int nelemd, np, nlev, t1, nelemd_todo, np_todo; - hvcoord_t hvcoord; - hybrid_t hybrid; - Real dt; - - // Inputs/Outputs - element_t *elem; - Real *dp; - - IopDomainRelaxationData(Int nelemd_, Int np_, Int nlev_, Int t1_, Int nelemd_todo_, Int np_todo_, Real dt_) : - PhysicsTestData({{ np_, np_, nlev_ }}, {{ &dp }}), nelemd(nelemd_), np(np_), nlev(nlev_), t1(t1_), nelemd_todo(nelemd_todo_), np_todo(np_todo_), dt(dt_) {} - - PTD_STD_DEF(IopDomainRelaxationData, 7, nelemd, np, nlev, t1, nelemd_todo, np_todo, dt); -}; - -struct CrmResolvedTurbData : public PhysicsTestData { - // Inputs - Int plev, nelemd, t1, nelemd_todo, np_todo; - hvcoord_t hvcoord; - hybrid_t hybrid; - - // Inputs/Outputs - element_t *elem; - - CrmResolvedTurbData(Int plev_, Int nelemd_, Int t1_, Int nelemd_todo_, Int np_todo_) : - PhysicsTestData({}, {}), plev(plev_), nelemd(nelemd_), t1(t1_), nelemd_todo(nelemd_todo_), np_todo(np_todo_) {} - - PTD_STD_DEF(CrmResolvedTurbData, 5, plev, nelemd, t1, nelemd_todo, np_todo); -}; - -struct IopDefaultOptsData { - // Inputs - Int plev; - - // Outputs - Real scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out; - std::string iopfile_out; - bool single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out; - - void randomize(std::mt19937_64& engine) {} - - IopDefaultOptsData() = default; -}; - -struct IopSetoptsData { - // Inputs - Int plev; - Real scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in; - std::string iopfile_in; - bool single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in; - - void randomize(std::mt19937_64& engine) {} - - IopSetoptsData() = default; -}; - -struct SetiopupdateInitData { - // Inputs - Int plev; - - void randomize(std::mt19937_64& engine) {} -}; - -struct SetiopupdateData { - // Inputs - Int plev; - - void randomize(std::mt19937_64& engine) {} -}; - -struct ReadiopdataData : public PhysicsTestData { - // Inputs - Int plev; - bool iop_update_phase1; - Real *hyam, *hybm; - - ReadiopdataData(Int plev_, bool iop_update_phase1_) : - PhysicsTestData({{ plev_ }}, {{ &hyam, &hybm }}), plev(plev_), iop_update_phase1(iop_update_phase1_) {} - - PTD_STD_DEF(ReadiopdataData, 2, plev, iop_update_phase1); -}; - -struct IopInthtData { - // Inputs - Int plev; - - void randomize(std::mt19937_64& engine) {} -}; - -// Glue functions to call fortran from from C++ with the Data struct - -void advance_iop_forcing(AdvanceIopForcingData& d); -void advance_iop_nudging(AdvanceIopNudgingData& d); -void advance_iop_subsidence(AdvanceIopSubsidenceData& d); -void iop_setinitial(IopSetinitialData& d); -void iop_broadcast(IopBroadcastData& d); -void apply_iop_forcing(ApplyIopForcingData& d); -void iop_domain_relaxation(IopDomainRelaxationData& d); -void crm_resolved_turb(CrmResolvedTurbData& d); -void iop_default_opts(IopDefaultOptsData& d); -void iop_setopts(IopSetoptsData& d); -void setiopupdate_init(SetiopupdateInitData& d); -void setiopupdate(SetiopupdateData& d); -void readiopdata(ReadiopdataData& d); -void iop_intht(IopInthtData& d); -extern "C" { // _f function decls - -void advance_iop_forcing_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, bool have_u, bool have_v, bool dp_crm, bool use_3dfrc, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* t_phys_frc, Real* divt3d, Real* divq3d, Real* divt, Real* divq, Real* wfld, Real* uobs, Real* vobs, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* u_update, Real* v_update, Real* t_update, Real* q_update); - -void advance_iop_nudging_f(Int plev, Real scm_dt, Real ps_in, Real* t_in, Real* q_in, Real* tobs, Real* qobs, - Real* hyai, Real* hyam, Real* hybi, Real* hybm, - Real* t_update, Real* q_update, Real* relaxt, Real* relaxq); - -void advance_iop_subsidence_f(Int plev, Int pcnst, Real scm_dt, Real ps_in, Real* u_in, Real* v_in, Real* t_in, Real* q_in, Real* hyai, Real* hyam, Real* hybi, Real* hybm, Real* wfld, Real* u_update, Real* v_update, Real* t_update, Real* q_update); - -void iop_setinitial_f(Int plev, Int pcnst, Int nelemd, Int np, Int nstep, Real psobs, bool use_replay, bool dynproc, bool have_t, bool have_q, bool have_ps, bool have_u, bool have_v, bool have_numliq, bool have_cldliq, bool have_numice, bool have_cldice, bool scm_zero_non_iop_tracers, bool is_first_restart_step, Real* qmin, Real* uobs, Real* vobs, Real* numliqobs, Real* numiceobs, Real* cldliqobs, Real* cldiceobs, Real* dx_short, tracer_t* tracers, element_t* elem, Real* dyn_dx_size, Real* tobs, Real* qobs); - -void iop_broadcast_f(); -void apply_iop_forcing_f(Int nelemd, element_t* elem, hvcoord_t* hvcoord, hybrid_t hybrid, timelevel_t tl, Int n, bool t_before_advance, Int nets, Int nete); -void iop_domain_relaxation_f(Int nelemd, Int np, Int nlev, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Real* dp, Int nelemd_todo, Int np_todo, Real dt); -void crm_resolved_turb_f(Int nelemd, element_t* elem, hvcoord_t hvcoord, hybrid_t hybrid, Int t1, Int nelemd_todo, Int np_todo); -void iop_default_opts_f(Real* scmlat_out, Real* scmlon_out, char** iopfile_out, bool* single_column_out, bool* scm_iop_srf_prop_out, bool* iop_nudge_tq_out, bool* iop_nudge_uv_out, Real* iop_nudge_tq_low_out, Real* iop_nudge_tq_high_out, Real* iop_nudge_tscale_out, bool* scm_observed_aero_out, bool* iop_dosubsidence_out, bool* scm_multcols_out, bool* dp_crm_out, Real* iop_perturb_high_out, bool* precip_off_out, bool* scm_zero_non_iop_tracers_out); -void iop_setopts_f(Real scmlat_in, Real scmlon_in, const char** iopfile_in, bool single_column_in, bool scm_iop_srf_prop_in, bool iop_nudge_tq_in, bool iop_nudge_uv_in, Real iop_nudge_tq_low_in, Real iop_nudge_tq_high_in, Real iop_nudge_tscale_in, bool scm_observed_aero_in, bool iop_dosubsidence_in, bool scm_multcols_in, bool dp_crm_in, Real iop_perturb_high_in, bool precip_off_in, bool scm_zero_non_iop_tracers_in); -void setiopupdate_init_f(); -void setiopupdate_f(); -void readiopdata_f(Int plev, bool iop_update_phase1, Real* hyam, Real* hybm); -void iop_intht_f(); -} // end _f function decls - -} // namespace dp -} // namespace scream - -#endif // SCREAM_DP_FUNCTIONS_F90_HPP diff --git a/components/eamxx/src/doubly-periodic/dp_iso_c.f90 b/components/eamxx/src/doubly-periodic/dp_iso_c.f90 deleted file mode 100644 index 29d2fe55d4b2..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_iso_c.f90 +++ /dev/null @@ -1,156 +0,0 @@ -module dp_iso_c - use iso_c_binding - implicit none - -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double -#else -# define c_real c_float -#endif - -! -! This file contains bridges from scream c++ to DP fortran. -! - -contains - subroutine advance_iop_forcing_c(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) bind(C) - !use dp, only : advance_iop_forcing - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in, t_phys_frc - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - - !call advance_iop_forcing(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) - end subroutine advance_iop_forcing_c - subroutine advance_iop_nudging_c(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) bind(C) - !use dp, only : advance_iop_nudging - - integer(kind=c_int) , value, intent(in) :: plev - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: t_in, q_in - real(kind=c_real) , intent(out), dimension(plev) :: t_update, q_update, relaxt, relaxq - - !call advance_iop_nudging(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) - end subroutine advance_iop_nudging_c - subroutine advance_iop_subsidence_c(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) bind(C) - !use dp, only : advance_iop_subsidence - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - - !call advance_iop_subsidence(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) - end subroutine advance_iop_subsidence_c - subroutine iop_setinitial_c(nelemd, elem) bind(C) - !use dp, only : iop_setinitial - - integer(kind=c_int) , value, intent(in) :: nelemd - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - - !call iop_setinitial(nelemd, elem) - end subroutine iop_setinitial_c - subroutine iop_broadcast_c() bind(C) - !use dp, only : iop_broadcast - - !call iop_broadcast() - end subroutine iop_broadcast_c - subroutine apply_iop_forcing_c(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) bind(C) - !use dp, only : apply_iop_forcing - - integer(kind=c_int) , value, intent(in) :: nelemd, n, nets, nete - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(inout) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - type(c_ptr) , intent(in) :: tl - logical(kind=c_bool) , value, intent(in) :: t_before_advance - - !call apply_iop_forcing(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) - end subroutine apply_iop_forcing_c - subroutine iop_domain_relaxation_c(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) bind(C) - !use dp, only : iop_domain_relaxation - - integer(kind=c_int) , value, intent(in) :: nelemd, np, nlev, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - real(kind=c_real) , intent(inout), dimension(np, np, nlev) :: dp - real(kind=c_real) , value, intent(in) :: dt - - !call iop_domain_relaxation(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) - end subroutine iop_domain_relaxation_c - subroutine crm_resolved_turb_c(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) bind(C) - !use dp, only : crm_resolved_turb - - integer(kind=c_int) , value, intent(in) :: nelemd, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - - !call crm_resolved_turb(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) - end subroutine crm_resolved_turb_c - subroutine iop_default_opts_c(scmlat_out, scmlon_out, iopfile_out, & - single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, & - iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, & - iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, & - scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, & - scm_zero_non_iop_tracers_out) bind(C) - !use dp, only : iop_default_opts - - real(kind=c_real) , intent(out) :: scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out - type(c_ptr) , intent(out) :: iopfile_out - logical(kind=c_bool) , intent(out) :: single_column_out, & - scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, & - scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, & - dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out - - !call iop_default_opts(scmlat_out, scmlon_out, iopfile_out, single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, scm_zero_non_iop_tracers_out) - end subroutine iop_default_opts_c - subroutine iop_setopts_c(scmlat_in, scmlon_in, iopfile_in, & - single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, & - iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, & - iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, & - scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, & - scm_zero_non_iop_tracers_in) bind(C) - !use dp, only : iop_setopts - - real(kind=c_real) , value, intent(in) :: scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in - type(c_ptr) , intent(in) :: iopfile_in - logical(kind=c_bool) , value, intent(in) :: single_column_in, & - scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, & - scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, & - dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in - - !call iop_setopts(scmlat_in, scmlon_in, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, scm_zero_non_iop_tracers_in) - end subroutine iop_setopts_c - subroutine setiopupdate_init_c() bind(C) - !use dp, only : setiopupdate_init - - ! call setiopupdate_init() - end subroutine setiopupdate_init_c - subroutine setiopupdate_c() bind(C) - !use dp, only : setiopupdate - - !call setiopupdate() - end subroutine setiopupdate_c - subroutine readiopdata_c(plev, iop_update_phase1, hyam, hybm) bind(C) - !use dp, only : readiopdata - - integer(kind=c_int) , value, intent(in) :: plev - logical(kind=c_bool) , value, intent(in) :: iop_update_phase1 - real(kind=c_real) , intent(in), dimension(plev) :: hyam, hybm - - !call readiopdata(plev, iop_update_phase1, hyam, hybm) - end subroutine readiopdata_c - subroutine iop_intht_c() bind(C) - !use dp, only : iop_intht - - !call iop_intht() - end subroutine iop_intht_c -end module dp_iso_c diff --git a/components/eamxx/src/doubly-periodic/dp_iso_f.f90 b/components/eamxx/src/doubly-periodic/dp_iso_f.f90 deleted file mode 100644 index 98c48961aa4a..000000000000 --- a/components/eamxx/src/doubly-periodic/dp_iso_f.f90 +++ /dev/null @@ -1,123 +0,0 @@ -module dp_iso_f - use iso_c_binding - implicit none - -#include "scream_config.f" -#ifdef SCREAM_DOUBLE_PRECISION -# define c_real c_double -#else -# define c_real c_float -#endif - -! -! This file contains bridges from DP fortran to scream c++. -! - -interface - - subroutine advance_iop_forcing_f(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, t_phys_frc, u_update, v_update, t_update, q_update) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in, t_phys_frc - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - end subroutine advance_iop_forcing_f - subroutine advance_iop_nudging_f(plev, scm_dt, ps_in, t_in, q_in, t_update, q_update, relaxt, relaxq) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: t_in, q_in - real(kind=c_real) , intent(out), dimension(plev) :: t_update, q_update, relaxt, relaxq - end subroutine advance_iop_nudging_f - subroutine advance_iop_subsidence_f(plev, pcnst, scm_dt, ps_in, u_in, v_in, t_in, q_in, u_update, v_update, t_update, q_update) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev, pcnst - real(kind=c_real) , value, intent(in) :: scm_dt, ps_in - real(kind=c_real) , intent(in), dimension(plev) :: u_in, v_in, t_in - real(kind=c_real) , intent(in), dimension(plev, pcnst) :: q_in - real(kind=c_real) , intent(out), dimension(plev) :: u_update, v_update, t_update - real(kind=c_real) , intent(out), dimension(plev, pcnst) :: q_update - end subroutine advance_iop_subsidence_f - subroutine iop_setinitial_f(nelemd, elem) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - end subroutine iop_setinitial_f - subroutine iop_broadcast_f() bind(C) - use iso_c_binding - - - end subroutine iop_broadcast_f - subroutine apply_iop_forcing_f(nelemd, elem, hvcoord, hybrid, tl, n, t_before_advance, nets, nete) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd, n, nets, nete - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(inout) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - type(c_ptr) , intent(in) :: tl - logical(kind=c_bool) , value, intent(in) :: t_before_advance - end subroutine apply_iop_forcing_f - subroutine iop_domain_relaxation_f(nelemd, np, nlev, elem, hvcoord, hybrid, t1, dp, nelemd_todo, np_todo, dt) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd, np, nlev, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - real(kind=c_real) , intent(inout), dimension(np, np, nlev) :: dp - real(kind=c_real) , value, intent(in) :: dt - end subroutine iop_domain_relaxation_f - subroutine crm_resolved_turb_f(nelemd, elem, hvcoord, hybrid, t1, nelemd_todo, np_todo) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: nelemd, t1, nelemd_todo, np_todo - type(c_ptr) , intent(inout), dimension(nelemd) :: elem - type(c_ptr) , intent(in) :: hvcoord - type(c_ptr) , intent(in) :: hybrid - end subroutine crm_resolved_turb_f - subroutine iop_default_opts_f(scmlat_out, scmlon_out, iopfile_out, single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, iop_perturb_high_out, precip_off_out, scm_zero_non_iop_tracers_out) bind(C) - use iso_c_binding - - real(kind=c_real) , intent(out) :: scmlat_out, scmlon_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, iop_perturb_high_out - type(c_ptr) , intent(out) :: iopfile_out - logical(kind=c_bool) , intent(out) :: single_column_out, scm_iop_srf_prop_out, iop_nudge_tq_out, iop_nudge_uv_out, scm_observed_aero_out, iop_dosubsidence_out, scm_multcols_out, dp_crm_out, precip_off_out, scm_zero_non_iop_tracers_out - end subroutine iop_default_opts_f - subroutine iop_setopts_f(scmlat_in, scmlon_in, iopfile_in, single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, iop_perturb_high_in, precip_off_in, scm_zero_non_iop_tracers_in) bind(C) - use iso_c_binding - - real(kind=c_real) , value, intent(in) :: scmlat_in, scmlon_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, iop_perturb_high_in - type(c_ptr) , intent(in) :: iopfile_in - logical(kind=c_bool) , value, intent(in) :: single_column_in, scm_iop_srf_prop_in, iop_nudge_tq_in, iop_nudge_uv_in, scm_observed_aero_in, iop_dosubsidence_in, scm_multcols_in, dp_crm_in, precip_off_in, scm_zero_non_iop_tracers_in - end subroutine iop_setopts_f - subroutine setiopupdate_init_f() bind(C) - use iso_c_binding - - - end subroutine setiopupdate_init_f - subroutine setiopupdate_f() bind(C) - use iso_c_binding - - - end subroutine setiopupdate_f - subroutine readiopdata_f(plev, iop_update_phase1, hyam, hybm) bind(C) - use iso_c_binding - - integer(kind=c_int) , value, intent(in) :: plev - logical(kind=c_bool) , value, intent(in) :: iop_update_phase1 - real(kind=c_real) , intent(in), dimension(plev) :: hyam, hybm - end subroutine readiopdata_f - subroutine iop_intht_f() bind(C) - use iso_c_binding - - - end subroutine iop_intht_f -end interface - -end module dp_iso_f diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp deleted file mode 100644 index a0f560796a4d..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_forcing.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_advance_iop_forcing_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing advance_iop_forcing on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp deleted file mode 100644 index 6e5fab7e6090..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_nudging.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_advance_iop_nudging_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing advance_iop_nudging on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp b/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp deleted file mode 100644 index 1fe6c79a160a..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_advance_iop_subsidence.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_advance_iop_subsidence_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing advance_iop_subsidence on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp b/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp deleted file mode 100644 index cb138eb9b048..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_apply_iop_forcing.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_apply_iop_forcing_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing apply_iop_forcing on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp b/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp deleted file mode 100644 index 6921dc81cc5a..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_crm_resolved_turb.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_crm_resolved_turb_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing crm_resolved_turb on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp deleted file mode 100644 index 083d19b97afd..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_broadcast.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_broadcast_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_broadcast on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp deleted file mode 100644 index 30fae390febb..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_default_opts.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_default_opts_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_default_opts on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp deleted file mode 100644 index 2b8aa9a09add..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_domain_relaxation.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_domain_relaxation_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_domain_relaxation on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp deleted file mode 100644 index b68587342a5c..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_intht.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_intht_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_intht on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp deleted file mode 100644 index 8641cfa0704a..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_setfield.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_setfield_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_setfield on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp deleted file mode 100644 index 9195f4f6cf3b..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_setinitial.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_setinitial_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_setinitial on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp b/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp deleted file mode 100644 index e46aff90bdc6..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_iop_setopts.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_iop_setopts_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing iop_setopts on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp b/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp deleted file mode 100644 index 4289e61b9848..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_readiopdata.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_readiopdata_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing readiopdata on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp deleted file mode 100644 index 56e7084e4760..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_setiopupdate_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing setiopupdate on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp b/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp deleted file mode 100644 index 45f26f184665..000000000000 --- a/components/eamxx/src/doubly-periodic/eti/dp_setiopupdate_init.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "impl/dp_setiopupdate_init_impl.hpp" - -namespace scream { -namespace dp { - -/* - * Explicit instantiation for doing setiopupdate_init on Reals using the - * default device. - */ - -template struct Functions; - -} // namespace dp -} // namespace scream diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp deleted file mode 100644 index 5220d14e290b..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_forcing_impl.hpp +++ /dev/null @@ -1,158 +0,0 @@ -#ifndef DP_ADVANCE_IOP_FORCING_IMPL_HPP -#define DP_ADVANCE_IOP_FORCING_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp advance_iop_forcing. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::plevs0( - // Input arguments - const Int& nver, - const Scalar& ps, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - // Kokkos stuff - const MemberType& team, - // Output arguments - const uview_1d& pint, - const uview_1d& pmid, - const uview_1d& pdel) -{ - const auto ps0 = C::P0; - const Int nver_pack = ekat::npack(nver); - - // Set interface pressures - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nver_pack), [&] (Int k) { - pint(k) = hyai(k)*ps0 + hybi(k)*ps; - pmid(k) = hyam(k)*ps0 + hybm(k)*ps; - }); - - // Set midpoint pressures and layer thicknesses - const auto pint_s = scalarize(pint); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, nver_pack), [&] (Int k) { - Spack spint, spint_1; - IntSmallPack range_pack1 = ekat::range(k*Spack::n); - auto range_pack2_p1_safe = range_pack1; - range_pack2_p1_safe.set(range_pack1 > nver-1, nver-1); - ekat::index_and_shift<1>(pint_s, range_pack2_p1_safe, spint, spint_1); - pdel(k) = spint_1 - spint; - }); -} - -template -KOKKOS_FUNCTION -void Functions::advance_iop_forcing( - // Input arguments - const Int& plev, - const Int& pcnst, - const bool& have_u, - const bool& have_v, - const bool& dp_crm, - const bool& use_3dfrc, - const Scalar& scm_dt, - const Scalar& ps_in, - const uview_1d& u_in, - const uview_1d& v_in, - const uview_1d& t_in, - const uview_2d& q_in, - const uview_1d& t_phys_frc, - const uview_1d& divt3d, - const uview_2d& divq3d, - const uview_1d& divt, - const uview_2d& divq, - const uview_1d& wfld, - const uview_1d& uobs, - const uview_1d& vobs, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& u_update, - const uview_1d& v_update, - const uview_1d& t_update, - const uview_2d& q_update) -{ - // Local variables - uview_1d - pmidm1, // pressure at model levels - pintm1, // pressure at model interfaces (dim=plev+1) - pdelm1; // pdel(k) = pint (k+1)-pint (k) - workspace.template take_many_contiguous_unsafe<3>( - {"pmidm1", "pintm1", "pdelm1"}, - {&pmidm1, &pintm1, &pdelm1}); - - // Get vertical level profiles - plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); - - //////////////////////////////////////////////////////////// - // Advance T and Q due to large scale forcing - - uview_1d t_lsf; // storage for temperature large scale forcing - uview_2d q_lsf; // storage for moisture large scale forcing - - if (use_3dfrc) { - t_lsf = divt3d; - q_lsf = divq3d; - } - else { - t_lsf = divt; - q_lsf = divq; - } - - const Int plev_pack = ekat::npack(plev); - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - // Initialize thermal expansion term to zero. This term is only - // considered if using the preq-x dycore and if three dimensional - // forcing is not provided by IOP forcing file. - Spack t_expan = 0; - t_update(k) = t_in(k) + t_expan + scm_dt*(t_phys_frc(k) + t_lsf(k)); - for (Int m = 0; m < pcnst; ++m) { - q_update(m, k) = q_in(m, k) + scm_dt*q_lsf(m, k); - } - }); - - //////////////////////////////////////////////////////////// - // Set U and V fields - - uview_1d u_src, v_src; - - if (have_v && have_u && !dp_crm) { - u_src = uobs; - v_src = vobs; - } - else { - u_src = u_in; - v_src = v_in; - } - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - u_update(k) = u_src(k); - v_update(k) = v_src(k); - }); - - workspace.template release_many_contiguous<3>( - {&pmidm1, &pintm1, &pdelm1}); -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp deleted file mode 100644 index cf0f160209a3..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_nudging_impl.hpp +++ /dev/null @@ -1,76 +0,0 @@ -#ifndef DP_ADVANCE_IOP_NUDGING_IMPL_HPP -#define DP_ADVANCE_IOP_NUDGING_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp advance_iop_nudging. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::advance_iop_nudging( - // Input arguments - const Int& plev, - const Scalar& scm_dt, - const Scalar& ps_in, - const uview_1d& t_in, - const uview_1d& q_in, - const uview_1d& tobs, - const uview_1d& qobs, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - // Kokkos stuff - const MemberType& team, - const Workspace& workspace, - // Output arguments - const uview_1d& t_update, - const uview_1d& q_update, - const uview_1d& relaxt, - const uview_1d& relaxq) -{ - // Local variables - uview_1d - pmidm1, // pressure at model levels - pintm1, // pressure at model interfaces (dim=plev+1) - pdelm1, // pdel(k) = pint (k+1)-pint (k) - rtau; - workspace.template take_many_contiguous_unsafe<4>( - {"pmidm1", "pintm1", "pdelm1", "rtau"}, - {&pmidm1, &pintm1, &pdelm1, &rtau}); - - // Get vertical level profiles - plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); - - const Int plev_pack = ekat::npack(plev); - constexpr Scalar iop_nudge_tq_low = DPC::iop_nudge_tq_low; - constexpr Scalar iop_nudge_tq_high = DPC::iop_nudge_tq_high; - constexpr Scalar iop_nudge_tscale = DPC::iop_nudge_tscale; - - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - relaxt(k) = 0; - relaxq(k) = 0; - - const auto condition = pmidm1(k) <= iop_nudge_tq_low*100 - && - pmidm1(k) >= iop_nudge_tq_high*100; - rtau(k).set(condition, ekat::impl::max(scm_dt, iop_nudge_tscale)); - relaxt(k).set(condition, (t_in(k) - tobs(k))/rtau(k)); - relaxq(k).set(condition, (q_in(k) - qobs(k))/rtau(k)); - - t_update(k).set(condition, t_in(k) + relaxt(k)*scm_dt); - q_update(k).set(condition, q_in(k) + relaxq(k)*scm_dt); - }); -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp deleted file mode 100644 index 3fc7a53320a8..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_advance_iop_subsidence_impl.hpp +++ /dev/null @@ -1,164 +0,0 @@ -#ifndef DP_ADVANCE_IOP_SUBSIDENCE_IMPL_HPP -#define DP_ADVANCE_IOP_SUBSIDENCE_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -#include "ekat/kokkos/ekat_subview_utils.hpp" - -namespace scream { -namespace dp { - -/* - * Implementation of dp advance_iop_subsidence. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_INLINE_FUNCTION -void Functions::do_advance_iop_subsidence_update( - const Int& k, - const Int& plev, - const Spack& fac, - const Spack& swfldint, - const Spack& swfldint_p1, - const uview_1d& in, - const uview_1d& in_s, - const uview_1d& update) -{ - Spack sin, sin_p1, sin_m1; - - auto range_pack1 = ekat::range(k*Spack::n); - auto range_pack2_m1_safe = range_pack1; - auto range_pack2_p1_safe = range_pack1; - range_pack2_m1_safe.set(range_pack1 < 1, 1); // don't want the shift to go below zero. we mask out that result anyway - range_pack2_p1_safe.set(range_pack1 > plev-2, plev-2); // don't want the shift to go beyond pack. - ekat::index_and_shift<-1>(in_s, range_pack2_m1_safe, sin, sin_m1); - ekat::index_and_shift< 1>(in_s, range_pack2_p1_safe, sin, sin_p1); - - update(k) = in(k) - fac*(swfldint_p1*(sin_p1 - sin) + swfldint*(sin - sin_m1)); -} - -template -KOKKOS_FUNCTION -void Functions::advance_iop_subsidence( - const Int& plev, - const Int& pcnst, - const Scalar& scm_dt, - const Scalar& ps_in, - const uview_1d& u_in, - const uview_1d& v_in, - const uview_1d& t_in, - const uview_2d& q_in, - const uview_1d& hyai, - const uview_1d& hyam, - const uview_1d& hybi, - const uview_1d& hybm, - const uview_1d& wfld, - const MemberType& team, - const Workspace& workspace, - const uview_1d& u_update, - const uview_1d& v_update, - const uview_1d& t_update, - const uview_2d& q_update) -{ - // Local variables - uview_1d - pmidm1, // pressure at model levels - pintm1, // pressure at model interfaces (dim=plev+1) - pdelm1, // pdel(k) = pint (k+1)-pint (k) - wfldint;// (dim=plev+1) - workspace.template take_many_contiguous_unsafe<4>( - {"pmidm1", "pintm1", "pdelm1", "wfldint"}, - {&pmidm1, &pintm1, &pdelm1, &wfldint}); - - const Int plev_pack = ekat::npack(plev); - - // Get vertical level profiles - plevs0(plev, ps_in, hyai, hyam, hybi, hybm, team, pintm1, pmidm1, pdelm1); - - // Scalarize a bunch of views that need shift operations - auto pmidm1_s = scalarize(pmidm1); - auto pintm1_s = scalarize(pintm1); - auto wfld_s = scalarize(wfld); - auto wfldint_s = scalarize(wfldint); - auto u_in_s = scalarize(u_in); - auto v_in_s = scalarize(v_in); - auto t_in_s = scalarize(t_in); - - wfldint_s(0) = 0; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - Spack spmidm1, spmidm1_m1, spintm1, spintm1_m1, swfld, swfld_m1; - auto range_pack1 = ekat::range(k*Spack::n); - auto range_pack2 = range_pack1; - range_pack2.set(range_pack1 < 1, 1); // don't want the shift to go below zero. we mask out that result anyway - ekat::index_and_shift<-1>(pmidm1_s, range_pack2, spmidm1, spmidm1_m1); - ekat::index_and_shift<-1>(pintm1_s, range_pack2, spintm1, spintm1_m1); - ekat::index_and_shift<-1>(wfld_s, range_pack2, swfld, swfld_m1); - Spack weight = (spintm1 - spintm1_m1) / (spmidm1 - spmidm1_m1); - wfldint(k) = (1 - weight)*swfld_m1 + weight*swfld; - }); - wfldint_s(plev) = 0; - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - Spack swfldint, swfldint_p1; - auto range_pack1 = ekat::range(k*Spack::n); - auto range_pack2 = range_pack1; - range_pack2.set(range_pack1 > plev-1, plev-1); - - ekat::index_and_shift<1>(wfldint_s, range_pack2, swfldint, swfldint_p1); - - Spack fac = scm_dt/(2 * pdelm1(k)); - - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, u_in, u_in_s, u_update); - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, v_in, v_in_s, v_update); - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, t_in, t_in_s, t_update); - - for (Int m = 0; m < pcnst; ++m) { - // Grab m-th subview of q stuff - auto q_update_sub = ekat::subview(q_update, m); - auto q_in_sub = ekat::subview(q_in, m); - auto q_in_sub_s = scalarize(q_in_sub); - do_advance_iop_subsidence_update(k, plev, fac, swfldint, swfldint_p1, q_in_sub, q_in_sub_s, q_update_sub); - } - }); - - // Top and bottom levels next - Kokkos::Array bot_top = {0, plev-1}; - for (Int i = 0; i < 2; ++i) { - const auto k = bot_top[i]; - const auto pack_idx = ekat::npack(k+1) - 1; - const auto s_idx = k % Spack::n; - const auto idx1 = k == 0 ? k+1 : k; - - Scalar fac = scm_dt/(2 * pdelm1(pack_idx)[s_idx]); - u_update(pack_idx)[s_idx] = u_in_s(k) - fac*(wfldint_s(idx1)*(u_in_s(idx1) - u_in_s(idx1-1))); - v_update(pack_idx)[s_idx] = v_in_s(k) - fac*(wfldint_s(idx1)*(v_in_s(idx1) - v_in_s(idx1-1))); - t_update(pack_idx)[s_idx] = t_in_s(k) - fac*(wfldint_s(idx1)*(t_in_s(idx1) - t_in_s(idx1-1))); - - for (Int m = 0; m < pcnst; ++m) { - auto q_update_sub = ekat::subview(q_update, m); - auto q_in_sub = ekat::subview(q_in, m); - auto q_in_sub_s = scalarize(q_in_sub); - - q_update_sub(pack_idx)[s_idx] = q_in_sub_s(k) - fac*(wfldint_s(idx1)*(q_in_sub_s(idx1) - q_in_sub_s(idx1-1))); - } - } - - // thermal expansion term due to LS vertical advection - constexpr Scalar rair = C::Rair; - constexpr Scalar cpair = C::Cpair; - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_pack), [&] (Int k) { - t_update(k) = t_update(k) + scm_dt*wfld(k)*t_in(k)*rair/(cpair*pmidm1(k)); - }); - - workspace.template release_many_contiguous<4>( - {&pmidm1, &pintm1, &pdelm1, &wfldint}); -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp deleted file mode 100644 index cacae8b46ddd..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_apply_iop_forcing_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_APPLY_IOP_FORCING_IMPL_HPP -#define DP_APPLY_IOP_FORCING_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp apply_iop_forcing. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::apply_iop_forcing(const Int& nelemd, const uview_1d& elem, hvcoord_t& hvcoord, const hybrid_t& hybrid, const timelevel_t& tl, const Int& n, const bool& t_before_advance, const Int& nets, const Int& nete) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp deleted file mode 100644 index 2e17f7a43c5b..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_crm_resolved_turb_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_CRM_RESOLVED_TURB_IMPL_HPP -#define DP_CRM_RESOLVED_TURB_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp crm_resolved_turb. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::crm_resolved_turb(const Int& nelemd, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const Int& nelemd_todo, const Int& np_todo) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp deleted file mode 100644 index ead8ec78a2b6..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_broadcast_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_BROADCAST_IMPL_HPP -#define DP_IOP_BROADCAST_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_broadcast. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_broadcast() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp deleted file mode 100644 index e5687d7558bb..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_default_opts_impl.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef DP_IOP_DEFAULT_OPTS_IMPL_HPP -#define DP_IOP_DEFAULT_OPTS_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_default_opts. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -void Functions::iop_default_opts(Spack& scmlat_out, Spack& scmlon_out, std::string& iopfile_out, bool& single_column_out, bool& scm_iop_srf_prop_out, bool& iop_nudge_tq_out, bool& iop_nudge_uv_out, Spack& iop_nudge_tq_low_out, Spack& iop_nudge_tq_high_out, Spack& iop_nudge_tscale_out, bool& scm_observed_aero_out, bool& iop_dosubsidence_out, bool& scm_multcols_out, bool& dp_crm_out, Spack& iop_perturb_high_out, bool& precip_off_out, bool& scm_zero_non_iop_tracers_out) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp deleted file mode 100644 index ad36a9a0ce51..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_domain_relaxation_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_DOMAIN_RELAXATION_IMPL_HPP -#define DP_IOP_DOMAIN_RELAXATION_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_domain_relaxation. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_domain_relaxation(const Int& nelemd, const Int& np, const Int& nlev, const uview_1d& elem, const hvcoord_t& hvcoord, const hybrid_t& hybrid, const Int& t1, const uview_1d& dp, const Int& nelemd_todo, const Int& np_todo, const Spack& dt) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp deleted file mode 100644 index e03631f94cb7..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_intht_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_INTHT_IMPL_HPP -#define DP_IOP_INTHT_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_intht. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_intht() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp deleted file mode 100644 index ee8991c472ba..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_setfield_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_IOP_SETFIELD_IMPL_HPP -#define DP_IOP_SETFIELD_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_setfield. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::iop_setfield(const Int& nelemd, const uview_1d& elem, const bool& iop_update_phase1) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp deleted file mode 100644 index 854ced7dd48d..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_setinitial_impl.hpp +++ /dev/null @@ -1,207 +0,0 @@ -#ifndef DP_IOP_SETINITIAL_IMPL_HPP -#define DP_IOP_SETINITIAL_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -#include "Context.hpp" -#include "TimeLevel.hpp" - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_setinitial. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -void Functions::iop_setinitial( - const Int& plev, - const Int& pcnst, - const Int& nelemd, - const Int& np, - const Int& nstep, - const bool& use_replay, - const bool& dynproc, - const bool& have_t, - const bool& have_q, - const bool& have_ps, - const bool& have_u, - const bool& have_v, - const bool& have_numliq, - const bool& have_cldliq, - const bool& have_numice, - const bool& have_cldice, - const bool& scm_zero_non_iop_tracers, - const bool& is_first_restart_step, - const uview_1d& qmin, - const uview_1d& uobs, - const uview_1d& vobs, - const uview_1d& numliqobs, - const uview_1d& numiceobs, - const uview_1d& cldliqobs, - const uview_1d& cldiceobs, - const Scalar& psobs, - const uview_1d& dx_short, - Scalar& dyn_dx_size, - tracer_t& tracers, - element_t& elem, - const uview_1d& tobs, - const uview_1d& qobs) -{ - // Made these up - - //FieldGroup g = ...; - //const auto& names = g.m_info->m_field_names; - constexpr Int inumliq = 1; - constexpr Int inumice = 2; - constexpr Int icldliq = 3; - constexpr Int icldice = 4; - - const Int plev_packs = ekat::npack(plev); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(1, plev_packs); - - assert(np <= NP); - //assert(plev_packs <= Homme::NUM_LEV); - assert(tracers.inited()); - assert(elem.inited()); - - // Get time info - const Int n0 = Homme::Context::singleton().get().n0; - - if (!use_replay && nstep == 0 && dynproc) { - - Kokkos::parallel_for( - "iop_setinitial loop", - policy, - KOKKOS_LAMBDA(const MemberType& team) { - - // We cannot support parallelism at the element level because the thelev - // computation of tobs is dependent on prior iterations that may have alterted tobs - for (Int ie = 0; ie < nelemd; ++ie) { - for (Int j = 0; j < np; ++j) { - for (Int i = 0; i < np; ++i) { - - // Find level where tobs is no longer zero - Int thelev=-1; - Kokkos::parallel_reduce( - Kokkos::TeamVectorRange(team, plev_packs), [&] (int plev_pack, Int& pmin) { - auto zmask = tobs(plev_pack) != 0; - if (zmask.any()) { - pmin = plev_pack; - } - }, Kokkos::Min(thelev)); - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, thelev+1), [&] (int k) { - auto zmask = k < thelev ? Smask(true) : tobs(k) == 0; - vector_simd for (Int p = 0; p < Spack::n; ++p) { - if (zmask[p]) { - tobs(k)[p] = elem.m_forcing.m_ft(ie,i,j,k)[p]; - qobs(k)[p] = tracers.Q(ie,0,i,j,k)[p]; // Tracer index 0 is qobs - } - } - }); - - if (scm_zero_non_iop_tracers) { - for (Int cix = 0; cix < pcnst; ++cix) { - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_packs), [&] (int k) { - tracers.Q(ie, cix, i, j, k) = qmin(cix); - }); - } - } - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, thelev, plev_packs), [&] (int k) { - auto zmask = k > thelev ? Smask(true) : tobs(k) != 0; - if (have_t) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - if (zmask[p]) { - elem.m_forcing.m_ft(ie,i,j,k)[p] = tobs(k)[p]; - } - } - } - if (have_q) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - if (zmask[p]) { - tracers.Q(ie,0,i,j,k)[p] = qobs(k)[p]; - } - } - } - }); - - if (have_ps) { - elem.m_state.m_ps_v(ie, n0, i, j) = psobs; - } - - Kokkos::parallel_for( - Kokkos::TeamVectorRange(team, plev_packs), [&] (int k) { - if (have_u) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - elem.m_state.m_v(ie, 0, 0, i, j, k)[p] = uobs(k)[p]; // [2] is 0 for u - } - } - if (have_v) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - elem.m_state.m_v(ie, 0, 1, i, j, k)[p] = vobs(k)[p]; // [2] is 1 for v - } - } - if (have_numliq) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, inumliq, i, j, k)[p] = numliqobs(k)[p]; - } - } - if (have_cldliq) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, icldliq, i, j, k)[p] = cldliqobs(k)[p]; - } - } - if (have_numice) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, inumice, i, j, k)[p] = numiceobs(k)[p]; - } - } - if (have_cldice) { - vector_simd for (Int p = 0; p < Spack::n; ++p) { - tracers.Q(ie, icldice, i, j, k)[p] = cldiceobs(k)[p]; - } - } - - // If DP-CRM mode we do NOT want to write over the dy-core vertical - // velocity with the large-scale one. wfld is used in forecast.F90 - // for the compuation of the large-scale subsidence. - elem.m_derived.m_omega_p(ie, i, j, k) = 0; - }); - } - } - } - }); - } - - // If DP-CRM mode then SHOC/CLUBB needs to know about grid - // length size. The calculations of this based on a sphere in the - // SHOC and CLUBB interefaces are not valid for a planar grid, thus - // save the grid length from the dycore. Note that planar dycore - // only supports uniform grids, thus we only save one value. - // Set this if it is the first time step or the first restart step - if ( (nstep == 0 || is_first_restart_step) && dynproc) { - // for (Int ie = 0; ie < nelemd; ++ie) { - // dyn_dx_size = dx_short(ie) * 1000; // why not just grab the last ie? - // } - Kokkos::parallel_reduce( - "iop_setinitial loop", - policy, - KOKKOS_LAMBDA(const MemberType& team, Scalar& dyn) { - const Int ie = team.league_rank(); - if (ie == nelemd-1) { - dyn = dx_short(nelemd-1) * 1000; - } - }, dyn_dx_size); - } -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp deleted file mode 100644 index 5483abdd6a41..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_iop_setopts_impl.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef DP_IOP_SETOPTS_IMPL_HPP -#define DP_IOP_SETOPTS_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp iop_setopts. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -void Functions::iop_setopts(const Spack& scmlat_in, const Spack& scmlon_in, const std::string& iopfile_in, const bool& single_column_in, const bool& scm_iop_srf_prop_in, const bool& iop_nudge_tq_in, const bool& iop_nudge_uv_in, const Spack& iop_nudge_tq_low_in, const Spack& iop_nudge_tq_high_in, const Spack& iop_nudge_tscale_in, const bool& scm_observed_aero_in, const bool& iop_dosubsidence_in, const bool& scm_multcols_in, const bool& dp_crm_in, const Spack& iop_perturb_high_in, const bool& precip_off_in, const bool& scm_zero_non_iop_tracers_in) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp deleted file mode 100644 index 3cd83bfcdb06..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_readiopdata_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_READIOPDATA_IMPL_HPP -#define DP_READIOPDATA_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp readiopdata. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::readiopdata(const Int& plev, const bool& iop_update_phase1, const uview_1d& hyam, const uview_1d& hybm) -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp deleted file mode 100644 index 78e1bfe7340c..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_SETIOPUPDATE_IMPL_HPP -#define DP_SETIOPUPDATE_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp setiopupdate. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::setiopupdate() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp b/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp deleted file mode 100644 index bc9ba8d41c61..000000000000 --- a/components/eamxx/src/doubly-periodic/impl/dp_setiopupdate_init_impl.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DP_SETIOPUPDATE_INIT_IMPL_HPP -#define DP_SETIOPUPDATE_INIT_IMPL_HPP - -#include "dp_functions.hpp" // for ETI only but harmless for GPU - -namespace scream { -namespace dp { - -/* - * Implementation of dp setiopupdate_init. Clients should NOT - * #include this file, but include dp_functions.hpp instead. - */ - -template -KOKKOS_FUNCTION -void Functions::setiopupdate_init() -{ - // TODO - // Note, argument types may need tweaking. Generator is not always able to tell what needs to be packed -} - -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt b/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt deleted file mode 100644 index 32e2883a22f6..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -INCLUDE (ScreamUtils) - -set(DP_TESTS_SRCS - dp_unit_tests.cpp - dp_advance_iop_forcing_tests.cpp - dp_advance_iop_nudging_tests.cpp - dp_advance_iop_subsidence_tests.cpp - dp_iop_setinitial_tests.cpp - dp_iop_broadcast_tests.cpp - dp_apply_iop_forcing_tests.cpp - dp_iop_domain_relaxation_tests.cpp - dp_crm_resolved_turb_tests.cpp - dp_iop_default_opts_tests.cpp - dp_iop_setopts_tests.cpp - dp_setiopupdate_init_tests.cpp - dp_setiopupdate_tests.cpp - dp_readiopdata_tests.cpp - dp_iop_intht_tests.cpp - ) # DP_TESTS_SRCS - -# NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) - CreateUnitTest(dp_tests "${DP_TESTS_SRCS}" - LIBS dp - THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} - ) -endif() diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp deleted file mode 100644 index d1f5ea89a42e..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_forcing_tests.cpp +++ /dev/null @@ -1,116 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAdvanceIopForcing { - - static void run_bfb() - { - auto engine = setup_random_test(); - - AdvanceIopForcingData f90_data[] = { - // plev, pcnst, scm_dt, ps_in, have_u, have_v, dp_crm, use_3dfrc - AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, true, true), - AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, true, false), - AdvanceIopForcingData(72, 10, 0.1, 1000.0, true, true, false, true), - - AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, true, true), - AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, true, false), - AdvanceIopForcingData(27, 7, 0.1, 1000.0, true, true, false, true), - - AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, true, true), - AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, true, false), - AdvanceIopForcingData(32, 7, 0.1, 1000.0, true, true, false, true), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopForcingData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - AdvanceIopForcingData cxx_data[num_runs] = { - AdvanceIopForcingData(f90_data[0]), - AdvanceIopForcingData(f90_data[1]), - AdvanceIopForcingData(f90_data[2]), - AdvanceIopForcingData(f90_data[3]), - AdvanceIopForcingData(f90_data[4]), - AdvanceIopForcingData(f90_data[5]), - AdvanceIopForcingData(f90_data[6]), - AdvanceIopForcingData(f90_data[7]), - AdvanceIopForcingData(f90_data[8]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - advance_iop_forcing(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.template transpose(); // _f expects data in fortran layout - advance_iop_forcing_f(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.have_u, d.have_v, d.dp_crm, d.use_3dfrc, d.u_in, d.v_in, d.t_in, d.q_in, d.t_phys_frc, d.divt3d, d.divq3d, d.divt, d.divq, d.wfld, d.uobs, d.vobs, d.hyai, d.hyam, d.hybi, d.hybm, d.u_update, d.v_update, d.t_update, d.q_update); - d.template transpose(); // go back to C layout - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - AdvanceIopForcingData& d_f90 = f90_data[i]; - AdvanceIopForcingData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.u_update); ++k) { - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.u_update)); - REQUIRE(d_f90.u_update[k] == d_cxx.u_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.v_update)); - REQUIRE(d_f90.v_update[k] == d_cxx.v_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.t_update)); - REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); - } - for (Int k = 0; k < d_f90.total(d_f90.q_update); ++k) { - REQUIRE(d_f90.total(d_f90.q_update) == d_cxx.total(d_cxx.q_update)); - REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); - } - - } - } -#endif - - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("advance_iop_forcing_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopForcing; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp deleted file mode 100644 index 02f9362eb7a3..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_nudging_tests.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAdvanceIopNudging { - - static void run_bfb() - { - auto engine = setup_random_test(); - - AdvanceIopNudgingData f90_data[] = { - // plev, scm_dt, ps_in - AdvanceIopNudgingData(72, 0.1, 1000), - AdvanceIopNudgingData(27, 0.1, 1000), - AdvanceIopNudgingData(32, 0.1, 1000), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopNudgingData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - AdvanceIopNudgingData cxx_data[] = { - AdvanceIopNudgingData(f90_data[0]), - AdvanceIopNudgingData(f90_data[1]), - AdvanceIopNudgingData(f90_data[2]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - advance_iop_nudging(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - advance_iop_nudging_f(d.plev, d.scm_dt, d.ps_in, d.t_in, d.q_in, d.tobs, d.qobs, - d.hyai, d.hyam, d.hybi, d.hybm, - d.t_update, d.q_update, d.relaxt, d.relaxq); - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - AdvanceIopNudgingData& d_f90 = f90_data[i]; - AdvanceIopNudgingData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.t_update); ++k) { - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.t_update)); - REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.q_update)); - REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.relaxt)); - REQUIRE(d_f90.relaxt[k] == d_cxx.relaxt[k]); - REQUIRE(d_f90.total(d_f90.t_update) == d_cxx.total(d_cxx.relaxq)); - REQUIRE(d_f90.relaxq[k] == d_cxx.relaxq[k]); - } - - } - } -#endif - } // run_bfb -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("advance_iop_nudging_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopNudging; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp deleted file mode 100644 index a430135f75be..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_advance_iop_subsidence_tests.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestAdvanceIopSubsidence { - - static void run_bfb() - { - auto engine = setup_random_test(); - - AdvanceIopSubsidenceData f90_data[] = { - // plev, pcnst, scm_dt, ps_in - AdvanceIopSubsidenceData(72, 10, 0.1, 1000.0), - AdvanceIopSubsidenceData(72, 7, 0.1, 1000.0), - AdvanceIopSubsidenceData(27, 10, 0.1, 1000.0), - AdvanceIopSubsidenceData(27, 7, 0.1, 1000.0), - AdvanceIopSubsidenceData(32, 10, 0.1, 1000.0), - AdvanceIopSubsidenceData(32, 7, 0.1, 1000.0), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(AdvanceIopSubsidenceData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - AdvanceIopSubsidenceData cxx_data[num_runs] = { - AdvanceIopSubsidenceData(f90_data[0]), - AdvanceIopSubsidenceData(f90_data[1]), - AdvanceIopSubsidenceData(f90_data[2]), - AdvanceIopSubsidenceData(f90_data[3]), - AdvanceIopSubsidenceData(f90_data[4]), - AdvanceIopSubsidenceData(f90_data[5]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - advance_iop_subsidence(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.template transpose(); // _f expects data in fortran layout - advance_iop_subsidence_f(d.plev, d.pcnst, d.scm_dt, d.ps_in, d.u_in, d.v_in, d.t_in, d.q_in, d.hyai, d.hyam, d.hybi, d.hybm, d.wfld, d.u_update, d.v_update, d.t_update, d.q_update); - d.template transpose(); // go back to C layout - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - AdvanceIopSubsidenceData& d_f90 = f90_data[i]; - AdvanceIopSubsidenceData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.u_update); ++k) { - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.u_update)); - REQUIRE(d_f90.u_update[k] == d_cxx.u_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.v_update)); - REQUIRE(d_f90.v_update[k] == d_cxx.v_update[k]); - REQUIRE(d_f90.total(d_f90.u_update) == d_cxx.total(d_cxx.t_update)); - REQUIRE(d_f90.t_update[k] == d_cxx.t_update[k]); - } - for (Int k = 0; k < d_f90.total(d_f90.q_update); ++k) { - REQUIRE(d_f90.total(d_f90.q_update) == d_cxx.total(d_cxx.q_update)); - REQUIRE(d_f90.q_update[k] == d_cxx.q_update[k]); - } - - } - } -#endif - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("advance_iop_subsidence_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestAdvanceIopSubsidence; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp deleted file mode 100644 index 3e85e30974bc..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_apply_iop_forcing_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestApplyIopForcing { - - static void run_bfb() - { - auto engine = setup_random_test(); - - ApplyIopForcingData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ApplyIopForcingData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - ApplyIopForcingData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - apply_iop_forcing(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - apply_iop_forcing_f(d.nelemd, d.elem, &d.hvcoord, d.hybrid, d.tl, d.n, d.t_before_advance, d.nets, d.nete); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - ApplyIopForcingData& d_f90 = f90_data[i]; - ApplyIopForcingData& d_cxx = cxx_data[i]; - //REQUIRE(d_f90.hvcoord == d_cxx.hvcoord); - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("apply_iop_forcing_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestApplyIopForcing; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp deleted file mode 100644 index f56d24880606..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_crm_resolved_turb_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestCrmResolvedTurb { - - static void run_bfb() - { - auto engine = setup_random_test(); - - CrmResolvedTurbData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(CrmResolvedTurbData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - CrmResolvedTurbData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - crm_resolved_turb(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - crm_resolved_turb_f(d.nelemd, d.elem, d.hvcoord, d.hybrid, d.t1, d.nelemd_todo, d.np_todo); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - CrmResolvedTurbData& d_f90 = f90_data[i]; - CrmResolvedTurbData& d_cxx = cxx_data[i]; - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("crm_resolved_turb_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestCrmResolvedTurb; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp deleted file mode 100644 index 0ffb8f63e5a5..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_broadcast_tests.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopBroadcast { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopBroadcastData f90_data[max_pack_size]; //= { - // // TODO - // }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopBroadcastData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - -#if 0 - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - iop_broadcast(d); - } - - // Get data from cxx. Run iop_broadcast from a kernel and copy results back to host - // Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - // const Int offset = i * Spack::n; - - - - - // Functions::iop_broadcast(); - - - // }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopBroadcastData& d_f90 = f90_data[i]; - IopBroadcastData& d_cxx = cxx_host[i]; - } - } -#endif - } // run_bfb -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_broadcast_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopBroadcast; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp deleted file mode 100644 index 622ca26a7264..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_default_opts_tests.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopDefaultOpts { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopDefaultOptsData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = 0; //sizeof(f90_data) / sizeof(IopDefaultOptsData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - using host_view = typename view_1d::host_mirror_type; - host_view cxx_host("cxx_host", max_pack_size); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - - // Get data from fortran - for (auto& d : f90_data) { - iop_default_opts(d); - } - - // Get data from cxx. Run iop_default_opts from a kernel and copy results back to host - for (Int i = 0; i < num_test_itrs; ++i) { - const Int offset = i * Spack::n; - - // Init outputs - Spack iop_nudge_tq_high_out(0), iop_nudge_tq_low_out(0), iop_nudge_tscale_out(0), iop_perturb_high_out(0), scmlat_out(0), scmlon_out(0); - - Functions::iop_default_opts(scmlat_out, scmlon_out, cxx_host(0).iopfile_out, cxx_host(0).single_column_out, cxx_host(0).scm_iop_srf_prop_out, cxx_host(0).iop_nudge_tq_out, cxx_host(0).iop_nudge_uv_out, iop_nudge_tq_low_out, iop_nudge_tq_high_out, iop_nudge_tscale_out, cxx_host(0).scm_observed_aero_out, cxx_host(0).iop_dosubsidence_out, cxx_host(0).scm_multcols_out, cxx_host(0).dp_crm_out, iop_perturb_high_out, cxx_host(0).precip_off_out, cxx_host(0).scm_zero_non_iop_tracers_out); - - // Copy spacks back into cxx_host view - for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { - cxx_host(vs).iop_nudge_tq_high_out = iop_nudge_tq_high_out[s]; - cxx_host(vs).iop_nudge_tq_low_out = iop_nudge_tq_low_out[s]; - cxx_host(vs).iop_nudge_tscale_out = iop_nudge_tscale_out[s]; - cxx_host(vs).iop_perturb_high_out = iop_perturb_high_out[s]; - cxx_host(vs).scmlat_out = scmlat_out[s]; - cxx_host(vs).scmlon_out = scmlon_out[s]; - } - } - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopDefaultOptsData& d_f90 = f90_data[i]; - IopDefaultOptsData& d_cxx = cxx_host[i]; - REQUIRE(d_f90.scmlat_out == d_cxx.scmlat_out); - REQUIRE(d_f90.scmlon_out == d_cxx.scmlon_out); - REQUIRE(d_f90.iopfile_out == d_cxx.iopfile_out); - REQUIRE(d_f90.single_column_out == d_cxx.single_column_out); - REQUIRE(d_f90.scm_iop_srf_prop_out == d_cxx.scm_iop_srf_prop_out); - REQUIRE(d_f90.iop_nudge_tq_out == d_cxx.iop_nudge_tq_out); - REQUIRE(d_f90.iop_nudge_uv_out == d_cxx.iop_nudge_uv_out); - REQUIRE(d_f90.iop_nudge_tq_low_out == d_cxx.iop_nudge_tq_low_out); - REQUIRE(d_f90.iop_nudge_tq_high_out == d_cxx.iop_nudge_tq_high_out); - REQUIRE(d_f90.iop_nudge_tscale_out == d_cxx.iop_nudge_tscale_out); - REQUIRE(d_f90.scm_observed_aero_out == d_cxx.scm_observed_aero_out); - REQUIRE(d_f90.iop_dosubsidence_out == d_cxx.iop_dosubsidence_out); - REQUIRE(d_f90.scm_multcols_out == d_cxx.scm_multcols_out); - REQUIRE(d_f90.dp_crm_out == d_cxx.dp_crm_out); - REQUIRE(d_f90.iop_perturb_high_out == d_cxx.iop_perturb_high_out); - REQUIRE(d_f90.precip_off_out == d_cxx.precip_off_out); - REQUIRE(d_f90.scm_zero_non_iop_tracers_out == d_cxx.scm_zero_non_iop_tracers_out); - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_default_opts_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopDefaultOpts; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp deleted file mode 100644 index ed570c6767a4..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_domain_relaxation_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopDomainRelaxation { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopDomainRelaxationData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopDomainRelaxationData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - IopDomainRelaxationData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - iop_domain_relaxation(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.transpose(); // _f expects data in fortran layout - iop_domain_relaxation_f(d.nelemd, d.np, d.nlev, d.elem, d.hvcoord, d.hybrid, d.t1, d.dp, d.nelemd_todo, d.np_todo, d.dt); - d.transpose(); // go back to C layout - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopDomainRelaxationData& d_f90 = f90_data[i]; - IopDomainRelaxationData& d_cxx = cxx_data[i]; - for (Int k = 0; k < d_f90.total(d_f90.dp); ++k) { - REQUIRE(d_f90.total(d_f90.dp) == d_cxx.total(d_cxx.dp)); - REQUIRE(d_f90.dp[k] == d_cxx.dp[k]); - } - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_domain_relaxation_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopDomainRelaxation; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp deleted file mode 100644 index f214ebb49961..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_intht_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopIntht { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopInthtData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopInthtData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - iop_intht(d); - } - - // Get data from cxx. Run iop_intht from a kernel and copy results back to host - Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - const Int offset = i * Spack::n; - - - - - Functions::iop_intht(); - - - }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopInthtData& d_f90 = f90_data[i]; - IopInthtData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_intht_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopIntht; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp deleted file mode 100644 index 04921cbcbfb8..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_setfield_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopSetfield { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopSetfieldData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetfieldData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - IopSetfieldData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - iop_setfield(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - iop_setfield_f(d.nelemd, d.elem, d.iop_update_phase1); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopSetfieldData& d_f90 = f90_data[i]; - IopSetfieldData& d_cxx = cxx_data[i]; - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_setfield_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetfield; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp deleted file mode 100644 index d7e1d6d7a7df..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_setinitial_tests.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopSetinitial { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopSetinitialData f90_data[] = { - // plev, pcnst, nelemd, np, nstep, psobs, use_replay, dynproc, have_t, have_q, have_ps, have_u, have_v, have_numliq, have_cldliq, have_numice, have_cldice, scm_zero_non_iop_tracers, is_first_restart_step - IopSetinitialData(72 , 5 , 10 , 2 , 10, 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 10, 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , false), - IopSetinitialData(72 , 5 , 10 , 2 , 1 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , false), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, true , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, false , true , true , true , true , true , true , true , true , true , true , true , true), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, true , true , true , true , true , true , true , true , true , true , true , false , true), - IopSetinitialData(72 , 5 , 10 , 2 , 0 , 0.1, false , true , true , true , true , true , true , true , true , true , true , false , true), - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetinitialData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - IopSetinitialData cxx_data[] = { - IopSetinitialData(f90_data[0]), - IopSetinitialData(f90_data[1]), - IopSetinitialData(f90_data[2]), - IopSetinitialData(f90_data[3]), - IopSetinitialData(f90_data[4]), - IopSetinitialData(f90_data[5]), - IopSetinitialData(f90_data[6]), - IopSetinitialData(f90_data[7]), - IopSetinitialData(f90_data[8]), - IopSetinitialData(f90_data[9]), - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - iop_setinitial(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - d.init(); - iop_setinitial_f(d.plev, d.pcnst, d.nelemd, d.np, d.nstep, d.psobs, d.use_replay, d.dynproc, d.have_t, d.have_q, d.have_ps, d.have_u, d.have_v, d.have_numliq, d.have_cldliq, d.have_numice, d.have_cldice, d.scm_zero_non_iop_tracers, d.is_first_restart_step, d.qmin, d.uobs, d.vobs, d.numliqobs, d.numiceobs, d.cldliqobs, d.cldiceobs, d.dx_short, &d.tracers, &d.elem, &d.dyn_dx_size, d.tobs, d.qobs); - } - - // We can't call into fortran. Due to all the dependencies it has, it's not possible - // to build it in standalone eamxx. Without fortran, we cannot do BFB tests. -#if 0 - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopSetinitialData& d_f90 = f90_data[i]; - IopSetinitialData& d_cxx = cxx_data[i]; - - } - } -#endif - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_setinitial_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetinitial; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp deleted file mode 100644 index d24f54fb634a..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_iop_setopts_tests.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestIopSetopts { - - static void run_bfb() - { - auto engine = setup_random_test(); - - IopSetoptsData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(IopSetoptsData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - using host_view = typename view_1d::host_mirror_type; - host_view cxx_host("cxx_host", max_pack_size); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - - // Get data from fortran - for (auto& d : f90_data) { - iop_setopts(d); - } - - // Get data from cxx. Run iop_setopts from a kernel and copy results back to host - for (Int i = 0; i < num_test_itrs; ++i) { - const Int offset = i * Spack::n; - - // Init pack inputs - Spack iop_nudge_tq_high_in, iop_nudge_tq_low_in, iop_nudge_tscale_in, iop_perturb_high_in, scmlat_in, scmlon_in; - for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { - iop_nudge_tq_high_in[s] = cxx_host(vs).iop_nudge_tq_high_in; - iop_nudge_tq_low_in[s] = cxx_host(vs).iop_nudge_tq_low_in; - iop_nudge_tscale_in[s] = cxx_host(vs).iop_nudge_tscale_in; - iop_perturb_high_in[s] = cxx_host(vs).iop_perturb_high_in; - scmlat_in[s] = cxx_host(vs).scmlat_in; - scmlon_in[s] = cxx_host(vs).scmlon_in; - } - - Functions::iop_setopts(scmlat_in, scmlon_in, cxx_host(0).iopfile_in, cxx_host(0).single_column_in, cxx_host(0).scm_iop_srf_prop_in, cxx_host(0).iop_nudge_tq_in, cxx_host(0).iop_nudge_uv_in, iop_nudge_tq_low_in, iop_nudge_tq_high_in, iop_nudge_tscale_in, cxx_host(0).scm_observed_aero_in, cxx_host(0).iop_dosubsidence_in, cxx_host(0).scm_multcols_in, cxx_host(0).dp_crm_in, iop_perturb_high_in, cxx_host(0).precip_off_in, cxx_host(0).scm_zero_non_iop_tracers_in); - } - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - IopSetoptsData& d_f90 = f90_data[i]; - IopSetoptsData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("iop_setopts_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestIopSetopts; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp deleted file mode 100644 index bef219811845..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_readiopdata_tests.cpp +++ /dev/null @@ -1,78 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestReadiopdata { - - static void run_bfb() - { - auto engine = setup_random_test(); - - ReadiopdataData f90_data[] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(ReadiopdataData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx. Needs to happen before fortran calls so that - // inout data is in original state - ReadiopdataData cxx_data[] = { - // TODO - }; - - // Assume all data is in C layout - - // Get data from fortran - for (auto& d : f90_data) { - // expects data in C layout - readiopdata(d); - } - - // Get data from cxx - for (auto& d : cxx_data) { - readiopdata_f(d.plev, d.iop_update_phase1, d.hyam, d.hybm); - } - - // Verify BFB results, all data should be in C layout - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - ReadiopdataData& d_f90 = f90_data[i]; - ReadiopdataData& d_cxx = cxx_data[i]; - - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("readiopdata_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestReadiopdata; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp deleted file mode 100644 index b60fc5f8d4cf..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_init_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestSetiopupdateInit { - - static void run_bfb() - { - auto engine = setup_random_test(); - - SetiopupdateInitData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(SetiopupdateInitData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - setiopupdate_init(d); - } - - // Get data from cxx. Run setiopupdate_init from a kernel and copy results back to host - Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - const Int offset = i * Spack::n; - - - - - Functions::setiopupdate_init(); - - - }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - SetiopupdateInitData& d_f90 = f90_data[i]; - SetiopupdateInitData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("setiopupdate_init_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestSetiopupdateInit; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp deleted file mode 100644 index 800aaab005bf..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_setiopupdate_tests.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "catch2/catch.hpp" - -#include "share/scream_types.hpp" -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "doubly-periodic/dp_functions.hpp" -#include "doubly-periodic/dp_functions_f90.hpp" - -#include "dp_unit_tests_common.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -template -struct UnitWrap::UnitTest::TestSetiopupdate { - - static void run_bfb() - { - auto engine = setup_random_test(); - - SetiopupdateData f90_data[max_pack_size] = { - // TODO - }; - - static constexpr Int num_runs = sizeof(f90_data) / sizeof(SetiopupdateData); - - // Generate random input data - // Alternatively, you can use the f90_data construtors/initializer lists to hardcode data - for (auto& d : f90_data) { - d.randomize(engine); - } - - // Create copies of data for use by cxx and sync it to device. Needs to happen before fortran calls so that - // inout data is in original state - view_1d cxx_device("cxx_device", max_pack_size); - const auto cxx_host = Kokkos::create_mirror_view(cxx_device); - std::copy(&f90_data[0], &f90_data[0] + max_pack_size, cxx_host.data()); - Kokkos::deep_copy(cxx_device, cxx_host); - - // Get data from fortran - for (auto& d : f90_data) { - setiopupdate(d); - } - - // Get data from cxx. Run setiopupdate from a kernel and copy results back to host - Kokkos::parallel_for(num_test_itrs, KOKKOS_LAMBDA(const Int& i) { - const Int offset = i * Spack::n; - - - - - Functions::setiopupdate(); - - - }); - - Kokkos::deep_copy(cxx_host, cxx_device); - - // Verify BFB results - if (SCREAM_BFB_TESTING) { - for (Int i = 0; i < num_runs; ++i) { - SetiopupdateData& d_f90 = f90_data[i]; - SetiopupdateData& d_cxx = cxx_host[i]; - } - } - } // run_bfb - -}; - -} // namespace unit_test -} // namespace dp -} // namespace scream - -namespace { - -TEST_CASE("setiopupdate_bfb", "[dp]") -{ - using TestStruct = scream::dp::unit_test::UnitWrap::UnitTest::TestSetiopupdate; - - TestStruct::run_bfb(); -} - -} // empty namespace diff --git a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp deleted file mode 100644 index bfd865a1d924..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "catch2/catch.hpp" - -#include "dp_unit_tests_common.hpp" - -#include "dp_functions.hpp" -#include "dp_functions_f90.hpp" - -#include "share/scream_types.hpp" - -#include "ekat/ekat_pack.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "ekat/util/ekat_arch.hpp" - -#include -#include -#include -#include - -namespace scream { -namespace dp { -namespace unit_test { - -}//namespace unit_test -}//namespace dp -}//namespace scream - diff --git a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp b/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp deleted file mode 100644 index b2a85e9e5db7..000000000000 --- a/components/eamxx/src/doubly-periodic/tests/dp_unit_tests_common.hpp +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef DP_UNIT_TESTS_COMMON_HPP -#define DP_UNIT_TESTS_COMMON_HPP - -#include "dp_functions.hpp" -#include "share/scream_types.hpp" -#include "ekat/kokkos/ekat_kokkos_utils.hpp" -#include "share/util/scream_setup_random_test.hpp" - -namespace scream { -namespace dp { -namespace unit_test { - -/* - * Unit test infrastructure for dp unit tests. - * - * dp entities can friend scream::dp::unit_test::UnitWrap to give unit tests - * access to private members. - * - * All unit test impls should be within an inner struct of UnitWrap::UnitTest for - * easy access to useful types. - */ - -struct UnitWrap { - - template - struct UnitTest : public KokkosTypes { - - using Device = D; - using MemberType = typename KokkosTypes::MemberType; - using TeamPolicy = typename KokkosTypes::TeamPolicy; - using RangePolicy = typename KokkosTypes::RangePolicy; - using ExeSpace = typename KokkosTypes::ExeSpace; - - template - using view_1d = typename KokkosTypes::template view_1d; - template - using view_2d = typename KokkosTypes::template view_2d; - template - using view_3d = typename KokkosTypes::template view_3d; - - template - using uview_1d = typename ekat::template Unmanaged >; - - using Functions = scream::dp::Functions; - using Scalar = typename Functions::Scalar; - using Spack = typename Functions::Spack; - using Pack = typename Functions::Pack; - using IntSmallPack = typename Functions::IntSmallPack; - using Smask = typename Functions::Smask; - using C = typename Functions::C; - - static constexpr Int max_pack_size = 16; - static constexpr Int num_test_itrs = max_pack_size / Spack::n; - - // Put struct decls here - struct TestAdvanceIopForcing; - struct TestAdvanceIopNudging; - struct TestAdvanceIopSubsidence; - struct TestIopSetinitial; - struct TestIopBroadcast; - struct TestApplyIopForcing; - struct TestIopDomainRelaxation; - struct TestCrmResolvedTurb; - struct TestIopDefaultOpts; - struct TestIopSetopts; - struct TestSetiopupdateInit; - struct TestSetiopupdate; - struct TestReadiopdata; - struct TestIopIntht; - }; - -}; - - -} // namespace unit_test -} // namespace dp -} // namespace scream - -#endif diff --git a/components/eamxx/src/dynamics/homme/CMakeLists.txt b/components/eamxx/src/dynamics/homme/CMakeLists.txt index 3cf34d3eb88a..b6a69a3605f9 100644 --- a/components/eamxx/src/dynamics/homme/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/CMakeLists.txt @@ -132,7 +132,7 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) # possible something that matters will be added in the future. SetCudaFlags(${hommeLibName} CUDA_LANG) endif() - + SetOmpFlags(${hommeLibName}) ##################################### @@ -146,6 +146,7 @@ macro (CreateDynamicsLib HOMME_TARGET NP PLEV QSIZE) ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_process_interface.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_fv_phys.cpp ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_rayleigh_friction.cpp + ${SCREAM_DYNAMICS_SRC_DIR}/eamxx_homme_iop.cpp ${SCREAM_DYNAMICS_SRC_DIR}/physics_dynamics_remapper.cpp ${SCREAM_DYNAMICS_SRC_DIR}/homme_grids_manager.cpp ${SCREAM_DYNAMICS_SRC_DIR}/interface/homme_context_mod.F90 diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp new file mode 100644 index 000000000000..09e2c7a17b2d --- /dev/null +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_iop.cpp @@ -0,0 +1,444 @@ +#include "eamxx_homme_process_interface.hpp" + +// EAMxx includes +#include "control/intensive_observation_period.hpp" +#include "dynamics/homme/homme_dimensions.hpp" +#include "dynamics/homme/homme_dynamics_helpers.hpp" +#include "physics/share/physics_constants.hpp" +#include "share/util/scream_column_ops.hpp" + +// Homme includes +#include "Context.hpp" +#include "ColumnOps.hpp" +#include "ElementOps.hpp" +#include "EquationOfState.hpp" +#include "HommexxEnums.hpp" +#include "HybridVCoord.hpp" +#include "KernelVariables.hpp" +#include "SimulationParams.hpp" +#include "Types.hpp" + +// EKAT includes +#include "ekat/ekat_workspace.hpp" +#include "ekat/kokkos/ekat_kokkos_types.hpp" + +namespace scream { + +// Compute effects of large scale subsidence on T, q, u, and v. +KOKKOS_FUNCTION +void HommeDynamics:: +advance_iop_subsidence(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& pmid, + const view_1d& pint, + const view_1d& pdel, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q) +{ + using ColOps = ColumnOps; + using C = physics::Constants; + constexpr Real Rair = C::Rair; + constexpr Real Cpair = C::Cpair; + + const auto n_q_tracers = Q.extent_int(0); + const auto nlev_packs = ekat::npack(nlevs); + + // Get some temporary views from WS + uview_1d omega_int, delta_u, delta_v, delta_T, tmp; + workspace.take_many_contiguous_unsafe<4>({"omega_int", "delta_u", "delta_v", "delta_T"}, + {&omega_int, &delta_u, &delta_v, &delta_T}); + const auto delta_Q_slot = workspace.take_macro_block("delta_Q", n_q_tracers); + uview_2d delta_Q(delta_Q_slot.data(), n_q_tracers, nlev_packs); + + auto s_pmid = ekat::scalarize(pmid); + auto s_omega = ekat::scalarize(omega); + auto s_delta_u = ekat::scalarize(delta_u); + auto s_delta_v = ekat::scalarize(delta_v); + auto s_delta_T = ekat::scalarize(delta_T); + auto s_delta_Q = ekat::scalarize(delta_Q); + auto s_omega_int = ekat::scalarize(omega_int); + + // Compute omega on the interface grid by using a weighted average in pressure + const int pack_begin = 1/Pack::n, pack_end = (nlevs-1)/Pack::n; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, pack_begin, pack_end+1), [&] (const int k){ + auto range_pack = ekat::range(k*Pack::n); + range_pack.set(range_pack<1, 1); + Pack pmid_k, pmid_km1, omega_k, omega_km1; + ekat::index_and_shift<-1>(s_pmid, range_pack, pmid_k, pmid_km1); + ekat::index_and_shift<-1>(s_omega, range_pack, omega_k, omega_km1); + + const auto weight = (pint(k) - pmid_km1)/(pmid_k - pmid_km1); + omega_int(k).set(range_pack>=1 and range_pack<=nlevs-1, + weight*omega_k + (1-weight)*omega_km1); + }); + omega_int(0)[0] = 0; + omega_int(nlevs/Pack::n)[nlevs%Pack::n] = 0; + + // Compute delta views for u, v, T, and Q (e.g., u(k+1) - u(k), k=0,...,nlevs-2) + ColOps::compute_midpoint_delta(team, nlevs-1, u, delta_u); + ColOps::compute_midpoint_delta(team, nlevs-1, v, delta_v); + ColOps::compute_midpoint_delta(team, nlevs-1, T, delta_T); + for (int iq=0; iq(k*Pack::n); + const auto at_top = range_pack==0; + const auto at_bot = range_pack==nlevs-1; + const auto at_mid = not at_top and not at_bot; + const bool any_at_top = at_top.any(); + const bool any_at_bot = at_bot.any(); + + // Get delta(k-1) packs. The range pack should not + // contain index 0 (so that we don't attempt to access + // k=-1 index) or index > nlevs-2 (since delta_* views + // are size nlevs-1). + auto range_pack_for_m1_shift = range_pack; + range_pack_for_m1_shift.set(range_pack<1, 1); + range_pack_for_m1_shift.set(range_pack>nlevs-2, nlevs-2); + Pack delta_u_k, delta_u_km1, + delta_v_k, delta_v_km1, + delta_T_k, delta_T_km1; + ekat::index_and_shift<-1>(s_delta_u, range_pack_for_m1_shift, delta_u_k, delta_u_km1); + ekat::index_and_shift<-1>(s_delta_v, range_pack_for_m1_shift, delta_v_k, delta_v_km1); + ekat::index_and_shift<-1>(s_delta_T, range_pack_for_m1_shift, delta_T_k, delta_T_km1); + + // At the top and bottom of the model, set the end points for + // delta_*_k and delta_*_km1 to be the first and last entries + // of delta_*, respectively. + if (any_at_top) { + delta_u_k.set(at_top, s_delta_u(0)); + delta_v_k.set(at_top, s_delta_v(0)); + delta_T_k.set(at_top, s_delta_T(0)); + } + if (any_at_bot) { + delta_u_km1.set(at_bot, s_delta_u(nlevs-2)); + delta_v_km1.set(at_bot, s_delta_v(nlevs-2)); + delta_T_km1.set(at_bot, s_delta_T(nlevs-2)); + } + + // Get omega_int(k+1) pack. The range pack should not + // contain index > nlevs-1 (since omega_int is size nlevs+1). + auto range_pack_for_p1_shift = range_pack; + range_pack_for_p1_shift.set(range_pack>nlevs-1, nlevs-1); + Pack omega_int_k, omega_int_kp1; + ekat::index_and_shift<1>(s_omega_int, range_pack, omega_int_k, omega_int_kp1); + + const auto fac = (dt/2)/pdel(k); + + // Update u + auto& u_k = u(k); + u_k.set(at_top, u_k - fac*omega_int_kp1*delta_u_k); + u_k.set(at_bot, u_k - fac*omega_int_k*delta_u_km1); + u_k.set(at_mid, u_k - fac*(omega_int_kp1*delta_u_k + omega_int_k*delta_u_km1)); + + // Update v + auto& v_k = v(k); + v_k.set(at_top, v_k - fac*omega_int_kp1*delta_v_k); + v_k.set(at_bot, v_k - fac*omega_int_k*delta_v_km1); + v_k.set(at_mid, v_k - fac*(omega_int_kp1*delta_v_k + omega_int_k*delta_v_km1)); + + // Before updating T, first scale using thermal + // expansion term due to LS vertical advection + auto& T_k = T(k); + T_k *= 1 + (dt*Rair/Cpair)*omega(k)/pmid(k); + + // Update T + T_k.set(at_top, T_k - fac*omega_int_kp1*delta_T_k); + T_k.set(at_bot, T_k - fac*omega_int_k*delta_T_km1); + T_k.set(at_mid, T_k - fac*(omega_int_kp1*delta_T_k + omega_int_k*delta_T_km1)); + + // Update Q + Pack delta_tracer_k, delta_tracer_km1; + for (int iq=0; iq(s_delta_tracer, range_pack_for_m1_shift, delta_tracer_k, delta_tracer_km1); + if (any_at_top) delta_tracer_k.set(at_top, s_delta_tracer(0)); + if (any_at_bot) delta_tracer_km1.set(at_bot, s_delta_tracer(nlevs-2)); + + auto& Q_k = Q(iq, k); + Q_k.set(at_top, Q_k - fac*omega_int_kp1*delta_tracer_k); + Q_k.set(at_bot, Q_k - fac*omega_int_k*delta_tracer_km1); + Q_k.set(at_mid, Q_k - fac*(omega_int_kp1*delta_tracer_k + omega_int_k*delta_tracer_km1)); + } + }); + + // Release WS views + workspace.release_macro_block(delta_Q_slot, n_q_tracers); + workspace.release_many_contiguous<4>({&omega_int, &delta_u, &delta_v, &delta_T}); +} + +// Apply large scale forcing for temperature and water vapor as provided by the IOP file +KOKKOS_FUNCTION +void HommeDynamics:: +advance_iop_forcing(const KT::MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv) +{ + const auto nlev_packs = ekat::npack(nlevs); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_packs), [&] (const int k) { + T(k) += dt*divT(k); + qv(k) += dt*divq(k); + }); +} + +void HommeDynamics:: +apply_iop_forcing(const Real dt) +{ + using ESU = ekat::ExeSpaceUtils; + + using EOS = Homme::EquationOfState; + using ElementOps = Homme::ElementOps; + using KV = Homme::KernelVariables; + + using ColOps = ColumnOps; + using C = physics::Constants; + constexpr Real Rair = C::Rair; + + // Homme objects + const auto& c = Homme::Context::singleton(); + const auto& hvcoord = c.get(); + const auto& params = c.get(); + + // Dimensions + constexpr int NGP = HOMMEXX_NP; + constexpr int NLEV = HOMMEXX_NUM_LEV; + constexpr int NLEVI = HOMMEXX_NUM_LEV_P; + const auto nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); + const auto total_levels = m_dyn_grid->get_num_vertical_levels(); + const auto qsize = params.qsize; + + // Sanity checks since we will be switching between ekat::Pack + // and Homme::Scalar view types + EKAT_ASSERT_MSG(NLEV == ekat::npack(total_levels), + "Error! Dimension for vectorized Homme levels does not match level dimension " + "of the packed views used here. Check that Pack typedef is using a pack size " + "consistent with Homme's vector size.\n"); + EKAT_ASSERT_MSG(NLEVI == ekat::npack(total_levels+1), + "Error! Dimension for vectorized Homme levels does not match level dimension " + "of the packed views used here. Check that Pack typedef is using a pack size " + "consistent with Homme's vector size.\n"); + + // Hybrid coord values + const auto ps0 = hvcoord.ps0; + const auto hyam = m_dyn_grid->get_geometry_data("hyam").get_view(); + const auto hybm = m_dyn_grid->get_geometry_data("hybm").get_view(); + const auto hyai = m_dyn_grid->get_geometry_data("hyai").get_view(); + const auto hybi = m_dyn_grid->get_geometry_data("hybi").get_view(); + + // Homme element states and EOS/EO classes + auto ps_dyn = get_internal_field("ps_dyn").get_view(); + auto dp3d_dyn = get_internal_field("dp3d_dyn").get_view(); + auto vtheta_dp_dyn = get_internal_field("vtheta_dp_dyn").get_view(); + auto phi_int_dyn = get_internal_field("phi_int_dyn").get_view(); + auto v_dyn = get_internal_field("v_dyn").get_view(); + auto Q_dyn = m_helper_fields.at("Q_dyn").get_view(); + auto Qdp_dyn = get_internal_field("Qdp_dyn").get_view(); + + EOS eos; + eos.init(params.theta_hydrostatic_mode, hvcoord); + + ElementOps elem_ops; + elem_ops.init(hvcoord); + const bool use_moisture = (params.moisture == Homme::MoistDry::MOIST); + + // Load data from IOP files, if necessary + m_iop->read_iop_file_data(timestamp()); + + // Define local IOP param values and views + const auto iop_dosubsidence = m_iop->get_params().get("iop_dosubsidence"); + const auto use_3d_forcing = m_iop->get_params().get("use_3d_forcing"); + const auto omega = m_iop->get_iop_field("omega").get_view(); + const auto divT = use_3d_forcing ? m_iop->get_iop_field("divT3d").get_view() + : m_iop->get_iop_field("divT").get_view(); + const auto divq = use_3d_forcing ? m_iop->get_iop_field("divq3d").get_view() + : m_iop->get_iop_field("divq").get_view(); + + // Team policy and workspace manager for both homme and scream + // related loops. We need separate policies since hommexx functions used here + // assume they are called inside nested loops for elements and Gaussian points, + // whereas EAMxx function we use expects a single level of parallelism + // for elements and Guassian points. + // TODO: scream::ColumnOps functions could take an arbitary loop boundary + // (TeamVectorRange, TeamThreadRange, ThreadVectorRange) so that + // all 3 kernel launches here could be combined. + const auto policy_homme = ESU::get_default_team_policy(nelem, NLEV); + const auto policy_eamxx = ESU::get_default_team_policy(nelem*NGP*NGP, NLEV); + + // TODO: Create a memory buffer for this class + // and add the below WSM and views + WorkspaceMgr eamxx_wsm(NLEVI, 7+qsize, policy_eamxx); + WorkspaceMgr homme_wsm(NLEV, 32, policy_homme); + view_Nd + temperature("temperature", nelem, NGP, NGP, NLEV), + exner("exner", nelem, NGP, NGP, NLEV); + + // Preprocess some homme states to get temperature and exner + Kokkos::parallel_for("compute_t_and_exner", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); + + // Get temp views from workspace + auto ws = homme_wsm.get_workspace(team); + auto pnh_slot = ws.take_macro_block("pnh" , NGP*NGP); + auto rstar_slot = ws.take_macro_block("rstar", NGP*NGP); + uview_2d + pnh (reinterpret_cast(pnh_slot.data()), NGP*NGP, NLEV), + rstar(reinterpret_cast(rstar_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto phi_int_i = ekat::subview(phi_int_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto pnh_i = ekat::subview(pnh, idx); + auto rstar_i = ekat::subview(rstar, idx); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged dp3d_scalar(reinterpret_cast(dp3d_i.data()), NLEV); + Homme::ExecViewUnmanaged vtheta_dp_scalar(reinterpret_cast(vtheta_dp_i.data()), NLEV); + Homme::ExecViewUnmanaged phi_int_scalar(reinterpret_cast(phi_int_i.data()), NLEVI); + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged pnh_scalar(reinterpret_cast(pnh_i.data()), NLEV); + Homme::ExecViewUnmanaged exner_scalar(reinterpret_cast(exner_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + Homme::ExecViewUnmanaged temperature_scalar(reinterpret_cast(temperature_i.data()), NLEV); + + // Compute exner from EOS + if (params.theta_hydrostatic_mode) { + auto hydro_p_int = ws.take("hydro_p_int"); + Homme::ExecViewUnmanaged hydro_p_int_scalar(reinterpret_cast(hydro_p_int.data()), NLEVI); + elem_ops.compute_hydrostatic_p(kv, dp3d_scalar, hydro_p_int_scalar, pnh_scalar); + eos.compute_exner(kv, pnh_scalar, exner_scalar); + ws.release(hydro_p_int); + } else { + eos.compute_pnh_and_exner(kv, vtheta_dp_scalar, phi_int_scalar, pnh_scalar, exner_scalar); + } + + // Get the temperature from dynamics states + elem_ops.get_temperature(kv, eos, use_moisture, dp3d_scalar, exner_scalar, vtheta_dp_scalar, qv_scalar, rstar_scalar, temperature_scalar); + }); + + // Release WS views + ws.release_macro_block(rstar_slot, NGP*NGP); + ws.release_macro_block(pnh_slot, NGP*NGP); + }); + Kokkos::fence(); + + // Apply IOP forcing + Kokkos::parallel_for("apply_iop_forcing", policy_eamxx, KOKKOS_LAMBDA (const KT::MemberType& team) { + const int ie = team.league_rank()/(NGP*NGP); + const int igp = (team.league_rank()/NGP)%NGP; + const int jgp = team.league_rank()%NGP; + + // Get temp views from workspace + auto ws = eamxx_wsm.get_workspace(team); + uview_1d pmid, pint, pdel; + ws.take_many_contiguous_unsafe<3>({"pmid", "pint", "pdel"}, + {&pmid, &pint, &pdel}); + + auto ps_i = ps_dyn(ie, igp, jgp); + auto u_i = ekat::subview(v_dyn, ie, 0, igp, jgp); + auto v_i = ekat::subview(v_dyn, ie, 1, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + + // Compute reference pressures and layer thickness. + // TODO: Allow geometry data to allocate packsize + auto s_pmid = ekat::scalarize(pmid); + auto s_pint = ekat::scalarize(pint); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, total_levels+1), [&](const int& k) { + s_pint(k) = hyai(k)*ps0 + hybi(k)*ps_i; + if (k < total_levels) { + s_pmid(k) = hyam(k)*ps0 + hybm(k)*ps_i; + } + }); + team.team_barrier(); + ColOps::compute_midpoint_delta(team, total_levels, pint, pdel); + team.team_barrier(); + + if (iop_dosubsidence) { + // Compute subsidence due to large-scale forcing + advance_iop_subsidence(team, total_levels, dt, ps_i, pmid, pint, pdel, omega, ws, u_i, v_i, temperature_i, Q_i); + } + + // Update T and qv according to large scale forcing as specified in IOP file. + advance_iop_forcing(team, total_levels, dt, divT, divq, temperature_i, qv_i); + + // Release WS views + ws.release_many_contiguous<3>({&pmid, &pint, &pdel}); + }); + Kokkos::fence(); + + // Postprocess homme states Qdp and vtheta_dp + Kokkos::parallel_for("compute_qdp_and_vtheta_dp", policy_homme, KOKKOS_LAMBDA (const KT::MemberType& team) { + KV kv(team); + const int ie = team.league_rank(); + + // Get temp views from workspace + auto ws = homme_wsm.get_workspace(team); + auto rstar_slot = ws.take_macro_block("rstar", NGP*NGP); + uview_2d + rstar(reinterpret_cast(rstar_slot.data()), NGP*NGP, NLEV); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(kv.team, NGP*NGP), [&] (const int idx) { + const int igp = idx/NGP; + const int jgp = idx%NGP; + + auto dp3d_i = ekat::subview(dp3d_dyn, ie, igp, jgp); + auto vtheta_dp_i = ekat::subview(vtheta_dp_dyn, ie, igp, jgp); + auto qv_i = ekat::subview(Q_dyn, ie, 0, igp, jgp); + auto Q_i = Kokkos::subview(Q_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto Qdp_i = Kokkos::subview(Qdp_dyn, ie, Kokkos::ALL(), igp, jgp, Kokkos::ALL()); + auto rstar_i = ekat::subview(rstar, idx); + auto exner_i = ekat::subview(exner, ie, igp, jgp); + auto temperature_i = ekat::subview(temperature, ie, igp, jgp); + + // Reinterperate into views of Homme::Scalar for calling Hommexx function. + Homme::ExecViewUnmanaged qv_scalar(reinterpret_cast(qv_i.data()), NLEV); + Homme::ExecViewUnmanaged rstar_scalar(reinterpret_cast(rstar_i.data()), NLEV); + + // Compute Qdp from updated Q + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV*qsize), [&] (const int k) { + const int ilev = k/qsize; + const int q = k%qsize; + + Qdp_i(q, ilev) = Q_i(q, ilev)*dp3d_i(ilev); + // For BFB on restarts, Q needs to be updated after we compute Qdp + Q_i(q, ilev) = Qdp_i(q, ilev)/dp3d_i(ilev); + }); + + // Convert updated temperature back to potential temperature + elem_ops.get_R_star(kv, use_moisture, qv_scalar, rstar_scalar); + Kokkos::parallel_for(Kokkos::ThreadVectorRange(team, NLEV), [&] (const int k) { + vtheta_dp_i(k) = temperature_i(k)*rstar_i(k)*dp3d_i(k)/(Rair*exner_i(k)); + }); + }); + + // Release WS views + ws.release_macro_block(rstar_slot, NGP*NGP); + }); +} + +} // namespace scream diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp index c535829fbfab..f33c453c117c 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.cpp @@ -423,6 +423,11 @@ void HommeDynamics::initialize_impl (const RunType run_type) if (run_type==RunType::Initial) { initialize_homme_state (); } else { + if (m_iop) { + // We need to reload IOP data after restarting + m_iop->read_iop_file_data(timestamp()); + } + restart_homme_state (); } @@ -547,16 +552,13 @@ void HommeDynamics::homme_pre_process (const double dt) { // T and uv tendencies are backed out on the ref grid. // Homme takes care of turning the FT tendency into a tendency for VTheta_dp. - constexpr int N = sizeof(Homme::Scalar) / sizeof(Real); - using Pack = RPack; - using namespace Homme; const auto& c = Context::singleton(); const auto& params = c.get(); const int ncols = m_phys_grid->get_num_local_dofs(); const int nlevs = m_phys_grid->get_num_vertical_levels(); - const int npacks = ekat::PackInfo::num_packs(nlevs); + const int npacks = ekat::npack(nlevs); const auto& pgn = m_phys_grid->name(); @@ -664,6 +666,10 @@ void HommeDynamics::homme_post_process (const double dt) { get_internal_field("w_int_dyn").get_header().get_alloc_properties().reset_subview_idx(tl.n0); } + if (m_iop) { + apply_iop_forcing(dt); + } + if (fv_phys_active()) { fv_phys_post_process(); // Apply Rayleigh friction to update temperature and horiz_winds @@ -675,8 +681,6 @@ void HommeDynamics::homme_post_process (const double dt) { // Remap outputs to ref grid m_d2p_remapper->remap(true); - constexpr int N = HOMMEXX_PACK_SIZE; - using Pack = RPack; using ColOps = ColumnOps; using PF = PhysicsFunctions; @@ -697,7 +701,7 @@ void HommeDynamics::homme_post_process (const double dt) { const auto ncols = m_phys_grid->get_num_local_dofs(); const auto nlevs = m_phys_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); using ESU = ekat::ExeSpaceUtils; const auto policy = ESU::get_thread_range_parallel_scan_team_policy(ncols,npacks); @@ -797,14 +801,13 @@ void HommeDynamics::init_homme_views () { constexpr int QSZ = HOMMEXX_QSIZE_D; constexpr int NVL = HOMMEXX_NUM_LEV; constexpr int NVLI = HOMMEXX_NUM_LEV_P; - constexpr int N = HOMMEXX_PACK_SIZE; const int nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const int qsize = tracers.num_tracers(); const auto ncols = m_phys_grid->get_num_local_dofs(); const auto nlevs = m_phys_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); using ESU = ekat::ExeSpaceUtils; const auto default_policy = ESU::get_default_team_policy(ncols,npacks); @@ -931,8 +934,6 @@ void HommeDynamics::restart_homme_state () { " - field name: " + f.get_header().get_identifier().name() + "\n"); } - constexpr int N = HOMMEXX_PACK_SIZE; - using Pack = RPack; using ESU = ekat::ExeSpaceUtils; using PF = PhysicsFunctions; @@ -957,7 +958,7 @@ void HommeDynamics::restart_homme_state () { const int nlevs = m_phys_grid->get_num_vertical_levels(); const int ncols = m_phys_grid->get_num_local_dofs(); const int nelem = m_dyn_grid->get_num_local_dofs() / (NGP*NGP); - const int npacks = ekat::PackInfo::num_packs(nlevs); + const int npacks = ekat::npack(nlevs); const int qsize = params.qsize; // NOTE: when restarting stuff like T_prev, and other "previous steps" quantities that HommeDynamics @@ -1060,12 +1061,10 @@ void HommeDynamics::restart_homme_state () { void HommeDynamics::initialize_homme_state () { // Some types - using Pack = RPack; using ColOps = ColumnOps; using PF = PhysicsFunctions; using ESU = ekat::ExeSpaceUtils; using EOS = Homme::EquationOfState; - using WS = ekat::WorkspaceManager; const auto& rgn = m_cgll_grid->name(); @@ -1080,8 +1079,8 @@ void HommeDynamics::initialize_homme_state () { constexpr int NGP = HOMMEXX_NP; const int nelem = m_dyn_grid->get_num_local_dofs()/(NGP*NGP); const int qsize = params.qsize; - const int npacks_mid = ekat::PackInfo::num_packs(nlevs); - const int npacks_int = ekat::PackInfo::num_packs(nlevs+1); + const int npacks_mid = ekat::npack(nlevs); + const int npacks_int = ekat::npack(nlevs+1); // Bootstrap dp on phys grid, and let the ic remapper transfer dp on dyn grid // NOTE: HybridVCoord already stores hyai and hybi deltas as packed views, @@ -1138,7 +1137,7 @@ void HommeDynamics::initialize_homme_state () { const auto hyai0 = hvcoord.hybrid_ai0; // Need two temporaries, for pi_mid and pi_int const auto policy = ESU::get_thread_range_parallel_scan_team_policy(nelem*NGP*NGP,npacks_mid); - WS wsm(npacks_int,2,policy); + WorkspaceMgr wsm(npacks_int,2,policy); Kokkos::parallel_for(policy, KOKKOS_LAMBDA (const KT::MemberType& team) { const int ie = team.league_rank() / (NGP*NGP); const int igp = (team.league_rank() / NGP) % NGP; @@ -1149,8 +1148,8 @@ void HommeDynamics::initialize_homme_state () { // Compute p_mid auto ws = wsm.get_workspace(team); - ekat::Unmanaged > p_int, p_mid; - ws.template take_many_and_reset<2>({"p_int", "p_mid"}, {&p_int, &p_mid}); + ekat::Unmanaged > p_int, p_mid; + ws.take_many_and_reset<2>({"p_int", "p_mid"}, {&p_int, &p_mid}); ColOps::column_scan(team,nlevs,dp,p_int,ps0*hyai0); team.team_barrier(); @@ -1281,12 +1280,11 @@ copy_dyn_states_to_all_timelevels () { // TODO item to consolidate how we update the pressure during initialization and run, but // for now we have two locations where we do this. void HommeDynamics::update_pressure(const std::shared_ptr& grid) { - using Pack = RPack; using ColOps = ColumnOps; const auto ncols = grid->get_num_local_dofs(); const auto nlevs = grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); const auto& c = Homme::Context::singleton(); const auto& hvcoord = c.get(); diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp index 6f6da5321484..2749d268c9a7 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_process_interface.hpp @@ -6,12 +6,12 @@ #include "ekat/ekat_parameter_list.hpp" #include "ekat/ekat_pack.hpp" +#include "ekat/ekat_workspace.hpp" #include namespace scream { - /* * The class responsible to handle the atmosphere dynamics * @@ -23,6 +23,25 @@ namespace scream */ class HommeDynamics : public AtmosphereProcess { + // Define some types needed by class + using Pack = ekat::Pack; + using IntPack = ekat::Pack; + + using KT = KokkosTypes; + template + using view_1d = typename KT::template view_1d; + template + using view_2d = typename KT::template view_2d; + template + using view_Nd = typename KT::template view_ND; + template + using uview_1d = ekat::Unmanaged>; + template + using uview_2d = ekat::Unmanaged>; + + using WorkspaceMgr = ekat::WorkspaceManager; + using Workspace = WorkspaceMgr::Workspace; + public: // Constructor(s) and Destructor @@ -89,6 +108,33 @@ class HommeDynamics : public AtmosphereProcess void rayleigh_friction_init (); void rayleigh_friction_apply (const Real dt) const; + // IOP functions + void apply_iop_forcing(const Real dt); + + KOKKOS_FUNCTION + static void advance_iop_subsidence(const KT::MemberType& team, + const int nlevs, + const Real dt, + const Real ps, + const view_1d& pmid, + const view_1d& pint, + const view_1d& pdel, + const view_1d& omega, + const Workspace& workspace, + const view_1d& u, + const view_1d& v, + const view_1d& T, + const view_2d& Q); + + KOKKOS_FUNCTION + static void advance_iop_forcing(const KT::MemberType& team, + const int nlevs, + const Real dt, + const view_1d& divT, + const view_1d& divq, + const view_1d& T, + const view_1d& qv); + public: // Fast boolean function returning whether Physics PGN is being used. bool fv_phys_active() const; @@ -134,15 +180,8 @@ class HommeDynamics : public AtmosphereProcess std::shared_ptr m_phys_grid; // Column parameterizations grid std::shared_ptr m_cgll_grid; // Unique CGLL - template - using RPack = ekat::Pack; - - using KT = KokkosTypes; - template - using view_1d = typename KT::template view_1d; - // Rayleigh friction decay rate profile - view_1d> m_otau; + view_1d m_otau; // Rayleigh friction paramaters int m_rayk0; // Vertical level at which rayleigh friction term is centered. diff --git a/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp b/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp index 120dce865db6..3b4cbdb711e6 100644 --- a/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp +++ b/components/eamxx/src/dynamics/homme/eamxx_homme_rayleigh_friction.cpp @@ -22,12 +22,9 @@ void HommeDynamics::rayleigh_friction_init() // to 0-based for computations m_rayk0 -= 1; - constexpr int N = SCREAM_PACK_SIZE; - using Pack = RPack; - // Calculate decay rate profile, otau. const int nlevs = m_dyn_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); m_otau = decltype(m_otau)("otau", npacks); // Local paramters @@ -48,7 +45,7 @@ void HommeDynamics::rayleigh_friction_init() Kokkos::parallel_for(KT::RangePolicy(0, npacks), KOKKOS_LAMBDA (const int ilev) { - const auto range_pack = ekat::range(ilev*N); + const auto range_pack = ekat::range(ilev*Pack::n); const Pack x = (rayk0 - range_pack)/krange; otau(ilev) = otau0*(1.0 + ekat::tanh(x))/2.0; }); @@ -56,9 +53,7 @@ void HommeDynamics::rayleigh_friction_init() void HommeDynamics::rayleigh_friction_apply(const Real dt) const { - constexpr int N = HOMMEXX_PACK_SIZE; using PF = PhysicsFunctions; - using Pack = RPack; using ESU = ekat::ExeSpaceUtils; // If m_raytau0==0, then no Rayleigh friction is applied. Return. @@ -66,7 +61,7 @@ void HommeDynamics::rayleigh_friction_apply(const Real dt) const const auto ncols = m_phys_grid->get_num_local_dofs(); const auto nlevs = m_phys_grid->get_num_vertical_levels(); - const auto npacks= ekat::PackInfo::num_packs(nlevs); + const auto npacks= ekat::npack(nlevs); const auto horiz_winds_view = get_field_out("horiz_winds").get_view(); const auto T_mid_view = get_field_out("T_mid").get_view(); diff --git a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp index 08dfe0ccb831..23b47a59aa20 100644 --- a/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp +++ b/components/eamxx/src/dynamics/homme/homme_grids_manager.cpp @@ -267,6 +267,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { for (auto f : {hyai, hybi, hyam, hybm}) { auto f_d = get_grid("Dynamics")->get_geometry_data(f.name()); f.deep_copy(f_d); + f.sync_to_host(); } } @@ -284,7 +285,7 @@ build_physics_grid (const ci_string& type, const ci_string& rebalance) { void HommeGridsManager:: initialize_vertical_coordinates (const nonconstgrid_ptr_type& dyn_grid) { - using view_1d_host = AtmosphereInput::view_1d_host; + using view_1d_host = AtmosphereInput::view_1d_host; using vos_t = std::vector; using namespace ShortFieldTagsNames; @@ -337,7 +338,7 @@ initialize_vertical_coordinates (const nonconstgrid_ptr_type& dyn_grid) { hybi.sync_to_dev(); hyam.sync_to_dev(); hybm.sync_to_dev(); - + // Pass host views data to hvcoord init function const auto ps0 = Homme::PhysicalConstants::p0; diff --git a/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt b/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt index 8b500fa239bc..cc09289ff564 100644 --- a/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt +++ b/components/eamxx/src/dynamics/homme/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include (ScreamUtils) # Get or create the dynamics lib diff --git a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp index 0819dd87fd39..a2534c049c26 100644 --- a/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp +++ b/components/eamxx/src/dynamics/homme/tests/dyn_grid_io.cpp @@ -151,12 +151,12 @@ TEST_CASE("dyn_grid_io") ekat::ParameterList out_params; out_params.set("Averaging Type","Instant"); out_params.set("filename_prefix","dyn_grid_io"); + out_params.set("MPI Ranks in Filename",true); out_params.sublist("Fields").sublist("Dynamics").set>("Field Names",fnames); out_params.sublist("Fields").sublist("Dynamics").set("IO Grid Name","Physics GLL"); out_params.sublist("output_control").set("Frequency",1); out_params.sublist("output_control").set("frequency_units","nsteps"); - out_params.sublist("output_control").set("MPI Ranks in Filename",true); out_params.set("Floating Point Precision","real"); OutputManager output; diff --git a/components/eamxx/src/physics/cosp/cosp_c2f.F90 b/components/eamxx/src/physics/cosp/cosp_c2f.F90 index 351207ae336d..957c546c4aa7 100644 --- a/components/eamxx/src/physics/cosp/cosp_c2f.F90 +++ b/components/eamxx/src/physics/cosp/cosp_c2f.F90 @@ -292,23 +292,30 @@ subroutine cosp_c2f_run(npoints, ncolumns, nlevels, ntau, nctp, & nptsperit = npoints - tca(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) - cca(:npoints,:nlevels) = 0 - mr_lsliq(:npoints,:nlevels) = 0 - mr_ccliq(:npoints,:nlevels) = 0 - mr_lsice(:npoints,:nlevels) = 0 - mr_ccice(:npoints,:nlevels) = 0 - dtau_c(:npoints,:nlevels) = 0 - dtau_s(:npoints,:nlevels) = dtau067(:npoints,:nlevels) - dem_c (:npoints,:nlevels) = 0 - dem_s (:npoints,:nlevels) = 1._wp - exp(-dtau105(:npoints,:nlevels)) - mr_lsrain(:npoints,:nlevels) = 0 - mr_ccrain(:npoints,:nlevels) = 0 - mr_lssnow(:npoints,:nlevels) = 0 - mr_lssnow(:npoints,:nlevels) = 0 - mr_ccsnow(:npoints,:nlevels) = 0 - mr_lsgrpl(:npoints,:nlevels) = 0 - reff = 0 ! FIXME + ! In-cloud values are assumed. If ncolumns = 1, then convert in-cloud values to gridbox + if (ncolumns == 1) then + tca(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) + dtau_s(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) * dtau067(:npoints,:nlevels) + dem_s (:npoints,:nlevels) = 1._wp - exp(-cldfrac(:npoints,:nlevels) * dtau105(:npoints,:nlevels)) + else + tca(:npoints,:nlevels) = cldfrac(:npoints,:nlevels) + dtau_s(:npoints,:nlevels) = dtau067(:npoints,:nlevels) + dem_s (:npoints,:nlevels) = 1._wp - exp(-dtau105(:npoints,:nlevels)) + end if + ! Fields not currently being used (will need to be revisted when turning on additional simulators) + cca(:npoints,:nlevels) = 0 ! Cloud fraction of convective clouds; not present or used in our model + dtau_c(:npoints,:nlevels) = 0 ! Optical depth of convective clouds; not present or used in our model + dem_c (:npoints,:nlevels) = 0 ! Emissivity of convective clouds; not present or used in our model + mr_lsliq(:npoints,:nlevels) = 0 ! Mixing ratio of cloud liquid; will be needed for radar/lidar + mr_ccliq(:npoints,:nlevels) = 0 ! Mixing ratio of cloud liquid for convective clouds; not present or used in our model + mr_lsice(:npoints,:nlevels) = 0 ! Mixing ratio of cloud ice; will be needed for radar/lidar + mr_ccice(:npoints,:nlevels) = 0 ! Mixing ratio of cloud ice for convective clouds; not present or used in our model + mr_lsrain(:npoints,:nlevels) = 0 ! Mixing ratio of rain; will be needed for radar/lidar + mr_ccrain(:npoints,:nlevels) = 0 ! Mixing ratio of rain for convective clouds; not present or used in our model + mr_lssnow(:npoints,:nlevels) = 0 ! Mixing ratio of snow; will be needed for radar/lidar + mr_ccsnow(:npoints,:nlevels) = 0 ! Mixing ratio of snow for convective clouds; will be needed for radar/lidar + mr_lsgrpl(:npoints,:nlevels) = 0 ! Mixing ratio of graupel; will be needed for radar/lidar + reff = 0 ! Effective radii; will be needed for MODIS start_idx = 1 end_idx = npoints diff --git a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp index 7aaa4be3b2b0..478fce989276 100644 --- a/components/eamxx/src/physics/cosp/eamxx_cosp.cpp +++ b/components/eamxx/src/physics/cosp/eamxx_cosp.cpp @@ -69,7 +69,7 @@ void Cosp::set_grids(const std::shared_ptr grids_manager) add_field("qv", scalar3d_layout_mid, Q, grid_name, "tracers"); add_field("qc", scalar3d_layout_mid, Q, grid_name, "tracers"); add_field("qi", scalar3d_layout_mid, Q, grid_name, "tracers"); - add_field("cldfrac_tot_for_analysis", scalar3d_layout_mid, nondim, grid_name); + add_field("cldfrac_rad", scalar3d_layout_mid, nondim, grid_name); // Optical properties, should be computed in radiation interface add_field("dtau067", scalar3d_layout_mid, nondim, grid_name); // 0.67 micron optical depth add_field("dtau105", scalar3d_layout_mid, nondim, grid_name); // 10.5 micron optical depth @@ -140,7 +140,7 @@ void Cosp::run_impl (const double dt) get_field_in("T_mid").sync_to_host(); get_field_in("p_mid").sync_to_host(); get_field_in("p_int").sync_to_host(); - get_field_in("cldfrac_tot_for_analysis").sync_to_host(); + get_field_in("cldfrac_rad").sync_to_host(); get_field_in("eff_radius_qc").sync_to_host(); get_field_in("eff_radius_qi").sync_to_host(); get_field_in("dtau067").sync_to_host(); @@ -154,7 +154,7 @@ void Cosp::run_impl (const double dt) auto T_mid = get_field_in("T_mid").get_view(); auto p_mid = get_field_in("p_mid").get_view(); auto p_int = get_field_in("p_int").get_view(); - auto cldfrac = get_field_in("cldfrac_tot_for_analysis").get_view(); + auto cldfrac = get_field_in("cldfrac_rad").get_view(); auto reff_qc = get_field_in("eff_radius_qc").get_view(); auto reff_qi = get_field_in("eff_radius_qi").get_view(); auto dtau067 = get_field_in("dtau067").get_view(); diff --git a/components/eamxx/src/physics/mam/CMakeLists.txt b/components/eamxx/src/physics/mam/CMakeLists.txt index a4e095bf5f2a..53eb4049f11b 100644 --- a/components/eamxx/src/physics/mam/CMakeLists.txt +++ b/components/eamxx/src/physics/mam/CMakeLists.txt @@ -1,11 +1,53 @@ +if (SCREAM_CIME_BUILD) + # PROJECT_SOURCE_DIR is SCREAM_ROOT/components + set(EXTERNALS_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../externals") +else() + # PROJECT_SOURCE_DIR is SCREAM_ROOT/components/eamxx + set(EXTERNALS_SOURCE_DIR "${PROJECT_SOURCE_DIR}/../../externals") +endif() + +# Haero (Kokkos-based aerosol interface library) +set(HAERO_ENABLE_GPU ${EAMXX_ENABLE_GPU}) +set(HAERO_ENABLE_MPI ON) +set(HAERO_ENABLE_TESTS OFF) +if (SCREAM_DOUBLE_PRECISION) + set(HAERO_PRECISION "double") +else() + set(HAERO_PRECISION "single") +endif() +list(APPEND CMAKE_MODULE_PATH + ${EXTERNALS_SOURCE_DIR}/haero/cmake +) +add_subdirectory(${EXTERNALS_SOURCE_DIR}/haero ${CMAKE_BINARY_DIR}/externals/haero) + +# mam4xx (C++ port of MAM4) +set(ENABLE_TESTS OFF) +set(ENABLE_SKYWALKER OFF) +set(NUM_VERTICAL_LEVELS ${SCREAM_NUM_VERTICAL_LEV}) +set(HAERO_C_STANDARD ${CMAKE_C_STANDARD}) +set(HAERO_C_COMPILER ${CMAKE_C_COMPILER}) +set(HAERO_CXX_STANDARD ${CMAKE_CXX_STANDARD}) +set(HAERO_CXX_COMPILER ${CMAKE_CXX_COMPILER}) +list(APPEND CMAKE_MODULE_PATH + ${EXTERNALS_SOURCE_DIR}/mam4xx/cmake +) +#include_directories( + # ${EXTERNALS_SOURCE_DIR}/ekat/src + # ${EXTERNALS_SOURCE_DIR}/haero + # ${CMAKE_BINARY_DIR}/externals/ekat/src + # ${CMAKE_BINARY_DIR}/externals/haero + #) +add_subdirectory(${EXTERNALS_SOURCE_DIR}/mam4xx ${CMAKE_BINARY_DIR}/externals/mam4xx) + +# EAMxx mam4xx-based atmospheric processes add_library(mam eamxx_mam_microphysics_process_interface.cpp eamxx_mam_optics_process_interface.cpp) target_compile_definitions(mam PUBLIC EAMXX_HAS_MAM) -add_dependencies(mam mam4xx_proj) +add_dependencies(mam mam4xx) target_include_directories(mam PUBLIC - ${PROJECT_BINARY_DIR}/externals/haero/include - ${PROJECT_BINARY_DIR}/externals/mam4xx/include + ${EXTERNALS_SOURCE_DIR}/haero + ${EXTERNALS_SOURCE_DIR}/mam4xx/src ) target_link_libraries(mam PUBLIC physics_share scream_share mam4xx haero) diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp index 1a89e1767522..7bfe86ef713f 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.cpp @@ -1,11 +1,18 @@ #include +#include #include #include #include "scream_config.h" // for SCREAM_CIME_BUILD +#include // for serial NetCDF file reads on MPI root #include +// NOTE: see the impl/ directory for the contents of the impl namespace +#include "impl/compute_o3_column_density.cpp" +#include "impl/compute_water_content.cpp" +#include "impl/gas_phase_chemistry.cpp" + namespace scream { @@ -13,7 +20,8 @@ MAMMicrophysics::MAMMicrophysics( const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params), - aero_config_(), nucleation_(new mam4::NucleationProcess(aero_config_)) { + aero_config_() { + configure(params); } AtmosphereProcessType MAMMicrophysics::type() const { @@ -24,33 +32,94 @@ std::string MAMMicrophysics::name() const { return "mam4_micro"; } +namespace { + +void set_data_file(const char *name, const char *path, char location[MAX_FILENAME_LEN]) { + EKAT_REQUIRE_MSG(strlen(SCREAM_DATA_DIR) + strlen(path) < MAX_FILENAME_LEN, + "Error! " << name << " path is too long (must be < " << MAX_FILENAME_LEN << " characters)"); + sprintf(location, "%s/%s", SCREAM_DATA_DIR, path); +} + +} + +#define set_file_location(data_file, path) set_data_file(#data_file, path, data_file) + +void MAMMicrophysics::set_defaults_() { + config_.amicphys.do_cond = true; + config_.amicphys.do_rename = true; + config_.amicphys.do_newnuc = true; + config_.amicphys.do_coag = true; + + config_.amicphys.nucleation = {}; + config_.amicphys.nucleation.dens_so4a_host = 1770.0; + config_.amicphys.nucleation.mw_so4a_host = 115.0; + config_.amicphys.nucleation.newnuc_method_user_choice = 2; + config_.amicphys.nucleation.pbl_nuc_wang2008_user_choice = 1; + config_.amicphys.nucleation.adjust_factor_pbl_ratenucl = 1.0; + config_.amicphys.nucleation.accom_coef_h2so4 = 1.0; + config_.amicphys.nucleation.newnuc_adjust_factor_dnaitdt = 1.0; + + // these parameters guide the coupling between parameterizations + // NOTE: mam4xx was ported with these parameters fixed, so it's probably not + // NOTE: safe to change these without code modifications. + config_.amicphys.gaexch_h2so4_uptake_optaa = 2; + config_.amicphys.newnuc_h2so4_conc_optaa = 2; + + //=========================================================== + // default data file locations (relative to SCREAM_DATA_DIR) + //=========================================================== + + // many of these paths were extracted from + // e3smv2/bld/namelist_files/namelist_defaults_eam.xml + + // photolysis + set_file_location(config_.photolysis.rsf_file, "../waccm/phot/RSF_GT200nm_v3.0_c080811.nc"); + set_file_location(config_.photolysis.xs_long_file, "../waccm/phot/temp_prs_GT200nm_JPL10_c130206.nc"); + + // stratospheric chemistry + set_file_location(config_.linoz.chlorine_loading_file, "../cam/chem/trop_mozart/ub/Linoz_Chlorine_Loading_CMIP6_0003-2017_c20171114.nc"); + + // dry deposition + set_file_location(config_.drydep.srf_file, "../cam/chem/trop_mam/atmsrf_ne4pg2_200527.nc"); +} + +void MAMMicrophysics::configure(const ekat::ParameterList& params) { + set_defaults_(); + // FIXME: implement "namelist" parsing +} + void MAMMicrophysics::set_grids(const std::shared_ptr grids_manager) { using namespace ekat::units; - // The units of mixing ratio q are technically non-dimensional. - // Nevertheless, for output reasons, we like to see 'kg/kg'. - auto q_unit = kg/kg; + auto q_unit = kg/kg; // mass mixing ratios [kg stuff / kg air] q_unit.set_string("kg/kg"); - auto n_unit = 1/kg; + auto n_unit = 1/kg; // number mixing ratios [# / kg air] n_unit.set_string("#/kg"); Units nondim(0,0,0,0,0,0,0); + const auto m2 = m*m; + const auto s2 = s*s; grid_ = grids_manager->get_grid("Physics"); const auto& grid_name = grid_->name(); - ncol_ = grid_->get_num_local_dofs(); // Number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // Number of levels per column + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column - // Define the different field layouts that will be used for this process + // get column geometry and locations + col_areas_ = grid_->get_geometry_data("area").get_view(); + col_latitudes_ = grid_->get_geometry_data("lat").get_view(); + col_longitudes_ = grid_->get_geometry_data("lon").get_view(); + + // define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; - // Layout for 2D (1d horiz X 1d vertical) variable + // layout for 2D (1d horiz X 1d vertical) variable FieldLayout scalar2d_layout_col{ {COL}, {ncol_} }; - // Layout for 3D (2d horiz X 1d vertical) variables + // layout for 3D (2d horiz X 1d vertical) variables FieldLayout scalar3d_layout_mid{ {COL, LEV}, {ncol_, nlev_} }; - // Define fields needed in mam4xx. + // define fields needed in mam4xx // atmospheric quantities add_field("omega", scalar3d_layout_mid, Pa/s, grid_name); // vertical pressure velocity @@ -60,19 +129,31 @@ void MAMMicrophysics::set_grids(const std::shared_ptr grids_ add_field("qi", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // ice wet mixing ratio add_field("ni", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // ice number mixing ratio add_field("pbl_height", scalar2d_layout_col, m, grid_name); // planetary boundary layer height - add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // pdel, hydrostatic pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); // p_del, hydrostatic pressure + add_field("phis", scalar2d_layout_col, m2/s2, grid_name); add_field("cldfrac_tot", scalar3d_layout_mid, nondim, grid_name); // cloud fraction // droplet activation can alter cloud liquid and number mixing ratios add_field("qc", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // cloud liquid wet mixing ratio add_field("nc", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // cloud liquid wet number mixing ratio - // aerosol tracers of interest: mass (q) and number (n) mixing ratios - add_field("q_aitken_so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // sulfate mixing ratio for aitken mode - add_field("n_aitken", scalar3d_layout_mid, n_unit, grid_name, "tracers"); // number mixing ratio of aitken mode + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, grid_name, "tracers"); + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } + } + } // aerosol-related gases: mass mixing ratios - add_field("q_h2so4", scalar3d_layout_mid, q_unit, grid_name, "tracers"); // wet mixing ratio of sulfuric acid gas + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, grid_name, "tracers"); + } // Tracers group -- do we need this in addition to the tracers above? In any // case, this call should be idempotent, so it can't hurt. @@ -89,184 +170,99 @@ set_computed_group_impl(const FieldGroup& group) { EKAT_REQUIRE_MSG(group.m_info->m_bundled, "Error! MAM4 expects bundled fields for tracers.\n"); - // How many aerosol/gas tracers do we expect? Recall that we maintain - // both cloudborne and interstitial aerosol tracers. For now, though, we have - // 3 aerosol tracers: - // * H2SO4 gas mass mixing ratio - // * interstitial aitken-mode sulfate number + mass mixing ratios - int num_aero_tracers = 3; - /* - aero_config_.num_gas_ids() + // gas tracers - 2 * aero_config_.num_modes(); // modal number mixing ratio tracers - for (int m = 0; m < aero_config_.num_modes(); ++m) { - auto m_index = static_cast(m); - for (int a = 0; a < aero_config_.num_aerosol_ids(); ++a) { - auto a_id = static_cast(a); - if (mam4::aerosol_index_for_mode(m_index, a_id) != -1) { - num_aero_tracers += 2; // aerosol mass mixing ratios (interstitial, cloudborne) - } - } - } - */ - - EKAT_REQUIRE_MSG(group.m_info->size() >= num_aero_tracers, - "Error! MAM4 requires at least " << num_aero_tracers << " aerosol tracers."); + // how many aerosol/gas tracers do we expect? + int num_tracers = 2 * (mam_coupling::num_aero_modes() + + mam_coupling::num_aero_tracers()) + + mam_coupling::num_aero_gases(); + EKAT_REQUIRE_MSG(group.m_info->size() >= num_tracers, + "Error! MAM4 requires at least " << num_tracers << " aerosol tracers."); } size_t MAMMicrophysics::requested_buffer_size_in_bytes() const { - // number of Reals needed by local views in interface - const size_t request = sizeof(Real) * - (Buffer::num_2d_mid * ncol_ * nlev_ + - Buffer::num_2d_iface * ncol_ * (nlev_+1)); - - // FIXME: Need to figure out whether we need this stuff - /* - // Number of Reals needed by the WorkspaceManager passed to shoc_main - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(m_num_cols, nlev_packs); - const int n_wind_slots = ekat::npack(2)*Spack::n; - const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const size_t wsm_request= WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy); - */ - - return request;// + wsm_request; + return mam_coupling::buffer_size(ncol_, nlev_); } void MAMMicrophysics::init_buffers(const ATMBufferManager &buffer_manager) { EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), "Error! Insufficient buffer size.\n"); - Real* mem = reinterpret_cast(buffer_manager.get_memory()); - - // set view pointers for midpoint fields - using view_2d_t = decltype(buffer_.z_mid); - view_2d_t* view_2d_mid_ptrs[Buffer::num_2d_mid] = { - &buffer_.z_mid, - &buffer_.dz, - &buffer_.qv_dry, - &buffer_.qc_dry, - &buffer_.n_qc_dry, - &buffer_.qi_dry, - &buffer_.n_qi_dry, - &buffer_.w_updraft, - &buffer_.q_h2so4_tend, - &buffer_.n_aitken_tend, - &buffer_.q_aitken_so4_tend, - }; - for (int i = 0; i < Buffer::num_2d_mid; ++i) { - *view_2d_mid_ptrs[i] = view_2d_t(mem, ncol_, nlev_); - mem += view_2d_mid_ptrs[i]->size(); - } - - // set view pointers for interface fields - view_2d_t* view_2d_iface_ptrs[Buffer::num_2d_iface] = {&buffer_.z_iface}; - for (int i = 0; i < Buffer::num_2d_iface; ++i) { - *view_2d_iface_ptrs[i] = view_2d_t(mem, ncol_, nlev_+1); - mem += view_2d_iface_ptrs[i]->size(); - } - - // WSM data - buffer_.wsm_data = mem; - - /* FIXME: this corresponds to the FIXME in the above function - // Compute workspace manager size to check used memory - // vs. requested memory - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - const int n_wind_slots = ekat::npack(2)*Spack::n; - const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; - const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); - mem += wsm_size; - */ - - size_t used_mem = (mem - buffer_manager.get_memory())*sizeof(Real); + size_t used_mem = mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error! Used memory != requested memory for MAMMicrophysics."); } + void MAMMicrophysics::initialize_impl(const RunType run_type) { - const auto& T_mid = get_field_in("T_mid").get_view(); - const auto& p_mid = get_field_in("p_mid").get_view(); - const auto& qv = get_field_in("qv").get_view(); - const auto& pblh = get_field_in("pbl_height").get_view(); - const auto& p_del = get_field_in("pseudo_density").get_view(); - const auto& cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? - const auto& qc = get_field_out("qc").get_view(); - const auto& n_qc = get_field_out("nc").get_view(); - const auto& qi = get_field_in("qi").get_view(); - const auto& n_qi = get_field_in("ni").get_view(); - const auto& omega = get_field_in("omega").get_view(); - const auto& q_h2so4 = get_field_out("q_h2so4").get_view(); - const auto& n_aitken = get_field_out("n_aitken").get_view(); - const auto& q_aitken_so4 = get_field_out("q_aitken_so4").get_view(); + step_ = 0; + + // populate the wet and dry atmosphere states with views from fields and + // the buffer + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_out("qc").get_view(); + wet_atm_.nc = get_field_out("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot").get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.dz = buffer_.dz; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + dry_atm_.z_surf = 0.0; // FIXME: for now const auto& tracers = get_group_out("tracers"); const auto& tracers_info = tracers.m_info; - int num_tracers = tracers_info->size(); - - // Alias local variables from temporary buffer - auto z_mid = buffer_.z_mid; - auto dz = buffer_.dz; - auto z_iface = buffer_.z_iface; - auto qv_dry = buffer_.qv_dry; - auto qc_dry = buffer_.qc_dry; - auto n_qc_dry = buffer_.n_qc_dry; - auto qi_dry = buffer_.qi_dry; - auto n_qi_dry = buffer_.n_qi_dry; - auto w_updraft = buffer_.w_updraft; - - // Perform any initialization work. - if (run_type==RunType::Initial){ - /* e.g. - Kokkos::deep_copy(sgs_buoy_flux,0.0); - Kokkos::deep_copy(tk,0.0); - Kokkos::deep_copy(tke,0.0004); - Kokkos::deep_copy(tke_copy,0.0004); - Kokkos::deep_copy(cldfrac_liq,0.0); - */ + + // perform any initialization work + if (run_type==RunType::Initial) { } - // set atmosphere state data - T_mid_ = T_mid; - p_mid_ = p_mid; - qv_ = qv; - qc_ = qc; - n_qc_ = n_qc; - qi_ = qi; - n_qi_ = n_qi; - pdel_ = p_del; - cloud_f_ = cldfrac; - pblh_ = pblh; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - n_aitken_ = n_aitken; - - // FIXME: For now, set z_surf to zero. - const Real z_surf = 0.0; - - // Determine indices of aerosol/gas tracers for wet<->dry conversion - auto q_aitken_so4_index = tracers_info->m_subview_idx.at("q_aitken_so4"); - auto q_h2so4_index = tracers_info->m_subview_idx.at("q_h2so4"); - int num_aero_tracers = 2; // for now, just 1 gas + aitken so4 - view_1d_int convert_wet_dry_idx_d("convert_wet_dry_idx_d", num_aero_tracers); - auto convert_wet_dry_idx_h = Kokkos::create_mirror_view(convert_wet_dry_idx_d); - for (int it=0, iq=0; it < num_tracers; ++it) { - if ((it == q_aitken_so4_index) || (it == q_h2so4_index)) { - convert_wet_dry_idx_h(iq) = it; - ++iq; + // set wet/dry aerosol state data (interstitial aerosols only) + for (int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char* int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + for (int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char* int_mmr_field_name = mam_coupling::int_aero_mmr_field_name(m, a); + if (strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } } } - Kokkos::deep_copy(convert_wet_dry_idx_d, convert_wet_dry_idx_h); - // hand views to our preprocess/postprocess functors - preprocess_.set_variables(ncol_, nlev_, z_surf, convert_wet_dry_idx_d, T_mid, - p_mid, qv, qv_dry, qc, n_qc, qc_dry, n_qc_dry, qi, n_qi, - qi_dry, n_qi_dry, z_mid, z_iface, dz, pdel_, cldfrac, omega, w_updraft, pblh, - q_h2so4_, q_aitken_so4_, n_aitken_); - postprocess_.set_variables(ncol_, nlev_, convert_wet_dry_idx_d, qv_dry, - q_h2so4_, q_aitken_so4_, n_aitken_); + // set wet/dry aerosol-related gas state data + for (int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char* mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // create our photolysis rate calculation table + photo_table_ = impl::read_photo_table(get_comm(), + config_.photolysis.rsf_file, + config_.photolysis.xs_long_file); + + // FIXME: read relevant land use data from drydep surface file - // Set field property checks for the fields in this process + // set up our preprocess/postprocess functors + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, dry_aero_); + + // set field property checks for the fields in this process /* e.g. using Interval = FieldWithinIntervalCheck; using LowerBound = FieldLowerBoundCheck; @@ -276,21 +272,10 @@ void MAMMicrophysics::initialize_impl(const RunType run_type) { add_postcondition_check(get_field_out("tke"),m_grid,0); */ - // Setup WSM for internal local variables - // FIXME: do we need this? + // set up WSM for internal local variables + // (we'll probably need this later, but we'll just use ATMBufferManager for now) //const auto default_policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); //workspace_mgr_.setup(buffer_.wsm_data, nlev_+1, 13+(n_wind_slots+n_trac_slots), default_policy); - - // configure the nucleation parameterization - mam4::NucleationProcess::ProcessConfig nuc_config{}; - nuc_config.dens_so4a_host = 1770.0; - nuc_config.mw_so4a_host = 115.0; - nuc_config.newnuc_method_user_choice = 2; - nuc_config.pbl_nuc_wang2008_user_choice = 1; - nuc_config.adjust_factor_pbl_ratenucl = 1.0; - nuc_config.accom_coef_h2so4 = 1.0; - nuc_config.newnuc_adjust_factor_dnaitdt = 1.0; - nucleation_->init(nuc_config); } void MAMMicrophysics::run_impl(const double dt) { @@ -302,85 +287,219 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::parallel_for("preprocess", scan_policy, preprocess_); Kokkos::fence(); - // Reset internal WSM variables. + // reset internal WSM variables //workspace_mgr_.reset_internals(); - // FIXME: nothing depends on simulation time (yet), so we can just use zero for now + // NOTE: nothing depends on simulation time (yet), so we can just use zero for now double t = 0.0; - // Alias member variables - auto T_mid = T_mid_; - auto p_mid = p_mid_; - auto qv_dry = buffer_.qv_dry; - auto qc = qc_; - auto n_qc = n_qc_; - auto qi = qi_; - auto n_qi = n_qi_; - auto z_mid = buffer_.z_mid; - auto cldfrac = cloud_f_; - auto pdel = pdel_; - auto w_updraft = buffer_.w_updraft; - - // Compute nucleation tendencies on all local columns and accumulate them - // into our tracer state. + // here's where we store per-column photolysis rates + using View2D = haero::DeviceType::view_2d; + View2D photo_rates("photo_rates", nlev_, mam4::mo_photo::phtcnt); + + // climatology data for linear stratospheric chemistry + auto linoz_o3_clim = buffer_.scratch[0]; // ozone (climatology) [vmr] + auto linoz_o3col_clim = buffer_.scratch[1]; // column o3 above box (climatology) [Dobson Units (DU)] + auto linoz_t_clim = buffer_.scratch[2]; // temperature (climatology) [K] + auto linoz_PmL_clim = buffer_.scratch[3]; // P minus L (climatology) [vmr/s] + auto linoz_dPmL_dO3 = buffer_.scratch[4]; // sensitivity of P minus L to O3 [1/s] + auto linoz_dPmL_dT = buffer_.scratch[5]; // sensitivity of P minus L to T3 [K] + auto linoz_dPmL_dO3col = buffer_.scratch[6]; // sensitivity of P minus L to overhead O3 column [vmr/DU] + auto linoz_cariolle_psc = buffer_.scratch[7]; // Cariolle parameter for PSC loss of ozone [1/s] + + // it's a bit wasteful to store this for all columns, but simpler from an + // allocation perspective + auto o3_col_dens = buffer_.scratch[8]; + + // FIXME: read relevant linoz climatology data from file(s) based on time + + // FIXME: read relevant chlorine loading data from file based on time + + // loop over atmosphere columns and compute aerosol microphyscs Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const Int icol = team.league_rank(); // column index - - // extract column-specific atmosphere state data - haero::Atmosphere atm(nlev_, ekat::subview(T_mid_, icol), - ekat::subview(p_mid_, icol), - ekat::subview(qv_dry, icol), - ekat::subview(qc_, icol), - ekat::subview(n_qc_, icol), - ekat::subview(qi_, icol), - ekat::subview(n_qi_, icol), - ekat::subview(z_mid, icol), - ekat::subview(pdel, icol), - ekat::subview(cldfrac, icol), - ekat::subview(w_updraft, icol), - pblh_(icol)); + const int icol = team.league_rank(); // column index + + Real col_lat = col_latitudes_(icol); // column latitude (degrees?) + + // fetch column-specific atmosphere state data + auto atm = mam_coupling::atmosphere_for_column(dry_atm_, icol); + auto z_iface = ekat::subview(dry_atm_.z_iface, icol); + Real z_surf = dry_atm_.z_surf; + Real phis = dry_atm_.phis(icol); // set surface state data haero::Surface sfc{}; - // extract column-specific subviews into aerosol prognostics - using ModeIndex = mam4::ModeIndex; - using AeroId = mam4::AeroId; - using GasId = mam4::GasId; - - mam4::Prognostics progs(nlev_); - int iait = static_cast(ModeIndex::Aitken); - progs.n_mode_i[iait] = ekat::subview(n_aitken_, icol); - int iso4 = mam4::aerosol_index_for_mode(ModeIndex::Aitken, AeroId::SO4); - progs.q_aero_i[iait][iso4] = ekat::subview(q_aitken_so4_, icol); - int ih2so4 = static_cast(GasId::H2SO4); - progs.q_gas[ih2so4] = ekat::subview(q_h2so4_, icol); - - // nucleation doesn't use any diagnostics, so it's okay to leave this alone - // for now + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = mam_coupling::interstitial_aerosols_for_column(dry_aero_, icol); + + // set up diagnostics mam4::Diagnostics diags(nlev_); - // grab views from the buffer to store tendencies - mam4::Tendencies tends(nlev_); - tends.q_gas[ih2so4] = ekat::subview(buffer_.q_h2so4_tend, icol); - tends.n_mode_i[iait] = ekat::subview(buffer_.n_aitken_tend, icol); - tends.q_aero_i[iait][iso4] = ekat::subview(buffer_.q_aitken_so4_tend, icol); - - // run the nucleation process to obtain tendencies - nucleation_->compute_tendencies(team, t, dt, atm, sfc, progs, diags, tends); -#ifndef NDEBUG - const int lev_idx = 0; - if (icol == 0) { - m_atm_logger->debug("tends.q_gas[ih2so4] = {}, tends.n_mode_i[iait] = {}, tends.q_aero_i[iait][iso4] = {}", - tends.q_gas[ih2so4](lev_idx), tends.n_mode_i[iait](lev_idx), tends.q_aero_i[iait][iso4](lev_idx)); - } -#endif + // calculate o3 column densities (first component of col_dens in Fortran code) + auto o3_col_dens_i = ekat::subview(o3_col_dens, icol); + impl::compute_o3_column_density(team, atm, progs, o3_col_dens_i); + + // set up photolysis work arrays for this column. + mam4::mo_photo::PhotoTableWorkArrays photo_work_arrays; + // FIXME: set views here + + // ... look up photolysis rates from our table + // NOTE: the table interpolation operates on an entire column of data, so we + // NOTE: must do it before dispatching to individual vertical levels + Real zenith_angle = 0.0; // FIXME: need to get this from EAMxx [radians] + Real surf_albedo = 0.0; // FIXME: surface albedo + Real esfact = 0.0; // FIXME: earth-sun distance factor + mam4::ColumnView lwc; // FIXME: liquid water cloud content: where do we get this? + mam4::mo_photo::table_photo(photo_rates, atm.pressure, atm.hydrostatic_dp, + atm.temperature, o3_col_dens_i, zenith_angle, surf_albedo, lwc, + atm.cloud_fraction, esfact, photo_table_, photo_work_arrays); + + // compute external forcings at time t(n+1) [molecules/cm^3/s] + constexpr int extcnt = mam4::gas_chemistry::extcnt; + view_2d extfrc; // FIXME: where to allocate? (nlev, extcnt) + mam4::mo_setext::Forcing forcings[extcnt]; // FIXME: forcings seem to require file data + mam4::mo_setext::extfrc_set(forcings, extfrc); + + // compute aerosol microphysics on each vertical level within this column + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int k) { + + constexpr int num_modes = mam4::AeroConfig::num_modes(); + constexpr int num_aero_ids = mam4::AeroConfig::num_aerosol_ids(); + constexpr int gas_pcnst = mam_coupling::gas_pcnst(); + constexpr int nqtendbb = mam_coupling::nqtendbb(); + + // extract atm state variables (input) + Real temp = atm.temperature(k); + Real pmid = atm.pressure(k); + Real pdel = atm.hydrostatic_dp(k); + Real zm = atm.height(k); + Real zi = z_iface(k); + Real pblh = atm.planetary_boundary_layer_height; + Real qv = atm.vapor_mixing_ratio(k); + Real cldfrac = atm.cloud_fraction(k); + + // extract aerosol state variables into "working arrays" (mass mixing ratios) + // (in EAM, this is done in the gas_phase_chemdr subroutine defined within + // mozart/mo_gas_phase_chemdr.F90) + Real q[gas_pcnst] = {}; + Real qqcw[gas_pcnst] = {}; + mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); + + // convert mass mixing ratios to volume mixing ratios (VMR), equivalent + // to tracer mixing ratios (TMR)) + Real vmr[gas_pcnst], vmrcw[gas_pcnst]; + mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); + + // aerosol/gas species tendencies (output) + Real vmr_tendbb[gas_pcnst][nqtendbb] = {}; + Real vmrcw_tendbb[gas_pcnst][nqtendbb] = {}; + + // create work array copies to retain "pre-chemistry" values + Real vmr_pregaschem[gas_pcnst] = {}; + Real vmr_precldchem[gas_pcnst] = {}; + Real vmrcw_precldchem[gas_pcnst] = {}; + for (int i = 0; i < gas_pcnst; ++i) { + vmr_pregaschem[i] = vmr[i]; + vmr_precldchem[i] = vmr[i]; + vmrcw_precldchem[i] = vmrcw[i]; + } + + //--------------------- + // Gas Phase Chemistry + //--------------------- + Real photo_rates_k[mam4::mo_photo::phtcnt]; + for (int i = 0; i < mam4::mo_photo::phtcnt; ++i) { + photo_rates_k[i] = photo_rates(k, i); + } + Real extfrc_k[extcnt]; + for (int i = 0; i < extcnt; ++i) { + extfrc_k[i] = extfrc(k, i); + } + constexpr int nfs = mam4::gas_chemistry::nfs; // number of "fixed species" + // NOTE: we compute invariants here and pass them out to use later with + // NOTE: setsox + Real invariants[nfs]; + impl::gas_phase_chemistry(zm, zi, phis, temp, pmid, pdel, dt, + photo_rates_k, extfrc_k, vmr, invariants); + + //---------------------- + // Aerosol microphysics + //---------------------- + // the logic below is taken from the aero_model_gasaerexch subroutine in + // eam/src/chemistry/modal_aero/aero_model.F90 + + // aqueous chemistry ... + const int loffset = 8; // offset of first tracer in work arrays + // (taken from mam4xx setsox validation test) + const Real mbar = haero::Constants::molec_weight_dry_air; + constexpr int indexm = 0; // FIXME: index of xhnm in invariants array (??) + Real cldnum = 0.0; // FIXME: droplet number concentration: where do we get this? + setsox_single_level(loffset, dt, pmid, pdel, temp, mbar, lwc(k), + cldfrac, cldnum, invariants[indexm], config_.setsox, vmrcw, vmr); + + // calculate aerosol water content using water uptake treatment + // * dry and wet diameters [m] + // * wet densities [kg/m3] + // * aerosol water mass mixing ratio [kg/kg] + Real dgncur_a[num_modes] = {}; + Real dgncur_awet[num_modes] = {}; + Real wetdens[num_modes] = {}; + Real qaerwat[num_modes] = {}; + impl::compute_water_content(progs, k, qv, temp, pmid, dgncur_a, dgncur_awet, wetdens, qaerwat); + + // do aerosol microphysics (gas-aerosol exchange, nucleation, coagulation) + impl::modal_aero_amicphys_intr(config_.amicphys, step_, dt, t, pmid, pdel, + zm, pblh, qv, cldfrac, vmr, vmrcw, vmr_pregaschem, + vmr_precldchem, vmrcw_precldchem, vmr_tendbb, + vmrcw_tendbb, dgncur_a, dgncur_awet, + wetdens, qaerwat); + + //----------------- + // LINOZ chemistry + //----------------- + + // the following things are diagnostics, which we're not + // including in the first rev + Real do3_linoz, do3_linoz_psc, ss_o3, o3col_du_diag, o3clim_linoz_diag, + zenith_angle_degrees; + + // FIXME: Need to get chlorine loading data from file + Real chlorine_loading = 0.0; + + Real rlats = col_lat * M_PI / 180.0; // convert column latitude to radians + int o3_ndx = 0; // index of "O3" in solsym array (in EAM) + mam4::lin_strat_chem::lin_strat_chem_solve_kk(o3_col_dens_i(k), temp, + zenith_angle, pmid, dt, rlats, + linoz_o3_clim(icol, k), linoz_t_clim(icol, k), linoz_o3col_clim(icol, k), + linoz_PmL_clim(icol, k), linoz_dPmL_dO3(icol, k), linoz_dPmL_dT(icol, k), + linoz_dPmL_dO3col(icol, k), linoz_cariolle_psc(icol, k), + chlorine_loading, config_.linoz.psc_T, vmr[o3_ndx], + do3_linoz, do3_linoz_psc, ss_o3, + o3col_du_diag, o3clim_linoz_diag, zenith_angle_degrees); + + // update source terms above the ozone decay threshold + if (k > nlev_ - config_.linoz.o3_lbl - 1) { + Real do3_mass; // diagnostic, not needed + mam4::lin_strat_chem::lin_strat_sfcsink_kk(dt, pdel, vmr[o3_ndx], config_.linoz.o3_sfc, + config_.linoz.o3_tau, do3_mass); + } + + // ... check for negative values and reset to zero + for (int i = 0; i < gas_pcnst; ++i) { + if (vmr[i] < 0.0) vmr[i] = 0.0; + } - // accumulate tendencies into prognostics - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, nlev_), [&](const int klev) { - progs.q_gas[ih2so4](klev) += dt * tends.q_gas[ih2so4](klev); - progs.n_mode_i[iait](klev) += dt * tends.n_mode_i[iait](klev); - progs.q_aero_i[iait][iso4](klev) += dt * tends.q_aero_i[iait][iso4](klev); + //---------------------- + // Dry deposition (gas) + //---------------------- + + // FIXME: C++ port in progress! + //mam4::drydep::drydep_xactive(...); + + // transfer updated prognostics from work arrays + mam_coupling::convert_work_arrays_to_mmr(vmr, vmrcw, q, qqcw); + mam_coupling::transfer_work_arrays_to_prognostics(q, qqcw, progs, k); }); }); @@ -389,8 +508,7 @@ void MAMMicrophysics::run_impl(const double dt) { Kokkos::fence(); } -void MAMMicrophysics::finalize_impl() -{ +void MAMMicrophysics::finalize_impl() { } } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp index ff3652459f1e..5f5a44b846be 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_microphysics_process_interface.hpp @@ -1,9 +1,11 @@ #ifndef EAMXX_MAM_MICROPHYSICS_HPP #define EAMXX_MAM_MICROPHYSICS_HPP +#include #include #include -#include + +#include "impl/mam4_amicphys.cpp" // mam4xx top-level microphysics function(s) #include #include @@ -22,8 +24,8 @@ namespace scream { -// The process responsible for handling MAM4 aerosols. The AD stores exactly ONE -// instance of this class in its list of subcomponents. +// The process responsible for handling MAM4 aerosol microphysics. The AD +// stores exactly ONE instance of this class in its list of subcomponents. class MAMMicrophysics final : public scream::AtmosphereProcess { using PF = scream::PhysicsFunctions; using KT = ekat::KokkosTypes; @@ -60,6 +62,9 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { AtmosphereProcessType type() const override; std::string name() const override; + // set aerosol microphysics configuration parameters (called by constructor) + void configure(const ekat::ParameterList& params); + // grid void set_grids(const std::shared_ptr grids_manager) override; @@ -80,284 +85,144 @@ class MAMMicrophysics final : public scream::AtmosphereProcess { // number of horizontal columns and vertical levels int ncol_, nlev_; + // configuration data (for the moment, we plan to be able to move this to + // the device, so we can't use C++ strings) + struct Config { + // photolysis parameters + struct { + char rsf_file[MAX_FILENAME_LEN]; + char xs_long_file[MAX_FILENAME_LEN]; + } photolysis; + + // stratospheric chemistry parameters + struct { + int o3_lbl; // number of layers with ozone decay from the surface + int o3_sfc; // set from namelist input linoz_sfc + int o3_tau; // set from namelist input linoz_tau + Real psc_T; // set from namelist input linoz_psc_T + char chlorine_loading_file[MAX_FILENAME_LEN]; + } linoz; + + // aqueous chemistry parameters + mam4::mo_setsox::Config setsox; + + // aero microphysics configuration (see impl/mam4_amicphys.cpp) + impl::AmicPhysConfig amicphys; + + // dry deposition parameters + struct { + char srf_file[MAX_FILENAME_LEN]; + } drydep; + }; + Config config_; + // Atmosphere processes often have a pre-processing step that constructs // required variables from the set of fields stored in the field manager. // This functor implements this step, which is called during run_impl. struct Preprocess { Preprocess() = default; + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + KOKKOS_INLINE_FUNCTION void operator()(const Kokkos::TeamPolicy::member_type& team) const { const int i = team.league_rank(); // column index - // Compute vertical layer heights - const auto dz_i = ekat::subview(dz_, i); - auto z_iface_i = ekat::subview(z_iface_, i); - auto z_mid_i = ekat::subview(z_mid_, i); - PF::calculate_z_int(team, nlev_, dz_i, z_surf_, z_iface_i); - team.team_barrier(); // TODO: is this barrier necessary? - PF::calculate_z_mid(team, nlev_, z_iface_i, z_mid_i); - // barrier here allows the kernels that follow to use layer heights - team.team_barrier(); - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_), - [&] (const int k) { - //-------------------------- - // Vertical velocity from pressure to height - //-------------------------- - const auto rho = PF::calculate_density(pdel_(i,k), dz_i(k)); - w_updraft_(i,k) = PF::calculate_vertical_velocity(omega_(i,k), rho); - - //-------------------------- - // Wet to dry mixing ratios - //-------------------------- - // - // Since tracers from the host model (or AD) are wet mixing ratios, and - // MAM4 expects these tracers in dry mixing ratios, we convert the wet - // mixing ratios to dry mixing ratios for all the tracers. - // - // The function calculate_drymmr_from_wetmmr takes 2 arguments: - // 1. wet mmr - // 2. "wet" water vapor mixing ratio - // - // Units of all tracers become [kg/kg(dry-air)] for mass mixing ratios and - // [#/kg(dry-air)] for number mixing ratios after the following - // conversion. - const auto qv_ik = qv_(i,k); - // const fields need separate storage for "dry" values - qv_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qv_(i,k), qv_ik); - qc_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qc_(i,k), qv_ik); - n_qc_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(n_qc_(i,k), qv_ik); - qi_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(qi_(i,k), qv_ik); - n_qi_dry_(i,k) = PF::calculate_drymmr_from_wetmmr(n_qi_(i,k), qv_ik); - - // non-const fields can be overwritten; we'll convert back to moist - // air ratios during postprocess - q_h2so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_h2so4_(i,k), qv_ik); - q_aitken_so4_(i,k) = PF::calculate_drymmr_from_wetmmr(q_aitken_so4_(i,k), qv_ik); - n_aitken_(i,k) = PF::calculate_drymmr_from_wetmmr(n_aitken_(i,k), qv_ik); - }); - // make sure all operations are done before exiting kernel + compute_vertical_layer_heights(team, dry_atm_, i); + team.team_barrier(); // allows kernels below to use layer heights + compute_updraft_velocities(team, wet_atm_, dry_atm_, i); + compute_dry_mixing_ratios(team, wet_atm_, dry_atm_, i); + compute_dry_mixing_ratios(team, wet_atm_, wet_aero_, dry_aero_, i); team.team_barrier(); } // operator() // number of horizontal columns and vertical levels int ncol_, nlev_; - // height of bottom of atmosphere - Real z_surf_; - - // used for converting between wet and dry mixing ratios - view_1d_int convert_wet_dry_idx_d_; - - // local atmospheric state column variables - const_view_2d T_mid_; // temperature at grid midpoints [K] - const_view_2d p_mid_; // total pressure at grid midpoints [Pa] - const_view_2d qv_; // water vapor specific humidity [kg vapor / kg moist air] - view_2d qv_dry_; // water vapor mixing ratio [kg vapor / kg dry air] - const_view_2d qc_; // cloud liquid water mass mixing ratio [kg vapor/kg moist air] - view_2d qc_dry_; - const_view_2d n_qc_; // cloud liquid water number mixing ratio [kg cloud water / kg moist air] - view_2d n_qc_dry_; // cloud liquid water number mixing ratio (dry air) - const_view_2d qi_; // cloud ice water mass mixing ratio - view_2d qi_dry_; // [kg vapor/kg dry air] - const_view_2d n_qi_; // cloud ice water number mixing ratio - view_2d n_qi_dry_; - view_2d z_mid_; // height at layer midpoints [m] - view_2d z_iface_; // height at layer interfaces [m] - view_2d dz_; // layer thickness [m] - const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - const_view_2d cloud_f_; // cloud fraction [-] - const_view_2d omega_; // vertical pressure velocity [Pa/s] - view_2d w_updraft_; // updraft velocity [m/s] - const_view_1d pblh_; // planetary boundary layer height [m] - - // local aerosol-related gases - view_2d q_h2so4_; // H2SO4 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] - view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] - - // assigns local variables - void set_variables(const int ncol, const int nlev, const Real z_surf, - const view_1d_int& convert_wet_dry_idx_d, - const const_view_2d& T_mid, - const const_view_2d& p_mid, - const const_view_2d& qv, - const view_2d& qv_dry, - const view_2d& qc, - const view_2d& n_qc, - const view_2d& qc_dry, - const view_2d& n_qc_dry, - const const_view_2d& qi, - const const_view_2d& n_qi, - const view_2d& qi_dry, - const view_2d& n_qi_dry, - const view_2d& z_mid, - const view_2d& z_iface, - const view_2d& dz, - const const_view_2d& pdel, - const const_view_2d& cf, - const const_view_2d& omega, - const view_2d& w_updraft, - const const_view_1d& pblh, - const view_2d& q_h2so4, - const view_2d& q_aitken_so4, - const view_2d& n_aitken) { - ncol_ = ncol; - nlev_ = nlev; - z_surf_ = z_surf; - convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; - T_mid_ = T_mid; - p_mid_ = p_mid; - qv_ = qv; - qv_dry_ = qv_dry; - qc_ = qc; - n_qc_ = n_qc; - qc_dry_ = qc_dry; - n_qc_dry_ = n_qc_dry; - qi_ = qi; - n_qi_ = n_qi; - qi_dry_ = qi_dry; - n_qi_dry_ = n_qi_dry; - z_mid_ = z_mid; - z_iface_ = z_iface; - dz_ = dz; - pdel_ = pdel; - cloud_f_ = cf; - omega_ = omega; - w_updraft_ = w_updraft; - pblh_ = pblh; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - n_aitken_ = n_aitken; - } // set_variables + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + }; // MAMMicrophysics::Preprocess // Postprocessing functor struct Postprocess { Postprocess() = default; + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere& wet_atm, + const mam_coupling::AerosolState& wet_aero, + const mam_coupling::DryAtmosphere& dry_atm, + const mam_coupling::AerosolState& dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + KOKKOS_INLINE_FUNCTION void operator()(const Kokkos::TeamPolicy::member_type& team) const { - // After these updates, all non-const tracers are converted from dry mmr to wet mmr - const int i = team.league_rank(); - - Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev_), - [&] (const int k) { - const auto qv_ik = qv_dry_(i,k); - q_h2so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_h2so4_(i,k), qv_ik); - q_aitken_so4_(i,k) = PF::calculate_wetmmr_from_drymmr(q_aitken_so4_(i,k), qv_ik); - n_aitken_(i,k) = PF::calculate_wetmmr_from_drymmr(n_aitken_(i,k), qv_ik); - }); + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_, dry_aero_, wet_aero_, i); team.team_barrier(); } // operator() - // Local variables + // number of horizontal columns and vertical levels int ncol_, nlev_; - view_2d qv_dry_; - - // used for converting between wet and dry mixing ratios - view_1d_int convert_wet_dry_idx_d_; - - // local aerosol-related gases - view_2d q_h2so4_; // H2SO4 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d q_aitken_so4_; // SO4 aerosol in aitken mode [kg/kg dry air] - // modal quantities - view_2d n_aitken_; - - // assigns local variables - void set_variables(const int ncol, - const int nlev, - const view_1d_int& convert_wet_dry_idx_d, - const view_2d& qv_dry, - const view_2d& q_h2so4, - const view_2d& q_aitken_so4, - const view_2d& n_aitken) { - ncol_ = ncol; - nlev_ = nlev; - convert_wet_dry_idx_d_ = convert_wet_dry_idx_d; - qv_dry_ = qv_dry; - q_h2so4_ = q_h2so4; - q_aitken_so4_ = q_aitken_so4; - n_aitken_ = n_aitken; - } // set_variables + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; }; // MAMMicrophysics::Postprocess - // storage for local variables, initialized with ATMBufferManager - struct Buffer { - // number of local fields stored at column midpoints - static constexpr int num_2d_mid = 11; - - // local column midpoint fields - uview_2d z_mid; // height at midpoints - uview_2d dz; // layer thickness - uview_2d qv_dry; // water vapor mixing ratio (dry air) - uview_2d qc_dry; // cloud water mass mixing ratio - uview_2d n_qc_dry; // cloud water number mixing ratio - uview_2d qi_dry; // cloud ice mass mixing ratio - uview_2d n_qi_dry; // cloud ice number mixing ratio - uview_2d w_updraft; // vertical wind velocity - uview_2d q_h2so4_tend; // tendency for H2SO4 gas - uview_2d n_aitken_tend; // tendency for aitken aerosol mode - uview_2d q_aitken_so4_tend; // tendency for aitken mode sulfate aerosol - - // number of local fields stored at column interfaces - static constexpr int num_2d_iface = 1; - - // local column interface fields - uview_2d z_iface; // height at interfaces - - // storage - Real* wsm_data; - }; - // MAM4 aerosol particle size description mam4::AeroConfig aero_config_; - // aerosol processes - std::unique_ptr nucleation_; - - // pre- and postprocessing scratch pads + // pre- and postprocessing scratch pads (for wet <-> dry conversions) Preprocess preprocess_; Postprocess postprocess_; - // local atmospheric state column variables - const_view_2d T_mid_; // temperature at grid midpoints [K] - const_view_2d p_mid_; // total pressure at grid midpoints [Pa] - const_view_2d qv_; // water vapor specific humidity [kg h2o vapor / kg moist air] - // we keep the specific humidity to use in the PostProcess step. - view_2d qc_; // cloud liquid mass mixing ratio - // must be converted from wet to dry [kg cloud water /kg dry air] - view_2d n_qc_; // cloud liquid number mixing ratio - // must be converted from wet to dry [1 /kg dry air] - const_view_2d qi_; // cloud ice mass mixing ratio - // must be converted from wet to dry [kg cloud water /kg dry air] - const_view_2d n_qi_; // cloud ice number mixing ratio - // must be converted from wet to dry [1 /kg dry air] - const_view_2d pdel_; // hydrostatic "pressure thickness" at grid - // interfaces [Pa] - const_view_2d cloud_f_; // cloud fraction [-] - const_view_1d pblh_; // planetary boundary layer height [m] - - // local aerosol-related gases - view_2d q_h2so4_; // H2SO3 gas [kg/kg dry air] - - // local aerosols (more to appear as we improve this atm process) - view_2d n_aitken_; // aitken mode number mixing ratio [1/kg dry air] - view_2d q_aitken_so4_; // SO4 mass mixing ratio in aitken mode [kg/kg dry air] + // atmospheric and aerosol state variables + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + + // photolysis rate table (column-independent) + mam4::mo_photo::PhotoTableData photo_table_; + + // column areas, latitudes, longitudes + const_view_1d col_areas_, col_latitudes_, col_longitudes_; + + // time step number + int step_; // workspace manager for internal local variables //ekat::WorkspaceManager workspace_mgr_; - Buffer buffer_; + mam_coupling::Buffer buffer_; // physics grid for column information std::shared_ptr grid_; + + // sets defaults for "namelist parameters" + void set_defaults_(); + }; // MAMMicrophysics } // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp index 85b51ff3f3dc..a827347c5385 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.cpp @@ -1,107 +1,518 @@ +#include #include #include #include -#include "scream_config.h" // for SCREAM_CIME_BUILD - -#include +#include "scream_config.h" // for SCREAM_CIME_BUILD +#include "share/grid/point_grid.hpp" +#include "share/io/scorpio_input.hpp" -namespace scream -{ +namespace scream { -MAMOptics::MAMOptics( - const ekat::Comm& comm, - const ekat::ParameterList& params) - : AtmosphereProcess(comm, params), - aero_config_() { -} +MAMOptics::MAMOptics(const ekat::Comm &comm, const ekat::ParameterList ¶ms) + : AtmosphereProcess(comm, params), aero_config_() { + } AtmosphereProcessType MAMOptics::type() const { return AtmosphereProcessType::Physics; } -std::string MAMOptics::name() const { - return "mam4_optics"; -} +std::string MAMOptics::name() const { return "mam4_optics"; } -void MAMOptics::set_grids(const std::shared_ptr grids_manager) { +void MAMOptics::set_grids( + const std::shared_ptr grids_manager) { using namespace ekat::units; - grid_ = grids_manager->get_grid("Physics"); - const auto& grid_name = grid_->name(); + grid_ = grids_manager->get_grid("Physics"); + const auto &grid_name = grid_->name(); + auto q_unit = kg / kg; // mass mixing ratios [kg stuff / kg air] + q_unit.set_string("kg/kg"); + auto n_unit = 1 / kg; // number mixing ratios [# / kg air] + n_unit.set_string("#/kg"); + const auto m2 = m * m; + const auto s2 = s * s; - ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank - nlev_ = grid_->get_num_vertical_levels(); // number of levels per column - nswbands_ = 14; // number of shortwave bands - nlwbands_ = 16; // number of longwave bands + ncol_ = grid_->get_num_local_dofs(); // number of columns on this rank + nlev_ = grid_->get_num_vertical_levels(); // number of levels per column + nswbands_ = mam4::modal_aer_opt::nswbands; // number of shortwave bands + nlwbands_ = mam4::modal_aer_opt::nlwbands; // number of longwave bands // Define the different field layouts that will be used for this process using namespace ShortFieldTagsNames; // Define aerosol optics fields computed by this process. auto nondim = Units::nondimensional(); - FieldLayout scalar3d_swband_layout { {COL, SWBND, LEV}, {ncol_, nswbands_, nlev_} }; - FieldLayout scalar3d_lwband_layout { {COL, LWBND, LEV}, {ncol_, nlwbands_, nlev_} }; - - // shortwave aerosol scattering asymmetry parameter [-] - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol single-scattering albedo [-] - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name); - // shortwave aerosol optical depth [-] - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name); - // longwave aerosol optical depth [-] + // 3D layout for shortwave aerosol fields: columns, number of shortwave, and nlev + FieldLayout scalar3d_swband_layout{{COL, SWBND, LEV}, + {ncol_, nswbands_, nlev_}}; + + // 3D layout for longwave aerosol fields: columns, number of shortwave, and nlev +1 s + FieldLayout scalar3d_lwband_layout{{COL, LWBND, LEV}, + {ncol_, nlwbands_, nlev_}}; + + FieldLayout scalar3d_layout_int{{COL, ILEV}, {ncol_, nlev_ + 1}}; + + // layout for 2D (1d horiz X 1d vertical) variable + FieldLayout scalar2d_layout_col{{COL}, {ncol_}}; + + // layout for 3D (2d horiz X 1d vertical) variables + FieldLayout scalar3d_layout_mid{{COL, LEV}, {ncol_, nlev_}}; + add_field("omega", scalar3d_layout_mid, Pa / s, + grid_name); // vertical pressure velocity + add_field("T_mid", scalar3d_layout_mid, K, + grid_name); // Temperature + add_field("p_mid", scalar3d_layout_mid, Pa, + grid_name); // total pressure + + add_field("p_int", scalar3d_layout_int, Pa, + grid_name); // total pressure + add_field("pseudo_density", scalar3d_layout_mid, Pa, grid_name); + add_field("pseudo_density_dry", scalar3d_layout_mid, Pa, grid_name); + + add_field("qv", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // specific humidity + add_field("qi", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // ice wet mixing ratio + add_field("ni", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // ice number mixing ratio + + // droplet activation can alter cloud liquid and number mixing ratios + add_field("qc", scalar3d_layout_mid, q_unit, grid_name, + "tracers"); // cloud liquid wet mixing ratio + add_field("nc", scalar3d_layout_mid, n_unit, grid_name, + "tracers"); // cloud liquid wet number mixing ratio + + add_field("phis", scalar2d_layout_col, m2 / s2, grid_name); + add_field("cldfrac_tot", scalar3d_layout_mid, nondim, + grid_name); // cloud fraction + add_field("pbl_height", scalar2d_layout_col, m, + grid_name); // planetary boundary layer height + // shortwave aerosol scattering asymmetry parameter [unitless] + add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name); + // shortwave aerosol single-scattering albedo [unitless] + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, + grid_name); + // shortwave aerosol extinction optical depth + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, + grid_name); + //longwave aerosol extinction optical depth [unitless] add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name); - // FIXME: this field doesn't belong here, but this is a convenient place to - // FIXME: put it for now. - // number mixing ratio for CCN - using Spack = ekat::Pack; - using Pack = ekat::Pack; - constexpr int ps = Pack::n; - FieldLayout scalar3d_layout_mid { {COL, LEV}, {ncol_, nlev_} }; - add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); + add_field("aodvis", scalar2d_layout_col, nondim, grid_name); + + // (interstitial) aerosol tracers of interest: mass (q) and number (n) mixing + // ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + + add_field(int_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name, "tracers"); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + + if(strlen(int_mmr_field_name) > 0) { + add_field(int_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } + } + } + // (cloud) aerosol tracers of interest: mass (q) and number (n) mixing ratios + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + + add_field(cld_nmr_field_name, scalar3d_layout_mid, n_unit, + grid_name); + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + + if(strlen(cld_mmr_field_name) > 0) { + add_field(cld_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name); + } + } + } + + // aerosol-related gases: mass mixing ratios + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *gas_mmr_field_name = mam_coupling::gas_mmr_field_name(g); + add_field(gas_mmr_field_name, scalar3d_layout_mid, q_unit, + grid_name, "tracers"); + } } -void MAMOptics::initialize_impl(const RunType run_type) { +size_t MAMOptics::requested_buffer_size_in_bytes() const { + return mam_coupling::buffer_size(ncol_, nlev_); } -void MAMOptics::run_impl(const double dt) { +void MAMOptics::init_buffers(const ATMBufferManager &buffer_manager) { + EKAT_REQUIRE_MSG( + buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), + "Error! Insufficient buffer size.\n"); - const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); - - // get the aerosol optics fields - auto aero_g_sw = get_field_out("aero_g_sw").get_view(); - auto aero_ssa_sw = get_field_out("aero_ssa_sw").get_view(); - auto aero_tau_sw = get_field_out("aero_tau_sw").get_view(); - auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); - - auto aero_nccn = get_field_out("nccn").get_view(); // FIXME: get rid of this - - // Compute optical properties on all local columns. - // (Strictly speaking, we don't need this parallel_for here yet, but we leave - // it in anticipation of column-specific aerosol optics to come.) - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(const ThreadTeam& team) { - const Int icol = team.league_rank(); // column index - - auto g_sw = ekat::subview(aero_g_sw, icol); - auto ssa_sw = ekat::subview(aero_ssa_sw, icol); - auto tau_sw = ekat::subview(aero_tau_sw, icol); - auto tau_lw = ekat::subview(aero_tau_lw, icol); - - // populate these fields with reasonable representative values - Kokkos::deep_copy(g_sw, 0.5); - Kokkos::deep_copy(ssa_sw, 0.7); - Kokkos::deep_copy(tau_sw, 0.0); - Kokkos::deep_copy(tau_lw, 0.0); - - // FIXME: Get rid of this - auto nccn = ekat::subview(aero_nccn, icol); - Kokkos::deep_copy(nccn, 50.0); - }); + size_t used_mem = + mam_coupling::init_buffer(buffer_manager, ncol_, nlev_, buffer_); + EKAT_REQUIRE_MSG(used_mem == requested_buffer_size_in_bytes(), + "Error! Used memory != requested memory for MAMMOptics."); } -void MAMOptics::finalize_impl() -{ +void MAMOptics::initialize_impl(const RunType run_type) { + // populate the wet and dry atmosphere states with views from fields and + // the buffer + wet_atm_.qv = get_field_in("qv").get_view(); + wet_atm_.qc = get_field_in("qc").get_view(); + wet_atm_.nc = get_field_in("nc").get_view(); + wet_atm_.qi = get_field_in("qi").get_view(); + wet_atm_.ni = get_field_in("ni").get_view(); + wet_atm_.omega = get_field_in("omega").get_view(); + + constexpr int ntot_amode = mam4::AeroConfig::num_modes(); + + dry_atm_.T_mid = get_field_in("T_mid").get_view(); + dry_atm_.p_mid = get_field_in("p_mid").get_view(); + dry_atm_.p_int = get_field_in("p_int").get_view(); + dry_atm_.p_del = get_field_in("pseudo_density_dry").get_view(); + p_del_ = get_field_in("pseudo_density").get_view(); + dry_atm_.cldfrac = get_field_in("cldfrac_tot") + .get_view(); // FIXME: tot or liq? + dry_atm_.pblh = get_field_in("pbl_height").get_view(); + dry_atm_.phis = get_field_in("phis").get_view(); + + dry_atm_.z_mid = buffer_.z_mid; + dry_atm_.dz = buffer_.dz; + dry_atm_.z_iface = buffer_.z_iface; + dry_atm_.qv = buffer_.qv_dry; + dry_atm_.qc = buffer_.qc_dry; + dry_atm_.nc = buffer_.nc_dry; + dry_atm_.qi = buffer_.qi_dry; + dry_atm_.ni = buffer_.ni_dry; + dry_atm_.w_updraft = buffer_.w_updraft; + // The surface height is zero by definition. + // see eam/src/physics/cam/geopotential.F90 + dry_atm_.z_surf = 0.0; + + // set wet/dry aerosol state data (interstitial aerosols only) + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *int_nmr_field_name = mam_coupling::int_aero_nmr_field_name(m); + wet_aero_.int_aero_nmr[m] = + get_field_out(int_nmr_field_name).get_view(); + dry_aero_.int_aero_nmr[m] = buffer_.dry_int_aero_nmr[m]; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *int_mmr_field_name = + mam_coupling::int_aero_mmr_field_name(m, a); + if(strlen(int_mmr_field_name) > 0) { + wet_aero_.int_aero_mmr[m][a] = + get_field_out(int_mmr_field_name).get_view(); + dry_aero_.int_aero_mmr[m][a] = buffer_.dry_int_aero_mmr[m][a]; + } + } + } + + // set wet/dry aerosol state data (cloud aerosols only) + for(int m = 0; m < mam_coupling::num_aero_modes(); ++m) { + const char *cld_nmr_field_name = mam_coupling::cld_aero_nmr_field_name(m); + wet_aero_.cld_aero_nmr[m] = + get_field_out(cld_nmr_field_name).get_view(); + dry_aero_.cld_aero_nmr[m] = buffer_.dry_cld_aero_nmr[m]; + for(int a = 0; a < mam_coupling::num_aero_species(); ++a) { + const char *cld_mmr_field_name = + mam_coupling::cld_aero_mmr_field_name(m, a); + if(strlen(cld_mmr_field_name) > 0) { + wet_aero_.cld_aero_mmr[m][a] = + get_field_out(cld_mmr_field_name).get_view(); + dry_aero_.cld_aero_mmr[m][a] = buffer_.dry_cld_aero_mmr[m][a]; + } + } + } + + // set wet/dry aerosol-related gas state data + for(int g = 0; g < mam_coupling::num_aero_gases(); ++g) { + const char *mmr_field_name = mam_coupling::gas_mmr_field_name(g); + wet_aero_.gas_mmr[g] = get_field_out(mmr_field_name).get_view(); + dry_aero_.gas_mmr[g] = buffer_.dry_gas_mmr[g]; + } + + // prescribed volcanic aerosols. + ssa_cmip6_sw_ = + mam_coupling::view_3d("ssa_cmip6_sw", ncol_, nlev_, nswbands_); + af_cmip6_sw_ = mam_coupling::view_3d("af_cmip6_sw", ncol_, nlev_, nswbands_); + ext_cmip6_sw_ = + mam_coupling::view_3d("ext_cmip6_sw", ncol_, nswbands_, nlev_); + ext_cmip6_lw_ = + mam_coupling::view_3d("ext_cmip6_lw_", ncol_, nlev_, nlwbands_); + // FIXME: We need to get ssa_cmip6_sw_, af_cmip6_sw_, ext_cmip6_sw_, + // ext_cmip6_lw_ from a netcdf file. + // We will rely on eamxx to interpolate/map data from netcdf files. + // The io interface in eamxx is being upgraded. + // Thus, I will wait until changes in the eamxx's side are completed. + Kokkos::deep_copy(ssa_cmip6_sw_, 0.0); + Kokkos::deep_copy(af_cmip6_sw_, 0.0); + Kokkos::deep_copy(ext_cmip6_sw_, 0.0); + Kokkos::deep_copy(ext_cmip6_lw_, 0.0); + + // set up our preprocess/postprocess functors + preprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + postprocess_.initialize(ncol_, nlev_, wet_atm_, wet_aero_, dry_atm_, + dry_aero_); + + const int work_len = mam4::modal_aer_opt::get_work_len_aerosol_optics(); + work_ = mam_coupling::view_2d("work", ncol_, work_len); + + // shortwave aerosol scattering asymmetry parameter [unitless] + tau_ssa_g_sw_ = mam_coupling::view_3d("tau_ssa_g_sw_", ncol_, nswbands_, nlev_ + 1); + // shortwave aerosol single-scattering albedo [unitless] + tau_ssa_sw_ = mam_coupling::view_3d("tau_ssa_sw_", ncol_, nswbands_, nlev_ + 1); + // shortwave aerosol extinction optical depth [unitless] + tau_sw_ = mam_coupling::view_3d("tau_sw_", ncol_, nswbands_, nlev_ + 1); + //aerosol forward scattered fraction * tau * w + tau_f_sw_= mam_coupling::view_3d("tau_f_sw_", ncol_, nswbands_, nlev_ + 1); + + // read table info + { + using namespace ShortFieldTagsNames; + + using view_1d_host = typename KT::view_1d::HostMirror; + + // Note: these functions do not set values for aerosol_optics_device_data_. + mam4::modal_aer_opt::set_complex_views_modal_aero( + aerosol_optics_device_data_); + mam4::modal_aer_opt::set_aerosol_optics_data_for_modal_aero_sw_views( + aerosol_optics_device_data_); + mam4::modal_aer_opt::set_aerosol_optics_data_for_modal_aero_lw_views( + aerosol_optics_device_data_); + + mam_coupling::AerosolOpticsHostData aerosol_optics_host_data; + + std::map layouts; + std::map host_views; + ekat::ParameterList rrtmg_params; + + mam_coupling::set_parameters_table(aerosol_optics_host_data, rrtmg_params, + layouts, host_views); + + for(int imode = 0; imode < ntot_amode; imode++) { + const auto key = "mam4_mode" + std::to_string(imode+1) + + "_physical_properties_file"; + const auto& fname = m_params.get(key); + mam_coupling::read_rrtmg_table(fname, + imode, // mode No + rrtmg_params, grid_, host_views, layouts, + aerosol_optics_host_data, + aerosol_optics_device_data_); + } + + std::string table_name_water = + m_params.get("mam4_water_refindex_file"); + + // it will syn data to device. + mam_coupling::read_water_refindex(table_name_water, grid_, + aerosol_optics_device_data_.crefwlw, + aerosol_optics_device_data_.crefwsw); + // + { + // make a list of host views + std::map host_views_aero; + // defines layouts + std::map layouts_aero; + ekat::ParameterList params_aero; + std::string surname_aero = "aer"; + mam_coupling::set_refindex_names(surname_aero, params_aero, host_views_aero, + layouts_aero); + + constexpr int maxd_aspectype = mam4::ndrop::maxd_aspectype; + auto specrefndxsw_host = mam_coupling::complex_view_2d::HostMirror( + "specrefndxsw_host", nswbands_, maxd_aspectype); + + auto specrefndxlw_host = mam_coupling::complex_view_2d::HostMirror( + "specrefndxlw_host", nlwbands_, maxd_aspectype); + + // read physical properties data for aerosol species + std::map map_table_name_species_id; + map_table_name_species_id["soa"] = 4; // soa:s-organic + map_table_name_species_id["dust"] = 7; // dst:dust: + map_table_name_species_id["nacl"] = 6; // ncl:seasalt + map_table_name_species_id["so4"] = 0; // so4:sulfate + map_table_name_species_id["pom"] = 3; // pom:p-organic + map_table_name_species_id["bc"] = 5; // bc :black-c + map_table_name_species_id["mom"] = 8; // mom:m-organic + + for (const auto& item : map_table_name_species_id) { + const auto spec_name = item.first; + const int species_id = item.second; + const auto table_name = "mam4_" + spec_name + "_physical_properties_file"; + const auto& fname = m_params.get(table_name); + // read data + // need to update table name + params_aero.set("Filename", fname); + AtmosphereInput refindex_aerosol(params_aero, grid_, host_views_aero, + layouts_aero); + refindex_aerosol.read_variables(); + refindex_aerosol.finalize(); + // copy data to device + mam_coupling::set_refindex_aerosol( + species_id, host_views_aero, + specrefndxsw_host, // complex refractive index for water visible + specrefndxlw_host); + } // done ispec + // reshape specrefndxsw_host and copy it to device + mam4::modal_aer_opt::set_device_specrefindex( + aerosol_optics_device_data_.specrefindex_sw, "short_wave", + specrefndxsw_host); + mam4::modal_aer_opt::set_device_specrefindex( + aerosol_optics_device_data_.specrefindex_lw, "long_wave", + specrefndxlw_host); + } + } + //FIXME: We are hard-coding the band ordering in RRTMGP. + //TODO: We can update optics file using the ordering below (rrtmg_to_rrtmgp_swbands_). + // Mapping from old RRTMG sw bands to new band ordering in RRTMGP + // rrtmg_swband (old) = 2925, 3625, 4325, 4900, 5650, 6925, 7875, 10450, 14425, 19325, + // 25825, 33500, 44000, 1710 ; + // rrtmgp_swband (new) = 1710, 2925, 3625, 4325, 4900, 5650, 6925, 7875, 10450, 14425, + // 19325, 25825, 33500, 44000 ; + // given the rrtmg index return the rrtmgp index + std::vector temporal = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0}; + auto get_idx_rrtmgp_from_rrtmg_swbands_host =mam_coupling::view_int_1d::HostMirror(temporal.data(),nswbands_); + get_idx_rrtmgp_from_rrtmg_swbands_ = mam_coupling::view_int_1d("rrtmg_to_rrtmgp_swbands",nswbands_); + Kokkos::deep_copy(get_idx_rrtmgp_from_rrtmg_swbands_,get_idx_rrtmgp_from_rrtmg_swbands_host); } +void MAMOptics::run_impl(const double dt) { + + constexpr Real zero=0.0; + constexpr Real one=1.0; + + const auto policy = + ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const auto scan_policy = ekat::ExeSpaceUtils< + KT::ExeSpace>::get_thread_range_parallel_scan_team_policy(ncol_, nlev_); + + // preprocess input -- needs a scan for the calculation of atm height + Kokkos::parallel_for("preprocess", scan_policy, preprocess_); + Kokkos::fence(); + + //tau_w_g : aerosol asymmetry parameter * tau * w + const auto tau_ssa_g_sw = tau_ssa_g_sw_; + //tau_w : aerosol single scattering albedo * tau + const auto tau_ssa_sw =tau_ssa_sw_; + // tau : aerosol extinction optical depth + const auto tau_sw = tau_sw_; + // get_field_out("aero_tau_sw_mam4").get_view(); + // aero_tau_lw ( or odap_aer) : absorption optical depth, per layer + const auto aero_tau_lw = get_field_out("aero_tau_lw").get_view(); + + const auto aero_g_sw_eamxx = get_field_out("aero_g_sw").get_view(); + + const auto aero_ssa_sw_eamxx = + get_field_out("aero_ssa_sw").get_view(); + const auto aero_tau_sw_eamxx = + get_field_out("aero_tau_sw").get_view(); + //tau_w_f : aerosol forward scattered fraction * tau * w + const auto tau_f_sw = tau_f_sw_; + const auto aodvis = get_field_out("aodvis").get_view(); + + // NOTE! we need a const mam_coupling::DryAtmosphere dry_atm for gpu access. + // We cannot use member of this class inside of the parallel_for + const mam_coupling::DryAtmosphere &dry_atm = dry_atm_; + const auto &p_del = p_del_; + const auto &ssa_cmip6_sw = ssa_cmip6_sw_; + const auto &af_cmip6_sw = af_cmip6_sw_; + const auto &ext_cmip6_sw = ext_cmip6_sw_; + const auto &ext_cmip6_lw = ext_cmip6_lw_; + const auto &work = work_; + const auto &dry_aero = dry_aero_; + const auto &aerosol_optics_device_data = aerosol_optics_device_data_; + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const ThreadTeam &team) { + const Int icol = team.league_rank(); // column index + // absorption optical depth, per layer [unitless] + auto odap_aer_icol = ekat::subview(aero_tau_lw, icol); + const auto atm = mam_coupling::atmosphere_for_column(dry_atm, icol); + + // FIXME: interface pressure [Pa] + auto pint = ekat::subview(dry_atm.p_int, icol); + // FIXME: dry mass pressure interval [Pa] + auto zi = ekat::subview(dry_atm.z_iface, icol); + auto pdel = ekat::subview(p_del, icol); + auto pdeldry = ekat::subview(dry_atm.p_del, icol); + + auto ssa_cmip6_sw_icol = ekat::subview(ssa_cmip6_sw, icol); + auto af_cmip6_sw_icol = ekat::subview(af_cmip6_sw, icol); + auto ext_cmip6_sw_icol = ekat::subview(ext_cmip6_sw, icol); + auto ext_cmip6_lw_icol = ekat::subview(ext_cmip6_lw, icol); + + // tau_w: aerosol single scattering albedo * tau + auto tau_w_icol = ekat::subview(tau_ssa_sw, icol); + // tau_w_g: aerosol assymetry + // parameter * tau * w + auto tau_w_g_icol = ekat::subview(tau_ssa_g_sw, icol); + // tau_w_f: aero_tau_forward + // forward scattered fraction * tau * w + auto tau_w_f_icol = ekat::subview(tau_f_sw, icol); + // tau: aerosol + // aerosol extinction optical depth + auto tau_icol = ekat::subview(tau_sw, icol); + + auto work_icol = ekat::subview(work, icol); + + // fetch column-specific subviews into aerosol prognostics + mam4::Prognostics progs = + mam_coupling::aerosols_for_column(dry_aero, icol); + + mam4::aer_rad_props::aer_rad_props_sw( + team, dt, progs, atm, zi, pint, pdel, pdeldry, ssa_cmip6_sw_icol, + af_cmip6_sw_icol, ext_cmip6_sw_icol, tau_icol, tau_w_icol, + tau_w_g_icol, tau_w_f_icol, aerosol_optics_device_data, aodvis(icol), work_icol); + + team.team_barrier(); + mam4::aer_rad_props::aer_rad_props_lw( + team, dt, progs, atm, pint, zi, pdel, pdeldry, ext_cmip6_lw_icol, + aerosol_optics_device_data, odap_aer_icol); + }); + Kokkos::fence(); + // TODO: We will need to generate optical inputs files with band ordering that is consistent with + // RRTMGP + // Optical files depend on the band ordering in RRTMG. As a temporary fix, + // we are correcting the band ordering of mam4xx's ouputs, so that they are consistent with + // inputs in RRTMGP. + // Mapping from old RRTMG sw bands to new band ordering in RRTMGP + // than rrtmgp mam4 layout: (ncols, nswlands, nlevs +1 ) rrtmgp in emaxx: + // (ncols, nswlands, nlevs) Here, we copy data from kk=1 in mam4xx + // Here, we are following: E3SM/components/eam/src/physics/rrtmgp + ///cam_optics.F90 + const auto& get_idx_rrtmgp_from_rrtmg_swbands = get_idx_rrtmgp_from_rrtmg_swbands_; + // postprocess output + Kokkos::parallel_for("postprocess", policy, postprocess_); + Kokkos::fence(); + + // nswbands loop is using rrtmg indexing. + Kokkos::parallel_for( + "copying data from mam4xx to eamxx", + Kokkos::MDRangePolicy >({0, 0, 0}, + {ncol_, nswbands_, nlev_}), + KOKKOS_LAMBDA(const int icol, const int iswband, const int kk) { + // Extract single scattering albedo from the product-defined fields + if (tau_sw(icol, iswband, kk + 1) > zero) { + aero_ssa_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = + tau_ssa_sw(icol, iswband, kk + 1)/tau_sw(icol, iswband, kk + 1); + } else { + aero_ssa_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = one; + } + // Extract assymmetry parameter from the product-defined fields + if (tau_ssa_sw(icol, iswband, kk + 1) > zero ) { + aero_g_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = + tau_ssa_g_sw(icol, iswband, kk + 1)/tau_ssa_sw(icol, iswband, kk + 1) ; + } else { + aero_g_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = zero; + } + // Copy cloud optical depth over directly + aero_tau_sw_eamxx(icol, get_idx_rrtmgp_from_rrtmg_swbands(iswband), kk) = + tau_sw(icol, iswband, kk + 1); + }); + Kokkos::fence(); +} + +void MAMOptics::finalize_impl() {} -} // namespace scream +} // namespace scream diff --git a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp index ebd69437dee1..466cdcb9febe 100644 --- a/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp +++ b/components/eamxx/src/physics/mam/eamxx_mam_optics_process_interface.hpp @@ -1,14 +1,14 @@ #ifndef EAMXX_MAM_OPTICS_HPP #define EAMXX_MAM_OPTICS_HPP -#include -#include -#include - #include #include #include - +#include +#include +#include +#include +#include #include #ifndef KOKKOS_ENABLE_CUDA @@ -19,8 +19,7 @@ #define private_except_cuda private #endif -namespace scream -{ +namespace scream { // The process responsible for handling MAM4 aerosol optical properties. The AD // stores exactly ONE instance of this class in its list of subcomponents. @@ -34,30 +33,121 @@ class MAMOptics final : public scream::AtmosphereProcess { // a thread team dispatched to a single vertical column using ThreadTeam = mam4::ThreadTeam; -public: - + public: // Constructor - MAMOptics(const ekat::Comm& comm, const ekat::ParameterList& params); + MAMOptics(const ekat::Comm &comm, const ekat::ParameterList ¶ms); -protected_except_cuda: + protected_except_cuda : - // -------------------------------------------------------------------------- - // AtmosphereProcess overrides (see share/atm_process/atmosphere_process.hpp) - // -------------------------------------------------------------------------- + // -------------------------------------------------------------------------- + // AtmosphereProcess overrides (see + // share/atm_process/atmosphere_process.hpp) + // -------------------------------------------------------------------------- - // process metadata - AtmosphereProcessType type() const override; + // process metadata + AtmosphereProcessType + type() const override; std::string name() const override; // grid - void set_grids(const std::shared_ptr grids_manager) override; + void set_grids( + const std::shared_ptr grids_manager) override; + + // management of common atm process memory + size_t requested_buffer_size_in_bytes() const override; + void init_buffers(const ATMBufferManager &buffer_manager) override; // process behavior void initialize_impl(const RunType run_type) override; void run_impl(const double dt) override; void finalize_impl() override; -private_except_cuda: + private_except_cuda : + // FIXME: duplicate code from microphysics: ask it can be moved to place + // where other process can see it. Atmosphere processes often have a + // pre-processing step that constructs required variables from the set of + // fields stored in the field manager. This functor implements this step, + // which is called during run_impl. + struct Preprocess { + Preprocess() = default; + + // on host: initializes preprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + // first, compute dry fields + compute_dry_mixing_ratios(team, wet_atm_, dry_atm_, i); + compute_dry_mixing_ratios(team, wet_atm_, wet_aero_, dry_aero_, i); + team.team_barrier(); + // second, we can use dry fields to compute dz, zmin, zint + compute_vertical_layer_heights(team, dry_atm_, i); + compute_updraft_velocities(team, wet_atm_, dry_atm_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + + }; // MAMMicrophysics::Preprocess + + // Postprocessing functor + struct Postprocess { + Postprocess() = default; + + // on host: initializes postprocess functor with necessary state data + void initialize(const int ncol, const int nlev, + const mam_coupling::WetAtmosphere &wet_atm, + const mam_coupling::AerosolState &wet_aero, + const mam_coupling::DryAtmosphere &dry_atm, + const mam_coupling::AerosolState &dry_aero) { + ncol_ = ncol; + nlev_ = nlev; + wet_atm_ = wet_atm; + wet_aero_ = wet_aero; + dry_atm_ = dry_atm; + dry_aero_ = dry_aero; + } + + KOKKOS_INLINE_FUNCTION + void operator()( + const Kokkos::TeamPolicy::member_type &team) const { + const int i = team.league_rank(); // column index + compute_wet_mixing_ratios(team, dry_atm_, dry_aero_, wet_aero_, i); + } // operator() + + // number of horizontal columns and vertical levels + int ncol_, nlev_; + + // local atmospheric and aerosol state data + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + }; // MAMMicrophysics::Postprocess + + // pre- and postprocessing scratch pads (for wet <-> dry conversions) + Preprocess preprocess_; + Postprocess postprocess_; + + // state variable + // mam_coupling::view_3d state_q_, qqcw_;// odap_aer_, // number of horizontal columns and vertical levels int ncol_, nlev_; @@ -65,16 +155,33 @@ class MAMOptics final : public scream::AtmosphereProcess { // number of shortwave and longwave radiation bands int nswbands_, nlwbands_; + // FIXME: move these values to mam_coupling + mam_coupling::const_view_2d p_int_, p_del_; + // MAM4 aerosol particle size description mam4::AeroConfig aero_config_; - // aerosol processes - //std::unique_ptr optics_; + // atmospheric and aerosol state variables + // atmospheric and aerosol state variables + mam_coupling::WetAtmosphere wet_atm_; + mam_coupling::DryAtmosphere dry_atm_; + mam_coupling::AerosolState wet_aero_, dry_aero_; + mam_coupling::view_3d ssa_cmip6_sw_, af_cmip6_sw_, ext_cmip6_sw_; + // long wave extinction in the units of [1/km] + mam_coupling::view_3d ext_cmip6_lw_; + mam4::modal_aer_opt::AerosolOpticsDeviceData aerosol_optics_device_data_; // physics grid for column information std::shared_ptr grid_; -}; // MAMOptics + mam_coupling::view_2d work_; + mam_coupling::view_3d tau_ssa_g_sw_, tau_ssa_sw_, tau_sw_, tau_f_sw_; + //Mapping from old RRTMG sw bands to new band ordering in RRTMGP + // given old index swband (RRTMG) return new index swband RRTMGP + mam_coupling::view_int_1d get_idx_rrtmgp_from_rrtmg_swbands_; + + mam_coupling::Buffer buffer_; +}; // MAMOptics -} // namespace scream +} // namespace scream -#endif // EAMXX_MAM_OPTICS_HPP +#endif // EAMXX_MAM_OPTICS_HPP diff --git a/components/eamxx/src/physics/mam/impl/README.md b/components/eamxx/src/physics/mam/impl/README.md new file mode 100644 index 000000000000..f05a484a82aa --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/README.md @@ -0,0 +1,11 @@ +# MAM4 Integration Code + +This folder contains C++ implementations of the higher-level MAM4 interface +routines for aerosol microphysics, cloud-aerosol interactions, and optical +properties. We've retained the overall structure of the original Fortran code +to make it easier to understand for folks who are more familiar with the +original implementation of MAM4. + +## Contents + +* `mam4_amicphys.cpp` - high-level MAM4 microphysics interface code diff --git a/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp b/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp new file mode 100644 index 000000000000..b31537eef83d --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/compute_o3_column_density.cpp @@ -0,0 +1,44 @@ +namespace scream::impl { + +KOKKOS_INLINE_FUNCTION +void compute_o3_column_density(const ThreadTeam& team, const haero::Atmosphere& atm, + const mam4::Prognostics &progs, ColumnView o3_col_dens) { + constexpr int nabscol = mam4::gas_chemistry::nabscol; // number of absorbing densities + constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species + constexpr int nfs = mam4::gas_chemistry::nfs; // number of "fixed species" + constexpr Real mwdry = 1.0/haero::Constants::molec_weight_dry_air; + + Real o3_col_deltas[mam4::nlev+1] = {}; // o3 column density above model [1/cm^2] + // NOTE: if we need o2 column densities, set_ub_col and setcol must be changed + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, atm.num_levels()), [&](const int k) { + + Real temp = atm.temperature(k); + Real pmid = atm.pressure(k); + Real pdel = atm.hydrostatic_dp(k); + Real qv = atm.vapor_mixing_ratio(k); + + // ... map incoming mass mixing ratios to working array + Real q[gas_pcnst], qqcw[gas_pcnst]; + mam_coupling::transfer_prognostics_to_work_arrays(progs, k, q, qqcw); + + // ... set atmosphere mean mass to the molecular weight of dry air + // and compute water vapor vmr + Real mbar = mwdry; + Real h2ovmr = mam4::conversions::vmr_from_mmr(qv, mbar); + + // ... Xform from mmr to vmr + Real vmr[gas_pcnst], vmrcw[gas_pcnst]; + mam_coupling::convert_work_arrays_to_vmr(q, qqcw, vmr, vmrcw); + + // ... compute invariants for this level + Real invariants[nfs]; + // setinv(invariants, temp, h2ovmr, vmr, pmid); FIXME: not yet ported + + // compute the change in o3 density for this column above its neighbor + mam4::mo_photo::set_ub_col(o3_col_deltas[k+1], vmr, invariants, pdel); + }); + // sum the o3 column deltas to densities + mam4::mo_photo::setcol(o3_col_deltas, o3_col_dens); +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/compute_water_content.cpp b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp new file mode 100644 index 000000000000..ea7190afff91 --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/compute_water_content.cpp @@ -0,0 +1,99 @@ +#include + +namespace scream::impl { + +KOKKOS_INLINE_FUNCTION +void compute_water_content(const mam4::Prognostics &progs, int k, + Real qv, Real temp, Real pmid, + Real dgncur_a[mam4::AeroConfig::num_modes()], + Real dgncur_awet[mam4::AeroConfig::num_modes()], + Real wetdens[mam4::AeroConfig::num_modes()], + Real qaerwat[mam4::AeroConfig::num_modes()]) { + constexpr int num_modes = mam4::AeroConfig::num_modes(); + constexpr int num_aero_ids = mam4::AeroConfig::num_aerosol_ids(); + + // get some information about aerosol species + // FIXME: this isn't great! + constexpr int maxd_aspectype = mam4::water_uptake::maxd_aspectype; + int nspec_amode[num_modes], lspectype_amode[maxd_aspectype][num_modes]; + Real specdens_amode[maxd_aspectype], spechygro[maxd_aspectype]; + mam4::water_uptake::get_e3sm_parameters(nspec_amode, lspectype_amode, + specdens_amode, spechygro); + + // extract aerosol tracers for this level into state_q, which is needed + // for computing dry aerosol properties below + // FIXME: we should eliminate this index translation stuff + constexpr int nvars = aero_model::pcnst; + Real state_q[nvars]; // aerosol tracers for level k + for (int imode = 0; imode < num_modes; ++imode) { + int la, lc; // interstitial and cloudborne indices within state_q + + // number mixing ratios + mam4::convproc::assign_la_lc(imode, -1, la, lc); + state_q[la] = progs.n_mode_i[imode](k); + state_q[lc] = progs.n_mode_c[imode](k); + // aerosol mass mixing ratios + for (int iaero = 0; iaero < num_aero_ids; ++iaero) { + mam4::convproc::assign_la_lc(imode, iaero, la, lc); + auto mode = static_cast(imode); + auto aero = static_cast(iaero); + int ispec = mam4::aerosol_index_for_mode(mode, aero); + if (ispec != -1) { + state_q[la] = progs.q_aero_i[imode][ispec](k); + state_q[lc] = progs.q_aero_c[imode][ispec](k); + } + } + } + + // compute the dry volume for each mode, and from it the current dry + // geometric nominal particle diameter. + // FIXME: We have to do some gymnastics here to set up the calls to + // FIXME: calcsize. This could be improved. + Real inv_densities[num_modes][num_aero_ids] = {}; + for (int imode = 0; imode < num_modes; ++imode) { + const int n_spec = mam4::num_species_mode(imode); + for (int ispec = 0; ispec < n_spec; ++ispec) { + const int iaer = static_cast(mam4::mode_aero_species(imode, ispec)); + const Real density = mam4::aero_species(iaer).density; + inv_densities[imode][ispec] = 1.0 / density; + } + } + for (int imode = 0; imode < num_modes; ++imode) { + Real dryvol_i, dryvol_c; // interstitial and cloudborne dry volumes + mam4::calcsize::compute_dry_volume_k(k, imode, inv_densities, progs, + dryvol_i, dryvol_c); + + // NOTE: there's some disagreement over whether vol2num should be called + // NOTE: num2vol here, so I'm just adopting the nomenclature used by + // NOTE: the following call to calcsize) + const mam4::Mode& mode = mam4::modes(imode); + Real vol2num_min = 1.0/mam4::conversions::mean_particle_volume_from_diameter( + mode.max_diameter, mode.mean_std_dev); + Real vol2num_max = 1.0/mam4::conversions::mean_particle_volume_from_diameter( + mode.min_diameter, mode.mean_std_dev); + Real vol2num; + mam4::calcsize::update_diameter_and_vol2num(dryvol_i, + progs.n_mode_i[imode](k), vol2num_min, vol2num_max, + mode.min_diameter, mode.max_diameter, mode.mean_std_dev, + dgncur_a[imode], vol2num); + } + + // calculate dry aerosol properties + Real hygro[num_modes], naer[num_modes], dryrad[num_modes], + dryvol[num_modes], drymass[num_modes], + rhcrystal[num_modes], rhdeliques[num_modes], specdens_1[num_modes]; + mam4::water_uptake::modal_aero_water_uptake_dryaer(nspec_amode, specdens_amode, + spechygro, lspectype_amode, state_q, dgncur_a, hygro, + naer, dryrad, dryvol, drymass, rhcrystal, rhdeliques, specdens_1); + + // calculate wet aerosol properties + Real rh = mam4::conversions::relative_humidity_from_vapor_mixing_ratio(qv, temp, pmid); + Real wetrad[num_modes], wetvol[num_modes], wtrvol[num_modes]; + mam4::water_uptake::modal_aero_water_uptake_wetaer(rhcrystal, rhdeliques, dgncur_a, + dryrad, hygro, rh, naer, dryvol, wetrad, wetvol, wtrvol, dgncur_awet, + qaerwat); + mam4::water_uptake::modal_aero_water_uptake_wetdens(wetvol, wtrvol, + drymass, specdens_1, wetdens); +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp b/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp new file mode 100644 index 000000000000..98e33b3c2b7c --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/gas_phase_chemistry.cpp @@ -0,0 +1,387 @@ +#include + +namespace scream::impl { + +using mam4::utils::min_max_bound; + +using HostView1D = haero::DeviceType::view_1d::HostMirror; +using HostViewInt1D = haero::DeviceType::view_1d::HostMirror; + +//------------------------------------------------------------------------- +// Reading the photolysis table +//------------------------------------------------------------------------- +// This logic is currently implemented using serial NetCDF calls for +// clarity of purpose. We should probably read the data for the photolysis +// table using SCREAM's SCORPIO interface instead, but I wanted to make +// clear what we're trying to do in terms of "elementary" operations first. + +// ON HOST (MPI root rank only), reads the dimension of a NetCDF variable from +// the file with the given ID +int nc_dimension(const char *file, int nc_id, const char *dim_name) { + int dim_id; + int result = nc_inq_dimid(nc_id, dim_name, &dim_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch " << dim_name << + " dimension ID from NetCDF file '" << file << "'\n"); + size_t dim; + result = nc_inq_dimlen(nc_id, dim_id, &dim); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch " << dim_name << + " dimension from NetCDF file '" << file << "'\n"); + return static_cast(dim); +} + +// ON HOST (MPI root rank only), reads data from the given NetCDF variable from +// the file with the given ID into the given Kokkos host View +template +void read_nc_var(const char *file, int nc_id, const char *var_name, V host_view) { + int var_id; + int result = nc_inq_varid(nc_id, var_name, &var_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch ID for variable '" << var_name << + "' from NetCDF file '" << file << "'\n"); + result = nc_get_var(nc_id, var_id, host_view.data()); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't read data for variable '" << var_name << + "' from NetCDF file '" << file << "'\n"); +} + +// ON HOST (MPI root rank only), reads data from the NetCDF variable with the +// given ID, from the file with the given ID, into the given Kokkos host View +template +void read_nc_var(const char *file, int nc_id, int var_id, V host_view) { + int result = nc_get_var(nc_id, var_id, host_view.data()); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't read data for variable with ID " << + var_id << " from NetCDF file '" << file << "'\n"); +} + +// ON HOST (MPI root only), sets the lng_indexer and pht_alias_mult_1 host views +// according to parameters in our (hardwired) chemical mechanism +void set_lng_indexer_and_pht_alias_mult_1(const char *file, int nc_id, + HostViewInt1D lng_indexer, + HostView1D pht_alias_mult_1) { + // NOTE: it seems that the chemical mechanism we're using + // NOTE: 1. sets pht_alias_lst to a blank string [1] + // NOTE: 2. sets pht_alias_mult_1 to 1.0 [1] + // NOTE: 3. sets rxt_tag_lst to ['jh2o2', 'usr_HO2_HO2', 'usr_SO2_OH', 'usr_DMS_OH'] [2] + // NOTE: References: + // NOTE: [1] (https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/pp_linoz_mam4_resus_mom_soag/mo_sim_dat.F90#L117) + // NOTE: [2] (https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/pp_linoz_mam4_resus_mom_soag/mo_sim_dat.F90#L99) + + // populate lng_indexer (see https://github.com/eagles-project/e3sm_mam4_refactor/blob/refactor-maint-2.0/components/eam/src/chemistry/mozart/mo_jlong.F90#L180) + static const char *var_names[4] = {"jh2o2", "usr_HO2_HO2", "usr_SO2_OH", "usr_DMS_OH"}; + for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { + int var_id; + int result = nc_inq_varid(nc_id, var_names[m], &var_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't fetch ID for variable '" + << var_names[m] << "' from NetCDF file '" << file << "'\n"); + lng_indexer(m) = var_id; + } + + // set pht_alias_mult_1 to 1 + Kokkos::deep_copy(pht_alias_mult_1, 1.0); +} + +// ON HOST (MPI root only), populates the etfphot view using rebinned +// solar data from our solar_data_file +void populate_etfphot(HostView1D we, HostView1D etfphot) { + // FIXME: It looks like EAM is relying on a piece of infrastructure that + // FIXME: we just don't have in EAMxx (eam/src/chemistry/utils/solar_data.F90). + // FIXME: I have no idea whether EAMxx has a plan for supporting this + // FIXME: solar irradiance / photon flux data, and I'm not going to recreate + // FIXME: that capability here. So this is an unplugged hole. + // FIXME: + // FIXME: If we are going to do this the way EAM does it, the relevant logic + // FIXME: is the call to rebin() in eam/src/chemistry/mozart/mo_jlong.F90, + // FIXME: around line 104. + + // FIXME: zero the photon flux for now + Kokkos::deep_copy(etfphot, 0); +} + +// ON HOST, reads the photolysis table (used for gas phase chemistry) from the +// files with the given names +mam4::mo_photo::PhotoTableData read_photo_table(const ekat::Comm& comm, + const char *rsf_file, + const char* xs_long_file) { + // NOTE: at the time of development, SCREAM's SCORPIO interface seems intended + // NOTE: for domain-decomposed grid data. The files we're reading here are not + // NOTE: spatial data, and should be the same everywhere, so we read them + // NOTE: using serial NetCDF calls on MPI rank 0 and broadcast to other ranks. + const int mpi_root = 0; + int rsf_id, xs_long_id; // NetCDF file IDs (used only on MPI root) + int nw, nump, numsza, numcolo3, numalb, nt, np_xs; // table dimensions + if (comm.rank() == mpi_root) { // read dimension data from files and broadcast + // open files + int result = nc_open(rsf_file, NC_NOWRITE, &rsf_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't open rsf_file '" << rsf_file << "'\n"); + result = nc_open(xs_long_file, NC_NOWRITE, &xs_long_id); + EKAT_REQUIRE_MSG(result == 0, "Error! Couldn't open xs_long_file '" << xs_long_file << "'\n"); + + // read and broadcast dimension data + nump = nc_dimension(rsf_file, rsf_id, "numz"); + numsza = nc_dimension(rsf_file, rsf_id, "numsza"); + numalb = nc_dimension(rsf_file, rsf_id, "numalb"); + numcolo3 = nc_dimension(rsf_file, rsf_id, "numcolo3fact"); + nt = nc_dimension(xs_long_file, xs_long_id, "numtemp"); + nw = nc_dimension(xs_long_file, xs_long_id, "numwl"); + np_xs = nc_dimension(xs_long_file, xs_long_id, "numprs"); + + int dim_data[7] = {nump, numsza, numcolo3, numalb, nt, nw, np_xs}; + comm.broadcast(dim_data, 7, mpi_root); + } else { // receive broadcasted dimension data from root rank + int dim_data[7]; + comm.broadcast(dim_data, 7, mpi_root); + nump = dim_data[0]; + numsza = dim_data[1]; + numcolo3 = dim_data[2]; + numalb = dim_data[3]; + nt = dim_data[4]; + nw = dim_data[5]; + np_xs = dim_data[6]; + } + + // set up the lng_indexer and pht_alias_mult_1 views based on our + // (hardwired) chemical mechanism + HostViewInt1D lng_indexer_h("lng_indexer(host)", mam4::mo_photo::phtcnt); + HostView1D pht_alias_mult_1_h("pht_alias_mult_1(host)", 2); + if (comm.rank() == mpi_root) { + set_lng_indexer_and_pht_alias_mult_1(xs_long_file, xs_long_id, + lng_indexer_h, pht_alias_mult_1_h); + } + comm.broadcast(lng_indexer_h.data(), mam4::mo_photo::phtcnt, mpi_root); + comm.broadcast(pht_alias_mult_1_h.data(), 2, mpi_root); + + // compute the size of the foremost dimension of xsqy using lng_indexer + int numj = 0; + for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { + if (lng_indexer_h(m) > 0) { + for (int mm = 0; mm < m; ++mm) { + if (lng_indexer_h(mm) == lng_indexer_h(m)) { + break; + } + ++numj; + } + } + } + + // allocate the photolysis table + auto table = mam4::mo_photo::create_photo_table_data(nw, nt, np_xs, numj, + nump, numsza, numcolo3, + numalb); + + // allocate host views for table data + auto rsf_tab_h = Kokkos::create_mirror_view(table.rsf_tab); + auto xsqy_h = Kokkos::create_mirror_view(table.xsqy); + auto sza_h = Kokkos::create_mirror_view(table.sza); + auto alb_h = Kokkos::create_mirror_view(table.alb); + auto press_h = Kokkos::create_mirror_view(table.press); + auto colo3_h = Kokkos::create_mirror_view(table.colo3); + auto o3rat_h = Kokkos::create_mirror_view(table.o3rat); + auto etfphot_h = Kokkos::create_mirror_view(table.etfphot); + auto prs_h = Kokkos::create_mirror_view(table.prs); + + if (comm.rank() == mpi_root) { // read data from files and broadcast + // read file data into our host views + read_nc_var(rsf_file, rsf_id, "pm", press_h); + read_nc_var(rsf_file, rsf_id, "sza", sza_h); + read_nc_var(rsf_file, rsf_id, "alb", alb_h); + read_nc_var(rsf_file, rsf_id, "colo3fact", o3rat_h); + read_nc_var(rsf_file, rsf_id, "colo3", colo3_h); + read_nc_var(rsf_file, rsf_id, "RSF", rsf_tab_h); + + read_nc_var(xs_long_file, xs_long_id, "pressure", prs_h); + + // read xsqy data (using lng_indexer_h for the first index) + int ndx = 0; + for (int m = 0; m < mam4::mo_photo::phtcnt; ++m) { + if (lng_indexer_h(m) > 0) { + auto xsqy_ndx_h = ekat::subview(xsqy_h, ndx); + read_nc_var(xs_long_file, xs_long_id, lng_indexer_h(m), xsqy_ndx_h); + ++ndx; + } + } + + // populate etfphot by rebinning solar data + HostView1D wc_h("wc", nw), wlintv_h("wlintv", nw), we_h("we", nw+1); + read_nc_var(rsf_file, rsf_id, "wc", wc_h); + read_nc_var(rsf_file, rsf_id, "wlintv", wlintv_h); + for (int i = 0; i < nw; ++i) { + we_h(i) = wc_h(i) - 0.5 * wlintv_h(i); + } + we_h(nw) = wc_h(nw-1) - 0.5 * wlintv_h(nw-1); + populate_etfphot(we_h, etfphot_h); + + // close the files + nc_close(rsf_id); + nc_close(xs_long_id); + } + + // broadcast host views from MPI root to others + comm.broadcast(rsf_tab_h.data(), nw*numalb*numcolo3*numsza*nump, mpi_root); + comm.broadcast(xsqy_h.data(), numj*nw*nt*np_xs, mpi_root); + comm.broadcast(sza_h.data(), numsza, mpi_root); + comm.broadcast(alb_h.data(), numalb, mpi_root); + comm.broadcast(press_h.data(), nump, mpi_root); + comm.broadcast(o3rat_h.data(), numcolo3, mpi_root); + comm.broadcast(colo3_h.data(), nump, mpi_root); + comm.broadcast(etfphot_h.data(), nw, mpi_root); + comm.broadcast(prs_h.data(), np_xs, mpi_root); + + // copy host photolysis table into place on device + Kokkos::deep_copy(table.rsf_tab, rsf_tab_h); + Kokkos::deep_copy(table.xsqy, xsqy_h); + Kokkos::deep_copy(table.sza, sza_h); + Kokkos::deep_copy(table.alb, alb_h); + Kokkos::deep_copy(table.press, press_h); + Kokkos::deep_copy(table.colo3, colo3_h); + Kokkos::deep_copy(table.o3rat, o3rat_h); + Kokkos::deep_copy(table.etfphot, etfphot_h); + Kokkos::deep_copy(table.prs, prs_h); + Kokkos::deep_copy(table.pht_alias_mult_1, pht_alias_mult_1_h); + Kokkos::deep_copy(table.lng_indexer, lng_indexer_h); + + // compute gradients (on device) + Kokkos::parallel_for("del_p", nump-1, KOKKOS_LAMBDA(int i) { + table.del_p(i) = 1.0/::abs(table.press(i)- table.press(i+1)); + }); + Kokkos::parallel_for("del_sza", numsza-1, KOKKOS_LAMBDA(int i) { + table.del_sza(i) = 1.0/(table.sza(i+1) - table.sza(i)); + }); + Kokkos::parallel_for("del_alb", numalb-1, KOKKOS_LAMBDA(int i) { + table.del_alb(i) = 1.0/(table.alb(i+1) - table.alb(i)); + }); + Kokkos::parallel_for("del_o3rat", numcolo3-1, KOKKOS_LAMBDA(int i) { + table.del_o3rat(i) = 1.0/(table.o3rat(i+1) - table.o3rat(i)); + }); + Kokkos::parallel_for("dprs", np_xs-1, KOKKOS_LAMBDA(int i) { + table.dprs(i) = 1.0/(table.prs(i) - table.prs(i+1)); + }); + + return table; +} + +// performs gas phase chemistry calculations on a single level of a single +// atmospheric column +KOKKOS_INLINE_FUNCTION +void gas_phase_chemistry(Real zm, Real zi, Real phis, Real temp, Real pmid, Real pdel, Real dt, + const Real photo_rates[mam4::mo_photo::phtcnt], // in + const Real extfrc[mam4::gas_chemistry::extcnt], // in + Real q[mam4::gas_chemistry::gas_pcnst], // VMRs, inout + Real invariants[mam4::gas_chemistry::nfs]) { // out + constexpr Real rga = 1.0/haero::Constants::gravity; + constexpr Real m2km = 0.01; // converts m -> km + + // The following things are chemical mechanism dependent! See mam4xx/src/mam4xx/gas_chem_mechanism.hpp) + constexpr int gas_pcnst = mam4::gas_chemistry::gas_pcnst; // number of gas phase species + constexpr int rxntot = mam4::gas_chemistry::rxntot; // number of chemical reactions + constexpr int extcnt = mam4::gas_chemistry::extcnt; // number of species with external forcing + constexpr int nabscol = mam4::gas_chemistry::nabscol; // number of "absorbing column densities" + constexpr int indexm = 0; // FIXME: index of total atm density in invariants array + + constexpr int phtcnt = mam4::mo_photo::phtcnt; // number of photolysis reactions + + constexpr int itermax = mam4::gas_chemistry::itermax; + constexpr int clscnt4 = mam4::gas_chemistry::clscnt4; + + // NOTE: vvv these arrays were copied from mam4xx/gas_chem_mechanism.hpp vvv + constexpr int permute_4[gas_pcnst] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29}; + constexpr int clsmap_4[gas_pcnst] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; + + // These indices for species are fixed by the chemical mechanism + // std::string solsym[] = {"O3", "H2O2", "H2SO4", "SO2", "DMS", "SOAG", + // "so4_a1", "pom_a1", "soa_a1", "bc_a1", "dst_a1", + // "ncl_a1", "mom_a1", "num_a1", "so4_a2", "soa_a2", + // "ncl_a2", "mom_a2", "num_a2", "dst_a3", "ncl_a3", + // "so4_a3", "bc_a3", "pom_a3", "soa_a3", "mom_a3", + // "num_a3", "pom_a4", "bc_a4", "mom_a4", "num_a4"}; + constexpr int ndx_h2so4 = 2; + constexpr int o3_ndx = 0; + // std::string extfrc_list[] = {"SO2", "so4_a1", "so4_a2", "pom_a4", "bc_a4", + // "num_a1", "num_a2", "num_a3", "num_a4", "SOAG"}; + constexpr int synoz_ndx = -1; + + // fetch the zenith angle (not its cosine!) in degrees for this column. + // FIXME: For now, we fix the zenith angle. At length, we need to compute it + // FIXME: from EAMxx's current set of orbital parameters, which requires some + // FIXME: conversation with the EAMxx team. + Real zen_angle = 0.0; // [deg] + + // xform geopotential height from m to km and pressure from Pa to mb + Real zsurf = rga * phis; + Real zintr = m2km * zi; + Real zmid = m2km * (zm + zsurf); + Real zint = m2km * (zi + zsurf); + + // ... compute the column's invariants + Real h2ovmr = q[0]; + // setinv(invariants, temp, h2ovmr, q, pmid); FIXME: not ported yet + + // ... set rates for "tabular" and user specified reactions + Real reaction_rates[rxntot]; + mam4::gas_chemistry::setrxt(reaction_rates, temp); + + // set reaction rates based on chemical invariants + // (indices (ndxes?) are taken from mam4 validation data and translated from + // 1-based indices to 0-based indices) + int usr_HO2_HO2_ndx = 1, usr_DMS_OH_ndx = 5, + usr_SO2_OH_ndx = 3, inv_h2o_ndx = 3; + mam4::gas_chemistry::usrrxt(reaction_rates, temp, invariants, invariants[indexm], + usr_HO2_HO2_ndx, usr_DMS_OH_ndx, + usr_SO2_OH_ndx, inv_h2o_ndx); + mam4::gas_chemistry::adjrxt(reaction_rates, invariants, invariants[indexm]); + + //=================================== + // Photolysis rates at time = t(n+1) + //=================================== + + // compute the rate of change from forcing + Real extfrc_rates[extcnt]; // [1/cm^3/s] + for (int mm = 0; mm < extcnt; ++mm) { + if (mm != synoz_ndx) { + extfrc_rates[mm] = extfrc[mm] / invariants[indexm]; + } + } + + // ... Form the washout rates + Real het_rates[gas_pcnst]; + // FIXME: not ported yet + //sethet(het_rates, pmid, zmid, phis, temp, cmfdqr, prain, nevapr, delt, + // invariants[indexm], q); + + int ltrop_sol = 0; // apply solver to all levels + + // save h2so4 before gas phase chem (for later new particle nucleation) + Real del_h2so4_gasprod = q[ndx_h2so4]; + + //=========================== + // Class solution algorithms + //=========================== + + // copy photolysis rates into reaction_rates (assumes photolysis rates come first) + for (int i = 0; i < phtcnt; ++i) { + reaction_rates[i] = photo_rates[i]; + } + + // ... solve for "Implicit" species + bool factor[itermax]; + for (int i = 0; i < itermax; ++i) { + factor[i] = true; + } + + // initialize error tolerances + Real epsilon[clscnt4]; + mam4::gas_chemistry::imp_slv_inti(epsilon); + + // solve chemical system implicitly + Real prod_out[clscnt4], loss_out[clscnt4]; + mam4::gas_chemistry::imp_sol(q, reaction_rates, het_rates, extfrc_rates, dt, + permute_4, clsmap_4, factor, epsilon, prod_out, loss_out); + + // save h2so4 change by gas phase chem (for later new particle nucleation) + if (ndx_h2so4 > 0) { + del_h2so4_gasprod = q[ndx_h2so4] - del_h2so4_gasprod; + } +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp new file mode 100644 index 000000000000..71cee1eeb407 --- /dev/null +++ b/components/eamxx/src/physics/mam/impl/mam4_amicphys.cpp @@ -0,0 +1,1769 @@ +#include +#include +#include +#include +#include +#include + +namespace scream::impl { + +#define MAX_FILENAME_LEN 256 + +using namespace mam4; + +// number of constituents in gas chemistry "work arrays" +KOKKOS_INLINE_FUNCTION +constexpr int gas_pcnst() { + constexpr int gas_pcnst_ = mam4::gas_chemistry::gas_pcnst; + return gas_pcnst_; +} + +// number of aerosol/gas species tendencies +KOKKOS_INLINE_FUNCTION +constexpr int nqtendbb() { return 4; } + +// MAM4 aerosol microphysics configuration data +struct AmicPhysConfig { + // these switches activate various aerosol microphysics processes + bool do_cond; // condensation (a.k.a gas-aerosol exchange) + bool do_rename; // mode "renaming" + bool do_newnuc; // gas -> aerosol nucleation + bool do_coag; // aerosol coagulation + + // configurations for specific aerosol microphysics + mam4::GasAerExchProcess::ProcessConfig condensation; + mam4::NucleationProcess::ProcessConfig nucleation; + + // controls treatment of h2so4 condensation in mam_gasaerexch_1subarea + // 1 = sequential calc. of gas-chem prod then condensation loss + // 2 = simultaneous calc. of gas-chem prod and condensation loss + int gaexch_h2so4_uptake_optaa; + + // controls how nucleation interprets h2so4 concentrations + int newnuc_h2so4_conc_optaa; +}; + +namespace { + +KOKKOS_INLINE_FUNCTION constexpr int nqtendaa() { return 5; } +KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendaa() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int nqqcwtendbb() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_rnam() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_nnuc() { return 2; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_coag() { return 3; } +KOKKOS_INLINE_FUNCTION constexpr int iqtend_cond_only() { return 4; } +KOKKOS_INLINE_FUNCTION constexpr int iqqcwtend_rnam() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int maxsubarea() { return 2; } + +// conversion factors +KOKKOS_INLINE_FUNCTION Real fcvt_gas(int gas_id) { + static const Real fcvt_gas_[AeroConfig::num_gas_ids()] = {1, 1, 1}; + return fcvt_gas_[gas_id]; +} +KOKKOS_INLINE_FUNCTION Real fcvt_aer(int aero_id) { + static const Real fcvt_aer_[AeroConfig::num_aerosol_ids()] = {1, 1, 1, 1, 1, 1, 1}; + return fcvt_aer_[aero_id]; +} + +// leave number mix-ratios unchanged (#/kmol-air) +KOKKOS_INLINE_FUNCTION Real fcvt_num() { return 1.0; } +// factor for converting aerosol water mix-ratios from (kg/kg) to (mol/mol) +KOKKOS_INLINE_FUNCTION Real fcvt_wtr() { return 1.0; } + +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_nul() { return 0; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_gas() { return 1; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_aer() { return 2; } +KOKKOS_INLINE_FUNCTION constexpr int lmapcc_val_num() { return 3; } +KOKKOS_INLINE_FUNCTION int lmapcc_all(int index) { + static const int lmapcc_all_[gas_pcnst()] = { + lmapcc_val_nul(), lmapcc_val_gas(), lmapcc_val_nul(), lmapcc_val_nul(), + lmapcc_val_gas(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num(), lmapcc_val_aer(), lmapcc_val_aer(), + lmapcc_val_aer(), lmapcc_val_num()}; + return lmapcc_all_[index]; +} + +// Where lmapcc_val_num are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int numptr_amode(int mode) { + static const int numptr_amode_[AeroConfig::num_modes()] = {12, 17, 25, 29}; + return numptr_amode_[mode]; +} + +// Where lmapcc_val_gas are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int lmap_gas(int mode) { + static const int lmap_gas_[AeroConfig::num_modes()] = {4, 1}; + return lmap_gas_[mode]; +} +// Where lmapcc_val_aer are defined in lmapcc_all +KOKKOS_INLINE_FUNCTION int lmassptr_amode(int aero_id, int mode) { + static const int lmassptr_amode_[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] = { + {5, 13, 18, 26}, {6, 14, 19, 27}, {7, 15, 20, 28}, {8, 16, 21, -6}, + {9, -6, 22, -6}, {10, -6, 23, -6}, {11, -6, 24, -6}}; + return lmassptr_amode_[aero_id][mode]; +} + +KOKKOS_INLINE_FUNCTION +void subarea_partition_factors( + const Real + q_int_cell_avg, // in grid cell mean interstitial aerosol mixing ratio + const Real + q_cbn_cell_avg, // in grid cell mean cloud-borne aerosol mixing ratio + const Real fcldy, // in cloudy fraction of the grid cell + const Real fclea, // in clear fraction of the grid cell + Real &part_fac_q_int_clea, // out + Real &part_fac_q_int_cldy) // out +{ + // Calculate mixing ratios of each subarea + + // cloud-borne, cloudy subarea + const Real tmp_q_cbn_cldy = q_cbn_cell_avg / fcldy; + // interstitial, cloudy subarea + const Real tmp_q_int_cldy = + haero::max(0.0, ((q_int_cell_avg + q_cbn_cell_avg) - tmp_q_cbn_cldy)); + // interstitial, clear subarea + const Real tmp_q_int_clea = (q_int_cell_avg - fcldy * tmp_q_int_cldy) / fclea; + + // Calculate the corresponding paritioning factors for interstitial aerosols + // using the above-derived subarea mixing ratios plus the constraint that + // the cloud fraction weighted average of subarea mean need to match grid box + // mean. + + // *** question *** + // use same part_fac_q_int_clea/cldy for everything ? + // use one for number and one for all masses (based on total mass) ? + // use separate ones for everything ? + // maybe one for number and one for all masses is best, + // because number and mass have different activation fractions + // *** question *** + + Real tmp_aa = haero::max(1.e-35, tmp_q_int_clea * fclea) / + haero::max(1.e-35, q_int_cell_avg); + tmp_aa = haero::max(0.0, haero::min(1.0, tmp_aa)); + + part_fac_q_int_clea = tmp_aa / fclea; + part_fac_q_int_cldy = (1.0 - tmp_aa) / fcldy; +} + +KOKKOS_INLINE_FUNCTION +void construct_subareas_1gridcell( + const Real cld, // in + const Real relhumgcm, // in + const Real q_pregaschem[gas_pcnst()], // in q TMRs before + // gas-phase chemistry + const Real q_precldchem[gas_pcnst()], // in q TMRs before + // cloud chemistry + const Real qqcw_precldchem[gas_pcnst()], // in qqcw TMRs before + // cloud chemistry + const Real q[gas_pcnst()], // in current tracer mixing ratios (TMRs) + // *** MUST BE #/kmol-air for number + // *** MUST BE mol/mol-air for mass + const Real qqcw[gas_pcnst()], // in like q but for + // cloud-borner tracers + int &nsubarea, // out + int &ncldy_subarea, // out + int &jclea, // out + int &jcldy, // out + bool iscldy_subarea[maxsubarea()], // out + Real afracsub[maxsubarea()], // out + Real relhumsub[maxsubarea()], // out + Real qsub1[gas_pcnst()][maxsubarea()], // out interstitial + Real qsub2[gas_pcnst()][maxsubarea()], // out interstitial + Real qsub3[gas_pcnst()][maxsubarea()], // out interstitial + Real qqcwsub1[gas_pcnst()][maxsubarea()], // out cloud-borne + Real qqcwsub2[gas_pcnst()][maxsubarea()], // out cloud-borne + Real qqcwsub3[gas_pcnst()][maxsubarea()], // outcloud-borne + Real qaerwatsub3[AeroConfig::num_modes()] + [maxsubarea()], // out aerosol water mixing ratios (mol/mol) + Real qaerwat[AeroConfig::num_modes()] // in aerosol water mixing ratio + // (kg/kg, NOT mol/mol) +) { + static constexpr int num_modes = AeroConfig::num_modes(); + // cloud chemistry is only on when cld(i,k) >= 1.0e-5_wp + // it may be that the macrophysics has a higher threshold that this + const Real fcld_locutoff = 1.0e-5; + const Real fcld_hicutoff = 0.999; + + // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios (TMRs, + // mol/mol or #/kmol) + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // qgcm1, qgcm2, qgcm3 + // qqcwgcm2, qqcwgcm3 + // qaerwatgcm3 ! aerosol water mixing ratios (mol/mol) + + // -------------------------------------------------------------------------------------- + // Determine the number of sub-areas, their fractional areas, and relative + // humidities + // -------------------------------------------------------------------------------------- + // if cloud fraction ~= 0, the grid-cell has a single clear sub-area + // (nsubarea = 1) if cloud fraction ~= 1, the grid-cell has a single cloudy + // sub-area (nsubarea = 1) otherwise, the grid-cell has a + // clear and a cloudy sub-area (nsubarea = 2) + + Real zfcldy = 0; + nsubarea = 0; + ncldy_subarea = 0; + jclea = 0; + jcldy = 0; + + if (cld < fcld_locutoff) { + nsubarea = 1; + jclea = 1; + } else if (cld > fcld_hicutoff) { + zfcldy = 1.0; + nsubarea = 1; + ncldy_subarea = 1; + jcldy = 1; + } else { + zfcldy = cld; + nsubarea = 2; + ncldy_subarea = 1; + jclea = 1; + jcldy = 2; + } + + const Real zfclea = 1.0 - zfcldy; + for (int i = 0; i < maxsubarea(); ++i) + iscldy_subarea[i] = false; + if (jcldy > 0) + iscldy_subarea[jcldy - 1] = true; + for (int i = 0; i < maxsubarea(); ++i) + afracsub[i] = 0.0; + if (jclea > 0) + afracsub[jclea - 1] = zfclea; + if (jcldy > 0) + afracsub[jcldy - 1] = zfcldy; + + // cldy_rh_sameas_clear is just to match mam_refactor. Compiler should + // optimize away. + const int cldy_rh_sameas_clear = 0; + if (ncldy_subarea <= 0) { + for (int i = 0; i < maxsubarea(); ++i) + relhumsub[i] = relhumgcm; + } else if (cldy_rh_sameas_clear > 0) { + for (int i = 0; i < maxsubarea(); ++i) + relhumsub[i] = relhumgcm; + } else { + if (jcldy > 0) { + relhumsub[jcldy - 1] = 1.0; + if (jclea > 0) { + const Real tmpa = + (relhumgcm - afracsub[jcldy - 1]) / afracsub[jclea - 1]; + relhumsub[jclea - 1] = haero::max(0.0, haero::min(1.0, tmpa)); + } + } + } + + // ---------------------------------------------------------------------------- + // Copy grid cell mean mixing ratios. + // These values, together with cloud fraction and a few assumptions, are used + // in the remainder of the subroutine to calculate the sub-area mean mixing + // ratios. + // ---------------------------------------------------------------------------- + // Interstitial aerosols + Real qgcm1[gas_pcnst()], qgcm2[gas_pcnst()], qgcm3[gas_pcnst()]; + for (int i = 0; i < gas_pcnst(); ++i) { + qgcm1[i] = haero::max(0.0, q_pregaschem[i]); + qgcm2[i] = haero::max(0.0, q_precldchem[i]); + qgcm3[i] = haero::max(0.0, q[i]); + } + + // Cloud-borne aerosols + Real qqcwgcm2[gas_pcnst()], qqcwgcm3[gas_pcnst()]; + for (int i = 0; i < gas_pcnst(); ++i) { + qqcwgcm2[i] = haero::max(0.0, qqcw_precldchem[i]); + qqcwgcm3[i] = haero::max(0.0, qqcw[i]); + } + + // aerosol water + Real qaerwatgcm3[num_modes] = {}; + for (int i = 0; i < num_modes; ++i) { + qaerwatgcm3[i] = haero::max(0.0, qaerwat[i]); + } + + // ---------------------------------------------------------------------------- + // Initialize the subarea mean mixing ratios + // ---------------------------------------------------------------------------- + { + const int n = haero::min(maxsubarea(), nsubarea + 1); + for (int i = 0; i < n; ++i) { + for (int j = 0; j < gas_pcnst(); ++j) { + qsub1[j][i] = 0.0; + qsub2[j][i] = 0.0; + qsub3[j][i] = 0.0; + qqcwsub1[j][i] = 0.0; + qqcwsub2[j][i] = 0.0; + qqcwsub3[j][i] = 0.0; + } + for (int j = 0; j < num_modes; ++j) { + qaerwatsub3[j][i] = 0.0; + } + } + } + + // ************************************************************************************************* + // Calculate initial (i.e., before cond/rnam/nnuc/coag) tracer mixing + // ratios within the sub-areas + // - for all-clear or all-cloudy cases, the sub-area TMRs are equal to the + // grid-cell means + // - for partly cloudy case, they are different. This is primarily + // because the + // interstitial aerosol mixing ratios are assumed lower in the cloudy + // sub-area than in the clear sub-area, because much of the aerosol is + // activated in the cloudy sub-area. + // ************************************************************************************************* + // Category I: partly cloudy case + // ************************************************************************************************* + if ((jclea > 0) && (jcldy > 0) && (jclea + jcldy == 3) && (nsubarea == 2)) { + + // --------------------------------------------------------------------- + // Set GAS mixing ratios in sub-areas (for the condensing gases only!!) + // --------------------------------------------------------------------- + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (lmapcc_all(lmz) == lmapcc_val_gas()) { + + // assume gas in both sub-areas before gas-chem and cloud-chem equal + // grid-cell mean + for (int i = 0; i < nsubarea; ++i) { + qsub1[lmz][i] = qgcm1[lmz]; + qsub2[lmz][i] = qgcm2[lmz]; + } + // assume gas in clear sub-area after cloud-chem equals before + // cloud-chem value + qsub3[lmz][jclea - 1] = qsub2[lmz][jclea - 1]; + // gas in cloud sub-area then determined by grid-cell mean and clear + // values + qsub3[lmz][jcldy - 1] = + (qgcm3[lmz] - zfclea * qsub3[lmz][jclea - 1]) / zfcldy; + + // check that this does not produce a negative value + if (qsub3[lmz][jcldy - 1] < 0.0) { + qsub3[lmz][jcldy - 1] = 0.0; + qsub3[lmz][jclea - 1] = qgcm3[lmz] / zfclea; + } + } + } + // --------------------------------------------------------------------- + // Set CLOUD-BORNE AEROSOL mixing ratios in sub-areas. + // This is straightforward, as the same partitioning factors (0 or 1/f) + // are applied to all mass and number mixing ratios in all modes. + // --------------------------------------------------------------------- + // loop thru log-normal modes + for (int n = 0; n < num_modes; ++n) { + // number - then mass of individual species - of a mode + for (int l2 = -1; l2 < num_species_mode(n); ++l2) { + int lc; + if (l2 == -1) + lc = numptr_amode(n); + else + lc = lmassptr_amode(l2, n); + qqcwsub2[lc][jclea - 1] = 0.0; + qqcwsub2[lc][jcldy - 1] = qqcwgcm2[lc] / zfcldy; + qqcwsub3[lc][jclea - 1] = 0.0; + qqcwsub3[lc][jcldy - 1] = qqcwgcm3[lc] / zfcldy; + } + } + + // --------------------------------------------------------------------- + // Set INTERSTITIAL AEROSOL mixing ratios in sub-areas. + // --------------------------------------------------------------------- + for (int n = 0; n < num_modes; ++n) { + // ------------------------------------- + // Aerosol number + // ------------------------------------- + // grid cell mean, interstitial + Real tmp_q_cellavg_int = qgcm2[numptr_amode(n)]; + // grid cell mean, cloud-borne + Real tmp_q_cellavg_cbn = qqcwgcm2[numptr_amode(n)]; + + Real nmbr_part_fac_clea = 0; + Real nmbr_part_fac_cldy = 0; + subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, + zfclea, nmbr_part_fac_clea, nmbr_part_fac_cldy); + + // Apply the partitioning factors to calculate sub-area mean number + // mixing ratios + + const int la = numptr_amode(n); + + qsub2[la][jclea - 1] = qgcm2[la] * nmbr_part_fac_clea; + qsub2[la][jcldy - 1] = qgcm2[la] * nmbr_part_fac_cldy; + qsub3[la][jclea - 1] = qgcm3[la] * nmbr_part_fac_clea; + qsub3[la][jcldy - 1] = qgcm3[la] * nmbr_part_fac_cldy; + + //------------------------------------- + // Aerosol mass + //------------------------------------- + // For aerosol mass, we use the total grid cell mean + // interstitial/cloud-borne mass mixing ratios to come up with the same + // partitioning for all species in the mode. + + // Compute the total mixing ratios by summing up the individual species + + tmp_q_cellavg_int = 0.0; // grid cell mean, interstitial + tmp_q_cellavg_cbn = 0.0; // grid cell mean, cloud-borne + + for (int l2 = 0; l2 < num_species_mode(n); ++l2) { + tmp_q_cellavg_int += qgcm2[lmassptr_amode(l2, n)]; + tmp_q_cellavg_cbn += qqcwgcm2[lmassptr_amode(l2, n)]; + } + Real mass_part_fac_clea = 0; + Real mass_part_fac_cldy = 0; + // Calculate the partitioning factors + subarea_partition_factors(tmp_q_cellavg_int, tmp_q_cellavg_cbn, zfcldy, + zfclea, mass_part_fac_clea, mass_part_fac_cldy); + + // Apply the partitioning factors to calculate sub-area mean mass mixing + // ratios + + for (int l2 = 0; l2 < num_species_mode(n); ++l2) { + const int la = lmassptr_amode(l2, n); + + qsub2[la][jclea - 1] = qgcm2[la] * mass_part_fac_clea; + qsub2[la][jcldy - 1] = qgcm2[la] * mass_part_fac_cldy; + qsub3[la][jclea - 1] = qgcm3[la] * mass_part_fac_clea; + qsub3[la][jcldy - 1] = qgcm3[la] * mass_part_fac_cldy; + } + } + + // ************************************************************************************************* + // Category II: all clear, or cld < 1e-5 + // In this case, zfclea=1 and zfcldy=0 + // ************************************************************************************************* + } else if ((jclea == 1) && (jcldy == 0) && (nsubarea == 1)) { + // + // put all the gases and interstitial aerosols in the clear sub-area + // and set mix-ratios = 0 in cloudy sub-area + // for cloud-borne aerosol, do nothing + // because the grid-cell-mean cloud-borne aerosol will be left + // unchanged (i.e., this routine only changes qqcw when cld >= 1e-5) + // + + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (0 < lmapcc_all(lmz)) { + qsub1[lmz][jclea - 1] = qgcm1[lmz]; + qsub2[lmz][jclea - 1] = qgcm2[lmz]; + qsub3[lmz][jclea - 1] = qgcm3[lmz]; + qqcwsub2[lmz][jclea - 1] = qqcwgcm2[lmz]; + qqcwsub3[lmz][jclea - 1] = qqcwgcm3[lmz]; + } + } + // ************************************************************************************************* + // Category III: all cloudy, or cld > 0.999 + // in this case, zfcldy= and zfclea=0 + // ************************************************************************************************* + } else if ((jclea == 0) && (jcldy == 1) && (nsubarea == 1)) { + // + // put all the gases and interstitial aerosols in the cloudy sub-area + // and set mix-ratios = 0 in clear sub-area + // + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (0 < lmapcc_all(lmz)) { + qsub1[lmz][jcldy - 1] = qgcm1[lmz]; + qsub2[lmz][jcldy - 1] = qgcm2[lmz]; + qsub3[lmz][jcldy - 1] = qgcm3[lmz]; + qqcwsub2[lmz][jcldy - 1] = qqcwgcm2[lmz]; + qqcwsub3[lmz][jcldy - 1] = qqcwgcm3[lmz]; + } + } + // ************************************************************************************************* + } else { // this should not happen + EKAT_KERNEL_REQUIRE_MSG(false, "*** modal_aero_amicphys - bad jclea, jcldy, nsubarea!"); + } + // ************************************************************************************************* + + // ------------------------------------------------------------------------------------ + // aerosol water -- how to treat this in sub-areas needs more work/thinking + // currently modal_aero_water_uptake calculates qaerwat using + // the grid-cell mean interstital-aerosol mix-rats and the clear-area rh + for (int jsub = 0; jsub < nsubarea; ++jsub) + for (int i = 0; i < num_modes; ++i) + qaerwatsub3[i][jsub] = qaerwatgcm3[i]; + + // ------------------------------------------------------------------------------------ + if (nsubarea == 1) { + // the j=1 subarea is used for some diagnostics + // but is not used in actual calculations + const int j = 1; + for (int i = 0; i < gas_pcnst(); ++i) { + qsub1[i][j] = 0.0; + qsub2[i][j] = 0.0; + qsub3[i][j] = 0.0; + qqcwsub2[i][j] = 0.0; + qqcwsub3[i][j] = 0.0; + } + } +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1subarea_clear( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, + const int nsubarea, const bool iscldy_subarea, const Real afracsub, + const Real temp, const Real pmid, const Real pdel, const Real zmid, + const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qgas1[AeroConfig::num_gas_ids()], + const Real qgas3[AeroConfig::num_gas_ids()], + Real qgas4[AeroConfig::num_gas_ids()], + Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], + const Real qnum3[AeroConfig::num_modes()], + Real qnum4[AeroConfig::num_modes()], + Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], + const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] + [nqtendaa()], + const Real qwtr3[AeroConfig::num_modes()], + Real qwtr4[AeroConfig::num_modes()]) { + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + // Turn off nh3 for now. This is a future enhancement. + static constexpr int igas_nh3 = -999888777; // Same as mam_refactor + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int iaer_pom = static_cast(AeroId::POM); + + const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, + AeroId::None}; + + const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { + {true, true, true, true}, + {true, true, true, true}, + {false, false, false, false}}; + enum { NA, ANAL, IMPL }; + const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; + + // air molar density (kmol/m3) + // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] + const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor + const Real aircon = pmid / (1000 * r_universal * temp); + const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, + 0.58778666490211906, 0.47000362924573563}; + const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qgas3, qaer3, qnum3 are the current incoming TMRs + // qgas4, qaer4, qnum4 are the updated outgoing TMRs + // + // this routine calculates changes involving + // gas-aerosol exchange (condensation/evaporation) + // growth from smaller to larger modes (renaming) due to condensation + // new particle nucleation + // coagulation + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation and coagulation + // + // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios + // XXX=gas - gas species + // XXX=aer - aerosol mass species (excluding water) + // XXX=wat - aerosol water + // XXX=num - aerosol number + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - current incoming values (before gas-aerosol exchange, newnuc, + // coag) N=4 - updated outgoing values (after gas-aerosol exchange, + // newnuc, coag) + // + // qXXX_delaa are TMR changes (not tendencies) + // for different processes, which are used to produce history output + // for a clear sub-area, the processes are condensation/evaporation (and + // associated aging), renaming, coagulation, and nucleation + + Real qgas_cur[num_gas_ids]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_cur[i] = qgas3[i]; + Real qaer_cur[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_cur[i][j] = qaer3[i][j]; + + Real qnum_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnum_cur[j] = qnum3[j]; + Real qwtr_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qwtr_cur[j] = qwtr3[j]; + + // qgas_netprod_otrproc = gas net production rate from other processes + // such as gas-phase chemistry and emissions (mol/mol/s) + // this allows the condensation (gasaerexch) routine to apply production and + // condensation loss + // together, which is more accurate numerically + // NOTE - must be >= zero, as numerical method can fail when it is negative + // NOTE - currently only the values for h2so4 and nh3 should be non-zero + Real qgas_netprod_otrproc[num_gas_ids] = {}; + if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { + for (int igas = 0; igas < num_gas_ids; ++igas) { + if (igas == igas_h2so4 || igas == igas_nh3) { + // if config.gaexch_h2so4_uptake_optaa == 2, then + // if qgas increases from pre-gaschem to post-cldchem, + // start from the pre-gaschem mix-ratio and add in the production + // during the integration + // if it decreases, + // start from post-cldchem mix-ratio + // *** currently just do this for h2so4 and nh3 + qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; + if (qgas_netprod_otrproc[igas] >= 0.0) + qgas_cur[igas] = qgas1[igas]; + else + qgas_netprod_otrproc[igas] = 0.0; + } + } + } + Real qgas_del_cond[num_gas_ids] = {}; + Real qgas_del_nnuc[num_gas_ids] = {}; + Real qgas_del_cond_only[num_gas_ids] = {}; + Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; + Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; + Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; + Real qnum_del_cond[num_modes] = {}; + Real qnum_del_rnam[num_modes] = {}; + Real qnum_del_nnuc[num_modes] = {}; + Real qnum_del_coag[num_modes] = {}; + Real qnum_delsub_cond[num_modes] = {}; + Real qnum_delsub_coag[num_modes] = {}; + Real qnum_del_cond_only[num_modes] = {}; + Real dnclusterdt = 0.0; + + const int ntsubstep = 1; + Real dtsubstep = deltat; + if (ntsubstep > 1) + dtsubstep = deltat / ntsubstep; + Real del_h2so4_gasprod = + haero::max(qgas3[igas_h2so4] - qgas1[igas_h2so4], 0.0) / ntsubstep; + + // loop over multiple time sub-steps + for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { + // gas-aerosol exchange + Real uptkrate_h2so4 = 0.0; + Real del_h2so4_aeruptk = 0.0; + Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real qgas_avg[num_gas_ids] = {}; + Real qnum_sv1[num_modes] = {}; + Real qaer_sv1[num_aerosol_ids][num_modes] = {}; + Real qgas_sv1[num_gas_ids] = {}; + + if (config.do_cond) { + + const bool l_calc_gas_uptake_coeff = jtsubstep == 1; + Real uptkaer[num_gas_ids][num_modes] = {}; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + + // time sub-step + const Real dtsub_soa_fixed = -1.0; + // Integration order + const int nghq = 2; + const int ntot_soamode = 4; + int niter_out = 0; + Real g0_soa_out = 0; + gasaerexch::mam_gasaerexch_1subarea( + nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, + iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, + eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, + aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, + qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, + uptkrate_h2so4, niter_out, g0_soa_out); + + if (config.newnuc_h2so4_conc_optaa == 11) + qgas_avg[igas_h2so4] = + 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); + else if (config.newnuc_h2so4_conc_optaa == 12) + qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond[i] += + (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); + + for (int i = 0; i < num_modes; ++i) + qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + + // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation + // calculations + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond_only[i] = qgas_del_cond[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; + for (int i = 0; i < num_modes; ++i) + qnum_del_cond_only[i] = qnum_delsub_cond[i]; + del_h2so4_aeruptk = + qgas_cur[igas_h2so4] - + (qgas_sv1[igas_h2so4] + qgas_netprod_otrproc[igas_h2so4] * dtsubstep); + } else { + for (int i = 0; i < num_gas_ids; ++i) + qgas_avg[i] = qgas_cur[i]; + } + + // renaming after "continuous growth" + if (config.do_rename) { + constexpr int nmodes = AeroConfig::num_modes(); + constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); + const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT + const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; + + Real qnumcw_cur[num_modes] = {}; + Real qaercw_cur[num_aerosol_ids][num_modes] = {}; + Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real mean_std_dev[nmodes]; + Real fmode_dist_tail_fac[nmodes]; + Real v2n_lo_rlx[nmodes]; + Real v2n_hi_rlx[nmodes]; + Real ln_diameter_tail_fac[nmodes]; + int num_pairs = 0; + Real diameter_cutoff[nmodes]; + Real ln_dia_cutoff[nmodes]; + Real diameter_threshold[nmodes]; + Real mass_2_vol[naerosol_species] = {0.15, + 6.4971751412429377e-002, + 0.15, + 7.0588235294117650e-003, + 3.0789473684210526e-002, + 5.1923076923076926e-002, + 156.20986883198000}; + + rename::find_renaming_pairs(dest_mode_of_mode, // in + mean_std_dev, // out + fmode_dist_tail_fac, // out + v2n_lo_rlx, // out + v2n_hi_rlx, // out + ln_diameter_tail_fac, // out + num_pairs, // out + diameter_cutoff, // out + ln_dia_cutoff, diameter_threshold); + + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + Real dgnum_amode[nmodes]; + for (int m = 0; m < nmodes; ++m) { + dgnum_amode[m] = modes(m).nom_diameter; + } + + { + Real qmol_i_cur[num_modes][num_aerosol_ids]; + Real qmol_i_del[num_modes][num_aerosol_ids]; + Real qmol_c_cur[num_modes][num_aerosol_ids]; + Real qmol_c_del[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qmol_i_cur[i][j] = qaer_cur[j][i]; + qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; + qmol_c_cur[i][j] = qaercw_cur[j][i]; + qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; + } + Rename rename; + rename.mam_rename_1subarea_( + iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, + mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, + ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, + diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, + qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); + + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_cur[j][i] = qmol_i_cur[i][j]; + qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; + qaercw_cur[j][i] = qmol_c_cur[i][j]; + qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; + } + } + + for (int i = 0; i < num_modes; ++i) + qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; + } + + // new particle formation (nucleation) + if (config.do_newnuc) { + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + Real qaer_cur_tmp[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_sv1[j][i] = qaer_cur[j][i]; + qaer_cur_tmp[i][j] = qaer_cur[j][i]; + } + Real dnclusterdt_substep = 0; + Real dndt_ait = 0; + Real dmdt_ait = 0; + Real dso4dt_ait = 0; + Real dnh4dt_ait = 0; + Nucleation nucleation; + Nucleation::Config config; + config.dens_so4a_host = 1770; + config.mw_nh4a_host = 115; + config.mw_so4a_host = 115; + config.accom_coef_h2so4 = 0.65; + AeroConfig aero_config; + nucleation.init(aero_config, config); + nucleation.compute_tendencies_( + dtsubstep, temp, pmid, aircon, zmid, pblh, relhum, uptkrate_h2so4, + del_h2so4_gasprod, del_h2so4_aeruptk, qgas_cur, qgas_avg, qnum_cur, + qaer_cur_tmp, qwtr_cur, dndt_ait, dmdt_ait, dso4dt_ait, dnh4dt_ait, + dnclusterdt_substep); + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_cur[j][i] = qaer_cur_tmp[i][j]; + + //! Apply the tendencies to the prognostics. + const int nait = static_cast(ModeIndex::Aitken); + qnum_cur[nait] += dndt_ait * dtsubstep; + + if (dso4dt_ait > 0.0) { + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + + Real delta_q = dso4dt_ait * dtsubstep; + qaer_cur[iaer_so4][nait] += delta_q; + delta_q = haero::min(delta_q, qgas_cur[igas_h2so4]); + qgas_cur[igas_h2so4] -= delta_q; + } + + if (igas_nh3 > 0 && dnh4dt_ait > 0.0) { + static constexpr int iaer_nh4 = + -9999999; // static_cast(AeroId::NH4); + + Real delta_q = dnh4dt_ait * dtsubstep; + qaer_cur[iaer_nh4][nait] += delta_q; + delta_q = haero::min(delta_q, qgas_cur[igas_nh3]); + qgas_cur[igas_nh3] -= delta_q; + } + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_nnuc[i] += (qgas_cur[i] - qgas_sv1[i]); + for (int i = 0; i < num_modes; ++i) + qnum_del_nnuc[i] += (qnum_cur[i] - qnum_sv1[i]); + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_nnuc[j][i] += (qaer_cur[j][i] - qaer_sv1[j][i]); + + dnclusterdt = dnclusterdt + dnclusterdt_substep * (dtsubstep / deltat); + } + + // coagulation part + if (config.do_coag) { + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + coagulation::mam_coag_1subarea(dtsubstep, temp, pmid, aircon, dgn_a, + dgn_awet, wetdens, qnum_cur, qaer_cur, + qaer_delsub_coag_in); + for (int i = 0; i < num_modes; ++i) + qnum_delsub_coag[i] = qnum_cur[i] - qnum_sv1[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_delsub_coag[j][i] = qaer_cur[j][i] - qaer_sv1[j][i]; + } + + // primary carbon aging + + aging::mam_pcarbon_aging_1subarea( + dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, + qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); + + // accumulate sub-step q-dels + if (config.do_coag) { + for (int i = 0; i < num_modes; ++i) + qnum_del_coag[i] += qnum_delsub_coag[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_coag[j][i] += qaer_delsub_coag[j][i]; + } + if (config.do_cond) { + for (int i = 0; i < num_modes; ++i) + qnum_del_cond[i] += qnum_delsub_cond[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; + } + } + + // final mix ratios + for (int i = 0; i < num_gas_ids; ++i) + qgas4[i] = qgas_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer4[j][i] = qaer_cur[j][i]; + for (int i = 0; i < num_modes; ++i) + qnum4[i] = qnum_cur[i]; + for (int i = 0; i < num_modes; ++i) + qwtr4[i] = qwtr_cur[i]; + + // final mix ratio changes + for (int i = 0; i < num_gas_ids; ++i) { + qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; + qgas_delaa[i][iqtend_rnam()] = 0.0; + qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; + qgas_delaa[i][iqtend_coag()] = 0.0; + qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; + } + for (int i = 0; i < num_modes; ++i) { + qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; + qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; + qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; + qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; + qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; + } + for (int j = 0; j < num_aerosol_ids; ++j) { + for (int i = 0; i < num_modes; ++i) { + qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; + qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; + qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; + qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; + qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; + } + } +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1subarea_cloudy( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int jsub, + const int nsubarea, const bool iscldy_subarea, const Real afracsub, + const Real temp, const Real pmid, const Real pdel, const Real zmid, + const Real pblh, const Real relhum, Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qgas1[AeroConfig::num_gas_ids()], + const Real qgas3[AeroConfig::num_gas_ids()], + Real qgas4[AeroConfig::num_gas_ids()], + Real qgas_delaa[AeroConfig::num_gas_ids()][nqtendaa()], + const Real qnum3[AeroConfig::num_modes()], + Real qnum4[AeroConfig::num_modes()], + Real qnum_delaa[AeroConfig::num_modes()][nqtendaa()], + const Real qaer2[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + const Real qaer3[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer4[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()], + Real qaer_delaa[AeroConfig::num_aerosol_ids()][AeroConfig::num_modes()] + [nqtendaa()], + const Real qwtr3[AeroConfig::num_modes()], + Real qwtr4[AeroConfig::num_modes()], + const Real qnumcw3[AeroConfig::num_modes()], + Real qnumcw4[AeroConfig::num_modes()], + Real qnumcw_delaa[AeroConfig::num_modes()][nqqcwtendaa()], + const Real qaercw2[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + const Real qaercw3[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + Real qaercw4[AeroConfig::num_gas_ids()][AeroConfig::num_modes()], + Real qaercw_delaa[AeroConfig::num_gas_ids()][AeroConfig::num_modes()] + [nqqcwtendaa()]) { + + // + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qgas3, qaer3, qaercw3, qnum3, qnumcw3 are the current incoming TMRs + // qgas4, qaer4, qaercw4, qnum4, qnumcw4 are the updated outgoing TMRs + // + // when config.do_cond = false, this routine only calculates changes involving + // growth from smaller to larger modes (renaming) following cloud chemistry + // so gas TMRs are not changed + // when config.do_cond = true, this routine also calculates changes involving + // gas-aerosol exchange (condensation/evaporation) + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation + // currently this routine does not do + // new particle nucleation - because h2so4 gas conc. should be very low in + // cloudy air coagulation - because cloud-borne aerosol would need to be + // included + // + + // qXXXN (X=gas,aer,wat,num; N=1:4) are sub-area mixing ratios + // XXX=gas - gas species + // XXX=aer - aerosol mass species (excluding water) + // XXX=wat - aerosol water + // XXX=num - aerosol number + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - current incoming values (before gas-aerosol exchange, newnuc, + // coag) N=4 - updated outgoing values (after gas-aerosol exchange, + // newnuc, coag) + // + // qXXX_delaa are TMR changes (not tendencies) + // for different processes, which are used to produce history output + // for a clear sub-area, the processes are condensation/evaporation (and + // associated aging), + // renaming, coagulation, and nucleation + + // qxxx_del_yyyy are mix-ratio changes over full time step (deltat) + // qxxx_delsub_yyyy are mix-ratio changes over time sub-step (dtsubstep) + + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + static constexpr int igas_h2so4 = static_cast(GasId::H2SO4); + // Turn off nh3 for now. This is a future enhancement. + static constexpr int igas_nh3 = -999888777; // Same as mam_refactor + static constexpr int iaer_so4 = static_cast(AeroId::SO4); + static constexpr int iaer_pom = static_cast(AeroId::POM); + + const AeroId gas_to_aer[num_gas_ids] = {AeroId::SOA, AeroId::SO4, + AeroId::None}; + const bool l_gas_condense_to_mode[num_gas_ids][num_modes] = { + {true, true, true, true}, + {true, true, true, true}, + {false, false, false, false}}; + enum { NA, ANAL, IMPL }; + const int eqn_and_numerics_category[num_gas_ids] = {IMPL, ANAL, ANAL}; + // air molar density (kmol/m3) + // In order to try to match the results in mam_refactor + // set r_universal as [mJ/(mol)] as in mam_refactor. + // const Real r_universal = Constants::r_gas; // [mJ/(K mol)] + const Real r_universal = 8.314467591; // [mJ/(mol)] as in mam_refactor + const Real aircon = pmid / (1000 * r_universal * temp); + const Real alnsg_aer[num_modes] = {0.58778666490211906, 0.47000362924573563, + 0.58778666490211906, 0.47000362924573563}; + const Real uptk_rate_factor[num_gas_ids] = {0.81, 1.0, 1.0}; + + Real qgas_cur[num_gas_ids]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_cur[i] = qgas3[i]; + Real qaer_cur[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_cur[i][j] = qaer3[i][j]; + + Real qnum_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnum_cur[j] = qnum3[j]; + Real qwtr_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qwtr_cur[j] = qwtr3[j]; + + Real qnumcw_cur[num_modes]; + for (int j = 0; j < num_modes; ++j) + qnumcw_cur[j] = qnumcw3[j]; + + Real qaercw_cur[num_gas_ids][num_modes]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_cur[i][j] = qaercw3[i][j]; + + Real qgas_netprod_otrproc[num_gas_ids] = {}; + if (config.do_cond && config.gaexch_h2so4_uptake_optaa == 2) { + for (int igas = 0; igas < num_gas_ids; ++igas) { + if (igas == igas_h2so4 || igas == igas_nh3) { + // if gaexch_h2so4_uptake_optaa == 2, then + // if qgas increases from pre-gaschem to post-cldchem, + // start from the pre-gaschem mix-ratio and add in the production + // during the integration + // if it decreases, + // start from post-cldchem mix-ratio + // *** currently just do this for h2so4 and nh3 + qgas_netprod_otrproc[igas] = (qgas3[igas] - qgas1[igas]) / deltat; + if (qgas_netprod_otrproc[igas] >= 0.0) + qgas_cur[igas] = qgas1[igas]; + else + qgas_netprod_otrproc[igas] = 0.0; + } + } + } + Real qgas_del_cond[num_gas_ids] = {}; + Real qgas_del_nnuc[num_gas_ids] = {}; + Real qgas_del_cond_only[num_gas_ids] = {}; + Real qaer_del_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qaer_del_nnuc[num_aerosol_ids][num_modes] = {}; + Real qaer_del_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_cond[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_coag[num_aerosol_ids][num_modes] = {}; + Real qaer_del_cond_only[num_aerosol_ids][num_modes] = {}; + Real qaercw_del_rnam[num_aerosol_ids][num_modes] = {}; + Real qnum_del_cond[num_modes] = {}; + Real qnum_del_rnam[num_modes] = {}; + Real qnum_del_nnuc[num_modes] = {}; + Real qnum_del_coag[num_modes] = {}; + Real qnum_delsub_cond[num_modes] = {}; + Real qnum_delsub_coag[num_modes] = {}; + Real qnum_del_cond_only[num_modes] = {}; + Real qnumcw_del_rnam[num_modes] = {}; + Real qaer_delsub_coag_in[num_aerosol_ids][AeroConfig::max_agepair()] = {}; + + const int ntsubstep = 1; + Real dtsubstep = deltat; + if (ntsubstep > 1) + dtsubstep = deltat / ntsubstep; + + // loop over multiple time sub-steps + + for (int jtsubstep = 1; jtsubstep <= ntsubstep; ++jtsubstep) { + // gas-aerosol exchange + Real uptkrate_h2so4 = 0.0; + Real qgas_avg[num_gas_ids] = {}; + Real qgas_sv1[num_gas_ids] = {}; + Real qnum_sv1[num_modes] = {}; + Real qaer_sv1[num_aerosol_ids][num_modes] = {}; + Real qaer_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + + if (config.do_cond) { + + const bool l_calc_gas_uptake_coeff = jtsubstep == 1; + Real uptkaer[num_gas_ids][num_modes] = {}; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_sv1[i] = qgas_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + + const int nghq = 2; + const int ntot_soamode = 4; + int niter_out = 0; + Real g0_soa_out = 0; + // time sub-step + const Real dtsub_soa_fixed = -1.0; + gasaerexch::mam_gasaerexch_1subarea( + nghq, igas_h2so4, igas_nh3, ntot_soamode, gas_to_aer, iaer_so4, + iaer_pom, l_calc_gas_uptake_coeff, l_gas_condense_to_mode, + eqn_and_numerics_category, dtsubstep, dtsub_soa_fixed, temp, pmid, + aircon, num_gas_ids, qgas_cur, qgas_avg, qgas_netprod_otrproc, + qaer_cur, qnum_cur, dgn_awet, alnsg_aer, uptk_rate_factor, uptkaer, + uptkrate_h2so4, niter_out, g0_soa_out); + + if (config.newnuc_h2so4_conc_optaa == 11) + qgas_avg[igas_h2so4] = + 0.5 * (qgas_sv1[igas_h2so4] + qgas_cur[igas_h2so4]); + else if (config.newnuc_h2so4_conc_optaa == 12) + qgas_avg[igas_h2so4] = qgas_cur[igas_h2so4]; + + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond[i] += + (qgas_cur[i] - (qgas_sv1[i] + qgas_netprod_otrproc[i] * dtsubstep)); + + for (int i = 0; i < num_modes; ++i) + qnum_delsub_cond[i] = qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_cond[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + + // qaer_del_grow4rnam = change in qaer_del_cond during latest condensation + // calculations + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_delsub_grow4rnam[i][j] = qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + qgas_del_cond_only[i] = qgas_del_cond[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_cond_only[i][j] = qaer_delsub_cond[i][j]; + for (int i = 0; i < num_modes; ++i) + qnum_del_cond_only[i] = qnum_delsub_cond[i]; + + } else { + for (int i = 0; i < num_gas_ids; ++i) + qgas_avg[i] = qgas_cur[i]; + } + // renaming after "continuous growth" + if (config.do_rename) { + constexpr int nmodes = AeroConfig::num_modes(); + constexpr int naerosol_species = AeroConfig::num_aerosol_ids(); + const Real smallest_dryvol_value = 1.0e-25; // BAD_CONSTANT + const int dest_mode_of_mode[nmodes] = {-1, 0, -1, -1}; + + Real qnumcw_cur[num_modes] = {}; + Real qaercw_cur[num_aerosol_ids][num_modes] = {}; + Real qaercw_delsub_grow4rnam[num_aerosol_ids][num_modes] = {}; + Real mean_std_dev[nmodes]; + Real fmode_dist_tail_fac[nmodes]; + Real v2n_lo_rlx[nmodes]; + Real v2n_hi_rlx[nmodes]; + Real ln_diameter_tail_fac[nmodes]; + int num_pairs = 0; + Real diameter_cutoff[nmodes]; + Real ln_dia_cutoff[nmodes]; + Real diameter_threshold[nmodes]; + Real mass_2_vol[naerosol_species] = {0.15, + 6.4971751412429377e-002, + 0.15, + 7.0588235294117650e-003, + 3.0789473684210526e-002, + 5.1923076923076926e-002, + 156.20986883198000}; + + rename::find_renaming_pairs(dest_mode_of_mode, // in + mean_std_dev, // out + fmode_dist_tail_fac, // out + v2n_lo_rlx, // out + v2n_hi_rlx, // out + ln_diameter_tail_fac, // out + num_pairs, // out + diameter_cutoff, // out + ln_dia_cutoff, diameter_threshold); + + for (int i = 0; i < num_modes; ++i) + qnum_sv1[i] = qnum_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_sv1[j][i] = qaer_cur[j][i]; + Real dgnum_amode[nmodes]; + for (int m = 0; m < nmodes; ++m) { + dgnum_amode[m] = modes(m).nom_diameter; + } + + // qaercw_delsub_grow4rnam = change in qaercw from cloud chemistry + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_delsub_grow4rnam[i][j] = + (qaercw3[i][j] - qaercw2[i][j]) / ntsubstep; + Real qnumcw_sv1[num_modes]; + for (int i = 0; i < num_modes; ++i) + qnumcw_sv1[i] = qnumcw_cur[i]; + Real qaercw_sv1[num_aerosol_ids][num_modes]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_sv1[i][j] = qaercw_cur[i][j]; + + { + Real qmol_i_cur[num_modes][num_aerosol_ids]; + Real qmol_i_del[num_modes][num_aerosol_ids]; + Real qmol_c_cur[num_modes][num_aerosol_ids]; + Real qmol_c_del[num_modes][num_aerosol_ids]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qmol_i_cur[i][j] = qaer_cur[j][i]; + qmol_i_del[i][j] = qaer_delsub_grow4rnam[j][i]; + qmol_c_cur[i][j] = qaercw_cur[j][i]; + qmol_c_del[i][j] = qaercw_delsub_grow4rnam[j][i]; + } + + Rename rename; + rename.mam_rename_1subarea_( + iscldy_subarea, smallest_dryvol_value, dest_mode_of_mode, + mean_std_dev, fmode_dist_tail_fac, v2n_lo_rlx, v2n_hi_rlx, + ln_diameter_tail_fac, num_pairs, diameter_cutoff, ln_dia_cutoff, + diameter_threshold, mass_2_vol, dgnum_amode, qnum_cur, qmol_i_cur, + qmol_i_del, qnumcw_cur, qmol_c_cur, qmol_c_del); + + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) { + qaer_cur[j][i] = qmol_i_cur[i][j]; + qaer_delsub_grow4rnam[j][i] = qmol_i_del[i][j]; + qaercw_cur[j][i] = qmol_c_cur[i][j]; + qaercw_delsub_grow4rnam[j][i] = qmol_c_del[i][j]; + } + } + for (int i = 0; i < num_modes; ++i) + qnum_del_rnam[i] += qnum_cur[i] - qnum_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaer_del_rnam[i][j] += qaer_cur[i][j] - qaer_sv1[i][j]; + for (int i = 0; i < num_modes; ++i) + qnumcw_del_rnam[i] += qnumcw_cur[i] - qnumcw_sv1[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_del_rnam[i][j] += qaercw_cur[i][j] - qaercw_sv1[i][j]; + } + + // primary carbon aging + if (config.do_cond) { + aging::mam_pcarbon_aging_1subarea( + dgn_a, qnum_cur, qnum_delsub_cond, qnum_delsub_coag, qaer_cur, + qaer_delsub_cond, qaer_delsub_coag, qaer_delsub_coag_in); + } + // accumulate sub-step q-dels + if (config.do_cond) { + for (int i = 0; i < num_modes; ++i) + qnum_del_cond[i] += qnum_delsub_cond[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer_del_cond[j][i] += qaer_delsub_cond[j][i]; + } + } + // final mix ratios + for (int i = 0; i < num_gas_ids; ++i) + qgas4[i] = qgas_cur[i]; + for (int j = 0; j < num_aerosol_ids; ++j) + for (int i = 0; i < num_modes; ++i) + qaer4[j][i] = qaer_cur[j][i]; + for (int i = 0; i < num_modes; ++i) + qnum4[i] = qnum_cur[i]; + for (int i = 0; i < num_modes; ++i) + qwtr4[i] = qwtr_cur[i]; + for (int i = 0; i < num_modes; ++i) + qnumcw4[i] = qnumcw_cur[i]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw4[i][j] = qaercw_cur[i][j]; + + // final mix ratio changes + for (int i = 0; i < num_gas_ids; ++i) { + qgas_delaa[i][iqtend_cond()] = qgas_del_cond[i]; + qgas_delaa[i][iqtend_rnam()] = 0.0; + qgas_delaa[i][iqtend_nnuc()] = qgas_del_nnuc[i]; + qgas_delaa[i][iqtend_coag()] = 0.0; + qgas_delaa[i][iqtend_cond_only()] = qgas_del_cond_only[i]; + } + for (int i = 0; i < num_modes; ++i) { + qnum_delaa[i][iqtend_cond()] = qnum_del_cond[i]; + qnum_delaa[i][iqtend_rnam()] = qnum_del_rnam[i]; + qnum_delaa[i][iqtend_nnuc()] = qnum_del_nnuc[i]; + qnum_delaa[i][iqtend_coag()] = qnum_del_coag[i]; + qnum_delaa[i][iqtend_cond_only()] = qnum_del_cond_only[i]; + } + for (int j = 0; j < num_aerosol_ids; ++j) { + for (int i = 0; i < num_modes; ++i) { + qaer_delaa[j][i][iqtend_cond()] = qaer_del_cond[j][i]; + qaer_delaa[j][i][iqtend_rnam()] = qaer_del_rnam[j][i]; + qaer_delaa[j][i][iqtend_nnuc()] = qaer_del_nnuc[j][i]; + qaer_delaa[j][i][iqtend_coag()] = qaer_del_coag[j][i]; + qaer_delaa[j][i][iqtend_cond_only()] = qaer_del_cond_only[j][i]; + } + } + for (int i = 0; i < num_modes; ++i) + qnumcw_delaa[i][iqqcwtend_rnam()] = qnumcw_del_rnam[i]; + for (int i = 0; i < num_aerosol_ids; ++i) + for (int j = 0; j < num_modes; ++j) + qaercw_delaa[i][j][iqqcwtend_rnam()] = qaercw_del_rnam[i][j]; +} + +KOKKOS_INLINE_FUNCTION +void mam_amicphys_1gridcell( + const AmicPhysConfig& config, const int nstep, const Real deltat, const int nsubarea, + const int ncldy_subarea, const bool iscldy_subarea[maxsubarea()], + const Real afracsub[maxsubarea()], const Real temp, const Real pmid, + const Real pdel, const Real zmid, const Real pblh, + const Real relhumsub[maxsubarea()], Real dgn_a[AeroConfig::num_modes()], + Real dgn_awet[AeroConfig::num_modes()], + Real wetdens[AeroConfig::num_modes()], + const Real qsub1[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qsub2[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qqcwsub2[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qsub3[AeroConfig::num_gas_ids()][maxsubarea()], + const Real qqcwsub3[AeroConfig::num_gas_ids()][maxsubarea()], + Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()], + Real qsub4[AeroConfig::num_gas_ids()][maxsubarea()], + Real qqcwsub4[AeroConfig::num_gas_ids()][maxsubarea()], + Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()], + Real qsub_tendaa[AeroConfig::num_gas_ids()][nqtendaa()][maxsubarea()], + Real qqcwsub_tendaa[AeroConfig::num_gas_ids()][nqqcwtendaa()][maxsubarea()]) { + + // + // calculates changes to gas and aerosol sub-area TMRs (tracer mixing ratios) + // qsub3 and qqcwsub3 are the incoming current TMRs + // qsub4 and qqcwsub4 are the outgoing updated TMRs + // + // qsubN and qqcwsubN (N=1:4) are tracer mixing ratios (TMRs, mol/mol or + // #/kmol) in sub-areas + // currently there are just clear and cloudy sub-areas + // the N=1:4 have same meanings as for qgcmN + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) + // qsub_tendaa and qqcwsub_tendaa are TMR tendencies + // for different processes, which are used to produce history output + // the processes are condensation/evaporation (and associated aging), + // renaming, coagulation, and nucleation + + static constexpr int num_gas_ids = AeroConfig::num_gas_ids(); + static constexpr int num_modes = AeroConfig::num_modes(); + static constexpr int num_aerosol_ids = AeroConfig::num_aerosol_ids(); + + // the q--4 values will be equal to q--3 values unless they get changed + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < maxsubarea(); ++j) { + qsub4[i][j] = qsub3[i][j]; + qqcwsub4[i][j] = qqcwsub3[i][j]; + } + for (int i = 0; i < num_modes; ++i) + for (int j = 0; j < maxsubarea(); ++j) + qaerwatsub4[i][j] = qaerwatsub3[i][j]; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < nqtendaa(); ++j) + for (int k = 0; k < maxsubarea(); ++k) + qsub_tendaa[i][j][k] = 0; + for (int i = 0; i < num_gas_ids; ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + for (int k = 0; k < maxsubarea(); ++k) + qqcwsub_tendaa[i][j][k] = 0.0; + + for (int jsub = 0; jsub < nsubarea; ++jsub) { + AmicPhysConfig sub_config = config; + if (iscldy_subarea[jsub]) { + sub_config.do_cond = config.do_cond; + sub_config.do_rename = config.do_rename; + sub_config.do_newnuc = false; + sub_config.do_coag = false; + } + const bool do_map_gas_sub = sub_config.do_cond || sub_config.do_newnuc; + + // map incoming sub-area mix-ratios to gas/aer/num arrays + Real qgas1[num_gas_ids] = {}; + Real qgas3[num_gas_ids] = {}; + Real qgas4[num_gas_ids] = {}; + if (do_map_gas_sub) { + // for cldy subarea, only do gases if doing gaexch + for (int igas = 0; igas < 2; ++igas) { + const int l = lmap_gas(igas); + qgas1[igas] = qsub1[l][jsub] * fcvt_gas(igas); + qgas3[igas] = qsub3[l][jsub] * fcvt_gas(igas); + qgas4[igas] = qgas3[igas]; + } + } + Real qaer2[num_aerosol_ids][num_modes] = {}; + Real qaer3[num_aerosol_ids][num_modes] = {}; + Real qnum3[num_modes] = {}; + Real qaer4[num_aerosol_ids][num_modes] = {}; + Real qnum4[num_modes] = {}; + Real qwtr3[num_modes] = {}; + Real qwtr4[num_modes] = {}; + for (int n = 0; n < num_modes; ++n) { + qnum3[n] = qsub3[n][jsub] * fcvt_num(); + qnum4[n] = qnum3[n]; + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qaer2[iaer][n] = qsub2[iaer][jsub] * fcvt_aer(iaer); + qaer3[iaer][n] = qsub3[iaer][jsub] * fcvt_aer(iaer); + qaer4[iaer][n] = qaer3[iaer][n]; + } + qwtr3[n] = qaerwatsub3[n][jsub] * fcvt_wtr(); + qwtr4[n] = qwtr3[n]; + } + Real qaercw2[num_aerosol_ids][num_modes] = {}; + Real qaercw3[num_aerosol_ids][num_modes] = {}; + Real qnumcw3[num_modes] = {}; + Real qaercw4[num_aerosol_ids][num_modes] = {}; + Real qnumcw4[num_modes] = {}; + if (iscldy_subarea[jsub]) { + // only do cloud-borne for cloudy + for (int n = 0; n < num_modes; ++n) { + qnumcw3[n] = qqcwsub3[n][jsub] * fcvt_num(); + qnumcw4[n] = qnumcw3[n]; + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qaercw2[iaer][n] = qqcwsub2[n][jsub] * fcvt_aer(iaer); + qaercw3[iaer][n] = qqcwsub3[n][jsub] * fcvt_aer(iaer); + qaercw4[iaer][n] = qaercw3[iaer][n]; + } + } + } + + Real qgas_delaa[num_gas_ids][nqtendaa()] = {}; + Real qnum_delaa[num_modes][nqtendaa()] = {}; + Real qnumcw_delaa[num_modes][nqqcwtendaa()] = {}; + Real qaer_delaa[num_aerosol_ids][num_modes][nqtendaa()] = {}; + Real qaercw_delaa[num_aerosol_ids][num_modes][nqqcwtendaa()] = {}; + + if (iscldy_subarea[jsub]) { + mam_amicphys_1subarea_cloudy(sub_config, nstep, deltat, + jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, + pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, + qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer2, qaer3, + qaer4, qaer_delaa, qwtr3, qwtr4, qnumcw3, qnumcw4, qnumcw_delaa, + qaercw2, qaercw3, qaercw4, qaercw_delaa); + } else { + mam_amicphys_1subarea_clear(sub_config, nstep, deltat, + jsub, nsubarea, iscldy_subarea[jsub], afracsub[jsub], temp, pmid, + pdel, zmid, pblh, relhumsub[jsub], dgn_a, dgn_awet, wetdens, qgas1, + qgas3, qgas4, qgas_delaa, qnum3, qnum4, qnum_delaa, qaer3, qaer4, + qaer_delaa, qwtr3, qwtr4); + // map gas/aer/num arrays (mix-ratio and del=change) back to sub-area + // arrays + + if (do_map_gas_sub) { + for (int igas = 0; igas < 2; ++igas) { + const int l = lmap_gas(igas); + qsub4[l][jsub] = qgas4[igas] / fcvt_gas(igas); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[l][i][jsub] = + qgas_delaa[igas][i] / (fcvt_gas(igas) * deltat); + } + } + for (int n = 0; n < num_modes; ++n) { + qsub4[n][jsub] = qnum4[n] / fcvt_num(); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[n][i][jsub] = qnum_delaa[n][i] / (fcvt_num() * deltat); + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qsub4[iaer][jsub] = qaer4[iaer][n] / fcvt_aer(iaer); + for (int i = 0; i < nqtendaa(); ++i) + qsub_tendaa[iaer][i][jsub] = + qaer_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); + } + qaerwatsub4[n][jsub] = qwtr4[n] / fcvt_wtr(); + + if (iscldy_subarea[jsub]) { + qqcwsub4[n][jsub] = qnumcw4[n] / fcvt_num(); + for (int i = 0; i < nqqcwtendaa(); ++i) + qqcwsub_tendaa[n][i][jsub] = + qnumcw_delaa[n][i] / (fcvt_num() * deltat); + for (int iaer = 0; iaer < num_aerosol_ids; ++iaer) { + qqcwsub4[iaer][jsub] = qaercw4[iaer][n] / fcvt_aer(iaer); + for (int i = 0; i < nqqcwtendaa(); ++i) + qqcwsub_tendaa[iaer][i][jsub] = + qaercw_delaa[iaer][n][i] / (fcvt_aer(iaer) * deltat); + } + } + } + } + } +} + +} // anonymous namespace + +KOKKOS_INLINE_FUNCTION +void modal_aero_amicphys_intr( + const AmicPhysConfig& config, const int nstep, const Real deltat, const Real t, const Real pmid, const Real pdel, + const Real zm, const Real pblh, const Real qv, const Real cld, + Real q[gas_pcnst()], Real qqcw[gas_pcnst()], const Real q_pregaschem[gas_pcnst()], + const Real q_precldchem[gas_pcnst()], const Real qqcw_precldchem[gas_pcnst()], + Real q_tendbb[gas_pcnst()][nqtendbb()], Real qqcw_tendbb[gas_pcnst()][nqtendbb()], + Real dgncur_a[AeroConfig::num_modes()], + Real dgncur_awet[AeroConfig::num_modes()], + Real wetdens_host[AeroConfig::num_modes()], + Real qaerwat[AeroConfig::num_modes()]) { + + /* + nstep ! model time-step number + nqtendbb ! dimension for q_tendbb + nqqcwtendbb ! dimension f + deltat ! + q(ncol,pver,pcnstxx) ! current tracer mixing ratios (TMRs) + these values are updated (so out /= in) + *** MUST BE #/kmol-air for number + *** MUST BE mol/mol-air for mass + *** NOTE ncol dimension + qqcw(ncol,pver,pcnstxx) + like q but for cloud-borner tracers + these values are updated + q_pregaschem(ncol,pver,pcnstxx) ! q TMRs before gas-phase + chemistry q_precldchem(ncol,pver,pcnstxx) ! q TMRs before cloud + chemistry qqcw_precldchem(ncol,pver,pcnstxx) ! qqcw TMRs before cloud + chemistry q_tendbb(ncol,pver,pcnstxx,nqtendbb()) ! TMR tendencies for + box-model diagnostic output qqcw_tendbb(ncol,pver,pcnstx t(pcols,pver) ! + temperature at model levels (K) pmid(pcols,pver) ! pressure at model + level centers (Pa) pdel(pcols,pver) ! pressure thickness of levels + (Pa) zm(pcols,pver) ! altitude (above ground) at level centers (m) + pblh(pcols) ! planetary boundary layer depth (m) + qv(pcols,pver) ! specific humidity (kg/kg) + cld(ncol,pver) ! cloud fraction (-) *** NOTE ncol dimension + dgncur_a(pcols,pver,ntot_amode) + dgncur_awet(pcols,pver,ntot_amode) + ! dry & wet geo. mean dia. (m) of + number distrib. wetdens_host(pcols,pver,ntot_amode) ! interstitial + aerosol wet density (kg/m3) + + qaerwat(pcols,pver,ntot_amode aerosol water mixing ratio (kg/kg, + NOT mol/mol) + + */ + + // !DESCRIPTION: + // calculates changes to gas and aerosol TMRs (tracer mixing ratios) from + // gas-aerosol exchange (condensation/evaporation) + // growth from smaller to larger modes (renaming) due to both + // condensation and cloud chemistry + // new particle nucleation + // coagulation + // transfer of particles from hydrophobic modes to hydrophilic modes + // (aging) + // due to condensation and coagulation + // + // the incoming mixing ratios (q and qqcw) are updated before output + // + // !REVISION HISTORY: + // RCE 07.04.13: Adapted from earlier version of CAM5 modal aerosol + // routines + // for these processes + // + + static constexpr int num_modes = AeroConfig::num_modes(); + + // qgcmN and qqcwgcmN (N=1:4) are grid-cell mean tracer mixing ratios + // (TMRs, mol/mol or #/kmol) + // N=1 - before gas-phase chemistry + // N=2 - before cloud chemistry + // N=3 - incoming values (before gas-aerosol exchange, newnuc, coag) + // N=4 - outgoing values (after gas-aerosol exchange, newnuc, coag) + + // qsubN and qqcwsubN (N=1:4) are TMRs in sub-areas + // currently there are just clear and cloudy sub-areas + // the N=1:4 have same meanings as for qgcmN + + // q_coltendaa and qqcw_coltendaa are column-integrated tendencies + // for different processes, which are output to history + // the processes are condensation/evaporation (and associated aging), + // renaming, coagulation, and nucleation + + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendbb(); ++j) + q_tendbb[i][j] = 0.0, qqcw_tendbb[i][j] = 0.0; + + // get saturation mixing ratio + // call qsat( t(1:ncol,1:pver), pmid(1:ncol,1:pvnner), & + // ev_sat(1:ncol,1:pver), qv_sat(1:ncol,1:pver) ) + const Real epsqs = haero::Constants::weight_ratio_h2o_air; + // Saturation vapor pressure + const Real ev_sat = conversions::vapor_saturation_pressure_magnus(t, pmid); + // Saturation specific humidity + const Real qv_sat = epsqs * ev_sat / (pmid - (1 - epsqs) * ev_sat); + + const Real relhumgcm = haero::max(0.0, haero::min(1.0, qv / qv_sat)); + + // Set up cloudy/clear subareas inside a grid cell + int nsubarea, ncldy_subarea, jclea, jcldy; + bool iscldy_subarea[maxsubarea()]; + Real afracsub[maxsubarea()]; + Real relhumsub[maxsubarea()]; + Real qsub1[gas_pcnst()][maxsubarea()]; + Real qsub2[gas_pcnst()][maxsubarea()]; + Real qsub3[gas_pcnst()][maxsubarea()]; + Real qqcwsub1[gas_pcnst()][maxsubarea()]; + Real qqcwsub2[gas_pcnst()][maxsubarea()]; + Real qqcwsub3[gas_pcnst()][maxsubarea()]; + // aerosol water mixing ratios (mol/mol) + Real qaerwatsub3[AeroConfig::num_modes()][maxsubarea()]; + construct_subareas_1gridcell(cld, relhumgcm, // in + q_pregaschem, q_precldchem, // in + qqcw_precldchem, // in + q, qqcw, // in + nsubarea, ncldy_subarea, jclea, jcldy, // out + iscldy_subarea, afracsub, relhumsub, // out + qsub1, qsub2, qsub3, // out + qqcwsub1, qqcwsub2, qqcwsub3, qaerwatsub3, // out + qaerwat // in + ); + + // Initialize the "after-amicphys" values + Real qsub4[gas_pcnst()][maxsubarea()] = {}; + Real qqcwsub4[gas_pcnst()][maxsubarea()] = {}; + Real qaerwatsub4[AeroConfig::num_modes()][maxsubarea()] = {}; + + // + // start integration + // + Real dgn_a[num_modes], dgn_awet[num_modes], wetdens[num_modes]; + for (int n = 0; n < num_modes; ++n) { + dgn_a[n] = dgncur_a[n]; + dgn_awet[n] = dgncur_awet[n]; + wetdens[n] = haero::max(1000.0, wetdens_host[n]); + } + Real qsub_tendaa[gas_pcnst()][nqtendaa()][maxsubarea()] = {}; + Real qqcwsub_tendaa[gas_pcnst()][nqqcwtendaa()][maxsubarea()] = {}; + mam_amicphys_1gridcell(config, nstep, deltat, + nsubarea, ncldy_subarea, iscldy_subarea, afracsub, t, + pmid, pdel, zm, pblh, relhumsub, dgn_a, dgn_awet, + wetdens, qsub1, qsub2, qqcwsub2, qsub3, qqcwsub3, + qaerwatsub3, qsub4, qqcwsub4, qaerwatsub4, qsub_tendaa, + qqcwsub_tendaa); + + // + // form new grid-mean mix-ratios + Real qgcm4[gas_pcnst()]; + Real qgcm_tendaa[gas_pcnst()][nqtendaa()]; + Real qaerwatgcm4[num_modes]; + if (nsubarea == 1) { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] = qsub4[i][0]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = qsub_tendaa[i][j][0]; + for (int i = 0; i < num_modes; ++i) + qaerwatgcm4[i] = qaerwatsub4[i][0]; + } else { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] = 0.0; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = 0.0; + for (int n = 0; n < nsubarea; ++n) { + for (int i = 0; i < gas_pcnst(); ++i) + qgcm4[i] += qsub4[i][n] * afracsub[n]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqtendaa(); ++j) + qgcm_tendaa[i][j] = + qgcm_tendaa[i][j] + qsub_tendaa[i][j][n] * afracsub[n]; + } + for (int i = 0; i < num_modes; ++i) + // for aerosol water use the clear sub-area value + qaerwatgcm4[i] = qaerwatsub4[i][jclea - 1]; + } + Real qqcwgcm4[gas_pcnst()]; + Real qqcwgcm_tendaa[gas_pcnst()][nqqcwtendaa()]; + if (ncldy_subarea <= 0) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = haero::max(0.0, qqcw[i]); + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = 0.0; + } else if (nsubarea == 1) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = qqcwsub4[i][0]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = qqcwsub_tendaa[i][j][0]; + } else { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] = 0.0; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] = 0.0; + for (int n = 0; n < nsubarea; ++n) { + if (iscldy_subarea[n]) { + for (int i = 0; i < gas_pcnst(); ++i) + qqcwgcm4[i] += qqcwsub4[i][n] * afracsub[n]; + for (int i = 0; i < gas_pcnst(); ++i) + for (int j = 0; j < nqqcwtendaa(); ++j) + qqcwgcm_tendaa[i][j] += qqcwsub_tendaa[i][j][n] * afracsub[n]; + } + } + } + + for (int lmz = 0; lmz < gas_pcnst(); ++lmz) { + if (lmapcc_all(lmz) > 0) { + // HW, to ensure non-negative + q[lmz] = haero::max(qgcm4[lmz], 0.0); + if (lmapcc_all(lmz) >= lmapcc_val_aer()) { + // HW, to ensure non-negative + qqcw[lmz] = haero::max(qqcwgcm4[lmz], 0.0); + } + } + } + for (int i = 0; i < gas_pcnst(); ++i) { + if (iqtend_cond() < nqtendbb()) + q_tendbb[i][iqtend_cond()] = qgcm_tendaa[i][iqtend_cond()]; + if (iqtend_rnam() < nqtendbb()) + q_tendbb[i][iqtend_rnam()] = qgcm_tendaa[i][iqtend_rnam()]; + if (iqtend_nnuc() < nqtendbb()) + q_tendbb[i][iqtend_nnuc()] = qgcm_tendaa[i][iqtend_nnuc()]; + if (iqtend_coag() < nqtendbb()) + q_tendbb[i][iqtend_coag()] = qgcm_tendaa[i][iqtend_coag()]; + if (iqqcwtend_rnam() < nqqcwtendbb()) + qqcw_tendbb[i][iqqcwtend_rnam()] = qqcwgcm_tendaa[i][iqqcwtend_rnam()]; + } + for (int i = 0; i < num_modes; ++i) + qaerwat[i] = qaerwatgcm4[i]; +} + +} // namespace scream::impl diff --git a/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp b/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp new file mode 100644 index 000000000000..d1d54e97fda0 --- /dev/null +++ b/components/eamxx/src/physics/mam/mam_aerosol_optics_read_tables.hpp @@ -0,0 +1,383 @@ +#ifndef MAM_AEROSOL_OPTICS_READ_TABLES_HPP +#define MAM_AEROSOL_OPTICS_READ_TABLES_HPP + +#include "ekat/ekat_parameter_list.hpp" +#include "mam_coupling.hpp" +#include "share/field/field_manager.hpp" +#include "share/grid/abstract_grid.hpp" +#include "share/grid/grids_manager.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +// later to mam_coupling.hpp +namespace scream::mam_coupling { + +using view_1d_host = typename KT::view_1d::HostMirror; +using view_2d_host = typename KT::view_2d::HostMirror; +using view_5d_host = typename KT::view_ND::HostMirror; +using complex_view_1d = typename KT::view_1d>; + +constexpr int nlwbands = mam4::modal_aer_opt::nlwbands; +constexpr int nswbands = mam4::modal_aer_opt::nswbands; + +struct AerosolOpticsHostData { + // host views + view_2d_host refindex_real_sw_host; + view_2d_host refindex_im_sw_host; + view_2d_host refindex_real_lw_host; + view_2d_host refindex_im_lw_host; + + view_5d_host absplw_host; + view_5d_host abspsw_host; + view_5d_host asmpsw_host; + view_5d_host extpsw_host; +}; + +using AerosolOpticsDeviceData = mam4::modal_aer_opt::AerosolOpticsDeviceData; + +inline void set_parameters_table( + AerosolOpticsHostData &aerosol_optics_host_data, + ekat::ParameterList &rrtmg_params, + std::map &layouts, + std::map &host_views) { + // Set up input structure to read data from file. + using strvec_t = std::vector; + using namespace ShortFieldTagsNames; + + constexpr int refindex_real = mam4::modal_aer_opt::refindex_real; + constexpr int refindex_im = mam4::modal_aer_opt::refindex_im; + constexpr int coef_number = mam4::modal_aer_opt::coef_number; + + auto refindex_real_lw_host = + view_2d_host("refrtablw_real_host", nlwbands, refindex_real); + auto refindex_im_lw_host = + view_2d_host("refrtablw_im_host", nlwbands, refindex_im); + + auto refindex_real_sw_host = + view_2d_host("refrtabsw_real_host", nswbands, refindex_real); + auto refindex_im_sw_host = + view_2d_host("refrtabsw_im_host", nswbands, refindex_im); + + // absplw(lw_band, mode, refindex_im, refindex_real, coef_number) + auto absplw_host = view_5d_host("absplw_host", nlwbands, 1, refindex_im, + refindex_real, coef_number); + + auto asmpsw_host = view_5d_host("asmpsw_host", nswbands, 1, refindex_im, + refindex_real, coef_number); + auto extpsw_host = view_5d_host("extpsw_host", nswbands, 1, refindex_im, + refindex_real, coef_number); + auto abspsw_host = view_5d_host("abspsw_host", nswbands, 1, refindex_im, + refindex_real, coef_number); + + aerosol_optics_host_data.refindex_real_lw_host = refindex_real_lw_host; + aerosol_optics_host_data.refindex_im_lw_host = refindex_im_lw_host; + aerosol_optics_host_data.refindex_real_sw_host = refindex_real_sw_host; + aerosol_optics_host_data.refindex_im_sw_host = refindex_im_sw_host; + aerosol_optics_host_data.absplw_host = absplw_host; + aerosol_optics_host_data.asmpsw_host = asmpsw_host; + aerosol_optics_host_data.extpsw_host = extpsw_host; + aerosol_optics_host_data.abspsw_host = abspsw_host; + + FieldLayout scalar_refindex_real_lw_layout{{LWBND, NREFINDEX_REAL}, + {nlwbands, refindex_real}}; + + FieldLayout scalar_refindex_im_lw_layout{{LWBND, NREFINDEX_IM}, + {nlwbands, refindex_im}}; + FieldLayout scalar_refindex_real_sw_layout{{SWBND, NREFINDEX_REAL}, + {nswbands, refindex_real}}; + + FieldLayout scalar_refindex_im_sw_layout{{SWBND, NREFINDEX_IM}, + {nswbands, refindex_im}}; + + FieldLayout scalar_absplw_layout{ + {LWBND, MODE, NREFINDEX_IM, NREFINDEX_REAL, NCOEF_NUMBER}, + {nlwbands, 1, refindex_im, refindex_real, coef_number}}; + // use also for extpsw, abspsw + FieldLayout scalar_asmpsw_layout{ + {SWBND, MODE, NREFINDEX_IM, NREFINDEX_REAL, NCOEF_NUMBER}, + {nswbands, 1, refindex_im, refindex_real, coef_number}}; + + rrtmg_params.set( + "Field Names", + {"asmpsw", "extpsw", "abspsw", "absplw", "refindex_real_sw", + "refindex_im_sw", "refindex_real_lw", "refindex_im_lw"}); + + rrtmg_params.set("Skip_Grid_Checks", true); + + host_views["refindex_real_sw"] = + view_1d_host(refindex_real_sw_host.data(), refindex_real_sw_host.size()); + + host_views["refindex_im_sw"] = + view_1d_host(refindex_im_sw_host.data(), refindex_im_sw_host.size()); + + host_views["refindex_real_lw"] = + view_1d_host(refindex_real_lw_host.data(), refindex_real_lw_host.size()); + + host_views["refindex_im_lw"] = + view_1d_host(refindex_im_lw_host.data(), refindex_im_lw_host.size()); + + host_views["absplw"] = view_1d_host(absplw_host.data(), absplw_host.size()); + + host_views["asmpsw"] = view_1d_host(asmpsw_host.data(), asmpsw_host.size()); + + host_views["extpsw"] = view_1d_host(extpsw_host.data(), extpsw_host.size()); + + host_views["abspsw"] = view_1d_host(abspsw_host.data(), abspsw_host.size()); + + layouts.emplace("refindex_real_lw", scalar_refindex_real_lw_layout); + layouts.emplace("refindex_im_lw", scalar_refindex_im_lw_layout); + layouts.emplace("refindex_real_sw", scalar_refindex_real_sw_layout); + layouts.emplace("refindex_im_sw", scalar_refindex_im_sw_layout); + layouts.emplace("absplw", scalar_absplw_layout); + layouts.emplace("asmpsw", scalar_asmpsw_layout); + layouts.emplace("extpsw", scalar_asmpsw_layout); + layouts.emplace("abspsw", scalar_asmpsw_layout); +} +// KOKKOS_INLINE_FUNCTION +inline void read_rrtmg_table( + const std::string &table_filename, const int imode, + ekat::ParameterList ¶ms, + const std::shared_ptr &grid, + const std::map &host_views_1d, + const std::map &layouts, + const AerosolOpticsHostData &aerosol_optics_host_data, + const AerosolOpticsDeviceData &aerosol_optics_device_data) { + constexpr int refindex_real = mam4::modal_aer_opt::refindex_real; + constexpr int refindex_im = mam4::modal_aer_opt::refindex_im; + constexpr int coef_number = mam4::modal_aer_opt::coef_number; + + using view_3d_host = typename KT::view_3d::HostMirror; + + // temp views: + view_3d_host temp_lw_3d_host("temp_absplw_host", coef_number, refindex_real, + refindex_im); + + params.set("Filename", table_filename); + AtmosphereInput rrtmg(params, grid, host_views_1d, layouts); + rrtmg.read_variables(); + rrtmg.finalize(); + + // copy data from host to device for mode 1 + int d1 = imode; + for(int d3 = 0; d3 < nswbands; ++d3) { + auto real_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_real_sw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refrtabsw[d1][d3], + real_host_d3); + auto im_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_im_sw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refitabsw[d1][d3], im_host_d3); + } // d3 + + for(int d3 = 0; d3 < nlwbands; ++d3) { + auto real_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_real_lw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refrtablw[d1][d3], + real_host_d3); + auto im_host_d3 = ekat::subview( + aerosol_optics_host_data.refindex_im_lw_host, d3); + Kokkos::deep_copy(aerosol_optics_device_data.refitablw[d1][d3], im_host_d3); + } // d3 + + // NOTE: we need to reorder dimensions in absplw + // netcfd : (lw_band, mode, refindex_im, refindex_real, coef_number) + // mam4xx : (mode, lw_band, coef_number, refindex_real, refindex_im ) + // e3sm : (ntot_amode,coef_number,refindex_real,refindex_im,nlwbands) + + for(int d5 = 0; d5 < nlwbands; ++d5) { + // reshape data: + Kokkos::parallel_for( + "reshaping absplw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.absplw_host(d5, 0, d4, d3, d2); + }); + Kokkos::fence(); + + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.absplw[d1][d5], + temp_lw_3d_host); + } // d5 + // asmpsw, abspsw, extpsw + // netcfd : (sw_band, mode, refindex_im, refindex_real, coef_number) + // mam4xx : (mode, sw_band, coef_number, refindex_real, refindex_im ) + + for(int d5 = 0; d5 < nswbands; ++d5) { + // reshape data + Kokkos::parallel_for( + "reshaping asmpsw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.asmpsw_host(d5, 0, d4, d3, d2); + }); + Kokkos::fence(); + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.asmpsw[d1][d5], + temp_lw_3d_host); + // reshape data + Kokkos::parallel_for( + "reshaping abspsw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.abspsw_host(d5, 0, d4, d3, d2); + }); + Kokkos::fence(); + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.abspsw[d1][d5], + temp_lw_3d_host); + // reshape data + Kokkos::parallel_for( + "reshaping extpsw", + Kokkos::MDRangePolicy,Kokkos::DefaultHostExecutionSpace >({0, 0, 0}, + {coef_number, refindex_real, refindex_im}), + [&](const int d2, const int d3, const int d4) { + temp_lw_3d_host(d2, d3, d4) = + aerosol_optics_host_data.extpsw_host(d5, 0, d4, d3, d2); + }); + + Kokkos::fence(); + // syn data to device + Kokkos::deep_copy(aerosol_optics_device_data.extpsw[d1][d5], + temp_lw_3d_host); + + } // d5 +} + +inline void read_water_refindex(const std::string &table_filename, + const std::shared_ptr &grid, + const complex_view_1d &crefwlw, + const complex_view_1d &crefwsw) { + // refractive index for water read in read_water_refindex + // crefwsw(nswbands) ! complex refractive index for water visible + // crefwlw(nlwbands) ! complex refractive index for water infrared + + using namespace ShortFieldTagsNames; + using view_1d_host = typename KT::view_1d::HostMirror; + // Set up input structure to read data from file. + using strvec_t = std::vector; + + // here a made a list of variables that I want to read from netcdf files + ekat::ParameterList params; + params.set("Filename", table_filename); + params.set("Skip_Grid_Checks", true); + + params.set("Field Names", + {"refindex_im_water_lw", "refindex_im_water_sw", + "refindex_real_water_lw", "refindex_real_water_sw"}); + // make a list of host views + std::map host_views_water; + // fist allocate host views. + view_1d_host refindex_im_water_sw_host("refindex_im_water_sw_host", nswbands); + view_1d_host refindex_real_water_sw_host("refindex_real_water_sw_host", + nswbands); + view_1d_host refindex_im_water_lw_host("refindex_im_water_lw_host", nlwbands); + view_1d_host refindex_real_water_lw_host("refindex_real_water_lw_host", + nlwbands); + + host_views_water["refindex_im_water_sw"] = refindex_im_water_sw_host; + host_views_water["refindex_real_water_sw"] = refindex_real_water_sw_host; + host_views_water["refindex_im_water_lw"] = refindex_im_water_lw_host; + host_views_water["refindex_real_water_lw"] = refindex_real_water_lw_host; + + // defines layouts + std::map layouts_water; + FieldLayout scalar_refindex_water_sw_layout{{SWBND}, {nswbands}}; + FieldLayout scalar_refindex_water_lw_layout{{LWBND}, {nlwbands}}; + + layouts_water.emplace("refindex_im_water_sw", + scalar_refindex_water_sw_layout); + layouts_water.emplace("refindex_real_water_sw", + scalar_refindex_water_sw_layout); + layouts_water.emplace("refindex_im_water_lw", + scalar_refindex_water_lw_layout); + layouts_water.emplace("refindex_real_water_lw", + scalar_refindex_water_lw_layout); + + // create a object to read data + AtmosphereInput refindex_water(params, grid, host_views_water, layouts_water); + refindex_water.read_variables(); + refindex_water.finalize(); + + // maybe make a 1D vied of Kokkos::complex + const auto crefwlw_host = Kokkos::create_mirror_view(crefwlw); + const auto crefwsw_host = Kokkos::create_mirror_view(crefwsw); + for(int i = 0; i < nlwbands; ++i) { + // Kokkos::complex temp; + crefwlw_host(i).real() = refindex_real_water_lw_host(i); + crefwlw_host(i).imag() = haero::abs(refindex_im_water_lw_host(i)); + } + Kokkos::deep_copy(crefwlw, crefwlw_host); + // set complex representation of refractive indices as module data + for(int i = 0; i < nswbands; ++i) { + // Kokkos::complex temp; + crefwsw_host(i).real() = refindex_real_water_sw_host(i); + crefwsw_host(i).imag() = haero::abs(refindex_im_water_sw_host(i)); + } + Kokkos::deep_copy(crefwsw, crefwsw_host); +} +// read_refindex_aero + +inline void set_refindex_names(std::string surname, ekat::ParameterList ¶ms, + std::map &host_views, + std::map &layouts) { + // set variables names + using view_1d_host = typename KT::view_1d::HostMirror; + using strvec_t = std::vector; + using namespace ShortFieldTagsNames; + + std::string refindex_real_sw = "refindex_real_" + surname + "_sw"; + std::string refindex_im_sw = "refindex_im_" + surname + "_sw"; + std::string refindex_real_lw = "refindex_real_" + surname + "_lw"; + std::string refindex_im_lw = "refindex_im_" + surname + "_lw"; + + params.set("Skip_Grid_Checks", true); + params.set("Field Names", {refindex_real_sw, refindex_im_sw, + refindex_real_lw, refindex_im_lw}); + // allocate host views + host_views[refindex_real_sw] = view_1d_host(refindex_real_sw, nswbands); + host_views[refindex_im_sw] = view_1d_host(refindex_im_sw, nswbands); + host_views[refindex_real_lw] = view_1d_host(refindex_real_lw, nlwbands); + host_views[refindex_im_lw] = view_1d_host(refindex_im_lw, nlwbands); + + FieldLayout scalar_refindex_sw_layout{{SWBND}, {nswbands}}; + FieldLayout scalar_refindex_lw_layout{{LWBND}, {nlwbands}}; + + layouts.emplace(refindex_real_sw, scalar_refindex_sw_layout); + layouts.emplace(refindex_im_sw, scalar_refindex_sw_layout); + layouts.emplace(refindex_real_lw, scalar_refindex_lw_layout); + layouts.emplace(refindex_im_lw, scalar_refindex_lw_layout); + +} // set_refindex_aero + +inline void set_refindex_aerosol( + const int species_id, std::map &host_views, + mam_coupling::complex_view_2d::HostMirror + &specrefndxsw_host, // complex refractive index for water visible + mam_coupling::complex_view_2d::HostMirror &specrefndxlw_host) { + std::string sw_real_name = "refindex_real_aer_sw"; + std::string lw_real_name = "refindex_real_aer_lw"; + std::string sw_im_name = "refindex_im_aer_sw"; + std::string lw_im_name = "refindex_im_aer_lw"; + + for(int i = 0; i < nswbands; i++) { + specrefndxsw_host(i, species_id).real() = host_views[sw_real_name](i); + specrefndxsw_host(i, species_id).imag() = + haero::abs(host_views[sw_im_name](i)); + } + for(int i = 0; i < nlwbands; i++) { + specrefndxlw_host(i, species_id).real() = host_views[lw_real_name](i); + specrefndxlw_host(i, species_id).imag() = + haero::abs(host_views[lw_im_name](i)); + } + +} // copy_refindex_to_device + +} // namespace scream::mam_coupling + +#endif diff --git a/components/eamxx/src/physics/mam/mam_coupling.hpp b/components/eamxx/src/physics/mam/mam_coupling.hpp new file mode 100644 index 000000000000..e8a2d127c0d2 --- /dev/null +++ b/components/eamxx/src/physics/mam/mam_coupling.hpp @@ -0,0 +1,916 @@ +#ifndef MAM_COUPLING_HPP +#define MAM_COUPLING_HPP + +#include +#include +#include +#include +#include + +// These data structures and functions are used to move data between EAMxx +// and mam4xx. This file must be adjusted whenever the aerosol modes and +// species are modified. + +namespace scream::mam_coupling { + +using KT = ekat::KokkosTypes; + +// views for single- and multi-column data +using view_1d = typename KT::template view_1d; +using view_2d = typename KT::template view_2d; +using view_3d = typename KT::template view_3d; +using const_view_1d = typename KT::template view_1d; +using const_view_2d = typename KT::template view_2d; + +using complex_view_3d = typename KT::template view_3d>; +using complex_view_2d = typename KT::template view_2d>; + +// Kokkos thread team (league member) +using Team = Kokkos::TeamPolicy::member_type; + +// unmanaged views (for buffer and workspace manager) +using uview_1d = typename ekat::template Unmanaged>; +using uview_2d = typename ekat::template Unmanaged>; + +using PF = scream::PhysicsFunctions; + +using view_int_1d = typename KT::template view_1d; + +// number of constituents in gas chemistry "work arrays" +KOKKOS_INLINE_FUNCTION +constexpr int gas_pcnst() { + constexpr int gas_pcnst_ = mam4::gas_chemistry::gas_pcnst; + return gas_pcnst_; +} + +// number of aerosol/gas species tendencies +KOKKOS_INLINE_FUNCTION +constexpr int nqtendbb() { return 4; } + +// returns the number of distinct aerosol modes +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_modes() { + return mam4::AeroConfig::num_modes(); +} + +// returns the number of distinct aerosol species +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_species() { + return mam4::AeroConfig::num_aerosol_ids(); +} + +// returns the number of distinct aerosol-related gases +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_gases() { + return mam4::AeroConfig::num_gas_ids(); +} + +// returns the total number of aerosol tracers (i.e. the total number of +// distinct valid mode-species pairs) +KOKKOS_INLINE_FUNCTION +constexpr int num_aero_tracers() { + // see mam4::mode_aero_species() for valid per-mode aerosol species + // (in mam4xx/aero_modes.hpp) + return 7 + 4 + 7 + 3; +} + +// Given a MAM aerosol mode index, returns a string denoting the symbolic +// name of the mode. +KOKKOS_INLINE_FUNCTION +const char* aero_mode_name(const int mode) { + static const char *mode_names[num_aero_modes()] = { + "1", + "2", + "3", + "4", + }; + return mode_names[mode]; +} + +// Given a MAM aerosol species ID, returns a string denoting the symbolic +// name of the species. +KOKKOS_INLINE_FUNCTION +const char* aero_species_name(const int species_id) { + static const char *species_names[num_aero_species()] = { + "soa", + "so4", + "pom", + "bc", + "nacl", + "dst", + "mom", + }; + return species_names[species_id]; +} + +// Given a MAM aerosol-related gas ID, returns a string denoting the symbolic +// name of the gas species. +KOKKOS_INLINE_FUNCTION +const char* gas_species_name(const int gas_id) { + static const char *species_names[num_aero_gases()] = { + "O3", + "H2O2", + "H2SO4", + "SO2", + "DMS", + "SOAG" + }; + return species_names[gas_id]; +} + +// here we provide storage for names of fields generated by the functions below +namespace { + +KOKKOS_INLINE_FUNCTION +constexpr int max_field_name_len() { + return 128; +} + +KOKKOS_INLINE_FUNCTION +size_t gpu_strlen(const char* s) { + size_t l = 0; + while (s[l]) ++l; + return l; +} + +KOKKOS_INLINE_FUNCTION +void concat_2_strings(const char *s1, const char *s2, char *concatted) { + size_t len1 = gpu_strlen(s1); + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; + size_t len2 = gpu_strlen(s2); + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; + concatted[len1+len2] = 0; +} + +KOKKOS_INLINE_FUNCTION +void concat_3_strings(const char *s1, const char *s2, const char *s3, char *concatted) { + size_t len1 = gpu_strlen(s1); + for (size_t i = 0; i < len1; ++i) + concatted[i] = s1[i]; + size_t len2 = gpu_strlen(s2); + for (size_t i = 0; i < len2; ++i) + concatted[i + len1] = s2[i]; + size_t len3 = gpu_strlen(s3); + for (size_t i = 0; i < len3; ++i) + concatted[i + len1 + len2] = s3[i]; + concatted[len1+len2+len3] = 0; +} + +KOKKOS_INLINE_FUNCTION +char* int_aero_nmr_names(int mode) { + static char int_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; + return int_aero_nmr_names_[mode]; +} + +KOKKOS_INLINE_FUNCTION +char* cld_aero_nmr_names(int mode) { + static char cld_aero_nmr_names_[num_aero_modes()][max_field_name_len()] = {}; + return cld_aero_nmr_names_[mode]; +} + +KOKKOS_INLINE_FUNCTION +char* int_aero_mmr_names(int mode, int species) { + static char int_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; + return int_aero_mmr_names_[mode][species]; +} + +KOKKOS_INLINE_FUNCTION +char* cld_aero_mmr_names(int mode, int species) { + static char cld_aero_mmr_names_[num_aero_modes()][num_aero_species()][max_field_name_len()] = {}; + return cld_aero_mmr_names_[mode][species]; +} + +KOKKOS_INLINE_FUNCTION +char* gas_mmr_names(int gas_id) { + static char gas_mmr_names_[num_aero_gases()][max_field_name_len()] = {}; + return gas_mmr_names_[gas_id]; +} + +} // end anonymous namespace + +// Given a MAM aerosol mode index, returns the name of the related interstitial +// modal number mixing ratio field in EAMxx ("num_a<1-based-mode-index>") +KOKKOS_INLINE_FUNCTION +const char* int_aero_nmr_field_name(const int mode) { + if (!int_aero_nmr_names(mode)[0]) { + concat_2_strings("num_a", aero_mode_name(mode), int_aero_nmr_names(mode)); + } + return const_cast(int_aero_nmr_names(mode)); +} + +// Given a MAM aerosol mode index, returns the name of the related cloudborne +// modal number mixing ratio field in EAMxx ("num_c<1-based-mode-index>>") +KOKKOS_INLINE_FUNCTION +const char* cld_aero_nmr_field_name(const int mode) { + if (!cld_aero_nmr_names(mode)[0]) { + concat_2_strings("num_c", aero_mode_name(mode), cld_aero_nmr_names(mode)); + } + return const_cast(cld_aero_nmr_names(mode)); +} + +// Given a MAM aerosol mode index and the index of the MAM aerosol species +// within it, returns the name of the relevant interstitial mass mixing ratio +// field in EAMxx. The form of the field name is "_a<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). +KOKKOS_INLINE_FUNCTION +const char* int_aero_mmr_field_name(const int mode, const int species) { + if (!int_aero_mmr_names(mode, species)[0]) { + const auto aero_id = mam4::mode_aero_species(mode, species); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_a", aero_mode_name(mode), + int_aero_mmr_names(mode, species)); + } + } + return const_cast(int_aero_mmr_names(mode, species)); +}; + +// Given a MAM aerosol mode index and the index of the MAM aerosol species +// within it, returns the name of the relevant cloudborne mass mixing ratio +// field in EAMxx. The form of the field name is "_c<1-based-mode-index>". +// If the desired species is not present within the desire mode, returns a blank +// string (""). +KOKKOS_INLINE_FUNCTION +const char* cld_aero_mmr_field_name(const int mode, const int species) { + if (!cld_aero_mmr_names(mode, species)[0]) { + const auto aero_id = mam4::mode_aero_species(mode, species); + if (aero_id != mam4::AeroId::None) { + concat_3_strings(aero_species_name(static_cast(aero_id)), + "_c", aero_mode_name(mode), + cld_aero_mmr_names(mode, species)); + } + } + return const_cast(cld_aero_mmr_names(mode, species)); +}; + +// Given a MAM aerosol-related gas identifier, returns the name of its mass +// mixing ratio field in EAMxx +KOKKOS_INLINE_FUNCTION +const char* gas_mmr_field_name(const int gas) { + return const_cast(gas_species_name(gas)); +} + +// This type stores multi-column views related specifically to the wet +// atmospheric state used by EAMxx. +struct WetAtmosphere { + const_view_2d qv; // wet water vapor specific humidity [kg vapor / kg moist air] + const_view_2d qc; // wet cloud liquid water mass mixing ratio [kg cloud water/kg moist air] + const_view_2d nc; // wet cloud liquid water number mixing ratio [# / kg moist air] + const_view_2d qi; // wet cloud ice water mass mixing ratio [kg cloud ice water / kg moist air] + const_view_2d ni; // wet cloud ice water number mixing ratio [# / kg moist air] + const_view_2d omega; // vertical pressure velocity [Pa/s] +}; + +// This type stores multi-column views related to the dry atmospheric state +// used by MAM. +struct DryAtmosphere { + Real z_surf; // height of bottom of atmosphere [m] + const_view_2d T_mid; // temperature at grid midpoints [K] + const_view_2d p_mid; // total pressure at grid midpoints [Pa] + view_2d qv; // dry water vapor mixing ratio [kg vapor / kg dry air] + view_2d qc; // dry cloud liquid water mass mixing ratio [kg cloud water/kg dry air] + view_2d nc; // dry cloud liquid water number mixing ratio [# / kg dry air] + view_2d qi; // dry cloud ice water mass mixing ratio [kg cloud ice water / kg dry air] + view_2d ni; // dry cloud ice water number mixing ratio [# / kg dry air] + view_2d z_mid; // height at layer midpoints [m] + view_2d z_iface; // height at layer interfaces [m] + view_2d dz; // layer thickness [m] + const_view_2d p_del; // hydrostatic "pressure thickness" at grid interfaces [Pa] + const_view_2d p_int; // total pressure at grid interfaces [Pa] + const_view_2d cldfrac; // cloud fraction [-] + view_2d w_updraft; // updraft velocity [m/s] + const_view_1d pblh; // planetary boundary layer height [m] + const_view_1d phis; // surface geopotential [m2/s2] +}; + +// This type stores aerosol number and mass mixing ratios evolved by MAM. It +// can be used to represent wet and dry aerosols. When you declare an +// AerosolState, you must decide whether it's a dry or wet aerosol state (with +// mixing ratios in terms of dry or wet parcels of air, respectively). +// These mixing ratios are organized by mode (and species, for mass mixing ratio) +// in the same way as they are in mam4xx, and indexed using mam4::AeroConfig. +struct AerosolState { + view_2d int_aero_nmr[num_aero_modes()]; // modal interstitial aerosol number mixing ratios [# / kg air] + view_2d cld_aero_nmr[num_aero_modes()]; // modal cloudborne aerosol number mixing ratios [# / kg air] + view_2d int_aero_mmr[num_aero_modes()][num_aero_species()]; // interstitial aerosol mass mixing ratios [kg aerosol / kg air] + view_2d cld_aero_mmr[num_aero_modes()][num_aero_species()]; // cloudborne aerosol mass mixing ratios [kg aerosol / kg air] + view_2d gas_mmr[num_aero_gases()]; // gas mass mixing ratios [kg gas / kg air] +}; + +// storage for variables used within MAM atmosphere processes, initialized with +// ATMBufferManager +struct Buffer { + // ====================== + // column midpoint fields + // ====================== + + // number of "scratch" fields that hold process-specific data + // (e.g. gas-phase chemistry fields that are only needed by aerosol + // microphysics) + static constexpr int num_2d_scratch = 10; + + // number of local fields stored at column midpoints + static constexpr int num_2d_mid = 8 + // number of dry atm fields + 2 * (num_aero_modes() + num_aero_tracers()) + + num_aero_gases() + + num_2d_scratch; + + // (dry) atmospheric state + uview_2d z_mid; // height at midpoints + uview_2d dz; // layer thickness + uview_2d qv_dry; // dry water vapor mixing ratio (dry air) + uview_2d qc_dry; // dry cloud water mass mixing ratio + uview_2d nc_dry; // dry cloud water number mixing ratio + uview_2d qi_dry; // cloud ice mass mixing ratio + uview_2d ni_dry; // dry cloud ice number mixing ratio + uview_2d w_updraft; // vertical wind velocity + + // aerosol dry interstitial/cloudborne number/mass mixing ratios + // (because the number of species per mode varies, not all of these will + // be used) + uview_2d dry_int_aero_nmr[num_aero_modes()]; + uview_2d dry_cld_aero_nmr[num_aero_modes()]; + uview_2d dry_int_aero_mmr[num_aero_modes()][num_aero_species()]; + uview_2d dry_cld_aero_mmr[num_aero_modes()][num_aero_species()]; + + // aerosol-related dry gas mass mixing ratios + uview_2d dry_gas_mmr[num_aero_gases()]; + + // undedicated scratch fields for process-specific data + uview_2d scratch[num_2d_scratch]; + + // ======================= + // column interface fields + // ======================= + + // number of local fields stored at column interfaces + static constexpr int num_2d_iface = 1; + + uview_2d z_iface; // height at interfaces + + // storage + Real* wsm_data; +}; + +// ON HOST, returns the number of bytes of device memory needed by the above +// Buffer type given the number of columns and vertical levels +inline size_t buffer_size(const int ncol, const int nlev) { + return sizeof(Real) * + (Buffer::num_2d_mid * ncol * nlev + + Buffer::num_2d_iface * ncol * (nlev+1)); +} + +// ON HOST, initializeÑ• the Buffer type with sufficient memory to store +// intermediate (dry) quantities on the given number of columns with the given +// number of vertical levels. Returns the number of bytes allocated. +inline size_t init_buffer(const ATMBufferManager &buffer_manager, + const int ncol, const int nlev, + Buffer &buffer) { + Real* mem = reinterpret_cast(buffer_manager.get_memory()); + + // set view pointers for midpoint fields + uview_2d* view_2d_mid_ptrs[Buffer::num_2d_mid] = { + &buffer.z_mid, + &buffer.dz, + &buffer.qv_dry, + &buffer.qc_dry, + &buffer.nc_dry, + &buffer.qi_dry, + &buffer.ni_dry, + &buffer.w_updraft, + + // aerosol modes + &buffer.dry_int_aero_nmr[0], + &buffer.dry_int_aero_nmr[1], + &buffer.dry_int_aero_nmr[2], + &buffer.dry_int_aero_nmr[3], + &buffer.dry_cld_aero_nmr[0], + &buffer.dry_cld_aero_nmr[1], + &buffer.dry_cld_aero_nmr[2], + &buffer.dry_cld_aero_nmr[3], + + // the following requires knowledge of mam4's mode-species layout + // (see mode_aero_species() in mam4xx/aero_modes.hpp) + + // accumulation mode + &buffer.dry_int_aero_mmr[0][0], + &buffer.dry_int_aero_mmr[0][1], + &buffer.dry_int_aero_mmr[0][2], + &buffer.dry_int_aero_mmr[0][3], + &buffer.dry_int_aero_mmr[0][4], + &buffer.dry_int_aero_mmr[0][5], + &buffer.dry_int_aero_mmr[0][6], + &buffer.dry_cld_aero_mmr[0][0], + &buffer.dry_cld_aero_mmr[0][1], + &buffer.dry_cld_aero_mmr[0][2], + &buffer.dry_cld_aero_mmr[0][3], + &buffer.dry_cld_aero_mmr[0][4], + &buffer.dry_cld_aero_mmr[0][5], + &buffer.dry_cld_aero_mmr[0][6], + + // aitken mode + &buffer.dry_int_aero_mmr[1][0], + &buffer.dry_int_aero_mmr[1][1], + &buffer.dry_int_aero_mmr[1][2], + &buffer.dry_int_aero_mmr[1][3], + &buffer.dry_cld_aero_mmr[1][0], + &buffer.dry_cld_aero_mmr[1][1], + &buffer.dry_cld_aero_mmr[1][2], + &buffer.dry_cld_aero_mmr[1][3], + + // coarse mode + &buffer.dry_int_aero_mmr[2][0], + &buffer.dry_int_aero_mmr[2][1], + &buffer.dry_int_aero_mmr[2][2], + &buffer.dry_int_aero_mmr[2][3], + &buffer.dry_int_aero_mmr[2][4], + &buffer.dry_int_aero_mmr[2][5], + &buffer.dry_int_aero_mmr[2][6], + &buffer.dry_cld_aero_mmr[2][0], + &buffer.dry_cld_aero_mmr[2][1], + &buffer.dry_cld_aero_mmr[2][2], + &buffer.dry_cld_aero_mmr[2][3], + &buffer.dry_cld_aero_mmr[2][4], + &buffer.dry_cld_aero_mmr[2][5], + &buffer.dry_cld_aero_mmr[2][6], + + // primary carbon mode + &buffer.dry_int_aero_mmr[3][0], + &buffer.dry_int_aero_mmr[3][1], + &buffer.dry_int_aero_mmr[3][2], + &buffer.dry_cld_aero_mmr[3][0], + &buffer.dry_cld_aero_mmr[3][1], + &buffer.dry_cld_aero_mmr[3][2], + + // aerosol gases + &buffer.dry_gas_mmr[0], + &buffer.dry_gas_mmr[1], + &buffer.dry_gas_mmr[2], + &buffer.dry_gas_mmr[3], + &buffer.dry_gas_mmr[4], + &buffer.dry_gas_mmr[5] + }; + for (int i = 0; i < Buffer::num_2d_scratch; ++i) { + view_2d_mid_ptrs[Buffer::num_2d_mid+i-Buffer::num_2d_scratch] = &buffer.scratch[i]; + } + + for (int i = 0; i < Buffer::num_2d_mid; ++i) { + *view_2d_mid_ptrs[i] = view_2d(mem, ncol, nlev); + mem += view_2d_mid_ptrs[i]->size(); + } + + // set view pointers for interface fields + uview_2d* view_2d_iface_ptrs[Buffer::num_2d_iface] = { + &buffer.z_iface + }; + for (int i = 0; i < Buffer::num_2d_iface; ++i) { + *view_2d_iface_ptrs[i] = view_2d(mem, ncol, nlev+1); + mem += view_2d_iface_ptrs[i]->size(); + } + + // WSM data + buffer.wsm_data = mem; + + /* + // Compute workspace manager size to check used memory vs. requested memory + // (if needed) + const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(ncol_, nlev_); + const int n_wind_slots = ekat::npack(2)*Spack::n; + const int n_trac_slots = ekat::npack(m_num_tracers+3)*Spack::n; + const int wsm_size = WSM::get_total_bytes_needed(nlevi_packs, 13+(n_wind_slots+n_trac_slots), policy)/sizeof(Spack); + mem += wsm_size; + */ + + // return the number of bytes allocated + return (mem - buffer_manager.get_memory())*sizeof(Real); +} + +// Given a dry atmosphere state, creates a haero::Atmosphere object for the +// column with the given index. This object can be provided to mam4xx for the +// column. +KOKKOS_INLINE_FUNCTION +haero::Atmosphere atmosphere_for_column(const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(dry_atm.T_mid.data() != nullptr, + "T_mid not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_mid.data() != nullptr, + "p_mid not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.qv.data() != nullptr, + "qv not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.qc.data() != nullptr, + "qc not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.nc.data() != nullptr, + "nc not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.qi.data() != nullptr, + "qi not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.ni.data() != nullptr, + "ni not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.z_mid.data() != nullptr, + "z_mid not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_del.data() != nullptr, + "p_del not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.p_int.data() != nullptr, + "p_int not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.cldfrac.data() != nullptr, + "cldfrac not defined for dry atmosphere state!"); + EKAT_KERNEL_ASSERT_MSG(dry_atm.w_updraft.data() != nullptr, + "w_updraft not defined for dry atmosphere state!"); + return haero::Atmosphere(mam4::nlev, + ekat::subview(dry_atm.T_mid, column_index), + ekat::subview(dry_atm.p_mid, column_index), + ekat::subview(dry_atm.qv, column_index), + ekat::subview(dry_atm.qc, column_index), + ekat::subview(dry_atm.nc, column_index), + ekat::subview(dry_atm.qi, column_index), + ekat::subview(dry_atm.ni, column_index), + ekat::subview(dry_atm.z_mid, column_index), + ekat::subview(dry_atm.p_del, column_index), + ekat::subview(dry_atm.p_int, column_index), + ekat::subview(dry_atm.cldfrac, column_index), + ekat::subview(dry_atm.w_updraft, column_index), + dry_atm.pblh(column_index)); +} + +// Given an AerosolState with views for dry aerosol quantities, creates a +// mam4::Prognostics object for the column with the given index with +// ONLY INTERSTITIAL AEROSOL VIEWS DEFINED. This object can be provided to +// mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Prognostics interstitial_aerosols_for_column(const AerosolState& dry_aero, + const int column_index) { + constexpr int nlev = mam4::nlev; + mam4::Prognostics progs(nlev); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.int_aero_nmr[m].data(), + "int_aero_nmr not defined for dry aerosol state!"); + progs.n_mode_i[m] = ekat::subview(dry_aero.int_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + progs.q_aero_i[m][a] = ekat::subview(dry_aero.int_aero_mmr[m][a], column_index); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.gas_mmr[g].data(), + "gas_mmr not defined for dry aerosol state!"); + progs.q_gas[g] = ekat::subview(dry_aero.gas_mmr[g], column_index); + } + return progs; +} + +// Given a dry aerosol state, creates a mam4::Prognostics object for the column +// with the given index with interstitial and cloudborne aerosol views defined. +// This object can be provided to mam4xx for the column. +KOKKOS_INLINE_FUNCTION +mam4::Prognostics aerosols_for_column(const AerosolState& dry_aero, + const int column_index) { + auto progs = interstitial_aerosols_for_column(dry_aero, column_index); + for (int m = 0; m < num_aero_modes(); ++m) { + EKAT_KERNEL_ASSERT_MSG(dry_aero.cld_aero_nmr[m].data(), + "dry_cld_aero_nmr not defined for aerosol state!"); + progs.n_mode_c[m] = ekat::subview(dry_aero.cld_aero_nmr[m], column_index); + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.cld_aero_mmr[m][a].data()) { + progs.q_aero_c[m][a] = ekat::subview(dry_aero.cld_aero_mmr[m][a], column_index); + } + } + } + return progs; +} + +// Given a thread team and a dry atmosphere state, dispatches threads from the +// team to compute vertical layer heights and interfaces for the column with +// the given index. +KOKKOS_INLINE_FUNCTION +void compute_vertical_layer_heights(const Team& team, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + const auto dz = ekat::subview(dry_atm.dz, column_index); + const auto z_iface = ekat::subview(dry_atm.z_iface, column_index); + const auto z_mid = ekat::subview(dry_atm.z_mid, column_index); + const auto qv = ekat::subview(dry_atm.qv, column_index); + const auto p_mid = ekat::subview(dry_atm.p_mid, column_index); + const auto T_mid = ekat::subview(dry_atm.T_mid, column_index); + const auto pseudo_density = ekat::subview(dry_atm.p_del, column_index); + PF::calculate_dz(team, pseudo_density, p_mid, T_mid, qv, dz); + team.team_barrier(); + PF::calculate_z_int(team, mam4::nlev, dz, dry_atm.z_surf, z_iface); + team.team_barrier(); // likely necessary to have z_iface up to date + PF::calculate_z_mid(team, mam4::nlev, z_iface, z_mid); +} + +// Given a thread team and wet and dry atmospheres, dispatches threads from the +// team to compute the vertical updraft velocity for the column with the given +// index. +KOKKOS_INLINE_FUNCTION +void compute_updraft_velocities(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto rho = PF::calculate_density(dry_atm.p_del(i,k), dry_atm.dz(i,k)); + dry_atm.w_updraft(i,k) = PF::calculate_vertical_velocity(wet_atm.omega(i,k), rho); + }); +} + +// Given a thread team and a wet atmosphere state, dispatches threads +// from the team to compute mixing ratios for a dry atmosphere state in th +// column with the given index. +KOKKOS_INLINE_FUNCTION +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const DryAtmosphere& dry_atm, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + dry_atm.qv(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qv(i,k), qv_ik); + dry_atm.qc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qc(i,k), qv_ik); + dry_atm.nc(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.nc(i,k), qv_ik); + dry_atm.qi(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.qi(i,k), qv_ik); + dry_atm.ni(i,k) = PF::calculate_drymmr_from_wetmmr(wet_atm.ni(i,k), qv_ik); + }); +} + +// Given a thread team and wet atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given dry interstitial aerosol +// state for the column with the given index. +KOKKOS_INLINE_FUNCTION +void compute_dry_mixing_ratios(const Team& team, + const WetAtmosphere& wet_atm, + const AerosolState& wet_aero, + const AerosolState& dry_aero, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = wet_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + dry_aero.int_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_nmr[m](i,k), qv_ik); + if (dry_aero.cld_aero_nmr[m].data()) { + dry_aero.cld_aero_nmr[m](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_nmr[m](i,k), qv_ik); + } + for (int a = 0; a < num_aero_species(); ++a) { + if (dry_aero.int_aero_mmr[m][a].data()) { + dry_aero.int_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.int_aero_mmr[m][a](i,k), qv_ik); + } + if (dry_aero.cld_aero_mmr[m][a].data()) { + dry_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.cld_aero_mmr[m][a](i,k), qv_ik); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + dry_aero.gas_mmr[g](i,k) = PF::calculate_drymmr_from_wetmmr(wet_aero.gas_mmr[g](i,k), qv_ik); + } + }); +} + +// Given a thread team and dry atmospheric and aerosol states, dispatches threads +// from the team to compute mixing ratios for the given wet interstitial aerosol +// state for the column with the given index. +KOKKOS_INLINE_FUNCTION +void compute_wet_mixing_ratios(const Team& team, + const DryAtmosphere& dry_atm, + const AerosolState& dry_aero, + const AerosolState& wet_aero, + const int column_index) { + EKAT_KERNEL_ASSERT_MSG(column_index == team.league_rank(), + "Given column index does not correspond to given team!"); + + constexpr int nlev = mam4::nlev; + int i = column_index; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlev), [&] (const int k) { + const auto qv_ik = dry_atm.qv(i,k); + for (int m = 0; m < num_aero_modes(); ++m) { + wet_aero.int_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_nmr[m](i,k), qv_ik); + if (wet_aero.cld_aero_nmr[m].data()) { + wet_aero.cld_aero_nmr[m](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_nmr[m](i,k), qv_ik); + } + for (int a = 0; a < num_aero_species(); ++a) { + if (wet_aero.int_aero_mmr[m][a].data()) { + wet_aero.int_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.int_aero_mmr[m][a](i,k), qv_ik); + } + if (wet_aero.cld_aero_mmr[m][a].data()) { + wet_aero.cld_aero_mmr[m][a](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.cld_aero_mmr[m][a](i,k), qv_ik); + } + } + } + for (int g = 0; g < num_aero_gases(); ++g) { + wet_aero.gas_mmr[g](i,k) = PF::calculate_wetmmr_from_drymmr(dry_aero.gas_mmr[g](i,k), qv_ik); + } + }); +} + +// Because CUDA C++ doesn't allow us to declare and use constants outside of +// KOKKOS_INLINE_FUNCTIONS, we define this macro that allows us to (re)define +// these constants where needed within two such functions so we don't define +// them inconsistently. Yes, it's the 21st century and we're still struggling +// with these basic things. +#define DECLARE_PROG_TRANSFER_CONSTANTS \ + /* mapping of constituent indices to aerosol modes */ \ + const auto Accum = mam4::ModeIndex::Accumulation; \ + const auto Aitken = mam4::ModeIndex::Aitken; \ + const auto Coarse = mam4::ModeIndex::Coarse; \ + const auto PC = mam4::ModeIndex::PrimaryCarbon; \ + const auto NoMode = mam4::ModeIndex::None; \ + static const mam4::ModeIndex mode_for_cnst[gas_pcnst()] = { \ + NoMode, NoMode, NoMode, NoMode, NoMode, NoMode, /* gases (not aerosols) */ \ + Accum, Accum, Accum, Accum, Accum, Accum, Accum, Accum, /* 7 aero species + NMR */ \ + Aitken, Aitken, Aitken, Aitken, Aitken, /* 4 aero species + NMR */ \ + Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, Coarse, /* 7 aero species + NMR */ \ + PC, PC, PC, PC, /* 3 aero species + NMR */ \ + }; \ + /* mapping of constituent indices to aerosol species */ \ + const auto SOA = mam4::AeroId::SOA; \ + const auto SO4 = mam4::AeroId::SO4; \ + const auto POM = mam4::AeroId::POM; \ + const auto BC = mam4::AeroId::BC; \ + const auto NaCl = mam4::AeroId::NaCl; \ + const auto DST = mam4::AeroId::DST; \ + const auto MOM = mam4::AeroId::MOM; \ + const auto NoAero = mam4::AeroId::None; \ + static const mam4::AeroId aero_for_cnst[gas_pcnst()] = { \ + NoAero, NoAero, NoAero, NoAero, NoAero, NoAero, /* gases (not aerosols) */ \ + SO4, POM, SOA, BC, DST, NaCl, MOM, NoAero, /* accumulation mode */ \ + SO4, SOA, NaCl, MOM, NoAero, /* aitken mode */ \ + DST, NaCl, SO4, BC, POM, SOA, MOM, NoAero, /* coarse mode */ \ + POM, BC, MOM, NoAero, /* primary carbon mode */ \ + }; \ + /* mapping of constituent indices to gases */ \ + const auto O3 = mam4::GasId::O3; \ + const auto H2O2 = mam4::GasId::H2O2; \ + const auto H2SO4 = mam4::GasId::H2SO4; \ + const auto SO2 = mam4::GasId::SO2; \ + const auto DMS = mam4::GasId::DMS; \ + const auto SOAG = mam4::GasId::SOAG; \ + const auto NoGas = mam4::GasId::None; \ + static const mam4::GasId gas_for_cnst[gas_pcnst()] = { \ + O3, H2O2, H2SO4, SO2, DMS, SOAG, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, NoGas, \ + NoGas, NoGas, NoGas, NoGas, \ + }; + +// Given a Prognostics object, transfers data for interstitial aerosols to the +// chemistry work array q, and cloudborne aerosols to the chemistry work array +// qqcw, both at vertical level k. The input and output quantities are stored as +// number/mass mixing ratios. +// NOTE: this mapping is chemistry-mechanism-specific (see mo_sim_dat.F90 +// NOTE: in the relevant preprocessed chemical mechanism) +// NOTE: see mam4xx/aero_modes.hpp to interpret these mode/aerosol/gas +// NOTE: indices +KOKKOS_INLINE_FUNCTION +void transfer_prognostics_to_work_arrays(const mam4::Prognostics &progs, + const int k, + Real q[gas_pcnst()], + Real qqcw[gas_pcnst()]) { + DECLARE_PROG_TRANSFER_CONSTANTS + + // copy number/mass mixing ratios from progs to q and qqcw at level k, + // converting them to VMR + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + q[i] = progs.q_gas[g](k); + qqcw[i] = progs.q_gas[g](k); + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + q[i] = progs.q_aero_i[m][a](k); + qqcw[i] = progs.q_aero_c[m][a](k); + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + q[i] = progs.n_mode_i[m](k); + qqcw[i] = progs.n_mode_c[m](k); + } + } + } +} + +// converts the quantities in the work arrays q and qqcw from mass/number +// mixing ratios to volume/number mixing ratios +KOKKOS_INLINE_FUNCTION +void convert_work_arrays_to_vmr(const Real q[gas_pcnst()], + const Real qqcw[gas_pcnst()], + Real vmr[gas_pcnst()], + Real vmrcw[gas_pcnst()]) { + DECLARE_PROG_TRANSFER_CONSTANTS + + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + const Real mw = mam4::gas_species(g).molecular_weight; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + const Real mw = mam4::aero_species(a).molecular_weight; + vmr[i] = mam4::conversions::vmr_from_mmr(q[i], mw); + vmrcw[i] = mam4::conversions::vmr_from_mmr(qqcw[i], mw); + } else { // constituent is a modal number mixing ratio + vmr[i] = q[i]; + vmrcw[i] = qqcw[i]; + } + } + } +} + +// converts the quantities in the work arrays vmrs and vmrscw from mass/number +// mixing ratios to volume/number mixing ratios +KOKKOS_INLINE_FUNCTION +void convert_work_arrays_to_mmr(const Real vmr[gas_pcnst()], + const Real vmrcw[gas_pcnst()], + Real q[gas_pcnst()], + Real qqcw[gas_pcnst()]) { + DECLARE_PROG_TRANSFER_CONSTANTS + + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + const Real mw = mam4::gas_species(g).molecular_weight; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + const Real mw = mam4::aero_species(a).molecular_weight; + q[i] = mam4::conversions::mmr_from_vmr(vmr[i], mw); + qqcw[i] = mam4::conversions::mmr_from_vmr(vmrcw[i], mw); + } else { // constituent is a modal number mixing ratio + q[i] = vmr[i]; + qqcw[i] = vmrcw[i]; + } + } + } +} + +// Given work arrays with interstitial and cloudborne aerosol data, transfers +// them to the given Prognostics object at the kth vertical level. This is the +// "inverse operator" for transfer_prognostics_to_work_arrays, above. +KOKKOS_INLINE_FUNCTION +void transfer_work_arrays_to_prognostics(const Real q[gas_pcnst()], + const Real qqcw[gas_pcnst()], + mam4::Prognostics &progs, const int k) { + DECLARE_PROG_TRANSFER_CONSTANTS + + // copy number/mass mixing ratios from progs to q and qqcw at level k, + // converting them to VMR + for (int i = 0; i < gas_pcnst(); ++i) { + auto mode_index = mode_for_cnst[i]; + auto aero_id = aero_for_cnst[i]; + auto gas_id = gas_for_cnst[i]; + if (gas_id != NoGas) { // constituent is a gas + int g = static_cast(gas_id); + progs.q_gas[g](k) = q[i]; + } else { + int m = static_cast(mode_index); + if (aero_id != NoAero) { // constituent is an aerosol species + int a = aerosol_index_for_mode(mode_index, aero_id); + progs.q_aero_i[m][a](k) = q[i]; + progs.q_aero_c[m][a](k) = qqcw[i]; + } else { // constituent is a modal number mixing ratio + int m = static_cast(mode_index); + progs.n_mode_i[m](k) = q[i]; + progs.n_mode_c[m](k) = qqcw[i]; + } + } + } +} + +#undef DECLARE_PROG_TRANSFER_CONSTANTS + +} // namespace scream::mam_coupling + +#endif diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp index 1dff45c7d032..3fefd1c84123 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.cpp @@ -1,7 +1,13 @@ #include "eamxx_nudging_process_interface.hpp" + #include "share/util/scream_universal_constants.hpp" #include "share/grid/remap/refining_remapper_p2p.hpp" #include "share/grid/remap/do_nothing_remapper.hpp" +#include "share/util/scream_utils.hpp" + +#include +#include +#include namespace scream { @@ -10,10 +16,12 @@ namespace scream Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { - m_datafiles = m_params.get>("nudging_filename"); + m_datafiles = filename_glob(m_params.get>("nudging_filenames_patterns")); m_timescale = m_params.get("nudging_timescale",0); + m_fields_nudge = m_params.get>("nudging_fields"); m_use_weights = m_params.get("use_nudging_weights",false); + m_skip_vert_interpolation = m_params.get("skip_vert_interpolation",false); // If we are doing horizontal refine-remapping, we need to get the mapfile from user m_refine_remap_file = m_params.get( "nudging_refine_remap_mapfile", "no-file-given"); @@ -26,9 +34,17 @@ Nudging::Nudging (const ekat::Comm& comm, const ekat::ParameterList& params) m_src_pres_type = STATIC_1D_VERTICAL_PROFILE; // Check for a designated source pressure file, default to first nudging data source if not given. m_static_vertical_pressure_file = m_params.get("source_pressure_file",m_datafiles[0]); + EKAT_REQUIRE_MSG(m_skip_vert_interpolation == false, + "Error! It makes no sense to not interpolate if src press is uniform and constant "); } else { EKAT_ERROR_MSG("ERROR! Nudging::parameter_list - unsupported source_pressure_type provided. Current options are [TIME_DEPENDENT_3D_PROFILE,STATIC_1D_VERTICAL_PROFILE]. Please check"); } + int first_file_levs = scorpio::get_dimlen(m_datafiles[0],"lev"); + for (const auto& file : m_datafiles) { + int current_file_levs = scorpio::get_dimlen(file,"lev"); + EKAT_REQUIRE_MSG(current_file_levs == first_file_levs, + "Error! Inconsistent 'lev' dimension found in nudging data files."); + } // use nudging weights if (m_use_weights) m_weights_file = m_params.get("nudging_weights_file"); @@ -84,21 +100,26 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) } else { m_num_src_levs = scorpio::get_dimlen(m_static_vertical_pressure_file,"lev"); } + if (m_skip_vert_interpolation) { + EKAT_REQUIRE_MSG(m_num_src_levs == m_num_levs, + "Error! skip_vert_interpolation requires the vertical level to be " + << " the same as model vertical level "); + } /* Check for consistency between nudging files, map file, and remapper */ // Number of columns globally - auto m_num_cols_global = m_grid->get_num_global_dofs(); + auto num_cols_global = m_grid->get_num_global_dofs(); // Get the information from the first nudging data file int num_cols_src = scorpio::get_dimlen(m_datafiles[0],"ncol"); - if (num_cols_src != m_num_cols_global) { + if (num_cols_src != num_cols_global) { // If differing cols, check if remap file is provided EKAT_REQUIRE_MSG(m_refine_remap_file != "no-file-given", "Error! Nudging::set_grids - the number of columns in the nudging data file " << std::to_string(num_cols_src) << " does not match the number of columns in the " - << "model grid " << std::to_string(m_num_cols_global) << ". Please check the " + << "model grid " << std::to_string(num_cols_global) << ". Please check the " << "nudging data file and/or the model grid."); // If remap file is provided, check if it is consistent with the nudging data file // First get the data from the mapfile @@ -110,9 +131,9 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) << std::to_string(num_cols_src) << " does not match the number of columns in the " << "mapfile " << std::to_string(num_cols_remap_a) << ". Please check the " << "nudging data file and/or the mapfile."); - EKAT_REQUIRE_MSG(num_cols_remap_b == m_num_cols_global, + EKAT_REQUIRE_MSG(num_cols_remap_b == num_cols_global, "Error! Nudging::set_grids - the number of columns in the model grid " - << std::to_string(m_num_cols_global) << " does not match the number of columns in the " + << std::to_string(num_cols_global) << " does not match the number of columns in the " << "mapfile " << std::to_string(num_cols_remap_b) << ". Please check the " << "model grid and/or the mapfile."); EKAT_REQUIRE_MSG(m_use_weights == false, @@ -126,232 +147,189 @@ void Nudging::set_grids(const std::shared_ptr grids_manager) // If the number of columns is the same, we don't need to do any remapping, // but print a warning if the user provided a mapfile if (m_refine_remap_file != "no-file-given") { - std::cout << "Warning! Nudging::set_grids - the number of columns in the nudging data file " - << std::to_string(num_cols_src) << " matches the number of columns in the " - << "model grid " << std::to_string(m_num_cols_global) << ". The mapfile " - << m_refine_remap_file << " will NOT be used. Please check the " - << "nudging data file and/or the model grid." << std::endl; + m_atm_logger->warn( + "[Nudging::set_grids] Warning! Map file provided, but it is not needed.\n" + " - num cols in nudging data file: " + std::to_string(num_cols_src) + "\n" + " - num cols in model grid : " + std::to_string(num_cols_global) + "\n" + " Please, make sure the nudging data file and/or model grid are correct.\n" + " The map file is only needed if the above two numbers differ."); } // If the user gives us the vertical cutoff, warn them if (m_refine_remap_vert_cutoff > 0.0) { - std::cout << "Warning! Nudging::set_grids - the vertical cutoff " - << std::to_string(m_refine_remap_vert_cutoff) - << " is larger than zero, but we are not remapping. Please " - "check your settings." << std::endl; + m_atm_logger->warn( + "[Nudging::set_grids] Warning! Non-zero vertical cutoff provided, but it is not needed\n" + " - vertical cutoff: " + std::to_string(m_refine_remap_vert_cutoff) + "\n" + " Please, check your settings. This parameter is only needed if we are remapping."); } // Set m_refine_remap to false m_refine_remap = false; } } // ========================================================================================= -void Nudging::apply_tendency(Field& base, const Field& next, const Real dt) +void Nudging::apply_tendency(Field& state, const Field& nudge, const Real dt) const { // Calculate the weight to apply the tendency - const Real dtend = dt/Real(m_timescale); - EKAT_REQUIRE_MSG(dtend>=0,"Error! Nudging::apply_tendency - timescale tendency of " << std::to_string(dt) - << " / " << std::to_string(m_timescale) << " = " << std::to_string(dtend) - << " is invalid. Please check the timescale and/or dt"); - // Now apply the tendency. - Field tend = base.clone(); - // Use update internal to set tendency, will be (1.0*next - 1.0*base), note tend=base at this point. - tend.update(next,Real(1.0),Real(-1.0)); - base.update(tend,dtend,Real(1.0)); -} -// ========================================================================================= -void Nudging::apply_weighted_tendency(Field& base, const Field& next, const Field& weights, const Real dt) -{ - // Calculate the weight to apply the tendency - const Real dtend = dt/Real(m_timescale); - EKAT_REQUIRE_MSG(dtend>=0,"Error! Nudging::apply_tendency - timescale tendency of " << std::to_string(dt) - << " / " << std::to_string(m_timescale) << " = " << std::to_string(dtend) - << " is invalid. Please check the timescale and/or dt"); - // Now apply the tendency. - Field tend = base.clone(); - - // Use update internal to set tendency, will be (weights*next - weights*base), note tend=base at this point. - auto base_view = base.get_view(); - auto tend_view = tend.get_view< Real**>(); - auto next_view = next.get_view< Real**>(); - auto w_view = weights.get_view< Real**>(); - - const int num_cols = base_view.extent(0); - const int num_vert_packs = base_view.extent(1); - Kokkos::parallel_for(Kokkos::MDRangePolicy>({0, 0}, {num_cols, num_vert_packs}), KOKKOS_LAMBDA(int i, int j) { - tend_view(i,j) = next_view(i,j)*w_view(i,j) - base_view(i,j)*w_view(i,j); - }); - base.update(tend, dtend, Real(1.0)); -} -// ========================================================================================= -void Nudging::apply_vert_cutoff_tendency(Field &base, const Field &next, - const Field &p_mid, const Real cutoff, - const Real dt) { - // Calculate the weight to apply the tendency - const Real dtend = dt / Real(m_timescale); - EKAT_REQUIRE_MSG(dtend >= 0, - "Error! Nudging::apply_tendency - timescale tendency of " - << std::to_string(dt) << " / " - << std::to_string(m_timescale) << " = " - << std::to_string(dtend) - << " is invalid. Please check the timescale and/or dt"); - // Now apply the tendency. - Field tend = base.clone(); - - // Use update internal to set tendency, will be (weights*next - weights*base), - // note tend=base at this point. - auto base_view = base.get_view(); - auto tend_view = tend.get_view(); - auto next_view = next.get_view(); - auto pmid_view = p_mid.get_view(); - - const int num_cols = base_view.extent(0); - const int num_vert_packs = base_view.extent(1); - Kokkos::parallel_for( - Kokkos::MDRangePolicy>({0, 0}, - {num_cols, num_vert_packs}), - KOKKOS_LAMBDA(int i, int j) { - // If the pressure is above the cutoff, then we are closer to the surface - if (pmid_view(i, j) > cutoff) { - // Don't apply the tendency closer to the surface - tend_view(i, j) = 0.0; - } else { - // Apply the tendency farther up from the surface - tend_view(i, j) = next_view(i, j) - base_view(i, j); - } - }); - base.update(tend, dtend, Real(1.0)); + const Real dtend = dt / m_timescale; + + using cview_2d = decltype(state.get_view()); + + auto state_view = state.get_view(); + auto nudge_view = nudge.get_view(); + cview_2d w_view, pmid_view; + + if (m_use_weights) { + auto weights = get_helper_field("nudging_weights"); + w_view = weights.get_view(); + } + if (m_refine_remap_vert_cutoff>0) { + pmid_view = get_field_in("p_mid").get_view(); + } + + auto use_weights = m_use_weights; + auto cutoff = m_refine_remap_vert_cutoff; + auto policy = Kokkos::MDRangePolicy>({0, 0}, {m_num_cols, m_num_levs}); + auto update = KOKKOS_LAMBDA(const int& i, const int& j) { + if (cutoff>0 and pmid_view(i,j)>=cutoff) { + return; + } + + auto tend = nudge_view(i,j) - state_view(i,j); + if (use_weights) { + tend *= w_view(i,j); + } + state_view(i,j) += dtend * tend; + }; + Kokkos::parallel_for(policy,update); } // ============================================================================================================= void Nudging::initialize_impl (const RunType /* run_type */) { using namespace ShortFieldTagsNames; - // Set up pointers for grids - // external grid: from source data - std::shared_ptr grid_ext; - // temporary grid: after vertical interpolation - std::shared_ptr grid_tmp; - // internal grid: after horizontal interpolation - std::shared_ptr grid_int; - - // Initialize the refining remapper stuff at the outset, - // because we need to know the grid information; - // for now, we are doing the horizontal interpolation last, - // so we use the m_grid (model physics) as the target grid - grid_int = m_grid->clone(m_grid->name(), true); + + // The first thing we do is time interpolation. + // The second thing we do is horiz interpolation. The reason for doing horizontal + // before vertical is that we do not have a coarse p_mid (which would be needed + // as tgt pressure during vert remap). To get it, we'd have to "remap back" p_mid, + // but that seems overly complicated. For horiz remap we do not need anything + // on the target grid. + + // The "intermediate" grid, is the grid after horiz remap, and before vert remap + auto grid_tmp = m_grid->clone("after_horiz_before_vert",true); + grid_tmp->reset_num_vertical_lev(m_num_src_levs); + if (m_refine_remap) { // P2P remapper - m_refine_remapper = - std::make_shared(grid_int, m_refine_remap_file); - // Get grid from remapper, and clone it - auto grid_ext_const = m_refine_remapper->get_src_grid(); - grid_ext = grid_ext_const->clone(grid_ext_const->name(), true); - // Finally, grid_ext may have different levels - grid_ext->reset_num_vertical_lev(m_num_src_levs); + m_horiz_remapper = std::make_shared(grid_tmp, m_refine_remap_file); } else { - // We set up a DoNothingRemapper, which will do nothing - m_refine_remapper = std::make_shared(grid_int, grid_int); - // We clone physics grid, but maybe we have different levels - grid_ext = m_grid->clone(m_grid->name(), true); - grid_ext->reset_num_vertical_lev(m_num_src_levs); + // We set up an IdentityRemapper, specifying that tgt is an alias + // of src, so that the remap method will do nothing + auto r = std::make_shared(grid_tmp); + r->set_aliasing(IdentityRemapper::TgtAliasSrc); + m_horiz_remapper = r; } - // The temporary grid is the external grid, but with - // the same number of levels as the internal (physics) grid - grid_tmp = grid_ext->clone(grid_ext->name(), true); - grid_tmp->reset_num_vertical_lev(m_num_levs); - - // Declare the layouts for the helper fields (int: internal) - FieldLayout scalar2d_layout_mid { {LEV}, {m_num_levs} }; - FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; - // Get the number of external cols on current rank - auto m_num_cols_ext = grid_ext->get_num_local_dofs(); - // Declare the layouts for the helper fields (tmp: temporary)) - FieldLayout scalar2d_layout_mid_tmp { {LEV}, {m_num_levs}}; - FieldLayout scalar3d_layout_mid_tmp { {COL,LEV}, {m_num_cols_ext, m_num_levs} }; - // Declare the layouts for the helper fields (ext: external) - FieldLayout scalar2d_layout_mid_ext { {LEV}, {m_num_src_levs}}; - FieldLayout scalar3d_layout_mid_ext { {COL,LEV}, {m_num_cols_ext, m_num_src_levs} }; + // Now that we have the remapper, we can grab the grid where the input data lives + auto grid_ext = m_horiz_remapper->get_src_grid(); - // Initialize the time interpolator + // Initialize the time interpolator and horiz remapper m_time_interp = util::TimeInterpolation(grid_ext, m_datafiles); + m_time_interp.set_logger(m_atm_logger,"[EAMxx::Nudging] Reading nudging data"); - constexpr int ps = SCREAM_PACK_SIZE; - // To be extra careful, this should be the ext_grid - const auto& grid_ext_name = grid_ext->name(); - if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { - auto pmid_ext = create_helper_field("p_mid_ext", scalar3d_layout_mid_ext, grid_ext_name, ps); - m_time_interp.add_field(pmid_ext.alias("p_mid"),true); - } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { - // Load p_levs from source data file - ekat::ParameterList in_params; - in_params.set("Filename",m_static_vertical_pressure_file); - in_params.set("Skip_Grid_Checks",true); // We need to skip grid checks because multiple ranks may want the same column of source data. - std::map> host_views; - std::map layouts; - auto pmid_ext = create_helper_field("p_mid_ext", scalar2d_layout_mid_ext, grid_ext_name, ps); - auto pmid_ext_v = pmid_ext.get_view(); - in_params.set>("Field Names",{"p_levs"}); - host_views["p_levs"] = pmid_ext_v; - layouts.emplace("p_levs",scalar2d_layout_mid_ext); - AtmosphereInput src_input(in_params,grid_ext,host_views,layouts); - src_input.read_variables(-1); - src_input.finalize(); - pmid_ext.sync_to_dev(); - } - - // Open the registration! - m_refine_remapper->registration_begins(); - - // To create helper fields for later; we do both tmp and ext... + // NOTE: we are ASSUMING all fields are 3d and scalar! + const auto layout_ext = grid_ext->get_3d_scalar_layout(true); + const auto layout_tmp = grid_tmp->get_3d_scalar_layout(true); + const auto layout_atm = m_grid->get_3d_scalar_layout(true); + m_horiz_remapper->registration_begins(); for (auto name : m_fields_nudge) { std::string name_ext = name + "_ext"; std::string name_tmp = name + "_tmp"; - // Helper fields that will temporarily store the target state, which can then - // be used to back out a nudging tendency - auto grid_int_name = grid_int->name(); - auto grid_ext_name = grid_ext->name(); - auto grid_tmp_name = grid_tmp->name(); - auto field = get_field_out_wrap(name); - auto layout = field.get_header().get_identifier().get_layout(); - auto field_ext = create_helper_field(name_ext, scalar3d_layout_mid_ext, grid_ext_name, ps); - auto field_tmp = create_helper_field(name_tmp, scalar3d_layout_mid_tmp, grid_tmp_name, ps); - Field field_int; + + // First copy of the field: what's read from file, and time-interpolated. + auto field_ext = create_helper_field(name_ext, layout_ext, grid_ext->name()); + + // Second copy of the field: after horiz interp (alias "ext" if no remap) + Field field_tmp; if (m_refine_remap) { - field_int = create_helper_field(name, scalar3d_layout_mid, grid_int_name, ps); + field_tmp = create_helper_field(name_tmp, layout_tmp, grid_tmp->name()); } else { - field_int = field_tmp.alias(name); - m_helper_fields[name] = field_int; + field_tmp = field_ext.alias(name_tmp); + m_helper_fields[name_tmp] = field_tmp; } - // Register the fields with the remapper - m_refine_remapper->register_field(field_tmp, field_int); - // Add the fields to the time interpolator + // Add the field to the time interpolator m_time_interp.add_field(field_ext.alias(name), true); + + // Register the fields with the remapper + m_horiz_remapper->register_field(field_ext, field_tmp); + + if (m_timescale>0) { + // Third copy of the field: after vert interpolation. + // We cannot store directly in get_field_out(name), + // since we need to back out tendencies + create_helper_field(name, layout_atm, m_grid->name()); + } else { + // We do not need to back out any tendency; the input data is used + // to directly replace the atm state + m_helper_fields[name] = get_field_out_wrap(name); + } } - m_time_interp.initialize_data_from_files(); - // Close the registration! - m_refine_remapper->registration_ends(); + // A helper field, where we copy each field after horiz remap, padding it + // at top/bot, to allow vert lin interp to extrapolate outside the bounds of p_mid + FieldLayout layout_padded ({COL,LEV},{m_num_cols,m_num_src_levs+2}); + create_helper_field("padded_field",layout_padded,""); + + if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE && !m_skip_vert_interpolation) { + // If the pressure profile is 3d and time-dep, we need to interpolate (in time/horiz) + auto pmid_ext = create_helper_field("p_mid_ext", layout_ext, grid_ext->name()); + m_time_interp.add_field(pmid_ext.alias("p_mid"),true); + Field pmid_tmp; + if (m_refine_remap) { + pmid_tmp = create_helper_field("p_mid_tmp", layout_tmp, grid_tmp->name()); + } else { + pmid_tmp = pmid_ext.alias("p_mid_tmp"); + m_helper_fields["p_mid_tmp"] = pmid_tmp; + } + m_horiz_remapper->register_field(pmid_ext,pmid_tmp); + create_helper_field("padded_p_mid_tmp",layout_padded,""); + } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { + // For static 1D profile, we can read p_mid now + auto pmid_ext = create_helper_field("p_mid_ext", grid_ext->get_vertical_layout(true), grid_ext->name()); + AtmosphereInput src_input(m_static_vertical_pressure_file,grid_ext,{pmid_ext.alias("p_levs")},true); + src_input.read_variables(-1); + + // For static 1d profile, p_mid_tmp is an alias of p_mid_ext + m_helper_fields["p_mid_tmp"] = pmid_ext.alias("p_mid_tmp"); + + // The padded p_mid is also 1d + FieldLayout pmid1d_padded_layout({COL},{m_num_src_levs+2}); + create_helper_field("padded_p_mid_tmp",pmid1d_padded_layout,""); + } + + // Close the registration + m_time_interp.initialize_data_from_files(); + m_horiz_remapper->registration_ends(); // load nudging weights from file // NOTE: the regional nudging use the same grid as the run, no need to // do the interpolation. - if (m_use_weights) - { - auto grid_name = m_grid->name(); - auto nudging_weights = create_helper_field("nudging_weights", scalar3d_layout_mid, grid_name, ps); - std::vector fields; - fields.push_back(nudging_weights); - AtmosphereInput src_weights_input(m_weights_file, m_grid, fields); + if (m_use_weights) { + auto nudging_weights = create_helper_field("nudging_weights", layout_atm, m_grid->name()); + AtmosphereInput src_weights_input(m_weights_file, m_grid, {nudging_weights},true); src_weights_input.read_variables(); - src_weights_input.finalize(); - nudging_weights.sync_to_dev(); } } // ========================================================================================= void Nudging::run_impl (const double dt) { - using namespace scream::vinterp; + using KT = KokkosTypes; + using RangePolicy = typename KT::RangePolicy; + using MemberType = typename KT::MemberType; + using ESU = ekat::ExeSpaceUtils; + using PackT = ekat::Pack; + using view_1d = KT::view_1d; + using view_2d = KT::view_2d; // Have to add dt because first time iteration is at 0 seconds where you will // not have any data from the field. The timestamp is only iterated at the @@ -361,181 +339,204 @@ void Nudging::run_impl (const double dt) // Perform time interpolation m_time_interp.perform_time_interpolation(ts); - // Process data and nudge the atmosphere state - const auto& p_mid_v = get_field_in("p_mid").get_view(); - view_Nd p_mid_ext_p; - view_Nd p_mid_ext_1d; - if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { - p_mid_ext_p = get_helper_field("p_mid_ext").get_view(); - } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { - p_mid_ext_1d = get_helper_field("p_mid_ext").get_view(); - } + // If the input data contains "masked" values (sometimes also called "filled" values), + // the horiz remapping would smear them around. To prevent that, we need to "cure" + // these values. Masked values can only happen at top/bot of the model (with top + // being not common), and they must be a contiguous set of entries. So to cure them, + // we simply set all bot/top masked entries equal to the first non-masked value + // from the bot/top respectively. This corresponds to a constant extrapolation. + // NOTE: we need to do a tol check, since time interpolation may not return fillValue, + // even if both f(t_beg)/f(t_end) are equal to fillValue (due to rounding). + // NOTE: if f(t_beg)==fillValue!=f(t_end), or viceversa, the time-interpolated value can + // substantially differ from fillValue. Here, we assume it didn't happen. + auto correct_masked_values = [&](const Field f) { + const auto fl = f.get_header().get_identifier().get_layout(); + const auto v = f.get_view(); - for (auto name : m_fields_nudge) { - auto atm_state_field = get_field_out_wrap(name); // int horiz, int vert - auto int_state_field = get_helper_field(name); // int horiz, int vert - auto ext_state_field = get_helper_field(name+"_ext"); // ext horiz, ext vert - auto tmp_state_field = get_helper_field(name+"_tmp"); // ext horiz, int vert - auto ext_state_view = ext_state_field.get_view(); - auto tmp_state_view = tmp_state_field.get_view(); - auto atm_state_view = atm_state_field.get_view(); // TODO: Right now assume whatever field is defined on COLxLEV - auto int_state_view = int_state_field.get_view(); - auto int_mask_view = m_buffer.int_mask_view; - // Masked values in the source data can lead to strange behavior in the vertical interpolation. - // We pre-process the data and map any masked values (sometimes called "filled" values) to the - // nearest un-masked value. - // Here we are updating the ext_state_view, which is the time interpolated values taken from the nudging - // data. Real var_fill_value = constants::DefaultFillValue().value; // Query the helper field for the fill value, if not present use default - if (ext_state_field.get_header().has_extra_data("mask_value")) { - var_fill_value = ext_state_field.get_header().get_extra_data("mask_value"); + if (f.get_header().has_extra_data("mask_value")) { + var_fill_value = f.get_header().get_extra_data("mask_value"); } - const int num_cols = ext_state_view.extent(0); - const int num_vert_packs = ext_state_view.extent(1); - const int num_src_levs = m_num_src_levs; - const auto policy = ESU::get_default_team_policy(num_cols, num_vert_packs); - Kokkos::parallel_for("correct_for_masked_values", policy, - KOKKOS_LAMBDA(MemberType const& team) { - const int icol = team.league_rank(); - auto ext_state_view_1d = ekat::subview(ext_state_view,icol); - Real fill_value; - int fill_idx = -1; - // Scan top to surf and backfill all values near TOM that are masked. - for (int kk=0; kkthresh) { + // This entry is substantially different from var_fill_value, so it's good + first_good = ekat::impl::min(first_good,k); + last_good = ekat::impl::max(last_good,k); + } } - }); - - // Vertical Interpolation onto atmosphere state pressure levels - // Note that we are going from ext to tmp here - if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { - perform_vertical_interpolation(p_mid_ext_p, - p_mid_v, - ext_state_view, - tmp_state_view, - int_mask_view, - m_num_src_levs, - m_num_levs); - } else if (m_src_pres_type == STATIC_1D_VERTICAL_PROFILE) { - perform_vertical_interpolation(p_mid_ext_1d, - p_mid_v, - ext_state_view, - tmp_state_view, - int_mask_view, - m_num_src_levs, - m_num_levs); - } + EKAT_KERNEL_REQUIRE_MSG (first_good=0, + "[Nudging] Error! Could not locate a non-masked entry in a column.\n"); - // Check that none of the nudging targets are masked, if they are, set value to - // nearest unmasked value above. - // NOTE: We use an algorithm whichs scans from TOM to the surface. - // If TOM is masked we keep scanning until we hit an unmasked value, - // we then set all masked values above to the unmasked value. - // We continue scanning towards the surface until we hit an unmasked value, we - // then assign that masked value the most recent unmasked value, until we hit the - // surface. - // Here we change the int_state_view which represents the vertically interpolated fields onto - // the simulation grid. - const int num_levs = m_num_levs; - Kokkos::parallel_for("correct_for_masked_values", policy, - KOKKOS_LAMBDA(MemberType const& team) { - const int icol = team.league_rank(); - auto int_mask_view_1d = ekat::subview(int_mask_view,icol); - auto tmp_state_view_1d = ekat::subview(tmp_state_view,icol); - Real fill_value; - int fill_idx = -1; - // Scan top to surf and backfill all values near TOM that are masked. - for (int kk=0; kkremap(true); + // Perform horizontal remap (if needed) + m_horiz_remapper->remap(true); - for (auto name : m_fields_nudge) { - auto atm_state_field = get_field_out_wrap(name); // int horiz, int vert - auto int_state_field = get_helper_field(name); // int horiz, int vert - auto atm_state_view = atm_state_field.get_view(); // TODO: Right now assume whatever field is defined on COLxLEV - auto int_state_view = int_state_field.get_view(); - // Apply the nudging tendencies to the ATM state - if (m_timescale <= 0) { - // We do direct replacement - Kokkos::deep_copy(atm_state_view,int_state_view); + // bypass copy_and_pad and vert_interp for skip_vert_interpolation: + if (m_skip_vert_interpolation) { + for (const auto& name : m_fields_nudge) { + auto tmp_state_field = get_helper_field(name+"_tmp"); + + if (m_timescale > 0) { + auto atm_state_field = get_field_out_wrap(name); + apply_tendency(atm_state_field,tmp_state_field,dt); + } + } + return; + } + + // Copy remapper tgt fields into padded views, to allow extrapolation at top/bot, + // then call remapping routines + + const int ncols = m_num_cols; + const int nlevs_src = m_num_src_levs; + auto copy_and_pad = [&](const Field from, const Field to, const bool is_pmid) { + auto from_view = from.get_view(); + auto to_view = to.get_view(); + auto fl = from.get_header().get_identifier().get_layout(); + + auto copy_3d = KOKKOS_LAMBDA (const MemberType& team) { + int icol = team.league_rank(); + + auto copy_col = [&](const int k) { + to_view(icol,k+1) = from_view(icol,k); + }; + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,nlevs_src),copy_col); + + // Set the first/last entries of data, so that linear interp + // can extrapolate if the p_tgt is outside the p_src bounds + Kokkos::single(Kokkos::PerTeam(team),[&]{ + to_view(icol,0) = 0; // Does this make sense for *every field*? + if (is_pmid) { + // For pmid, we put a very large value, so that any p_mid_tgt + // that is larger than input p_mid bnds will end up in the + // last interval. + to_view(icol,nlevs_src+1) = 1e7; + } else { + // For data, we set last entry equal to second-to-last. + // This will cause constant extrapolation outside of + // the input p_mid bounds + to_view(icol,nlevs_src+1) = from_view(icol,nlevs_src-1); + } + }); + }; + + auto policy = ESU::get_default_team_policy(ncols,nlevs_src); + Kokkos::parallel_for("", policy, copy_3d); + }; + + // First, copy/pad p_mid, and extract the right copy (1d vs 3d) + if (m_src_pres_type==TIME_DEPENDENT_3D_PROFILE) { + copy_and_pad (get_helper_field("p_mid_tmp"),get_helper_field("padded_p_mid_tmp"),true); + } else { + // pmid is a 1d view. Just pad by hand + auto from = get_helper_field("p_mid_tmp"); + auto to = get_helper_field("padded_p_mid_tmp"); + auto from_v = from.get_view(); + auto to_v = to.get_view(); + auto lambda = KOKKOS_LAMBDA(const int& lev) { + to_v(lev+1) = from_v(lev); + if (lev==0) { + to_v(0) = 0; + } else if (lev==nlevs_src-1) { + to_v(lev+2) = to_v(lev+1); + } + }; + Kokkos::parallel_for(RangePolicy(0,nlevs_src),lambda); + } + const auto& p_mid_v = get_field_in("p_mid").get_view(); + view_2d p_mid_tmp_3d; + view_1d p_mid_tmp_1d; + bool src_pmid_3d; + if (m_src_pres_type == TIME_DEPENDENT_3D_PROFILE) { + p_mid_tmp_3d = get_helper_field("padded_p_mid_tmp").get_view(); + src_pmid_3d = true; + } else { + p_mid_tmp_1d = get_helper_field("padded_p_mid_tmp").get_view(); + src_pmid_3d = false; + } + + // Setup the linear interpolation object + using LI = ekat::LinInterp; + const int nlevs_tgt = m_num_levs; + LI vert_interp(ncols,nlevs_src+2,nlevs_tgt); + const auto policy_vinterp = ESU::get_default_team_policy(ncols, nlevs_tgt); + auto p_tgt = get_field_in("p_mid").get_view(); + Kokkos::parallel_for("nudging_vert_interp_setup_loop", policy_vinterp, + KOKKOS_LAMBDA(const MemberType& team) { + + const int icol = team.league_rank(); + + // Setup + if (src_pmid_3d) { + vert_interp.setup(team, ekat::subview(p_mid_tmp_3d,icol), + ekat::subview(p_tgt,icol)); } else { - // Back out a tendency and apply it. - if (m_use_weights) { - // get nudging weights field - // NOTES: do we really need the vertical interpolation for nudging weights? Since we are going to - // use the same grids as the case by providing the nudging weights file. - // I would not apply the vertical interpolation here, but it depends... - // - auto nudging_weights_field = get_helper_field("nudging_weights"); - // appply the nudging tendencies to the ATM states - apply_weighted_tendency(atm_state_field, int_state_field, nudging_weights_field, dt); - } else if (m_refine_remap_vert_cutoff > 0.0) { - // If we have a cutoff, we apply the tendency with p_mid cutoff - // First, get p_mid the field in the atm (i.e., int) state - auto p_mid_field = get_field_in("p_mid"); - // Then, call the tendency with a Heaviside-like cutoff - apply_vert_cutoff_tendency(atm_state_field, int_state_field, - p_mid_field, m_refine_remap_vert_cutoff, dt); + vert_interp.setup(team, p_mid_tmp_1d, + ekat::subview(p_tgt,icol)); + } + }); + Kokkos::fence(); + + // Then loop over fields, and do copy_and_pad + vremap + auto padded_field = get_helper_field("padded_field"); + for (const auto& name : m_fields_nudge) { + copy_and_pad(get_helper_field(name+"_tmp"),padded_field,false); + + auto field_after_vinterp = get_helper_field(name); + auto view_in = padded_field.get_view(); + auto view_out = field_after_vinterp.get_view(); + // Now use the interpolation object in || over all variables. + auto vinterp = KOKKOS_LAMBDA(const MemberType& team) { + const int icol = team.league_rank(); + + view_1d x1; + if (src_pmid_3d) { + x1 = ekat::subview(p_mid_tmp_3d,icol); } else { - apply_tendency(atm_state_field, int_state_field, dt); + x1 = p_mid_tmp_1d; } + auto x2 = ekat::subview(p_tgt,icol); + + auto y1 = ekat::subview(view_in, icol); + auto y2 = ekat::subview(view_out,icol); + + vert_interp.lin_interp(team, x1, x2, y1, y2, icol); + }; + Kokkos::parallel_for("nudging_vert_interp_loop", policy_vinterp, vinterp); + Kokkos::fence(); + + // If timescale==0, the call get_helper_field(name) returns the same + // fields as get_field_out_wrap(name) they are alias, so nothing to do. + // If timescale>0, then we need to back out a tendency. + if (m_timescale > 0) { + auto atm_state_field = get_field_out_wrap(name); + apply_tendency(atm_state_field,field_after_vinterp,dt); } } } @@ -552,16 +553,13 @@ Field Nudging::create_helper_field (const std::string& name, const int ps) { using namespace ekat::units; + // For helper fields we don't bother w/ units, so we set them to non-dimensional FieldIdentifier id(name,layout,Units::nondimensional(),grid_name); // Create the field. Init with NaN's, so we spot instances of uninited memory usage Field f(id); - if (ps>=0) { - f.get_header().get_alloc_properties().request_allocation(ps); - } else { - f.get_header().get_alloc_properties().request_allocation(); - } + f.get_header().get_alloc_properties().request_allocation(ps); f.allocate_view(); f.deep_copy(ekat::ScalarTraits::invalid()); @@ -569,23 +567,6 @@ Field Nudging::create_helper_field (const std::string& name, return m_helper_fields[name]; } -// ========================================================================================= -size_t Nudging::requested_buffer_size_in_bytes() const { - return m_buffer.num_2d_midpoint_mask_views*m_num_cols*m_num_levs*sizeof(mMask); -} - -// ========================================================================================= -void Nudging::init_buffers(const ATMBufferManager& buffer_manager) { - EKAT_REQUIRE_MSG(buffer_manager.allocated_bytes() >= requested_buffer_size_in_bytes(), - "Error, Nudging::init_buffers! Buffers size not sufficient.\n"); - mMask* mem = reinterpret_cast(buffer_manager.get_memory()); - - m_buffer.int_mask_view = decltype(m_buffer.int_mask_view)(mem,m_num_cols,m_num_levs); - mem += m_buffer.int_mask_view.size(); - - size_t used_mem = (reinterpret_cast(mem) - buffer_manager.get_memory())*sizeof(Real); - EKAT_REQUIRE_MSG(used_mem==requested_buffer_size_in_bytes(), "Error: Nudging::init_buffers! Used memory != requested memory."); -} // ========================================================================================= Field Nudging::get_field_out_wrap(const std::string& field_name) { if (field_name == "U" or field_name == "V") { diff --git a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp index 89d05728bcd8..604360b8dc2b 100644 --- a/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp +++ b/components/eamxx/src/physics/nudging/eamxx_nudging_process_interface.hpp @@ -1,20 +1,12 @@ #ifndef SCREAM_NUDGING_HPP #define SCREAM_NUDGING_HPP -#include "share/util/eamxx_time_interpolation.hpp" #include "share/atm_process/atmosphere_process.hpp" -#include "ekat/ekat_parameter_list.hpp" -#include "ekat/util/ekat_lin_interp.hpp" -#include "share/io/scream_output_manager.hpp" -#include "share/io/scorpio_output.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/grid/point_grid.hpp" -#include "share/util/scream_vertical_interpolation.hpp" -#include "share/util/scream_time_stamp.hpp" + +#include "share/util/eamxx_time_interpolation.hpp" #include "share/grid/remap/abstract_remapper.hpp" +#include #include namespace scream @@ -23,102 +15,56 @@ namespace scream /* * The class responsible to handle the nudging of variables */ - -// enum to track how the source pressure levels are defined -enum SourcePresType { - TIME_DEPENDENT_3D_PROFILE = 0, // DEFAULT - source data should include time/spatially varying p_mid with dimensions (time, col, lev) - STATIC_1D_VERTICAL_PROFILE = 1, // source data includes p_levs which is a static set of levels in both space and time, with dimensions (lev) -}; - class Nudging : public AtmosphereProcess { public: - using mPack = ekat::Pack; - using mMask = ekat::Mask<1>; - using KT = KokkosTypes; - - template - using view_1d = typename KT::template view_1d; - - template - using view_2d = typename KT::template view_2d; - - using uview_2d_mask = Unmanaged>; - - template - using view_Nd_host = typename KT::template view_ND::HostMirror; - - template - using view_1d_host = view_Nd_host; - - template - using view_2d_host = view_Nd_host; + // enum to track how the source pressure levels are defined + enum SourcePresType { + // DEFAULT - source data should include time/spatially varying p_mid with dimensions (time, col, lev) + TIME_DEPENDENT_3D_PROFILE, + // source data includes p_levs which is a static set of levels in both space and time, with dimensions (lev), + STATIC_1D_VERTICAL_PROFILE + }; // Constructors Nudging (const ekat::Comm& comm, const ekat::ParameterList& params); // The type of subcomponent - AtmosphereProcessType type () const { return AtmosphereProcessType::Physics; } + AtmosphereProcessType type () const override { return AtmosphereProcessType::Physics; } // The name of the subcomponent - std::string name () const { return "Nudging"; } + std::string name () const override { return "Nudging"; } // Set the grid - void set_grids (const std::shared_ptr grids_manager); - - // Internal function to apply nudging at specific timescale with weights - void apply_weighted_tendency(Field& base, const Field& next, const Field& weights, const Real dt); - - // Structure for storing local variables initialized using the ATMBufferManager - struct Buffer { - // 2D view - uview_2d_mask int_mask_view; - - // Total number of 2d views - static constexpr int num_2d_midpoint_mask_views = 1; - }; + void set_grids (const std::shared_ptr grids_manager) override; #ifndef KOKKOS_ENABLE_CUDA // Cuda requires methods enclosing __device__ lambda's to be public protected: #endif - void run_impl (const double dt); + void run_impl (const double dt) override; + + // Internal function to apply nudging at specific timescale + // NOTE: this method will handle weighted and cutoff cases as well + void apply_tendency (Field &state, const Field &nudge, const Real dt) const; - /* Nudge from coarse data */ - // See more details later in this file - // Must add this here to make it public for CUDA - // (refining) remapper vertically-weighted tendency application - void apply_vert_cutoff_tendency(Field &base, const Field &next, - const Field &p_mid, const Real cutoff, - const Real dt); protected: Field get_field_out_wrap(const std::string& field_name); // The two other main overrides for the subcomponent - void initialize_impl (const RunType run_type); - void finalize_impl (); - - // Computes total number of bytes needed for local variables - size_t requested_buffer_size_in_bytes() const; - - // Set local variables using memory provided by - // the ATMBufferManager - void init_buffers(const ATMBufferManager &buffer_manager); + void initialize_impl (const RunType run_type) override; + void finalize_impl () override; // Creates an helper field, not to be shared with the AD's FieldManager Field create_helper_field (const std::string& name, const FieldLayout& layout, const std::string& grid_name, - const int ps=0); + const int ps = 1); - // Query if a local field exists - bool has_helper_field (const std::string& name) const { return m_helper_fields.find(name)!=m_helper_fields.end(); } // Retrieve a helper field Field get_helper_field (const std::string& name) const { return m_helper_fields.at(name); } - // Internal function to apply nudging at specific timescale - void apply_tendency(Field& base, const Field& next, const Real dt); std::shared_ptr m_grid; // Keep track of field dimensions and the iteration count @@ -127,6 +73,7 @@ class Nudging : public AtmosphereProcess int m_num_src_levs; int m_timescale; bool m_use_weights; + bool m_skip_vert_interpolation; std::vector m_datafiles; std::string m_static_vertical_pressure_file; // add nudging weights for regional nudging update @@ -134,8 +81,6 @@ class Nudging : public AtmosphereProcess SourcePresType m_src_pres_type; - - // Some helper fields. std::map m_helper_fields; std::vector m_fields_nudge; @@ -146,13 +91,11 @@ class Nudging : public AtmosphereProcess // file containing coarse data mapping std::string m_refine_remap_file; // (refining) remapper object - std::shared_ptr m_refine_remapper; + std::shared_ptr m_horiz_remapper; // (refining) remapper vertical cutoff Real m_refine_remap_vert_cutoff; util::TimeInterpolation m_time_interp; - - Buffer m_buffer; }; // class Nudging } // namespace scream diff --git a/components/eamxx/src/physics/nudging/tests/CMakeLists.txt b/components/eamxx/src/physics/nudging/tests/CMakeLists.txt index 0187b85d2476..75f1c33d0473 100644 --- a/components/eamxx/src/physics/nudging/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/nudging/tests/CMakeLists.txt @@ -1,8 +1,19 @@ -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include(ScreamUtils) + CreateUnitTest (nudging_data "create_nudging_data.cpp" + LIBS scream_io + FIXTURES_SETUP nudging_data + ) + CreateUnitTest (nudging_map_files "create_map_file.cpp" + LIBS scream_io + FIXTURES_SETUP nudging_map_files + ) CreateUnitTest(nudging_tests "nudging_tests.cpp" - LIBS nudging scream_io - LABELS "physics_nudging" ) + LIBS nudging scream_io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} + LABELS physics nudging + FIXTURES_REQUIRED "nudging_map_files;nudging_data" + ) endif() diff --git a/components/eamxx/src/physics/nudging/tests/create_map_file.cpp b/components/eamxx/src/physics/nudging/tests/create_map_file.cpp new file mode 100644 index 000000000000..1a83abb5f7bb --- /dev/null +++ b/components/eamxx/src/physics/nudging/tests/create_map_file.cpp @@ -0,0 +1,63 @@ +#include + +#include "share/io/scream_scorpio_interface.hpp" + +TEST_CASE("create_map_file") +{ + using namespace scream; + + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + + // Add a dof in the middle of two coarse dofs + const int ngdofs_src = 12; + const int ngdofs_tgt = 2*ngdofs_src-1; + + std::string filename = "map_ncol" + std::to_string(ngdofs_src) + + "_to_" + std::to_string(ngdofs_tgt) + ".nc"; + + // Existing dofs are "copied", added dofs are averaged from neighbors + const int nnz = ngdofs_src + 2*(ngdofs_src-1); + + scorpio::register_file(filename, scorpio::FileMode::Write); + + scorpio::register_dimension(filename, "n_a", "n_a", ngdofs_src, false); + scorpio::register_dimension(filename, "n_b", "n_b", ngdofs_tgt, false); + scorpio::register_dimension(filename, "n_s", "n_s", nnz, false); + + scorpio::register_variable(filename, "col", "col", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "row", "row", "1", {"n_s"}, "int", "int", ""); + scorpio::register_variable(filename, "S", "S", "1", {"n_s"}, "double", "double", ""); + + std::vector dofs(nnz); + std::iota(dofs.begin(),dofs.end(),0); + scorpio::set_dof(filename,"col",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"row",dofs.size(),dofs.data()); + scorpio::set_dof(filename,"S", dofs.size(),dofs.data()); + + scorpio::eam_pio_enddef(filename); + + std::vector col(nnz), row(nnz); + std::vector S(nnz); + for (int i=0; iget_grid("Point Grid"); + + // Create a field manager, and init fields (since OM's need t0 values) + const auto fm1 = create_fm(grid); + const auto fm2 = create_fm(grid); + const auto fm3 = create_fm(grid); + compute_fields(fm1,t0,comm,0); + compute_fields(fm2,t0,comm,nlevs_filled); + compute_fields(fm3,t0,comm,0); + + // Create output manager + const auto om1 = create_om("nudging_data",fm1,gm,t0,comm); + const auto om2 = create_om("nudging_data_filled",fm2,gm,t0,comm); + const auto om3 = create_om("nudging_data_nonconst_p",fm3,gm,t0,comm); + + auto time = t0; + for (int istep=1; istep<=nsteps; ++istep) { + time += dt; + + // Compute fields, but keep p_mid constnat in fm1 and fm2, to avoid vinterp + compute_fields(fm1,time,comm,0,false); + compute_fields(fm2,time,comm,nlevs_filled,false); + compute_fields(fm3,time,comm,0); + + om1->run(time); + om2->run(time); + om3->run(time); + } + om1->finalize(); + om2->finalize(); + om3->finalize(); + + scorpio::eam_pio_finalize(); +} diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp index 9851f2c72714..0f0742875fd1 100644 --- a/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests.cpp @@ -1,390 +1,466 @@ #include "catch2/catch.hpp" + #include "physics/nudging/eamxx_nudging_process_interface.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/io/scream_output_manager.hpp" -#include "share/io/scorpio_output.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_scorpio_interface.hpp" +#include "nudging_tests_helpers.hpp" -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/grid/point_grid.hpp" +#include "share/field/field_utils.hpp" -#include "share/field/field_identifier.hpp" -#include "share/field/field_header.hpp" -#include "share/field/field.hpp" -#include "share/field/field_manager.hpp" +using namespace scream; -#include "share/util/scream_time_stamp.hpp" -#include "share/scream_types.hpp" -#include "scream_config.h" +std::shared_ptr +create_nudging (const ekat::Comm& comm, + const ekat::ParameterList& params, + const std::shared_ptr& fm, + const std::shared_ptr& gm, + const util::TimeStamp& t0) +{ + auto nudging = std::make_shared(comm,params); + nudging->set_grids(gm); + for (const auto& req : nudging->get_required_field_requests()) { + auto f = fm->get_field(req.fid.name()); + nudging->set_required_field(f); + } + for (const auto& req : nudging->get_computed_field_requests()) { + auto f = fm->get_field(req.fid.name()); + nudging->set_computed_field(f); + } + nudging->initialize(t0,RunType::Initial); -using namespace scream; + return nudging; +} + +TEST_CASE("nudging_tests") { + auto& catch_capture = Catch::getResultCapture(); -std::shared_ptr -create_gm (const ekat::Comm& comm, const int ncols, const int nlevs) { + using strvec_t = std::vector; - const int num_global_cols = ncols*comm.size(); + ekat::Comm comm(MPI_COMM_WORLD); - using vos_t = std::vector; - ekat::ParameterList gm_params; - gm_params.set("grids_names",vos_t{"Point Grid"}); - auto& pl = gm_params.sublist("Point Grid"); - pl.set("type","point_grid"); - pl.set("aliases",vos_t{"Physics"}); - pl.set("number_of_global_columns", num_global_cols); - pl.set("number_of_vertical_levels", nlevs); + auto root_print = [&](const std::string& msg) { + if (comm.am_i_root()) { + printf("%s",msg.c_str()); + } + }; + + // Init scorpio + scorpio::eam_init_pio_subsystem(comm); + + // A refined grid, with one extra node in between each of the coarse ones + const int ngcols_fine = 2*ngcols_data - 1; + const int nlevs_fine = 2*nlevs_data -1; + + // Files names + auto postfix = ".INSTANT.nsteps_x1." + get_t0().to_string() + ".nc"; + auto nudging_data = "nudging_data" + postfix; + auto nudging_data_filled = "nudging_data_filled" + postfix; + auto map_file = "map_ncol" + std::to_string(ngcols_data) + + "_to_" + std::to_string(ngcols_fine) + ".nc"; + + // For grids managers, depending on whether ncols/nlevs match the (coarse) + // values used to generate the data or are finer + auto gm_data = create_gm (comm,ngcols_data,nlevs_data); + auto gm_fine_h = create_gm (comm,ngcols_fine,nlevs_data); + auto gm_fine_v = create_gm (comm,ngcols_data,nlevs_fine); + auto gm_fine = create_gm (comm,ngcols_fine,nlevs_fine); + + auto grid_data = gm_data->get_grid("Point Grid"); + auto grid_fine_h = gm_fine_h->get_grid("Point Grid"); + auto grid_fine_v = gm_fine_v->get_grid("Point Grid"); + auto grid_fine = gm_fine->get_grid("Point Grid"); + + const int ncols_data = grid_data->get_num_local_dofs(); + + // First section tests nudging when there is no horiz-vert interp + SECTION ("no-horiz-no-vert") { + ekat::ParameterList params; + params.set("nudging_filenames_patterns",{nudging_data}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); - auto gm = create_mesh_free_grids_manager(comm,gm_params); - gm->build_grids(); + // Create fm. Init p_mid, since it's constant in this file + auto fm = create_fm(grid_data); + compute_field(fm->get_field("p_mid"),get_t0(),comm,0); - return gm; -} + auto U = fm->get_field("U"); + SECTION ("same-time") { + std::string msg = " -> Testing same time/horiz/vert grid as data ..........."; + root_print (msg + "\n"); + bool ok = true; + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_data,get_t0()); + + auto time = get_t0(); + + auto U_tgt = U.clone("U_tgt"); + for (int n=0; ok and nrun(dt_data); -TEST_CASE("nudging") { - - //This test makes a netcdf file with 3 columns and 34 levels - //with 4 fields, T_mid, p_mid, qv, and horiz_winds - //The p_mid field is filled based on the following equation: - //2j+1, where j is the level - //The T_mid, qv, and horiz_winds fields are filled based on the equations: - //(i-1)*10000+200*j+10*(dt/250.)*ii, where i is the column, j is the level, - //and ii in the timestep - //It then runs then nudging module with the netcdf file as input to nudge - //And then checks that the output fields of the nudging module match - //what should be in the netcdf file - - using namespace ekat::units; - using namespace ShortFieldTagsNames; - using FL = FieldLayout; - - // A time stamp - util::TimeStamp t0 ({2000,1,1},{0,0,0}); - - ekat::Comm io_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - const int packsize = SCREAM_PACK_SIZE; - Int num_levs = 34; - - // Initialize the pio_subsystem for this test: - // MPI communicator group used for I/O. - // In our simple test we use MPI_COMM_WORLD, however a subset could be used. - MPI_Fint fcomm = MPI_Comm_c2f(io_comm.mpi_comm()); - // Gather the initial PIO subsystem data creater by component coupler - scorpio::eam_init_pio_subsystem(fcomm); - - // First set up a field manager and grids manager to interact - // with the output functions - auto gm2 = create_gm(io_comm,3,num_levs); - auto grid2 = gm2->get_grid("Point Grid"); - int num_lcols = grid2->get_num_local_dofs(); - - IOControl io_control; // Needed for testing input. - io_control.timestamp_of_last_write = t0; - io_control.nsamples_since_last_write = 0; - io_control.frequency_units = "nsteps"; - std::vector output_stamps; - - const Int dt = 250; - const Int max_steps = 12; - { - util::TimeStamp time = t0; - util::TimeStamp time0(t0); - - // Re-create the fm anew, so the fields are re-inited for each output type - //auto field_manager = get_test_fm(grid); - - using namespace ShortFieldTagsNames; - using FR = FieldRequest; - - // Create a fm - auto fm = std::make_shared(grid2); - - const int num_levs = grid2->get_num_vertical_levels(); - - // Create some fields for this fm - std::vector tag_h = {COL}; - std::vector tag_v = {LEV}; - std::vector tag_2d = {COL,LEV}; - std::vector tag_2d_vec = {COL,CMP,LEV}; - - std::vector dims_h = {num_lcols}; - std::vector dims_v = {num_levs}; - std::vector dims_2d = {num_lcols,num_levs}; - std::vector dims_2d_vec = {num_lcols,2,num_levs}; - - const std::string& gn = grid2->name(); - - FieldIdentifier fid1("p_mid",FL{tag_2d,dims_2d},Pa,gn); - FieldIdentifier fid2("T_mid",FL{tag_2d,dims_2d},K,gn); - FieldIdentifier fid3("qv",FL{tag_2d,dims_2d},kg/kg,gn); - FieldIdentifier fidhw("horiz_winds",FL{tag_2d_vec,dims_2d_vec},m/s,gn); - - // Register fields with fm - fm->registration_begins(); - fm->register_field(FR{fid1,"output",packsize}); - fm->register_field(FR{fid2,"output",packsize}); - fm->register_field(FR{fid3,"output",packsize}); - fm->register_field(FR{fidhw,"output",packsize}); - fm->registration_ends(); - - // Initialize these fields - auto f1 = fm->get_field(fid1); - auto f1_host = f1.get_view(); - - auto f2 = fm->get_field(fid2); - auto f2_host = f2.get_view(); - - auto f3 = fm->get_field(fid3); - auto f3_host = f3.get_view(); - - auto fhw = fm->get_field(fidhw); - auto fhw_host = fhw.get_view(); - - for (int ii=0;iiinit_fields_time_stamp(time); - f1.sync_to_dev(); - f2.sync_to_dev(); - f3.sync_to_dev(); - fhw.sync_to_dev(); - - // Add subfields U and V to field manager - { - auto hw = fm->get_field("horiz_winds"); - const auto& fid = hw.get_header().get_identifier(); - const auto& layout = fid.get_layout(); - const int vec_dim = layout.get_vector_dim(); - const auto& units = fid.get_units(); - auto fU = hw.subfield("U",units,vec_dim,0); - auto fV = hw.subfield("V",units,vec_dim,1); - fm->add_field(fU); - fm->add_field(fV); + + // Test case where model times are in the middle of input data time intervals + SECTION ("half-time") { + std::string msg = " -> Testing same horiz/vert grid, different time grid ..."; + root_print (msg + "\n"); + bool ok = true; + + // Init time as t0-dt/2, so we're "half way" between data slices + auto time = get_t0() - dt_data/2; + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_data,time); + + auto tmp1 = U.clone(""); + auto tmp2 = U.clone(""); + + auto check_f = [&](const Field& f, + const util::TimeStamp& t_prev, + const util::TimeStamp& t_next) { + compute_field(tmp1,t_prev,comm,0); + compute_field(tmp2,t_next,comm,0); + tmp1.update(tmp2,0.5,0.5); + + // Since input data are integers, and the time-interp coeff is 0.5, + // we should be getting the exact answer + CHECK (views_are_equal(f,tmp1)); + ok &= catch_capture.lastAssertionPassed(); + }; + + auto t_prev = get_t0(); + for (int n=1; ok and nrun(dt_data); + + // Compare the two. Since we're exactly half way, we should get exact fp representation + check_f(U,t_prev,t_next); + + t_prev = t_next; + } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); } + } - // Set up parameter list control for output - ekat::ParameterList params; - params.set("filename_prefix","io_output_test"); - params.set("Averaging Type","Instant"); - params.set("Max Snapshots Per File",15); - std::vector fnames = {"T_mid","p_mid","qv","U","V"}; - params.set>("Field Names",fnames); - auto& params_sub = params.sublist("output_control"); - params_sub.set("frequency_units","nsteps"); - params_sub.set("Frequency",1); - params_sub.set("MPI Ranks in Filename",true); - io_control.frequency = params_sub.get("Frequency"); - - // Set up output manager. - OutputManager om; - om.setup(io_comm,params,fm,gm2,t0,t0,false); - io_comm.barrier(); - - const auto& out_fields = fm->get_groups_info().at("output"); - using namespace ShortFieldTagsNames; - // Time loop - for (Int ii=0;iim_fields_names) { - auto f = fm->get_field(fname); - f.sync_to_host(); - auto fl = f.get_header().get_identifier().get_layout(); - switch (fl.rank()) { - case 2: - { - auto v = f.get_view(); - for (int i=0; i(); - for (int i=0; i(); + auto data_h = data.get_view(); + const bool is_pmid = data.name()=="p_mid"; + for (int icol=0; icol("nudging_filenames_patterns",{nudging_data}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); + + std::string msg = " -> Testing same time/horiz grid, different vert grid"; + root_print (msg + "\n"); + SECTION ("pmid_in_bounds") { + std::string msg = " -> Target pressure within source pressure bounds ....."; + root_print (msg + "\n"); + bool ok = true; + + // Create fm + auto fm = create_fm(grid_fine_v); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_fine_v,get_t0()); + + // Compute pmid on data grid + auto layout_data = grid_data->get_3d_scalar_layout(true); + Field p_mid_data(FieldIdentifier("p_mid",layout_data,Pa,grid_data->name())); + p_mid_data.allocate_view(); + compute_field(p_mid_data,get_t0(),comm,0); + + manual_interp(p_mid_data,p_mid,true); + + auto time = get_t0(); + Field tmp_data = p_mid_data.clone("tmp data"); + Field tmp_fine = p_mid.clone("tmp fine"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp_data,time+dt_data,comm,0); + manual_interp(tmp_data,tmp_fine,true); + + CHECK (views_are_equal(tmp_fine,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); } - // Finalize the output manager (close files) - om.finalize(); - } - - // Create a grids manager - const int ncols = 3; - const int nlevs = 35; - auto gm = create_gm(io_comm,ncols,nlevs); - auto grid = gm->get_grid("Physics"); - - ekat::ParameterList params_mid; - std::string nudging_f = "io_output_test.INSTANT.nsteps_x1."\ - "np1.2000-01-01-00000.nc"; - params_mid.set>("nudging_filename",{nudging_f}); - params_mid.set>("nudging_fields",{"T_mid","qv","U","V"}); - params_mid.set("use_nudging_weights",false); - std::shared_ptr nudging_mid = std::make_shared(io_comm,params_mid); - - nudging_mid->set_grids(gm); - - std::map input_fields; - std::map output_fields; - for (const auto& req : nudging_mid->get_required_field_requests()) { - Field f(req.fid); - auto & f_ap = f.get_header().get_alloc_properties(); - f_ap.request_allocation(packsize); - f.allocate_view(); - const auto name = f.name(); - f.get_header().get_tracking().update_time_stamp(t0); - nudging_mid->set_required_field(f); - input_fields.emplace(name,f); - if (name != "p_mid"){ - nudging_mid->set_computed_field(f); - output_fields.emplace(name,f); + SECTION ("pmid_out_of_bounds") { + std::string msg = " -> Target pressure outside source pressure bounds ...."; + root_print (msg + "\n"); + bool ok = true; + + // Create fm + auto fm = create_fm(grid_fine_v); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_fine_v,get_t0()); + + // Compute pmid on data grid + auto layout_data = grid_data->get_3d_scalar_layout(true); + Field p_mid_data(FieldIdentifier("p_mid",layout_data,Pa,grid_data->name())); + p_mid_data.allocate_view(); + compute_field(p_mid_data,get_t0(),comm,0); + + manual_interp(p_mid_data,p_mid,false); + + auto time = get_t0(); + Field tmp_data = p_mid_data.clone("tmp data"); + Field tmp_fine = p_mid.clone("tmp fine"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp_data,time+dt_data,comm,0); + manual_interp(tmp_data,tmp_fine,false); + + CHECK (views_are_equal(tmp_fine,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; + } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); } } - //initialize - nudging_mid->initialize(t0,RunType::Initial); - Field p_mid = input_fields["p_mid"]; - Field T_mid = input_fields["T_mid"]; - Field qv = input_fields["qv"]; - Field hw = input_fields["horiz_winds"]; - Field T_mid_o = output_fields["T_mid"]; - Field qv_mid_o = output_fields["qv"]; - Field hw_o = output_fields["horiz_winds"]; - // Initialize memory buffer for all atm processes - auto memory_buffer = std::make_shared(); - memory_buffer->request_bytes(nudging_mid->requested_buffer_size_in_bytes()); - memory_buffer->allocate(); - nudging_mid->init_buffers(*memory_buffer); - - //fill data - //Don't fill T,qv,u,v because they will be nudged anyways - auto p_mid_v_h = p_mid.get_view(); - for (int icol=0; icol(); - auto qv_h_o = qv_mid_o.get_view(); - auto hw_h_o = hw_o.get_view(); - nudging_mid->run(100); - T_mid_o.sync_to_host(); - qv_mid_o.sync_to_host(); - hw_o.sync_to_host(); - - for (int icol=0; icol(); + auto glb_view_h = glb.get_view(); + for (int rank=0; rank(); + auto glb_data_h = glb_data.get_view(); + for (int icol=0; icolget_num_local_dofs(); + auto offset = ncols_fine; + comm.scan(&offset,1,MPI_SUM); + offset -= ncols_fine; + + auto fine_h = fine.get_view(); + for (int i=0; i("nudging_filenames_patterns",{nudging_data}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_refine_remap_mapfile",map_file); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); + + // Create fm + auto fm = create_fm(grid_fine_h); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_fine_h,get_t0()); + + // Compute pmid on data grid + auto layout_data = grid_data->get_3d_scalar_layout(true); + Field p_mid_data(FieldIdentifier("p_mid",layout_data,Pa,grid_data->name())); + p_mid_data.allocate_view(); + compute_field(p_mid_data,get_t0(),comm,0); + + manual_interp(p_mid_data,p_mid); + + auto time = get_t0(); + Field tmp_data = p_mid_data.clone("tmp data"); + Field tmp_fine = p_mid.clone("tmp fine"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp_data,time+dt_data,comm,0); + manual_interp(tmp_data,tmp_fine); + + CHECK (views_are_equal(tmp_fine,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); + } -} + SECTION ("filled-data") { + std::string msg = " -> Testing data with top/bot levels filled ............."; + root_print (msg + "\n"); + bool ok = true; + + // Helper lambda, to manually cure filled levels + auto manual_cure = [&](const Field& f) { + auto f_h = f.get_view(); + auto ncols = f.get_header().get_identifier().get_layout().dim(0); + auto nlevs = f.get_header().get_identifier().get_layout().dim(1); + auto first_good = nlevs_filled; + auto last_good = nlevs - nlevs_filled - 1; + for (int icol=0; icolfinalize(); + ekat::ParameterList params; + params.set("nudging_filenames_patterns",{nudging_data_filled}); + params.set("source_pressure_type","TIME_DEPENDENT_3D_PROFILE"); + params.set("nudging_fields",{"U"}); + params.get("log_level","warn"); + + // Create fm. Init p_mid, since it's constant in this file + auto fm = create_fm(grid_data); + auto U = fm->get_field("U"); + auto p_mid = fm->get_field("p_mid"); + compute_field(p_mid,get_t0(),comm,0); + + // Create and init nudging process + auto nudging = create_nudging(comm,params,fm,gm_data,get_t0()); + + auto time = get_t0(); + Field tmp = p_mid.clone("tmp"); + for (int n=0; ok and nrun(dt_data); + + // Compute data on fine grid, by manually interpolating + // (recall that nudging runs at t+dt) + compute_field(tmp,time+dt_data,comm,0); + manual_cure(tmp); + + CHECK (views_are_equal(tmp,U)); + ok &= catch_capture.lastAssertionPassed(); + time += dt_data; + } + root_print (msg + (ok ? " PASS\n" : " FAIL\n")); + } + // Clean up scorpio + scorpio::eam_pio_finalize(); } - diff --git a/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp new file mode 100644 index 000000000000..52e2b8ca0e30 --- /dev/null +++ b/components/eamxx/src/physics/nudging/tests/nudging_tests_helpers.hpp @@ -0,0 +1,167 @@ +#include "share/io/scream_output_manager.hpp" +#include "share/grid/mesh_free_grids_manager.hpp" +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/scream_types.hpp" + +namespace scream +{ + +constexpr int ngcols_data = 12; +constexpr int nlevs_data = 20; +constexpr int nsteps_data = 5; +constexpr int dt_data = 100; +constexpr int nlevs_filled = 2; +constexpr double fill_val = 1e30; + +util::TimeStamp get_t0 () { + return util::TimeStamp ({2000,1,1},{12,0,0}); +} + +std::shared_ptr +create_gm (const ekat::Comm& comm, const int ngcols, const int nlevs) { + + using vos_t = std::vector; + ekat::ParameterList gm_params; + gm_params.set("grids_names",vos_t{"Point Grid"}); + auto& pl = gm_params.sublist("Point Grid"); + pl.set("type","point_grid"); + pl.set("aliases",vos_t{"Physics"}); + pl.set("number_of_global_columns", ngcols); + pl.set("number_of_vertical_levels", nlevs); + + auto gm = create_mesh_free_grids_manager(comm,gm_params); + gm->build_grids(); + + return gm; +} + +std::shared_ptr +create_fm (const std::shared_ptr& grid) +{ + using namespace ShortFieldTagsNames; + using namespace ekat::units; + using FR = FieldRequest; + + auto fm = std::make_shared(grid); + + const std::string& gn = grid->name(); + + auto scalar3d = grid->get_3d_scalar_layout(true); + auto vector3d = grid->get_3d_vector_layout(true,CMP,2); + + FieldIdentifier fid1("p_mid",scalar3d,Pa,gn); + FieldIdentifier fid2("horiz_winds",vector3d,m/s,gn); + + // Register fields with fm + fm->registration_begins(); + fm->register_field(FR(fid1)); + fm->register_field(FR(fid2)); + fm->registration_ends(); + + auto U = fm->get_field("horiz_winds").subfield("U",1,0); + auto V = fm->get_field("horiz_winds").subfield("V",1,1); + fm->add_field(U); + fm->add_field(V); + + return fm; +} + +void compute_field (Field f, + const util::TimeStamp& time, + const ekat::Comm& comm, + const int num_masked_levs = 0) +{ + const auto& fl = f.get_header().get_identifier().get_layout(); + + const int ncols = fl.dim(0); + const int nlevs = fl.dim(1); + + int offset = ncols; + comm.scan(&offset,1,MPI_SUM); + offset -= ncols; + + const int step = time.get_num_steps(); + const int lev_beg = num_masked_levs; + const int lev_end = nlevs - num_masked_levs; + + if (fl.rank()==2) { + const auto f_h = f.get_view(); + for (int icol=0;icol(); + for (int icol=0;icol& fm, + const util::TimeStamp& time, + const ekat::Comm& comm, + const int num_masked_levs = 0, + const bool update_p_mid = true) +{ + if (update_p_mid) { + // Don't mask pressure + compute_field(fm->get_field("p_mid"),time,comm,0); + } + compute_field(fm->get_field("U"),time,comm,num_masked_levs); + compute_field(fm->get_field("V"),time,comm,num_masked_levs); + + // Not sure if we need it, since we don't handle horiz_winds directly, I think + fm->get_field("horiz_winds").get_header().get_tracking().update_time_stamp(time); +} + +std::shared_ptr +create_om (const std::string& filename_prefix, + const std::shared_ptr& fm, + const std::shared_ptr& gm, + const util::TimeStamp& t0, + const ekat::Comm& comm) +{ + using strvec_t = std::vector; + + // NOTE: ask "real" fp precision, so even when building in double precision + // we can retrieve exactly the nudging data (if no remapping happens) + ekat::ParameterList params; + params.set("Averaging Type","INSTANT"); + params.set("filename_prefix",filename_prefix); + params.set("Floating Point Precision","real"); + params.set("MPI Ranks in Filename", false); + params.set("Field Names",strvec_t{"p_mid","U","V"}); + params.set("fill_value",fill_val); + + auto& ctrl_pl = params.sublist("output_control"); + ctrl_pl.set("frequency_units","nsteps"); + ctrl_pl.set("Frequency",1); + ctrl_pl.set("save_grid_data",false); + + auto om = std::make_shared(); + om->setup(comm,params,fm,gm,t0,t0,false); + return om; +} + +} // namespace scream diff --git a/components/eamxx/src/physics/p3/CMakeLists.txt b/components/eamxx/src/physics/p3/CMakeLists.txt index fb90ca03be2c..7ea0aeeb8145 100644 --- a/components/eamxx/src/physics/p3/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/CMakeLists.txt @@ -78,7 +78,7 @@ if (SCREAM_SMALL_KERNELS) add_library(p3 ${P3_SRCS} ${P3_SK_SRCS}) else() add_library(p3 ${P3_SRCS}) - if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_BASELINES_ONLY) + if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(p3_sk ${P3_SRCS} ${P3_SK_SRCS}) # Always build shoc_sk with SCREAM_SMALL_KERNELS on target_compile_definitions(p3_sk PUBLIC "SCREAM_SMALL_KERNELS") diff --git a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp index e26d6fe18743..0d2aad5247dd 100644 --- a/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_ice_sed_impl_disp.cpp @@ -28,7 +28,8 @@ ::ice_sedimentation_disp( const view_ice_table& ice_table_vals, const uview_1d& precip_ice_surf, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -49,7 +50,7 @@ ::ice_sedimentation_disp( ekat::subview(inv_dz, i), team, workspace, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qi, i), ekat::subview(qi_incld, i), ekat::subview(ni, i), ekat::subview(ni_incld, i), ekat::subview(qm, i), ekat::subview(qm_incld, i), ekat::subview(bm, i), ekat::subview(bm_incld, i), ekat::subview(qi_tend, i), ekat::subview(ni_tend, i), - ice_table_vals, precip_ice_surf(i)); + ice_table_vals, precip_ice_surf(i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp index 5b075439aa91..c3692f3f814f 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_disp.cpp @@ -35,6 +35,7 @@ ::p3_main_init_disp( { using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + Kokkos::parallel_for("p3_main_init", policy, KOKKOS_LAMBDA(const MemberType& team) { @@ -108,7 +109,8 @@ ::p3_main_internal_disp( const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, - Int nk) + Int nk, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; @@ -231,7 +233,7 @@ ::p3_main_internal_disp( T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, - ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent); + ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, p3constants); // ------------------------------------------------------------------------------------------ // main k-loop (for processes): @@ -247,7 +249,7 @@ ::p3_main_internal_disp( nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, qv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, vap_liq_exchange, vap_ice_exchange, liq_ice_exchange, - pratot, prctot, nucleationPossible, hydrometeorsPresent); + pratot, prctot, nucleationPossible, hydrometeorsPresent, p3constants); //NOTE: At this point, it is possible to have negative (but small) nc, nr, ni. This is not // a problem; those values get clipped to zero in the sedimentation section (if necessary). @@ -273,14 +275,14 @@ ::p3_main_internal_disp( rho, inv_rho, rhofacr, cld_frac_r, inv_dz, qr_incld, workspace_mgr, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, qr, nr, nr_incld, mu_r, lamr, precip_liq_flux, qtend_ignore, ntend_ignore, - diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent); + diagnostic_outputs.precip_liq_surf, nucleationPossible, hydrometeorsPresent, p3constants); // Ice sedimentation: (adaptive substepping) ice_sedimentation_disp( rho, inv_rho, rhofaci, cld_frac_i, inv_dz, workspace_mgr, nj, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, qi, qi_incld, ni, ni_incld, qm, qm_incld, bm, bm_incld, qtend_ignore, ntend_ignore, - lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf, nucleationPossible, hydrometeorsPresent); + lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf, nucleationPossible, hydrometeorsPresent, p3constants); // homogeneous freezing f cloud and rain homogeneous_freezing_disp( @@ -296,7 +298,8 @@ ::p3_main_internal_disp( rho, inv_rho, rhofaci, qv, th, qc, nc, qr, nr, qi, ni, qm, bm, latent_heat_vapor, latent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, vap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, diag_eff_radius_qi, diag_diam_qi, - rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent); + rho_qi, diag_equiv_reflectivity, diag_eff_radius_qc, diag_eff_radius_qr, nucleationPossible, hydrometeorsPresent, + p3constants); // // merge ice categories with similar properties diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp index a9c0e5fe1445..3dae805255b0 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part1_disp.cpp @@ -62,7 +62,8 @@ ::p3_main_part1_disp( const uview_2d& ni_incld, const uview_2d& bm_incld, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -83,7 +84,7 @@ ::p3_main_part1_disp( ekat::subview(qv, i), ekat::subview(th_atm, i), ekat::subview(qc, i), ekat::subview(nc, i), ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(qi, i), ekat::subview(ni, i), ekat::subview(qm, i), ekat::subview(bm, i), ekat::subview(qc_incld, i), ekat::subview(qr_incld, i), ekat::subview(qi_incld, i), ekat::subview(qm_incld, i), ekat::subview(nc_incld, i), ekat::subview(nr_incld, i), ekat::subview(ni_incld, i), ekat::subview(bm_incld, i), - nucleationPossible(i), hydrometeorsPresent(i)); + nucleationPossible(i), hydrometeorsPresent(i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 02073bd4f5af..2b619d54bf33 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -89,11 +89,14 @@ ::p3_main_part2_disp( const uview_2d& pratot, const uview_2d& prctot, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); + + // p3_cloud_sedimentation loop Kokkos::parallel_for( "p3_main_part2_disp", @@ -122,7 +125,7 @@ ::p3_main_part2_disp( ekat::subview(cdist1, i), ekat::subview(cdistr, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(logn0r, i), ekat::subview(qv2qi_depos_tend, i), ekat::subview(precip_total_tend, i), ekat::subview(nevapr, i), ekat::subview(qr_evap_tend, i), ekat::subview(vap_liq_exchange, i), ekat::subview(vap_ice_exchange, i), ekat::subview(liq_ice_exchange, i), - ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk); + ekat::subview(pratot, i), ekat::subview(prctot, i), hydrometeorsPresent(i), nk, p3constants); if (!hydrometeorsPresent(i)) return; }); diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp index 8e442cf2fc53..6d591a7e9328 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part3_disp.cpp @@ -56,7 +56,8 @@ ::p3_main_part3_disp( const uview_2d& diag_eff_radius_qc, const uview_2d& diag_eff_radius_qr, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const auto policy = ekat::ExeSpaceUtils::get_default_team_policy(nj, nk_pack); @@ -82,7 +83,7 @@ ::p3_main_part3_disp( ekat::subview(mu_c, i), ekat::subview(nu, i), ekat::subview(lamc, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(vap_liq_exchange, i), ekat::subview(ze_rain, i), ekat::subview(ze_ice, i), ekat::subview(diag_vm_qi, i), ekat::subview(diag_eff_radius_qi, i), ekat::subview(diag_diam_qi, i), ekat::subview(rho_qi, i), ekat::subview(diag_equiv_reflectivity, i), ekat::subview(diag_eff_radius_qc, i), - ekat::subview(diag_eff_radius_qr, i)); + ekat::subview(diag_eff_radius_qr, i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp index 6a9a36b4b353..e77c91035174 100644 --- a/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_rain_sed_impl_disp.cpp @@ -27,7 +27,8 @@ ::rain_sedimentation_disp( const uview_2d& nr_tend, const uview_1d& precip_liq_surf, const uview_1d& nucleationPossible, - const uview_1d& hydrometeorsPresent) + const uview_1d& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; const Int nk_pack = ekat::npack(nk); @@ -49,7 +50,7 @@ ::rain_sedimentation_disp( team, workspace, vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, ekat::subview(qr, i), ekat::subview(nr, i), ekat::subview(nr_incld, i), ekat::subview(mu_r, i), ekat::subview(lamr, i), ekat::subview(precip_liq_flux, i), - ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i)); + ekat::subview(qr_tend, i), ekat::subview(nr_tend, i), precip_liq_surf(i), p3constants); }); } diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp index 04b304e2691c..d6264c38b8b7 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.cpp @@ -3,6 +3,7 @@ #include "share/property_checks/field_lower_bound_check.hpp" // Needed for p3_init, the only F90 code still used. #include "physics/p3/p3_functions.hpp" +#include "physics/share/physics_constants.hpp" #include "physics/p3/p3_f90.hpp" #include "ekat/ekat_assert.hpp" @@ -204,6 +205,12 @@ void P3Microphysics::initialize_impl (const RunType /* run_type */) { // Gather runtime options runtime_options.max_total_ni = m_params.get("max_total_ni"); + + // setting P3 constants in a struct + m_p3constants.set_p3_from_namelist(m_params); + m_p3constants.print_p3constants(m_atm_logger); + // done setting P3 constants + // Set property checks for fields in this process add_invariant_check(get_field_out("T_mid"),m_grid,100.0,500.0,false); add_invariant_check(get_field_out("qv"),m_grid,1e-13,0.2,true); diff --git a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp index 6a9c945e5176..47d543ded022 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_process_interface.hpp @@ -22,6 +22,7 @@ namespace scream class P3Microphysics : public AtmosphereProcess { using P3F = p3::Functions; + using CP3 = physics::P3_Constants; using Spack = typename P3F::Spack; using Smask = typename P3F::Smask; using Pack = ekat::Pack; @@ -53,6 +54,8 @@ class P3Microphysics : public AtmosphereProcess // Set the grid void set_grids (const std::shared_ptr grids_manager); + CP3 m_p3constants; + /*--------------------------------------------------------------------------------------------*/ // Most individual processes have a pre-processing step that constructs needed variables from // the set of fields stored in the field manager. A structure like this defines those operations, diff --git a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp index d4d1fdd6fdca..9ccfdaf20220 100644 --- a/components/eamxx/src/physics/p3/eamxx_p3_run.cpp +++ b/components/eamxx/src/physics/p3/eamxx_p3_run.cpp @@ -29,7 +29,7 @@ void P3Microphysics::run_impl (const double dt) get_field_out("micro_vap_ice_exchange").deep_copy(0.0); P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, m_num_cols, m_num_levs); + history_only, lookup_tables, workspace_mgr, m_num_cols, m_num_levs, m_p3constants); // Conduct the post-processing of the p3_main output. Kokkos::parallel_for( diff --git a/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp index a939a7ab4a5f..43b1f21b7a74 100644 --- a/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_autoconversion_impl.hpp @@ -3,6 +3,7 @@ #include "p3_functions.hpp" // for ETI only but harmless for GPU #include "p3_subgrid_variance_scaling_impl.hpp" +#include "physics/share/physics_constants.hpp" namespace scream { namespace p3 { @@ -13,18 +14,25 @@ void Functions ::cloud_water_autoconversion( const Spack& rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& inv_qc_relvar, Spack& qc2qr_autoconv_tend, Spack& nc2nr_autoconv_tend, Spack& ncautr, + const physics::P3_Constants & p3constants, const Smask& context) { + // Khroutdinov and Kogan (2000) const auto qc_not_small = qc_incld >= 1e-8 && context; constexpr Scalar CONS3 = C::CONS3; + + const Scalar p3_autoconversion_prefactor = p3constants.p3_autoconversion_prefactor; + +//printf(" hey inside AAAAAAAAAAAAAAAA %13.6f \n", p3_autoconversion_factor); + if(qc_not_small.any()){ Spack sgs_var_coef; // sgs_var_coef = subgrid_variance_scaling(inv_qc_relvar, sp(2.47) ); sgs_var_coef = 1; qc2qr_autoconv_tend.set(qc_not_small, - sgs_var_coef*1350*pow(qc_incld,sp(2.47))*pow(nc_incld*sp(1.e-6)*rho,sp(-1.79))); + sgs_var_coef*p3_autoconversion_prefactor*pow(qc_incld,sp(2.47))*pow(nc_incld*sp(1.e-6)*rho,sp(-1.79))); // note: ncautr is change in Nr; nc2nr_autoconv_tend is change in Nc ncautr.set(qc_not_small, qc2qr_autoconv_tend*CONS3); nc2nr_autoconv_tend.set(qc_not_small, qc2qr_autoconv_tend*nc_incld/qc_incld); diff --git a/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp index ea6597f43a75..2abcc3ef135e 100644 --- a/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_cldliq_imm_freezing_impl.hpp @@ -20,20 +20,21 @@ ::cldliq_immersion_freezing( const Spack& mu_c, const Spack& cdist1, const Spack& qc_incld, const Spack& inv_qc_relvar, Spack& qc2qi_hetero_freeze_tend, Spack& nc2ni_immers_freeze_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; - constexpr Scalar AIMM = C::AIMM; constexpr Scalar T_rainfrz = C::T_rainfrz; constexpr Scalar T_zerodegc = C::T_zerodegc; constexpr Scalar CONS5 = C::CONS5; constexpr Scalar CONS6 = C::CONS6; + const Scalar p3_a_imm = p3constants.p3_a_imm; const auto qc_not_small_and_t_freezing = (qc_incld >= qsmall) && (T_atm <= T_rainfrz) && context; if (qc_not_small_and_t_freezing.any()) { Spack expAimmDt, inv_lamc3; - expAimmDt.set(qc_not_small_and_t_freezing, exp(AIMM * (T_zerodegc-T_atm))); + expAimmDt.set(qc_not_small_and_t_freezing, exp(p3_a_imm * (T_zerodegc-T_atm))); inv_lamc3.set(qc_not_small_and_t_freezing, cube(1/lamc)); Spack sgs_var_coef; diff --git a/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp index 74e605c0e4f3..5d64cb593afb 100644 --- a/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_cloud_rain_acc_impl.hpp @@ -20,10 +20,13 @@ ::cloud_rain_accretion( const Spack& qc_incld, const Spack& nc_incld, const Spack& qr_incld, const Spack& inv_qc_relvar, Spack& qc2qr_accret_tend, Spack& nc_accret_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; + const Scalar p3_k_accretion = p3constants.p3_k_accretion;; + Spack sgs_var_coef; // sgs_var_coef = subgrid_variance_scaling(inv_qc_relvar, sp(1.15) ); sgs_var_coef = 1; @@ -32,7 +35,7 @@ ::cloud_rain_accretion( if (qr_and_qc_not_small.any()) { // Khroutdinov and Kogan (2000) qc2qr_accret_tend.set(qr_and_qc_not_small, - sgs_var_coef * sp(67.0) * pow(qc_incld * qr_incld, sp(1.15))); + sgs_var_coef * sp(p3_k_accretion) * pow(qc_incld * qr_incld, sp(1.15))); nc_accret_tend.set(qr_and_qc_not_small, qc2qr_accret_tend * nc_incld / qc_incld); qc2qr_accret_tend.set(nc_accret_tend == 0 && context, 0); diff --git a/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp index a503248ad855..c7c434c0daee 100644 --- a/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_dsd2_impl.hpp @@ -76,7 +76,8 @@ KOKKOS_FUNCTION void Functions:: get_rain_dsd2 ( const Spack& qr, Spack& nr, Spack& mu_r, - Spack& lamr, Spack& cdistr, Spack& logn0r, + Spack& lamr, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr auto nsmall = C::NSMALL; @@ -84,27 +85,24 @@ get_rain_dsd2 ( constexpr auto cons1 = C::CONS1; lamr.set(context , 0); - cdistr.set(context, 0); - logn0r.set(context, 0); const auto qr_gt_small = qr >= qsmall && context; + const Scalar mu_r_const = p3constants.p3_mu_r_constant; + if (qr_gt_small.any()) { - constexpr Scalar mu_r_const = C::mu_r_const; // use lookup table to get mu // mu-lambda relationship is from Cao et al. (2008), eq. (7) // find spot in lookup table // (scaled N/q for lookup table parameter space) const auto nr_lim = max(nr, nsmall); - Spack inv_dum(0); - inv_dum.set(qr_gt_small, cbrt(qr / (cons1 * nr_lim * 6))); // Apply constant mu_r: Recall the switch to v4 tables means constant mu_r mu_r.set(qr_gt_small, mu_r_const); // recalculate slope based on mu_r - lamr.set(qr_gt_small, cbrt(cons1 * nr_lim * (mu_r + 3) * - (mu_r + 2) * (mu_r + 1)/qr)); + const auto mass_to_d3_factor = cons1 * (mu_r + 3) * (mu_r + 2) * (mu_r + 1); + lamr.set(qr_gt_small, cbrt(mass_to_d3_factor * nr_lim / qr)); // check for slope const auto lammax = (mu_r+1.)*sp(1.e+5); @@ -121,15 +119,31 @@ get_rain_dsd2 ( lamr.set(lt, lammin); lamr.set(gt, lammax); ekat_masked_loop(either, s) { - nr[s] = std::exp(3*std::log(lamr[s]) + std::log(qr[s]) + - std::log(std::tgamma(mu_r[s] + 1)) - std::log(std::tgamma(mu_r[s] + 4))) - / cons1; + nr[s] = lamr[s]*lamr[s]*lamr[s] * qr[s] / mass_to_d3_factor[s]; } } + } +} + +template +KOKKOS_FUNCTION +void Functions:: +get_cdistr_logn0r ( + const Spack& qr, const Spack& nr, const Spack& mu_r, + const Spack& lamr, Spack& cdistr, Spack& logn0r, + const Smask& context) +{ + constexpr auto qsmall = C::QSMALL; + cdistr.set(context, 0); + logn0r.set(context, 0); + + const auto qr_gt_small = qr >= qsmall && context; + + if (qr_gt_small.any()) { cdistr.set(qr_gt_small, nr/tgamma(mu_r + 1)); // note: logn0r is calculated as log10(n0r) - logn0r.set(qr_gt_small, log10(nr) + (mu_r + 1) * log10(lamr) - log10(tgamma(mu_r+1))); + logn0r.set(qr_gt_small, log10(cdistr) + (mu_r + 1) * log10(lamr)); } } diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp index 6cb9e0aed049..2c3d5f6fc22c 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_collection_impl.hpp @@ -15,6 +15,7 @@ ::ice_cldliq_collection( const Spack& qi_incld, const Spack& qc_incld, const Spack& ni_incld, const Spack& nc_incld, Spack& qc2qi_collect_tend, Spack& nc_collect_tend, Spack& qc2qr_ice_shed_tend, Spack& ncshdc, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -27,17 +28,17 @@ ::ice_cldliq_collection( const auto both_ge_small = qi_incld_ge_small && qc_incld_ge_small && context; const auto both_ge_small_pos_t = both_ge_small && !t_is_negative; - constexpr auto eci = C::eci; + const Scalar p3_eci = p3constants.p3_eci; constexpr auto inv_dropmass = C::ONE/C::dropmass; qc2qi_collect_tend.set(both_ge_small && t_is_negative, - rhofaci*table_val_qc2qi_collect*qc_incld*eci*rho*ni_incld); - nc_collect_tend.set(both_ge_small, rhofaci*table_val_qc2qi_collect*nc_incld*eci*rho*ni_incld); + rhofaci*table_val_qc2qi_collect*qc_incld*p3_eci*rho*ni_incld); + nc_collect_tend.set(both_ge_small, rhofaci*table_val_qc2qi_collect*nc_incld*p3_eci*rho*ni_incld); // for T_atm > 273.15, assume cloud water is collected and shed as rain drops // sink for cloud water mass and number, note qcshed is source for rain mass - qc2qr_ice_shed_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*qc_incld*eci*rho*ni_incld); - nc_collect_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*nc_incld*eci*rho*ni_incld); + qc2qr_ice_shed_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*qc_incld*p3_eci*rho*ni_incld); + nc_collect_tend.set(both_ge_small_pos_t, rhofaci*table_val_qc2qi_collect*nc_incld*p3_eci*rho*ni_incld); // source for rain number, assume 1 mm drops are shed ncshdc.set(both_ge_small_pos_t, qc2qr_ice_shed_tend*inv_dropmass); } @@ -52,6 +53,7 @@ ::ice_rain_collection( const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Spack& qr2qi_collect_tend, Spack& nr_collect_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar qsmall = C::QSMALL; @@ -65,11 +67,11 @@ ::ice_rain_collection( const auto both_ge_small_neg_t = both_ge_small && t_is_negative; constexpr Scalar ten = 10.0; - constexpr auto eri = C::eri; + const Scalar p3_eri = p3constants.p3_eri; // note: table_val_qr2qi_collect and logn0r are already calculated as log_10 - qr2qi_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_qr2qi_collect+logn0r)*rho*rhofaci*eri*ni_incld); - nr_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_nr_collect+logn0r)*rho*rhofaci*eri*ni_incld); + qr2qi_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_qr2qi_collect+logn0r)*rho*rhofaci*p3_eri*ni_incld); + nr_collect_tend.set(both_ge_small_neg_t, pow(ten, table_val_nr_collect+logn0r)*rho*rhofaci*p3_eri*ni_incld); // rain number sink due to collection // for T_atm > 273.15, assume collected rain number is shed as @@ -78,7 +80,7 @@ ::ice_rain_collection( // rate of ice mass due to melting // collection of rain above freezing does not impact total rain mass nr_collect_tend.set(both_ge_small && !t_is_negative, - pow(ten, table_val_nr_collect + logn0r)*rho*rhofaci*eri*ni_incld); + pow(ten, table_val_nr_collect + logn0r)*rho*rhofaci*p3_eri*ni_incld); // for now neglect shedding of ice collecting rain above freezing, since snow is // not expected to shed in these conditions (though more hevaily rimed ice would be // expected to lead to shedding) diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp index 7f84b42d21ec..27313934fa7e 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_nucleation_impl.hpp @@ -13,6 +13,7 @@ ::ice_nucleation( const Spack& temp, const Spack& inv_rho, const Spack& ni, const Spack& ni_activated, const Spack& qv_supersat_i, const Scalar& inv_dt, const bool& do_predict_nc, const bool& do_prescribed_CCN, Spack& qv2qi_nucleat_tend, Spack& ni_nucleat_tend, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar nsmall = C::NSMALL; @@ -22,6 +23,8 @@ ::ice_nucleation( constexpr Scalar piov3 = C::PIOV3; constexpr Scalar mi0 = sp(4.0)*piov3*sp(900.0)*sp(1.e-18); + const Scalar p3_dep_nucleation_exponent = p3constants.p3_dep_nucleation_exponent; + const auto t_lt_T_icenuc = temp < T_icenuc; const auto qv_supersat_i_ge_005 = qv_supersat_i >= 0.05; @@ -33,7 +36,7 @@ ::ice_nucleation( Spack dum{0.0}, N_nuc{0.0}, Q_nuc{0.0}; if (any_if_not_log.any()) { - dum = sp(0.005)*exp(sp(0.304)*(tmelt-temp))*sp(1.0e3)*inv_rho; + dum = sp(0.005)*exp(sp(p3_dep_nucleation_exponent)*(tmelt-temp))*sp(1.0e3)*inv_rho; dum = min(dum, sp(1.0e5)*inv_rho); diff --git a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp index 3ede317333a1..29232c69e68f 100644 --- a/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_ice_sed_impl.hpp @@ -17,12 +17,13 @@ typename Functions::Spack Functions ::calc_bulk_rho_rime( const Spack& qi_tot, Spack& qi_rim, Spack& bi_rim, + const physics::P3_Constants & p3constants, const Smask& context) { constexpr Scalar bsmall = C::BSMALL; constexpr Scalar qsmall = C::QSMALL; - constexpr Scalar rho_rime_min = C::rho_rimeMin; - constexpr Scalar rho_rime_max = C::rho_rimeMax; + const Scalar p3_rho_rime_min = p3constants.p3_rho_rime_min; + const Scalar p3_rho_rime_max = p3constants.p3_rho_rime_max; Spack rho_rime(0); @@ -32,12 +33,12 @@ ::calc_bulk_rho_rime( rho_rime.set(bi_rim_gt_small, qi_rim / bi_rim); } - Smask rho_rime_lt_min = rho_rime < rho_rime_min; - Smask rho_rime_gt_max = rho_rime > rho_rime_max; + Smask rho_rime_lt_min = rho_rime < p3_rho_rime_min; + Smask rho_rime_gt_max = rho_rime > p3_rho_rime_max; // impose limits on rho_rime; adjust bi_rim if needed - rho_rime.set(bi_rim_gt_small && rho_rime_lt_min, rho_rime_min); - rho_rime.set(bi_rim_gt_small && rho_rime_gt_max, rho_rime_max); + rho_rime.set(bi_rim_gt_small && rho_rime_lt_min, p3_rho_rime_min); + rho_rime.set(bi_rim_gt_small && rho_rime_gt_max, p3_rho_rime_max); Smask adjust = bi_rim_gt_small && (rho_rime_gt_max || rho_rime_lt_min); if (adjust.any()) { bi_rim.set(adjust, qi_rim / rho_rime); @@ -85,7 +86,8 @@ ::ice_sedimentation( const uview_1d& qi_tend, const uview_1d& ni_tend, const view_ice_table& ice_table_vals, - Scalar& precip_ice_surf) + Scalar& precip_ice_surf, + const physics::P3_Constants & p3constants) { // Get temporary workspaces needed for the ice-sed calculation uview_1d V_qit, V_nit, flux_nit, flux_bir, flux_qir, flux_qit; @@ -102,6 +104,9 @@ ::ice_sedimentation( const auto sqi = scalarize(qi); constexpr Scalar qsmall = C::QSMALL; constexpr Scalar nsmall = C::NSMALL; + + const Scalar p3_ice_sed_knob = p3constants.p3_ice_sed_knob; + bool log_qxpresent; const Int k_qxtop = find_top(team, sqi, qsmall, kbot, ktop, kdir, log_qxpresent); @@ -140,7 +145,7 @@ ::ice_sedimentation( // impose lower limits to prevent log(<0) ni_incld(pk).set(qi_gt_small, max(ni_incld(pk), nsmall)); - const auto rhop = calc_bulk_rho_rime(qi_incld(pk), qm_incld(pk), bm_incld(pk), qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld(pk), qm_incld(pk), bm_incld(pk), p3constants, qi_gt_small); qm(pk).set(qi_gt_small, qm_incld(pk)*cld_frac_i(pk) ); bm(pk).set(qi_gt_small, bm_incld(pk)*cld_frac_i(pk) ); @@ -158,8 +163,8 @@ ::ice_sedimentation( ni_incld(pk).set(qi_gt_small, max(ni_incld(pk), table_val_ni_lammin * ni_incld(pk))); ni(pk).set(qi_gt_small, ni_incld(pk) * cld_frac_i(pk)); - V_qit(pk).set(qi_gt_small, table_val_qi_fallspd * rhofaci(pk)); // mass-weighted fall speed (with density factor) - V_nit(pk).set(qi_gt_small, table_val_ni_fallspd * rhofaci(pk)); // number-weighted fall speed (with density factor) + V_qit(pk).set(qi_gt_small, p3_ice_sed_knob * table_val_qi_fallspd * rhofaci(pk)); // mass-weighted fall speed (with density factor) + V_nit(pk).set(qi_gt_small, p3_ice_sed_knob * table_val_ni_fallspd * rhofaci(pk)); // number-weighted fall speed (with density factor) } const auto Co_max_local = max(qi_gt_small, 0, V_qit(pk) * dt_left * inv_dz(pk)); diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp index 22819e64aad1..b05b713aec29 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl.hpp @@ -83,7 +83,8 @@ ::p3_main_internal( const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, - Int nk) + Int nk, + const physics::P3_Constants & p3constants) { using ExeSpace = typename KT::ExeSpace; @@ -233,7 +234,7 @@ ::p3_main_internal( T_atm, rho, inv_rho, qv_sat_l, qv_sat_i, qv_supersat_i, rhofacr, rhofaci, acn, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, qc_incld, qr_incld, qi_incld, qm_incld, nc_incld, nr_incld, - ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent); + ni_incld, bm_incld, nucleationPossible, hydrometeorsPresent, p3constants); // There might not be any work to do for this team if (!(nucleationPossible || hydrometeorsPresent)) { @@ -253,7 +254,7 @@ ::p3_main_internal( nr_incld, ni_incld, bm_incld, mu_c, nu, lamc, cdist, cdist1, cdistr, mu_r, lamr, logn0r, oqv2qi_depos_tend, precip_total_tend, nevapr, qr_evap_tend, ovap_liq_exchange, ovap_ice_exchange, oliq_ice_exchange, - pratot, prctot, hydrometeorsPresent, nk); + pratot, prctot, hydrometeorsPresent, nk, p3constants); //NOTE: At this point, it is possible to have negative (but small) nc, nr, ni. This is not // a problem; those values get clipped to zero in the sedimentation section (if necessary). @@ -281,14 +282,14 @@ ::p3_main_internal( rho, inv_rho, rhofacr, ocld_frac_r, inv_dz, qr_incld, team, workspace, lookup_tables.vn_table_vals, lookup_tables.vm_table_vals, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, oqr, onr, nr_incld, mu_r, lamr, oprecip_liq_flux, qtend_ignore, ntend_ignore, - diagnostic_outputs.precip_liq_surf(i)); + diagnostic_outputs.precip_liq_surf(i), p3constants); // Ice sedimentation: (adaptive substepping) ice_sedimentation( rho, inv_rho, rhofaci, ocld_frac_i, inv_dz, team, workspace, nk, ktop, kbot, kdir, infrastructure.dt, inv_dt, oqi, qi_incld, oni, ni_incld, oqm, qm_incld, obm, bm_incld, qtend_ignore, ntend_ignore, - lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf(i)); + lookup_tables.ice_table_vals, diagnostic_outputs.precip_ice_surf(i), p3constants); // homogeneous freezing of cloud and rain homogeneous_freezing( @@ -304,7 +305,7 @@ ::p3_main_internal( rho, inv_rho, rhofaci, oqv, oth, oqc, onc, oqr, onr, oqi, oni, oqm, obm, olatent_heat_vapor, olatent_heat_sublim, mu_c, nu, lamc, mu_r, lamr, ovap_liq_exchange, ze_rain, ze_ice, diag_vm_qi, odiag_eff_radius_qi, diag_diam_qi, - orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr); + orho_qi, diag_equiv_reflectivity, odiag_eff_radius_qc, odiag_eff_radius_qr, p3constants); // // merge ice categories with similar properties @@ -343,7 +344,8 @@ ::p3_main( const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, - Int nk) + Int nk, + const physics::P3_Constants & p3constants) { #ifndef SCREAM_SMALL_KERNELS return p3_main_internal(runtime_options, @@ -354,7 +356,7 @@ ::p3_main( history_only, lookup_tables, workspace_mgr, - nj, nk); + nj, nk, p3constants); #else return p3_main_internal_disp(runtime_options, prognostic_state, @@ -364,7 +366,7 @@ ::p3_main( history_only, lookup_tables, workspace_mgr, - nj, nk); + nj, nk, p3constants); #endif } } // namespace p3 diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp index 28be9ab9040b..439b0544adf9 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part1.hpp @@ -65,7 +65,8 @@ ::p3_main_part1( const uview_1d& ni_incld, const uview_1d& bm_incld, bool& nucleationPossible, - bool& hydrometeorsPresent) + bool& hydrometeorsPresent, + const physics::P3_Constants & p3constants) { // Get access to saturation functions using physics = scream::physics::Functions; @@ -80,6 +81,8 @@ ::p3_main_part1( constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; + const Scalar p3_spa_to_nc = p3constants.p3_spa_to_nc; + nucleationPossible = false; hydrometeorsPresent = false; team.team_barrier(); @@ -131,7 +134,7 @@ ::p3_main_part1( // prescribe that value if (do_prescribed_CCN) { - nc(k).set(not_drymass, max(nc(k), nccn_prescribed(k)/inv_cld_frac_l(k))); + nc(k).set(not_drymass, max(nc(k), p3_spa_to_nc*nccn_prescribed(k)/inv_cld_frac_l(k))); } else if (predictNc) { nc(k).set(not_drymass, max(nc(k) + nc_nuceat_tend(k) * dt, 0.0)); } else { diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp index 29fc2fce11d6..c32fb7202ebb 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part2.hpp @@ -94,7 +94,8 @@ ::p3_main_part2( const uview_1d& liq_ice_exchange, const uview_1d& pratot, const uview_1d& prctot, - bool& hydrometeorsPresent, const Int& nk) + bool& hydrometeorsPresent, const Int& nk, + const physics::P3_Constants & p3constants) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar nsmall = C::NSMALL; @@ -215,7 +216,8 @@ ::p3_main_part2( lamc(k), cdist(k), cdist1(k), not_skip_micro); nc(k).set(not_skip_micro, nc_incld(k) * cld_frac_l(k)); - get_rain_dsd2(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), cdistr(k), logn0r(k), not_skip_micro); + get_rain_dsd2(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), p3constants, not_skip_micro); + get_cdistr_logn0r(qr_incld(k), nr_incld(k), mu_r(k), lamr(k), cdistr(k), logn0r(k), not_skip_micro); nr(k).set(not_skip_micro, nr_incld(k) * cld_frac_r(k)); impose_max_total_ni(ni_incld(k), max_total_ni, inv_rho(k), not_skip_micro); @@ -227,7 +229,7 @@ ::p3_main_part2( ni_incld(k).set(qi_gt_small, max(ni_incld(k), nsmall)); nr_incld(k).set(qi_gt_small, max(nr_incld(k), nsmall)); - const auto rhop = calc_bulk_rho_rime(qi_incld(k), qm_incld(k), bm_incld(k), qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld(k), qm_incld(k), bm_incld(k), p3constants, qi_gt_small); qm(k).set(qi_gt_small, qm_incld(k)*cld_frac_i(k) ); bm(k).set(qi_gt_small, bm_incld(k)*cld_frac_i(k) ); @@ -268,12 +270,12 @@ ::p3_main_part2( // collection of droplets ice_cldliq_collection( rho(k), T_atm(k), rhofaci(k), table_val_qc2qi_collect, qi_incld(k), qc_incld(k), ni_incld(k), nc_incld(k), - qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, not_skip_micro); + qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, p3constants, not_skip_micro); // collection of rain ice_rain_collection( rho(k), T_atm(k), rhofaci(k), logn0r(k), table_val_nr_collect, table_val_qr2qi_collect, qi_incld(k), ni_incld(k), qr_incld(k), - qr2qi_collect_tend, nr_collect_tend, not_skip_micro); + qr2qi_collect_tend, nr_collect_tend, p3constants, not_skip_micro); // collection between ice categories @@ -309,12 +311,12 @@ ::p3_main_part2( // contact and immersion freezing droplets cldliq_immersion_freezing( T_atm(k), lamc(k), mu_c(k), cdist1(k), qc_incld(k), inv_qc_relvar(k), - qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, not_skip_micro); + qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, p3constants, not_skip_micro); // for future: get rid of log statements below for rain freezing rain_immersion_freezing( T_atm(k), lamr(k), mu_r(k), cdistr(k), qr_incld(k), - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, not_skip_micro); + qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, p3constants, not_skip_micro); // rime splintering (Hallet-Mossop 1974) // PMC comment: Morrison and Milbrandt 2015 part 1 and 2016 part 3 both say @@ -343,13 +345,13 @@ ::p3_main_part2( // deposition/condensation-freezing nucleation ice_nucleation( T_atm(k), inv_rho(k), ni(k), ni_activated(k), qv_supersat_i(k), inv_dt, predictNc, do_prescribed_CCN, - qv2qi_nucleat_tend, ni_nucleat_tend, not_skip_all); + qv2qi_nucleat_tend, ni_nucleat_tend, p3constants, not_skip_all); // cloud water autoconversion // NOTE: cloud_water_autoconversion must be called before droplet_self_collection cloud_water_autoconversion( rho(k), qc_incld(k), nc_incld(k), inv_qc_relvar(k), - qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, not_skip_all); + qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, p3constants, not_skip_all); // self-collection of droplets droplet_self_collection( @@ -359,13 +361,13 @@ ::p3_main_part2( // accretion of cloud by rain cloud_rain_accretion( rho(k), inv_rho(k), qc_incld(k), nc_incld(k), qr_incld(k), inv_qc_relvar(k), - qc2qr_accret_tend, nc_accret_tend, not_skip_all); + qc2qr_accret_tend, nc_accret_tend, p3constants, not_skip_all); // self-collection and breakup of rain // (breakup following modified Verlinde and Cotton scheme) rain_self_collection( rho(k), qr_incld(k), nr_incld(k), - nr_selfcollect_tend, not_skip_all); + nr_selfcollect_tend, p3constants, not_skip_all); // Here we map the microphysics tendency rates back to CELL-AVERAGE quantities for updating // cell-average quantities. diff --git a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp index 6791248abca4..441bcb9feb9b 100644 --- a/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_main_impl_part3.hpp @@ -57,7 +57,8 @@ ::p3_main_part3( const uview_1d& rho_qi, const uview_1d& diag_equiv_reflectivity, const uview_1d& diag_eff_radius_qc, - const uview_1d& diag_eff_radius_qr) + const uview_1d& diag_eff_radius_qr, + const physics::P3_Constants & p3constants) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar inv_cp = C::INV_CP; @@ -107,7 +108,7 @@ ::p3_main_part3( auto nr_incld = nr(k)/cld_frac_r(k); //nr_incld is updated in get_rain_dsd2 but isn't used again get_rain_dsd2( - qr_incld, nr_incld, mu_r(k), lamr(k), ignore1, ignore2, qr_gt_small); + qr_incld, nr_incld, mu_r(k), lamr(k), p3constants, qr_gt_small); //Note that integrating over the drop-size PDF as done here should only be done to in-cloud //quantities but radar reflectivity is likely meant to be a cell ave. Thus nr in the next line @@ -142,7 +143,7 @@ ::p3_main_part3( auto qm_incld = qm(k)/cld_frac_i(k); auto bm_incld = bm(k)/cld_frac_i(k); - const auto rhop = calc_bulk_rho_rime(qi_incld, qm_incld, bm_incld, qi_gt_small); + const auto rhop = calc_bulk_rho_rime(qi_incld, qm_incld, bm_incld, p3constants, qi_gt_small); qm(k).set(qi_gt_small, qm_incld*cld_frac_i(k) ); bm(k).set(qi_gt_small, bm_incld*cld_frac_i(k) ); diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp index acee2805bf77..c960698e4b18 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_imm_freezing_impl.hpp @@ -17,26 +17,28 @@ void Functions ::rain_immersion_freezing(const Spack& T_atm, const Spack& lamr, const Spack& mu_r, const Spack& cdistr, const Spack& qr_incld, Spack& qr2qi_immers_freeze_tend, Spack& nr2ni_immers_freeze_tend, - const Smask& context) + const physics::P3_Constants & p3constants, + const Smask& context) { constexpr Scalar qsmall = C::QSMALL; constexpr Scalar T_rainfrz = C::T_rainfrz; constexpr Scalar T_zerodegc = C::T_zerodegc; - constexpr Scalar AIMM = C::AIMM; constexpr Scalar CONS5 = C::CONS5; constexpr Scalar CONS6 = C::CONS6; + const Scalar p3_a_imm = p3constants.p3_a_imm; + const auto qr_not_small_and_t_freezing = (qr_incld >= qsmall) && (T_atm <= T_rainfrz) && context; if (qr_not_small_and_t_freezing.any()) { qr2qi_immers_freeze_tend.set(qr_not_small_and_t_freezing, CONS6 * exp(log(cdistr) + log(tgamma(sp(7.)+mu_r)) - sp(6.)*log(lamr)) * - exp(AIMM*(T_zerodegc-T_atm))); + exp(p3_a_imm*(T_zerodegc-T_atm))); nr2ni_immers_freeze_tend.set(qr_not_small_and_t_freezing, CONS5 * exp(log(cdistr) + log(tgamma(sp(4.)+mu_r)) - sp(3.)*log(lamr)) * - exp(AIMM*(T_zerodegc-T_atm))); + exp(p3_a_imm*(T_zerodegc-T_atm))); } } diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp index d54db725caee..558264305f17 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_sed_impl.hpp @@ -18,11 +18,11 @@ ::compute_rain_fall_velocity( const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, const Spack& qr_incld, const Spack& rhofacr, Spack& nr_incld, Spack& mu_r, Spack& lamr, Spack& V_qr, Spack& V_nr, + const physics::P3_Constants & p3constants, const Smask& context) { Table3 table; - Spack tmp1, tmp2; //ignore - get_rain_dsd2(qr_incld, nr_incld, mu_r, lamr, tmp1, tmp2, context); + get_rain_dsd2(qr_incld, nr_incld, mu_r, lamr, p3constants, context); if (context.any()) { lookup(mu_r, lamr, table, context); @@ -55,7 +55,8 @@ ::rain_sedimentation( const uview_1d& precip_liq_flux, const uview_1d& qr_tend, const uview_1d& nr_tend, - Scalar& precip_liq_surf) + Scalar& precip_liq_surf, + const physics::P3_Constants & p3constants) { // Get temporary workspaces needed for the ice-sed calculation uview_1d V_qr, V_nr, flux_qx, flux_nx; @@ -111,7 +112,7 @@ ::rain_sedimentation( compute_rain_fall_velocity(vn_table_vals, vm_table_vals, qr_incld(pk), rhofacr(pk), nr_incld(pk), mu_r(pk), lamr(pk), - V_qr(pk), V_nr(pk), qr_gt_small); + V_qr(pk), V_nr(pk), p3constants, qr_gt_small); //in compute_rain_fall_velocity, get_rain_dsd2 keeps the drop-size //distribution within reasonable bounds by modifying nr_incld. diff --git a/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp b/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp index 2df0c5790311..807e928948cb 100644 --- a/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp +++ b/components/eamxx/src/physics/p3/impl/p3_rain_self_collection_impl.hpp @@ -11,6 +11,7 @@ KOKKOS_FUNCTION void Functions ::rain_self_collection( const Spack& rho, const Spack& qr_incld, const Spack& nr_incld, Spack& nr_selfcollect_tend, + const physics::P3_Constants & p3constants, const Smask& context) { // ------------------------------------------------------ @@ -21,18 +22,20 @@ ::rain_self_collection( constexpr Scalar rho_h2o = C::RHO_H2O; constexpr Scalar pi = C::Pi; + const Scalar p3_d_breakup_cutoff = p3constants.p3_d_breakup_cutoff; + const auto qr_incld_not_small = qr_incld >= qsmall && context; if (qr_incld_not_small.any()) { - const Real dum1 = 280.e-6; + const auto dum2 = cbrt((qr_incld)/(pi*rho_h2o*nr_incld)); Spack dum; - const auto dum2_lt_dum1 = dum2 < dum1 && qr_incld_not_small; - const auto dum2_gt_dum1 = dum2 >= dum1 && qr_incld_not_small; + const auto dum2_lt_dum1 = dum2 < p3_d_breakup_cutoff && qr_incld_not_small; + const auto dum2_gt_dum1 = dum2 >= p3_d_breakup_cutoff && qr_incld_not_small; dum.set(dum2_lt_dum1, 1); if (dum2_gt_dum1.any()) { - dum.set(dum2_gt_dum1, 2 - exp(2300 * (dum2-dum1))); + dum.set(dum2_gt_dum1, 2 - exp(2300 * (dum2-p3_d_breakup_cutoff))); } nr_selfcollect_tend.set(qr_incld_not_small, dum*sp(5.78)*nr_incld*qr_incld*rho); diff --git a/components/eamxx/src/physics/p3/p3_functions.hpp b/components/eamxx/src/physics/p3/p3_functions.hpp index 6a8fbc343b04..1749a4ae7346 100644 --- a/components/eamxx/src/physics/p3/p3_functions.hpp +++ b/components/eamxx/src/physics/p3/p3_functions.hpp @@ -72,6 +72,7 @@ struct Functions using KT = KokkosTypes; using C = scream::physics::Constants; + using CP3 = scream::physics::P3_Constants; template using view_1d = typename KT::template view_1d; @@ -460,7 +461,8 @@ struct Functions const uview_1d& precip_liq_flux, const uview_1d& qr_tend, const uview_1d& nr_tend, - Scalar& precip_liq_surf); + Scalar& precip_liq_surf, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void rain_sedimentation_disp( @@ -483,7 +485,8 @@ struct Functions const uview_2d& nr_tend, const uview_1d& precip_liq_surf, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif // TODO: comment @@ -508,7 +511,8 @@ struct Functions const uview_1d& qi_tend, const uview_1d& ni_tend, const view_ice_table& ice_table_vals, - Scalar& precip_ice_surf); + Scalar& precip_ice_surf, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void ice_sedimentation_disp( @@ -532,7 +536,8 @@ struct Functions const view_ice_table& ice_table_vals, const uview_1d& precip_ice_surf, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif // homogeneous freezing of cloud and rain @@ -620,7 +625,15 @@ struct Functions KOKKOS_FUNCTION static void get_rain_dsd2 ( const Spack& qr, Spack& nr, Spack& mu_r, - Spack& lamr, Spack& cdistr, Spack& logn0r, + Spack& lamr, + const physics::P3_Constants & p3constants, + const Smask& context = Smask(true) ); + + // Computes and returns additional rain size distribution parameters + KOKKOS_FUNCTION + static void get_cdistr_logn0r ( + const Spack& qr, const Spack& nr, const Spack& mu_r, + const Spack& lamr, Spack& cdistr, Spack& logn0r, const Smask& context = Smask(true) ); // Calculates rime density @@ -636,6 +649,7 @@ struct Functions static void cldliq_immersion_freezing(const Spack& T_atm, const Spack& lamc, const Spack& mu_c, const Spack& cdist1, const Spack& qc_incld, const Spack& inv_qc_relvar, Spack& qc2qi_hetero_freeze_tend, Spack& nc2ni_immers_freeze_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Computes the immersion freezing of rain @@ -643,6 +657,7 @@ struct Functions static void rain_immersion_freezing(const Spack& T_atm, const Spack& lamr, const Spack& mu_r, const Spack& cdistr, const Spack& qr_incld, Spack& qr2qi_immers_freeze_tend, Spack& nr2ni_immers_freeze_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Computes droplet self collection @@ -657,6 +672,7 @@ struct Functions static void cloud_rain_accretion(const Spack& rho, const Spack& inv_rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& qr_incld, const Spack& inv_qc_relvar, Spack& qc2qr_accret_tend, Spack& nc_accret_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Computes cloud water autoconversion process rate @@ -664,11 +680,13 @@ struct Functions static void cloud_water_autoconversion(const Spack& rho, const Spack& qc_incld, const Spack& nc_incld, const Spack& inv_qc_relvar, Spack& qc2qr_autoconv_tend, Spack& nc2nr_autoconv_tend, Spack& ncautr, - const Smask& context = Smask(true) ); + const physics::P3_Constants & p3constants, + const Smask& context = Smask(true)); // Computes rain self collection process rate KOKKOS_FUNCTION static void rain_self_collection(const Spack& rho, const Spack& qr_incld, const Spack& nr_incld, Spack& nr_selfcollect_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // Impose maximum ice number @@ -683,6 +701,7 @@ struct Functions KOKKOS_FUNCTION static Spack calc_bulk_rho_rime( const Spack& qi_tot, Spack& qi_rim, Spack& bi_rim, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true) ); // TODO - comment @@ -691,6 +710,7 @@ struct Functions const view_2d_table& vn_table_vals, const view_2d_table& vm_table_vals, const Spack& qr_incld, const Spack& rhofacr, Spack& nr_incld, Spack& mu_r, Spack& lamr, Spack& V_qr, Spack& V_nr, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true)); //--------------------------------------------------------------------------------- @@ -726,7 +746,8 @@ struct Functions const Spack& qi_incld, const Spack& qc_incld, const Spack& ni_incld, const Spack& nc_incld, Spack& qc2qi_collect_tend, Spack& nc_collect_tend, Spack& qc2qr_ice_shed_tend, Spack& ncshdc, - const Smask& context = Smask(true)); + const physics::P3_Constants & p3constants, + const Smask& context = Smask(true)); // TODO (comments) KOKKOS_FUNCTION @@ -736,6 +757,7 @@ struct Functions const Spack& qi_incld, const Spack& ni_incld, const Spack& qr_incld, Spack& qr2qi_collect_tend, Spack& nr_collect_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true)); // TODO (comments) @@ -820,6 +842,7 @@ struct Functions const Spack& qv_supersat_i, const Scalar& inv_dt, const bool& do_predict_nc, const bool& do_prescribed_CCN, Spack& qv2qi_nucleat_tend, Spack& ni_nucleat_tend, + const physics::P3_Constants & p3constants, const Smask& context = Smask(true)); KOKKOS_FUNCTION @@ -958,7 +981,8 @@ struct Functions const uview_1d& ni_incld, const uview_1d& bm_incld, bool& is_nucleat_possible, - bool& is_hydromet_present); + bool& is_hydromet_present, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void p3_main_part1_disp( @@ -1008,7 +1032,8 @@ struct Functions const uview_2d& ni_incld, const uview_2d& bm_incld, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif KOKKOS_FUNCTION @@ -1089,7 +1114,8 @@ struct Functions const uview_1d& pratot, const uview_1d& prctot, bool& is_hydromet_present, - const Int& nk); + const Int& nk, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void p3_main_part2_disp( @@ -1169,7 +1195,8 @@ struct Functions const uview_2d& pratot, const uview_2d& prctot, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif KOKKOS_FUNCTION @@ -1212,7 +1239,8 @@ struct Functions const uview_1d& rho_qi, const uview_1d& diag_equiv_reflectivity, const uview_1d& diag_eff_radius_qc, - const uview_1d& diag_eff_radius_qr); + const uview_1d& diag_eff_radius_qr, + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static void p3_main_part3_disp( @@ -1256,7 +1284,8 @@ struct Functions const uview_2d& diag_eff_radius_qc, const uview_2d& diag_eff_radius_qr, const uview_1d& is_nucleat_possible, - const uview_1d& is_hydromet_present); + const uview_1d& is_hydromet_present, + const physics::P3_Constants & p3constants); #endif // Return microseconds elapsed @@ -1270,7 +1299,8 @@ struct Functions const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk); // number of vertical cells per column + Int nk, // number of vertical cells per column + const physics::P3_Constants & p3constants); static Int p3_main_internal( const P3Runtime& runtime_options, @@ -1282,7 +1312,8 @@ struct Functions const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk); // number of vertical cells per column + Int nk, // number of vertical cells per column + const physics::P3_Constants & p3constants); #ifdef SCREAM_SMALL_KERNELS static Int p3_main_internal_disp( @@ -1295,7 +1326,8 @@ struct Functions const P3LookupTables& lookup_tables, const WorkspaceManager& workspace_mgr, Int nj, // number of columns - Int nk); // number of vertical cells per column + Int nk, // number of vertical cells per column + const physics::P3_Constants & p3constants); #endif KOKKOS_FUNCTION diff --git a/components/eamxx/src/physics/p3/p3_functions_f90.cpp b/components/eamxx/src/physics/p3/p3_functions_f90.cpp index 1f6a9d26d871..83aeb5b16d3d 100644 --- a/components/eamxx/src/physics/p3/p3_functions_f90.cpp +++ b/components/eamxx/src/physics/p3/p3_functions_f90.cpp @@ -1329,7 +1329,7 @@ void ice_sedimentation_f( nk, ktop, kbot, kdir, dt, inv_dt, qi_d, qi_incld_d, ni_d, ni_incld_d, qm_d, qm_incld_d, bm_d, bm_incld_d, qi_tend_d, ni_tend_d, ice_table_vals, - precip_ice_surf_k); + precip_ice_surf_k, physics::P3_Constants()); }, my_precip_ice_surf); *precip_ice_surf += my_precip_ice_surf; @@ -1402,7 +1402,7 @@ void rain_sedimentation_f( team, wsm.get_workspace(team), vn_table_vals, vm_table_vals, nk, ktop, kbot, kdir, dt, inv_dt, qr_d, nr_d, nr_incld_d, mu_r_d, lamr_d, precip_liq_flux_d, qr_tend_d, nr_tend_d, - precip_liq_surf_k); + precip_liq_surf_k, physics::P3_Constants()); }, my_precip_liq_surf); *precip_liq_surf += my_precip_liq_surf; @@ -1634,7 +1634,7 @@ void p3_main_part1_f( t_d, rho_d, inv_rho_d, qv_sat_l_d, qv_sat_i_d, qv_supersat_i_d, rhofacr_d, rhofaci_d, acn_d, qv_d, th_atm_d, qc_d, nc_d, qr_d, nr_d, qi_d, ni_d, qm_d, bm_d, qc_incld_d, qr_incld_d, qi_incld_d, qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, - bools_d(0), bools_d(1)); + bools_d(0), bools_d(1), physics::P3_Constants()); }); // Sync back to host @@ -1784,7 +1784,7 @@ void p3_main_part2_f( qm_incld_d, nc_incld_d, nr_incld_d, ni_incld_d, bm_incld_d, mu_c_d, nu_d, lamc_d, cdist_d, cdist1_d, cdistr_d, mu_r_d, lamr_d, logn0r_d, qv2qi_depos_tend_d, precip_total_tend_d, nevapr_d, qr_evap_tend_d, vap_liq_exchange_d, - vap_ice_exchange_d, liq_ice_exchange_d, pratot_d, prctot_d, bools_d(0),nk); + vap_ice_exchange_d, liq_ice_exchange_d, pratot_d, prctot_d, bools_d(0),nk, physics::P3_Constants()); }); // Sync back to host. Skip intent in variables. @@ -1902,7 +1902,7 @@ void p3_main_part3_f( latent_heat_sublim_d, mu_c_d, nu_d, lamc_d, mu_r_d, lamr_d, vap_liq_exchange_d, ze_rain_d, ze_ice_d, diag_vm_qi_d, diag_eff_radius_qi_d, diag_diam_qi_d, rho_qi_d, - diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d); + diag_equiv_reflectivity_d, diag_eff_radius_qc_d, diag_eff_radius_qr_d, physics::P3_Constants()); }); // Sync back to host @@ -2073,7 +2073,7 @@ Int p3_main_f( ekat::WorkspaceManager workspace_mgr(nk_pack, 52, policy); auto elapsed_microsec = P3F::p3_main(runtime_options, prog_state, diag_inputs, diag_outputs, infrastructure, - history_only, lookup_tables, workspace_mgr, nj, nk); + history_only, lookup_tables, workspace_mgr, nj, nk, physics::P3_Constants()); Kokkos::parallel_for(nj, KOKKOS_LAMBDA(const Int& i) { precip_liq_surf_temp_d(0, i / Spack::n)[i % Spack::n] = precip_liq_surf_d(i); diff --git a/components/eamxx/src/physics/p3/tests/CMakeLists.txt b/components/eamxx/src/physics/p3/tests/CMakeLists.txt index fd864e427815..4e8901069e05 100644 --- a/components/eamxx/src/physics/p3/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/p3/tests/CMakeLists.txt @@ -40,13 +40,13 @@ set(P3_TESTS_SRCS ) # P3_TESTS_SRCS if (SCREAM_DEBUG AND NOT SCREAM_TEST_SIZE STREQUAL "SHORT") - set (FORCE_RUN_DIFF_FAILS TRUE) + set (FORCE_RUN_DIFF_FAILS "WILL_FAIL") else () - set (FORCE_RUN_DIFF_FAILS FALSE) + set (FORCE_RUN_DIFF_FAILS "") endif() # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(p3_tests "${P3_TESTS_SRCS}" LIBS p3 THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} @@ -58,9 +58,9 @@ if (NOT SCREAM_BASELINES_ONLY) COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3;physics;fail" - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) + ${FORCE_RUN_DIFF_FAILS}) - if (NOT SCREAM_SMALL_KERNELS) + if (NOT SCREAM_SMALL_KERNELS) CreateUnitTest(p3_sk_tests "${P3_TESTS_SRCS}" LIBS p3_sk THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} @@ -72,12 +72,16 @@ if (NOT SCREAM_BASELINES_ONLY) COMPILER_CXX_DEFS SCREAM_FORCE_RUN_DIFF THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} LABELS "p3_sk;physics;fail" - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) + ${FORCE_RUN_DIFF_FAILS}) endif() endif() if (SCREAM_ENABLE_BASELINE_TESTS) - set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/p3_run_and_cmp.baseline") + if (SCREAM_ONLY_GENERATE_BASELINES) + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + else() + set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/p3_run_and_cmp.baseline") + endif() CreateUnitTestExec(p3_run_and_cmp "p3_run_and_cmp.cpp" LIBS p3 @@ -101,44 +105,15 @@ if (SCREAM_ENABLE_BASELINE_TESTS) EXE_ARGS "${BASELINE_FILE_ARG}" LABELS "p3;physics;fail" EXCLUDE_MAIN_CPP - PROPERTIES WILL_FAIL ${FORCE_RUN_DIFF_FAILS}) - - # - # Use fake tests to generate shell commands to generate baselines - # - CreateUnitTestFromExec(p3_baseline_f90_fake p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f -g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - - CreateUnitTestFromExec(p3_baseline_cxx_fake p3_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) + ${FORCE_RUN_DIFF_FAILS}) + # By default, baselines should be created using all fortran (ctest -L baseline_gen). If the user wants + # to use CXX to generate their baselines, they should use "ctest -L baseline_gen_cxx". + # Note: the baseline_gen label label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it if (SCREAM_TEST_MAX_THREADS GREATER 1) - get_test_property(p3_baseline_f90_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND P3_F90_GEN) - get_test_property(p3_baseline_cxx_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND P3_CXX_GEN) - else() - get_test_property(p3_baseline_f90_fake FULL_TEST_COMMAND P3_F90_GEN) - get_test_property(p3_baseline_cxx_fake FULL_TEST_COMMAND P3_CXX_GEN) + # ECUT only adds _ompX if we have more than one value of X, or if X>1 + set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) endif() - - if (P3_F90_GEN STREQUAL "NOTFOUND") - message(FATAL_ERROR "Could not get FULL_TEST_COMMAND for p3_baseline fake test") - endif() - - separate_arguments(P3_F90_GEN_ARGS UNIX_COMMAND "${P3_F90_GEN}") - separate_arguments(P3_CXX_GEN_ARGS UNIX_COMMAND "${P3_CXX_GEN}") - - add_custom_target(p3_baseline_f90 - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${P3_F90_GEN_ARGS}) - - add_custom_target(p3_baseline_cxx - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${P3_CXX_GEN_ARGS}) - - # By default, baselines should be created using all fortran (make baseline). If the user wants - # to use CXX to generate their baselines, they should use "make baseline_cxx". - add_dependencies(baseline p3_baseline_f90) - add_dependencies(baseline_cxx p3_baseline_cxx) + set_tests_properties (p3_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") + set_tests_properties (p3_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") endif() diff --git a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp index d9be4ed6b33e..aa9606e653e5 100644 --- a/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_autoconversion_unit_tests.cpp @@ -81,7 +81,7 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ } Functions::cloud_water_autoconversion(rho, qc_incld, nc_incld, - inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr); + inv_qc_relvar, qc2qr_autoconv_tend, nc2nr_autoconv_tend, ncautr, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -123,7 +123,8 @@ static void cloud_water_autoconversion_unit_bfb_tests(){ for(int si=0; si()); if((qc2qr_autoconv_tend < 0.0).any()){errors++;} } diff --git a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp index 4e2ccfbf5004..a9267449541c 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cldliq_imm_freezing_unit_tests.cpp @@ -94,7 +94,7 @@ static void run_bfb() Spack nc2ni_immers_freeze_tend{0.0}; Functions::cldliq_immersion_freezing(T_atm, lamc, mu_c, cdist1, qc_incld, inv_qc_relvar, - qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend); + qc2qi_hetero_freeze_tend, nc2ni_immers_freeze_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp index b494e108f056..6b9ed9f63181 100644 --- a/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_cloud_rain_acc_unit_tests.cpp @@ -97,7 +97,7 @@ static void run_bfb() Spack nc_accret_tend{0.0}; Functions::cloud_rain_accretion(rho, inv_rho, qc_incld, nc_incld, qr_incld, - inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend); + inv_qc_relvar, qc2qr_accret_tend, nc_accret_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp index 9adcc8078f24..0e00cf5acddb 100644 --- a/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_dsd2_unit_tests.cpp @@ -161,7 +161,8 @@ struct UnitWrap::UnitTest::TestDsd2 { } Spack mu_r(0.0), lamr(0.0), cdistr(0.0), logn0r(0.0); - Functions::get_rain_dsd2(qr, nr, mu_r, lamr, cdistr, logn0r); + Functions::get_rain_dsd2(qr, nr, mu_r, lamr, physics::P3_Constants()); + Functions::get_cdistr_logn0r(qr, nr, mu_r, lamr, cdistr, logn0r); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp index d9a0c6b1c636..bfad594acfb1 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_collection_unit_tests.cpp @@ -92,7 +92,7 @@ struct UnitWrap::UnitTest::TestIceCollection { Functions::ice_cldliq_collection(rho, temp, rhofaci, table_val_qc2qi_collect, qi_incld, qc_incld, ni_incld, nc_incld, - qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc); + qc2qi_collect_tend, nc_collect_tend, qc2qr_ice_shed_tend, ncshdc, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { @@ -181,7 +181,7 @@ struct UnitWrap::UnitTest::TestIceCollection { Spack qr2qi_collect_tend(0.0), nr_collect_tend(0.0); Functions::ice_rain_collection(rho, temp, rhofaci, logn0r, table_val_nr_collect, table_val_qr2qi_collect, qi_incld, ni_incld, qr_incld, - qr2qi_collect_tend, nr_collect_tend); + qr2qi_collect_tend, nr_collect_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp index ea30145ba001..cb26a7ee6c06 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_nucleation_unit_tests.cpp @@ -82,7 +82,7 @@ struct UnitWrap::UnitTest::TestIceNucleation { Spack qv2qi_nucleat_tend{0.0}; Spack ni_nucleat_tend{0.0}; Functions::ice_nucleation(temp, inv_rho, ni, ni_activated, qv_supersat_i, self_device(0).inv_dt, do_predict_nc, - do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend); + do_prescribed_CCN, qv2qi_nucleat_tend, ni_nucleat_tend, physics::P3_Constants()); for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { self_device(vs).qv2qi_nucleat_tend = qv2qi_nucleat_tend[s]; diff --git a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp index f5369cee2d9c..02167261bf86 100644 --- a/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_ice_sed_unit_tests.cpp @@ -98,7 +98,7 @@ static void run_bfb_calc_bulk_rhime() } Smask gt_small(qi_tot > qsmall); - Spack rho_rime = Functions::calc_bulk_rho_rime(qi_tot, qi_rim, bi_rim, gt_small); + Spack rho_rime = Functions::calc_bulk_rho_rime(qi_tot, qi_rim, bi_rim, physics::P3_Constants(), gt_small); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp index 51bd82ed3bd1..b74f734e87c4 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_imm_freezing_unit_tests.cpp @@ -92,7 +92,7 @@ static void run_bfb() Spack nr2ni_immers_freeze_tend{0.0}; Functions::rain_immersion_freezing(T_atm, lamr, mu_r, cdistr, qr_incld, - qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend); + qr2qi_immers_freeze_tend, nr2ni_immers_freeze_tend, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp index 8a1fa4969bdf..dfbf5f3a2f1f 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_sed_unit_tests.cpp @@ -96,7 +96,7 @@ static void run_bfb_rain_vel() Spack mu_r(0), lamr(0), V_qr(0), V_nr(0); Functions::compute_rain_fall_velocity( - vn_table_vals, vm_table_vals, qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr); + vn_table_vals, vm_table_vals, qr_incld, rhofacr, nr_incld, mu_r, lamr, V_qr, V_nr, physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp index 6902a1892775..5adc3ed71bab 100644 --- a/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp +++ b/components/eamxx/src/physics/p3/tests/p3_rain_self_collection_tests.cpp @@ -74,7 +74,8 @@ struct UnitWrap::UnitTest::TestRainSelfCollection { nr_selfcollect_tend_local[s] = dc_device(vs).nr_selfcollect_tend; } - Functions::rain_self_collection(rho_local, qr_incld_local, nr_incld_local, nr_selfcollect_tend_local); + Functions::rain_self_collection(rho_local, qr_incld_local, nr_incld_local, nr_selfcollect_tend_local, + physics::P3_Constants()); // Copy results back into views for (Int s = 0, vs = offset; s < Spack::n; ++s, ++vs) { diff --git a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt index cc4af5506d4a..5945189412c8 100644 --- a/components/eamxx/src/physics/rrtmgp/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/CMakeLists.txt @@ -2,6 +2,50 @@ include(EkatUtils) include(EkatSetCompilerFlags) include(ScreamUtils) +# Copied from EKAT, YAKL is an interface target so requires special +# handling. Get rid of this once RRTMGP is using kokkos. +macro (SetCudaFlagsYakl targetName) + if (Kokkos_ENABLE_CUDA) + # We must find CUDA + find_package(CUDA REQUIRED) + + # Still check if CUDA_FOUND is true, since we don't know if the particular + # FindCUDA.cmake module being used is checking _FIND_REQUIRED + if (NOT CUDA_FOUND) + message (FATAL_ERROR "Error! Unable to find CUDA.") + endif() + + set(options CUDA_LANG) + set(args1v) + set(argsMv FLAGS) + cmake_parse_arguments(SCF "${options}" "${args1v}" "${argsMv}" ${ARGN}) + + if (SCF_FLAGS) + set (FLAGS ${SCF_FLAGS}) + else () + # We need host-device lambdas + set (FLAGS --expt-extended-lambda) + + IsDebugBuild (SCF_DEBUG) + if (SCF_DEBUG) + # Turn off fused multiply add for debug so we can stay BFB with host + list (APPEND FLAGS --fmad=false) + endif() + endif() + + # Set the flags on the target + if (SCF_CUDA_LANG) + # User is setting the src files language to CUDA + target_compile_options (${targetName} INTERFACE + "$<$:${FLAGS}>") + else() + # We assume the user is setting the src files lang to CXX + target_compile_options (${targetName} INTERFACE + "$<$:${FLAGS}>") + endif() + endif() +endmacro() + ################################## # YAKL # ################################## @@ -17,8 +61,9 @@ if (TARGET yakl) else () # Prepare CUDA/HIP flags for YAKL if (CUDA_BUILD) + string(REPLACE ";" " " KOKKOS_CUDA_OPTIONS_STR "${KOKKOS_CUDA_OPTIONS}") set(YAKL_ARCH "CUDA") - set(YAKL_CUDA_FLAGS "-DYAKL_ARCH_CUDA --expt-extended-lambda --expt-relaxed-constexpr -ccbin ${CMAKE_CXX_COMPILER}") + set(YAKL_CUDA_FLAGS "-DYAKL_ARCH_CUDA ${KOKKOS_CUDA_OPTIONS_STR} --expt-relaxed-constexpr -ccbin ${CMAKE_CXX_COMPILER}") string (REPLACE " " ";" YAKL_CUDA_FLAGS_LIST ${YAKL_CUDA_FLAGS}) endif() if (HIP_BUILD) @@ -36,21 +81,20 @@ else () # EAMxx *requires* MPI, so simply look for it, then link against it find_package(MPI REQUIRED COMPONENTS C) - target_link_libraries (yakl MPI::MPI_C) - EkatDisableAllWarning(yakl) + target_link_libraries (yakl INTERFACE MPI::MPI_C) # For debug builds, set -DYAKL_DEBUG if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") - target_compile_definitions(yakl PUBLIC YAKL_DEBUG) + target_compile_definitions(yakl INTERFACE YAKL_DEBUG) endif() endif() # See eamxx/src/dynamics/homme/CMakeLists.txt for an explanation of this # workaround. if ((SCREAM_MACHINE STREQUAL "ascent" OR SCREAM_MACHINE STREQUAL "pm-gpu") AND CMAKE_BUILD_TYPE_ci STREQUAL "debug") - SetCudaFlags(yakl CUDA_LANG FLAGS -UNDEBUG) + SetCudaFlagsYakl(yakl CUDA_LANG FLAGS -UNDEBUG) else() - SetCudaFlags(yakl CUDA_LANG) + SetCudaFlagsYakl(yakl CUDA_LANG) endif() ################################## @@ -81,7 +125,7 @@ yakl_process_target(rrtmgp) # NOTE: cannot use 'PUBLIC' in target_link_libraries, # since yakl_process_target already used it # with the "plain" signature -target_link_libraries(rrtmgp yakl) +target_link_libraries(rrtmgp yakl Kokkos::kokkos) target_include_directories(rrtmgp PUBLIC ${SCREAM_BASE_DIR}/../../externals/YAKL ${EAM_RRTMGP_DIR}/external/cpp @@ -91,12 +135,6 @@ target_include_directories(rrtmgp PUBLIC ${EAM_RRTMGP_DIR}/external/cpp/rrtmgp/kernels ) -# The lines below are needed to ensure that kokkos_launch_compiler injects -# nvcc into compilations. rrtmgp uses YAKL, not kokkos, so the wrapper -# didn't know to add nvcc without these lines. -target_compile_definitions(rrtmgp PRIVATE KOKKOS_DEPENDENCE) -target_link_options(rrtmgp PRIVATE -DKOKKOS_DEPENDENCE) - # Build RRTMGP interface; note that we separate the SCREAM-specific RRTMGP interface # from the external core RRTMGP library because, ideally, the RRTMGP library has its # own build, and we would just use add_subdirectory() above to build it. Also, this @@ -125,7 +163,7 @@ yakl_process_target(scream_rrtmgp_yakl) # since yakl_process_target already used it # with the "plain" signature find_library(NETCDF_C netcdf HINTS ${NetCDF_C_PATH}/lib) -target_link_libraries(scream_rrtmgp_yakl ${NETCDF_C} rrtmgp scream_share) +target_link_libraries(scream_rrtmgp_yakl ${NETCDF_C} rrtmgp scream_share Kokkos::kokkos) target_include_directories(scream_rrtmgp_yakl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_include_directories(scream_rrtmgp_yakl SYSTEM PUBLIC @@ -142,7 +180,7 @@ set(SCREAM_RRTMGP_SOURCES ) add_library(scream_rrtmgp ${SCREAM_RRTMGP_SOURCES}) -target_link_libraries(scream_rrtmgp PUBLIC scream_share physics_share csm_share scream_rrtmgp_yakl) +target_link_libraries(scream_rrtmgp PUBLIC scream_share physics_share csm_share scream_rrtmgp_yakl Kokkos::kokkos) set_target_properties(scream_rrtmgp PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/modules ) diff --git a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp index 0d7a933ec3e0..16811885a3b9 100644 --- a/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp +++ b/components/eamxx/src/physics/rrtmgp/eamxx_rrtmgp_process_interface.cpp @@ -159,6 +159,7 @@ void RRTMGPRadiation::set_grids(const std::shared_ptr grids_ add_field("dtau067" , scalar3d_layout_mid, nondim, grid_name); add_field("dtau105" , scalar3d_layout_mid, nondim, grid_name); add_field("sunlit" , scalar2d_layout , nondim, grid_name); + add_field("cldfrac_rad" , scalar3d_layout_mid, nondim, grid_name); // Cloud-top diagnostics following AeroCOM recommendation add_field("T_mid_at_cldtop", scalar2d_layout, K, grid_name); add_field("p_mid_at_cldtop", scalar2d_layout, Pa, grid_name); @@ -468,6 +469,7 @@ void RRTMGPRadiation::run_impl (const double dt) { auto d_surf_lw_flux_up = get_field_in("surf_lw_flux_up").get_view(); // Output fields auto d_tmid = get_field_out("T_mid").get_view(); + auto d_cldfrac_rad = get_field_out("cldfrac_rad").get_view(); // Aerosol optics only exist if m_do_aerosol_rad is true, so declare views and copy from FM if so using view_3d = Field::view_dev_t; @@ -868,6 +870,7 @@ void RRTMGPRadiation::run_impl (const double dt) { } else { cldfrac_tot(i+1,k+1) = 0; } + d_cldfrac_rad(icol,k) = cldfrac_tot(i+1,k+1); }); }); } else { @@ -877,6 +880,7 @@ void RRTMGPRadiation::run_impl (const double dt) { const int icol = i + beg; Kokkos::parallel_for(Kokkos::TeamVectorRange(team, nlay), [&] (const int& k) { cldfrac_tot(i+1,k+1) = d_cldfrac_tot(icol,k); + d_cldfrac_rad(icol,k) = d_cldfrac_tot(icol,k); }); }); } diff --git a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt index 8a6a15948b4c..c6fcfae76f8d 100644 --- a/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/rrtmgp/tests/CMakeLists.txt @@ -1,6 +1,4 @@ -# NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) - +if (SCREAM_ONLY_GENERATE_BASELINES) # Build baseline code add_executable(generate_baseline generate_baseline.cpp) target_link_libraries(generate_baseline PUBLIC scream_rrtmgp rrtmgp_test_utils) @@ -8,23 +6,23 @@ if (NOT SCREAM_BASELINES_ONLY) # Generate allsky baseline with the usual cmake custom command-target pair pattern # Note: these "baselines" are not to compare scream with a previous version, but # rather to compare scream::rrtmgp with raw rrtmgp. - add_custom_command ( - OUTPUT ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc - COMMAND ${CMAKE_COMMAND} -E env $ - ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc - ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc - ) - add_custom_target(rrtmgp_allsky_baseline.nc - DEPENDS ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc + CreateUnitTestFromExec( + rrtmgp-allsky-baseline generate_baseline + LABELS baseline_gen rrtmgp + EXE_ARGS "${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" ) - CreateUnitTest(rrtmgp_tests rrtmgp_tests.cpp - LIBS scream_rrtmgp rrtmgp_test_utils - LABELS "rrtmgp;physics" - EXE_ARGS "-i ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc -b ${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc" - EXCLUDE_MAIN_CPP - ) - add_dependencies (rrtmgp_tests rrtmgp_allsky_baseline.nc) +else () + + if (SCREAM_ENABLE_BASELINE_TESTS) + # NOTE: tests inside this branch won't be built in a baselines-only build + CreateUnitTest(rrtmgp_tests rrtmgp_tests.cpp + LIBS scream_rrtmgp rrtmgp_test_utils + LABELS "rrtmgp;physics" + EXE_ARGS "-i ${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc -b ${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" + EXCLUDE_MAIN_CPP + ) + endif() CreateUnitTest(rrtmgp_unit_tests rrtmgp_unit_tests.cpp LIBS scream_rrtmgp rrtmgp_test_utils diff --git a/components/eamxx/src/physics/share/physics_constants.hpp b/components/eamxx/src/physics/share/physics_constants.hpp index 16750a791e3e..fd78dff8c86a 100644 --- a/components/eamxx/src/physics/share/physics_constants.hpp +++ b/components/eamxx/src/physics/share/physics_constants.hpp @@ -5,6 +5,7 @@ #include "ekat/util/ekat_string_utils.hpp" #include "ekat/ekat_scalar_traits.hpp" +#include "ekat/logging/ekat_logger.hpp" #include @@ -52,7 +53,6 @@ struct Constants static constexpr Scalar SXTH = 1.0/6.0; static constexpr Scalar PIOV3 = Pi*THIRD; static constexpr Scalar PIOV6 = Pi*SXTH; - static constexpr Scalar AIMM = 0.65; static constexpr Scalar BIMM = 2.0; static constexpr Scalar CONS1 = PIOV6*RHOW; static constexpr Scalar CONS2 = 4.*PIOV3*RHOW; @@ -76,13 +76,8 @@ struct Constants static constexpr Scalar INV_CP = 1.0/CP; // static constexpr Scalar Tol = ekat::is_single_precision::value ? 2e-5 : 1e-14; static constexpr Scalar macheps = std::numeric_limits::epsilon(); - static constexpr Scalar mu_r_const = 1.0; static constexpr Scalar dt_left_tol = 1.e-4; static constexpr Scalar bcn = 2.; - static constexpr Scalar rho_rimeMin = 50.; - static constexpr Scalar rho_rimeMax = 900.; - static constexpr Scalar eci = 0.5; - static constexpr Scalar eri = 1.0; static constexpr Scalar dropmass = 5.2e-7; static constexpr Scalar NCCNST = 200.0e+6; static constexpr Scalar incloud_limit = 5.1e-3; @@ -126,6 +121,121 @@ struct Constants static constexpr Scalar earth_ellipsoid3 = 1.175; // third expansion coefficient for WGS84 ellipsoid }; +template +struct P3_Constants +{ + public: + Scalar p3_autoconversion_prefactor = 1350.0; + Scalar p3_mu_r_constant = 1.0; + Scalar p3_spa_to_nc = 1.0; + Scalar p3_k_accretion = 67.0; + Scalar p3_eci = 0.5; + Scalar p3_eri = 1.0; + Scalar p3_rho_rime_min = 50.0; + Scalar p3_rho_rime_max = 900.0; + Scalar p3_a_imm = 0.65; + Scalar p3_dep_nucleation_exponent = 0.304; + Scalar p3_ice_sed_knob = 1.0; + Scalar p3_d_breakup_cutoff = 0.00028; + + void set_p3_from_namelist(ekat::ParameterList ¶ms){ + + std::string nname = "p3_autoconversion_prefactor"; + if(params.isParameter(nname)) + p3_autoconversion_prefactor = params.get(nname); + + nname = "p3_mu_r_constant"; + if(params.isParameter(nname)) + p3_mu_r_constant = params.get(nname); + + nname = "p3_spa_to_nc"; + if(params.isParameter(nname)) + p3_spa_to_nc = params.get(nname); + + nname = "p3_k_accretion"; + if(params.isParameter(nname)) + p3_k_accretion = params.get(nname); + + nname = "p3_eci"; + if(params.isParameter(nname)) + p3_eci = params.get(nname); + + nname = "p3_eri"; + if(params.isParameter(nname)) + p3_eri = params.get(nname); + + nname = "p3_rho_rime_min"; + if(params.isParameter(nname)) + p3_rho_rime_min = params.get(nname); + + nname = "p3_rho_rime_max"; + if(params.isParameter(nname)) + p3_rho_rime_max = params.get(nname); + + nname = "p3_a_imm"; + if(params.isParameter(nname)) + p3_a_imm = params.get(nname); + + nname = "p3_dep_nucleation_exponent"; + if(params.isParameter(nname)) + p3_dep_nucleation_exponent = params.get(nname); + + nname = "p3_ice_sed_knob"; + if(params.isParameter(nname)) + p3_ice_sed_knob = params.get(nname); + + nname = "p3_d_breakup_cutoff"; + if(params.isParameter(nname)) + p3_d_breakup_cutoff = params.get(nname); + + }; + + void print_p3constants(std::shared_ptr logger){ + logger->info("P3 Constants:"); + + std::string nname = "p3_autoconversion_prefactor"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_autoconversion_prefactor)); + + nname = "p3_mu_r_constant"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_mu_r_constant)); + + nname = "p3_spa_to_nc"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_spa_to_nc)); + + nname = "p3_k_accretion"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_k_accretion)); + + nname = "p3_eci"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_eci)); + + nname = "p3_eri"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_eri)); + + nname = "p3_rho_rime_min"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_rho_rime_min)); + + nname = "p3_rho_rime_max"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_rho_rime_max)); + + nname = "p3_a_imm"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_a_imm)); + + nname = "p3_dep_nucleation_exponent"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_dep_nucleation_exponent)); + + nname = "p3_ice_sed_knob"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_ice_sed_knob)); + + nname = "p3_d_breakup_cutoff"; + logger->info(std::string("P3 ") + nname + std::string(" = ") + std::to_string(p3_d_breakup_cutoff)); + + logger->info(" "); + }; + + //one can implement a check here too, for acceptable ranges + +}; // P3_Constants + // Gases // Define the molecular weight for each gas, which can then be // used to determine the volume mixing ratio for each gas. diff --git a/components/eamxx/src/physics/share/tests/CMakeLists.txt b/components/eamxx/src/physics/share/tests/CMakeLists.txt index f3849b7571bf..a196c8470581 100644 --- a/components/eamxx/src/physics/share/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/share/tests/CMakeLists.txt @@ -1,14 +1,18 @@ include(ScreamUtils) # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(physics_test_data physics_test_data_unit_tests.cpp LIBS physics_share THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC}) endif() if (SCREAM_ENABLE_BASELINE_TESTS) - set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/physics_saturation.baseline") + if (SCREAM_ONLY_GENERATE_BASELINES) + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/physics_saturation.baseline") + else() + set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/physics_saturation.baseline") + endif() # The comparison test. Expects baseline to exist. All thread configurations # will use the same baseline. @@ -17,23 +21,6 @@ if (SCREAM_ENABLE_BASELINE_TESTS) EXE_ARGS "${BASELINE_FILE_ARG}" LABELS "physics") - # - # Use fake tests to generate shell commands to generate baselines - # - CreateUnitTestFromExec(physics_saturation_baseline_fake physics_saturation_run_and_cmp - EXE_ARGS "-g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - - get_test_property(physics_saturation_baseline_fake FULL_TEST_COMMAND PHYSICS_SATURATION_GEN) - - if (PHYSICS_SATURATION_GEN STREQUAL "NOTFOUND") - message(FATAL_ERROR "Could not get FULL_TEST_COMMAND for physics_saturation_baseline fake test") - endif() - - separate_arguments(PHYSICS_SATURATION_GEN_ARGS UNIX_COMMAND "${PHYSICS_SATURATION_GEN}") - - add_custom_target(physics_saturation_baseline - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${PHYSICS_SATURATION_GEN_ARGS}) - - add_dependencies(baseline physics_saturation_baseline) + # Note: the baseline_gen label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it + set_tests_properties(physics_saturation_run_and_cmp PROPERTIES LABELS "baseline_gen;baseline_cmp") endif() diff --git a/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp b/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp index cbf9945d1445..2fcb68f6337c 100644 --- a/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp +++ b/components/eamxx/src/physics/share/tests/physics_saturation_run_and_cmp.cpp @@ -50,7 +50,7 @@ struct UnitWrap::UnitTest::TestSaturation sat_ice_fp = physics::polysvp1(temps, true, Smask(true))[0]; sat_liq_fp = physics::polysvp1(temps, false, Smask(true))[0]; - //Functions::qv_sat_dry(const Spack& t_atm, const Spack& p_atm_dry, const bool ice, const Smask& range_mask, + //Functions::qv_sat_dry(const Spack& t_atm, const Spack& p_atm_dry, const bool ice, const Smask& range_mask, // const SaturationFcn func_idx, const char* caller) mix_ice_fr = physics::qv_sat_dry(temps, pres, true, Smask(true), physics::Polysvp1)[0]; mix_liq_fr = physics::qv_sat_dry(temps, pres, false,Smask(true), physics::Polysvp1)[0]; @@ -109,24 +109,27 @@ struct UnitWrap::UnitTest::TestSaturation EKAT_REQUIRE_MSG( fid, "generate_baseline can't write " << filename); for (auto p : params_) { - OutputData d; ParamSet ps = p; - Kokkos::parallel_reduce(1, - KOKKOS_LAMBDA(const size_t&, OutputData& output) { + Kokkos::View d_dev("",1); + Kokkos::parallel_for(1, + KOKKOS_LAMBDA(const size_t&) { TestSaturation::saturation_tests( ps.temperature, ps.pressure, - output.sat_ice_fp, - output.sat_liq_fp, - output.mix_ice_fr, - output.mix_liq_fr, - output.sat_ice_mkp, - output.sat_liq_mkp, - output.mix_ice_mkr, - output.mix_liq_mkr); - }, d); - + d_dev[0].sat_ice_fp, + d_dev[0].sat_liq_fp, + d_dev[0].mix_ice_fr, + d_dev[0].mix_liq_fr, + d_dev[0].sat_ice_mkp, + d_dev[0].sat_liq_mkp, + d_dev[0].mix_ice_mkr, + d_dev[0].mix_liq_mkr); + }); Kokkos::fence(); - write(fid, d); // Save the fields to the baseline file. + + auto d_host = Kokkos::create_mirror_view(d_dev); + Kokkos::deep_copy(d_host,d_dev); + + write(fid, d_host[0]); // Save the fields to the baseline file. } return 0; @@ -139,28 +142,31 @@ struct UnitWrap::UnitTest::TestSaturation int case_num = 0; for (auto p : params_) { ++case_num; - OutputData ref, d; + OutputData ref; ParamSet ps = p; std::cout << "--- checking physics saturation case # " << case_num << std::endl; read(fid, ref); - Kokkos::parallel_reduce(1, - KOKKOS_LAMBDA(const size_t&, OutputData& output) { + Kokkos::View d_dev("",1); + Kokkos::parallel_for(1, + KOKKOS_LAMBDA(const size_t&) { TestSaturation::saturation_tests( ps.temperature, ps.pressure, - output.sat_ice_fp, - output.sat_liq_fp, - output.mix_ice_fr, - output.mix_liq_fr, - output.sat_ice_mkp, - output.sat_liq_mkp, - output.mix_ice_mkr, - output.mix_liq_mkr); - }, d); - + d_dev[0].sat_ice_fp, + d_dev[0].sat_liq_fp, + d_dev[0].mix_ice_fr, + d_dev[0].mix_liq_fr, + d_dev[0].sat_ice_mkp, + d_dev[0].sat_liq_mkp, + d_dev[0].mix_ice_mkr, + d_dev[0].mix_liq_mkr); + }); Kokkos::fence(); - ne = compare(tol, ref, d); + auto d_host = Kokkos::create_mirror_view(d_dev); + Kokkos::deep_copy(d_host,d_dev); + + ne = compare(tol, ref, d_host[0]); if (ne) std::cout << "Ref impl failed.\n"; nerr += ne; } @@ -189,6 +195,7 @@ struct UnitWrap::UnitTest::TestSaturation Scalar mix_ice_mkr; Scalar mix_liq_mkr; + KOKKOS_INLINE_FUNCTION OutputData& operator+=(const OutputData& rhs) { sat_ice_fp += rhs.sat_ice_fp; diff --git a/components/eamxx/src/physics/shoc/CMakeLists.txt b/components/eamxx/src/physics/shoc/CMakeLists.txt index 051f9635d179..e37379095d00 100644 --- a/components/eamxx/src/physics/shoc/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/CMakeLists.txt @@ -108,7 +108,7 @@ if (SCREAM_SMALL_KERNELS) add_library(shoc ${SHOC_SRCS} ${SHOC_SK_SRCS}) else() add_library(shoc ${SHOC_SRCS}) - if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_BASELINES_ONLY) + if (NOT SCREAM_LIBS_ONLY AND NOT SCREAM_ONLY_GENERATE_BASELINES) add_library(shoc_sk ${SHOC_SRCS} ${SHOC_SK_SRCS}) # Always build shoc_sk with SCREAM_SMALL_KERNELS on target_compile_definitions(shoc_sk PUBLIC "SCREAM_SMALL_KERNELS") diff --git a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp index c107e53b5b31..f0c368549400 100644 --- a/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp +++ b/components/eamxx/src/physics/shoc/eamxx_shoc_process_interface.cpp @@ -432,7 +432,7 @@ void SHOCMacrophysics::initialize_impl (const RunType run_type) if (m_grid->has_geometry_data("dx_short")) { // We must be running with IntensiveObservationPeriod on, with a planar geometry auto dx = m_grid->get_geometry_data("dx_short").get_view()(); - Kokkos::deep_copy(cell_length, dx); + Kokkos::deep_copy(cell_length, dx*1000); // convert km -> m } else { const auto area = m_grid->get_geometry_data("area").get_view(); const auto lat = m_grid->get_geometry_data("lat").get_view(); diff --git a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt index 485f625e1423..87e55959c528 100644 --- a/components/eamxx/src/physics/shoc/tests/CMakeLists.txt +++ b/components/eamxx/src/physics/shoc/tests/CMakeLists.txt @@ -72,7 +72,7 @@ set(SHOC_TESTS_SRCS ) # SHOC_TESTS_SRCS # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT SCREAM_BASELINES_ONLY) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) CreateUnitTest(shoc_tests "${SHOC_TESTS_SRCS}" LIBS shoc THREADS 1 ${SCREAM_TEST_MAX_THREADS} ${SCREAM_TEST_THREAD_INC} @@ -88,7 +88,11 @@ if (NOT SCREAM_BASELINES_ONLY) endif() if (SCREAM_ENABLE_BASELINE_TESTS) - set(BASELINE_FILE_ARG "-b ${SCREAM_TEST_DATA_DIR}/shoc_run_and_cmp.baseline") + if (SCREAM_ONLY_GENERATE_BASELINES) + set(BASELINE_FILE_ARG "-g -b ${SCREAM_BASELINES_DIR}/data/shoc_run_and_cmp.baseline") + else() + set(BASELINE_FILE_ARG "-b ${SCREAM_BASELINES_DIR}/data/shoc_run_and_cmp.baseline") + endif() CreateUnitTestExec(shoc_run_and_cmp "shoc_run_and_cmp.cpp" LIBS shoc @@ -104,42 +108,13 @@ if (SCREAM_ENABLE_BASELINE_TESTS) EXE_ARGS "-f ${BASELINE_FILE_ARG}" LABELS "shoc;physics") - # - # Use fake tests to generate shell commands to generate baselines - # - CreateUnitTestFromExec(shoc_baseline_f90_fake shoc_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-f -g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - - CreateUnitTestFromExec(shoc_baseline_cxx_fake shoc_run_and_cmp - THREADS ${SCREAM_TEST_MAX_THREADS} - EXE_ARGS "-g ${BASELINE_FILE_ARG}" - PROPERTIES DISABLED True) - + # By default, baselines should be created using all fortran (ctest -L baseline_gen). If the user wants + # to use CXX to generate their baselines, they should use "ctest -L baseline_gen_cxx". + # Note: the baseline_gen label is really only used if SCREAM_ONLY_GENERATE_BASELINES=ON, but no harm adding it if (SCREAM_TEST_MAX_THREADS GREATER 1) - get_test_property(shoc_baseline_f90_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND SHOC_F90_GEN) - get_test_property(shoc_baseline_cxx_fake_omp${SCREAM_TEST_MAX_THREADS} FULL_TEST_COMMAND SHOC_CXX_GEN) - else() - get_test_property(shoc_baseline_f90_fake FULL_TEST_COMMAND SHOC_F90_GEN) - get_test_property(shoc_baseline_cxx_fake FULL_TEST_COMMAND SHOC_CXX_GEN) + # ECUT only adds _ompX if we have more than one value of X, or if X>1 + set (TEST_SUFFIX _omp${SCREAM_TEST_MAX_THREADS}) endif() - - if (SHOC_F90_GEN STREQUAL "NOTFOUND") - message(FATAL_ERROR "Could not get FULL_TEST_COMMAND for shoc_baseline fake test") - endif() - - separate_arguments(SHOC_F90_GEN_ARGS UNIX_COMMAND "${SHOC_F90_GEN}") - separate_arguments(SHOC_CXX_GEN_ARGS UNIX_COMMAND "${SHOC_CXX_GEN}") - - add_custom_target(shoc_baseline_f90 - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${SHOC_F90_GEN_ARGS}) - - add_custom_target(shoc_baseline_cxx - COMMAND ${CMAKE_COMMAND} -E env OMP_NUM_THREADS=${SCREAM_TEST_MAX_THREADS} ${SHOC_CXX_GEN_ARGS}) - - # By default, baselines should be created using all fortran (make baseline). If the user wants - # to use CXX to generate their baselines, they should use "make baseline_cxx". - add_dependencies(baseline shoc_baseline_f90) - add_dependencies(baseline_cxx shoc_baseline_cxx) + set_tests_properties (shoc_run_and_cmp_f90${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;baseline_cmp") + set_tests_properties (shoc_run_and_cmp_cxx${TEST_SUFFIX} PROPERTIES LABELS "baseline_gen;cxx baseline_cmp") endif() diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp index 6b59ce16a877..f244a2262f9c 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.cpp @@ -3,7 +3,6 @@ #include "share/util/scream_time_stamp.hpp" #include "share/io/scream_scorpio_interface.hpp" #include "share/property_checks/field_within_interval_check.hpp" -#include "share/property_checks/field_lower_bound_check.hpp" #include "ekat/ekat_assert.hpp" #include "ekat/util/ekat_units.hpp" @@ -17,7 +16,8 @@ namespace scream SPA::SPA (const ekat::Comm& comm, const ekat::ParameterList& params) : AtmosphereProcess(comm, params) { - // Nothing to do here + EKAT_REQUIRE_MSG(m_params.isParameter("spa_data_file"), + "ERROR: spa_data_file is missing from SPA parameter list."); } // ========================================================================================= @@ -36,42 +36,109 @@ void SPA::set_grids(const std::shared_ptr grids_manager) const auto& grid_name = m_grid->name(); m_num_cols = m_grid->get_num_local_dofs(); // Number of columns on this rank m_num_levs = m_grid->get_num_vertical_levels(); // Number of levels per column - m_dofs_gids = m_grid->get_dofs_gids().get_view(); - m_min_global_dof = m_grid->get_global_min_dof_gid(); // Define the different field layouts that will be used for this process - // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces + // Layout for 3D (2d horiz X 1d vertical) variable defined at mid-level and interfaces FieldLayout scalar3d_layout_mid { {COL,LEV}, {m_num_cols, m_num_levs} }; FieldLayout scalar2d_layout { {COL}, {m_num_cols} }; FieldLayout scalar1d_layout_mid { {LEV}, {m_num_levs} }; // Use VAR field tag for gases for now; consider adding a tag? - FieldLayout scalar3d_swband_layout { {COL,SWBND, LEV}, {m_num_cols, m_nswbands, m_num_levs} }; - FieldLayout scalar3d_lwband_layout { {COL,LWBND, LEV}, {m_num_cols, m_nlwbands, m_num_levs} }; + FieldLayout scalar3d_swband_layout { {COL,SWBND, LEV}, {m_num_cols, m_nswbands, m_num_levs} }; + FieldLayout scalar3d_lwband_layout { {COL,LWBND, LEV}, {m_num_cols, m_nlwbands, m_num_levs} }; // Set of fields used strictly as input - constexpr int ps = Pack::n; + constexpr int ps = Spack::n; add_field("p_mid" , scalar3d_layout_mid, Pa, grid_name, ps); // Set of fields used strictly as output - add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name,ps); - add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name,ps); - add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name,ps); - add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name,ps); - add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name,ps); - - // Init output data structure + add_field("nccn", scalar3d_layout_mid, 1/kg, grid_name, ps); + add_field("aero_g_sw", scalar3d_swband_layout, nondim, grid_name, ps); + add_field("aero_ssa_sw", scalar3d_swband_layout, nondim, grid_name, ps); + add_field("aero_tau_sw", scalar3d_swband_layout, nondim, grid_name, ps); + add_field("aero_tau_lw", scalar3d_lwband_layout, nondim, grid_name, ps); + + // We can already create some of the spa structures + + // 1. Create SPAHorizInterp remapper + auto spa_data_file = m_params.get("spa_data_file"); + auto spa_map_file = m_params.get("spa_remap_file",""); + + // IOP cases cannot have a remap file. IOP file reader is itself a remaper, + // where a single column of data corresponding to the closest lat/lon pair to + // the IOP lat/lon parameters is read from file, and that column data is mapped + // to all columns of the IdentityRemapper source fields. + EKAT_REQUIRE_MSG(spa_map_file == "" or spa_map_file == "None" or not m_iop, + "Error! Cannot define spa_remap_file for cases with an Intensive Observation Period defined. " + "The IOP class defines it's own remap from file data -> model data.\n"); + + SPAHorizInterp = SPAFunc::create_horiz_remapper (m_grid,spa_data_file,spa_map_file, m_iop!=nullptr); + + // Grab a sw and lw field from the horiz interp, and check sw/lw dim against what we hardcoded in this class + auto nswbands_data = SPAHorizInterp->get_src_field(4).get_header().get_identifier().get_layout().dim(SWBND); + auto nlwbands_data = SPAHorizInterp->get_src_field(5).get_header().get_identifier().get_layout().dim(LWBND); + EKAT_REQUIRE_MSG (nswbands_data==m_nswbands, + "Error! Spa data file has a different number of sw bands than the model.\n" + " - spa data swbands: " + std::to_string(nswbands_data) + "\n" + " - model swbands : " + std::to_string(m_nswbands) + "\n"); + EKAT_REQUIRE_MSG (nlwbands_data==m_nlwbands, + "Error! Spa data file has a different number of lw bands than the model.\n" + " - spa data lwbands: " + std::to_string(nlwbands_data) + "\n" + " - model lwbands : " + std::to_string(m_nlwbands) + "\n"); + + const auto io_grid = SPAHorizInterp->get_src_grid(); + + // 2. Initialize the size of the SPAData structures. + // Note: add 2 to number of levels to allow extrapolation if model pressure is outside the data range + m_num_src_levs = io_grid->get_num_vertical_levels(); + SPAData_start = SPAFunc::SPAInput(m_num_cols, m_num_src_levs+2, m_nswbands, m_nlwbands); + SPAData_end = SPAFunc::SPAInput(m_num_cols, m_num_src_levs+2, m_nswbands, m_nlwbands); SPAData_out.init(m_num_cols,m_num_levs,m_nswbands,m_nlwbands,false); - // Note: only the number of levels associated with this data haven't been set. We can - // take this information directly from the spa data file. - m_spa_data_file = m_params.get("spa_data_file"); - scorpio::register_file(m_spa_data_file,scorpio::Read); - m_num_src_levs = scorpio::get_dimlen(m_spa_data_file,"lev"); - scorpio::eam_pio_closefile(m_spa_data_file); - SPAHorizInterp.m_comm = m_comm; + // 3 Read in hyam/hybm in start/end data, and pad them + Field hyam(FieldIdentifier("hyam",io_grid->get_vertical_layout(true),nondim,io_grid->name())); + Field hybm(FieldIdentifier("hybm",io_grid->get_vertical_layout(true),nondim,io_grid->name())); + hyam.allocate_view(); + hybm.allocate_view(); + + AtmosphereInput hvcoord_reader(spa_data_file,io_grid,{hyam,hybm},true); + hvcoord_reader.read_variables(); + hvcoord_reader.finalize(); + + // Do the copy on host, cause it's just easier, + // and this is just a cheap loop during init + auto hyam_h = hyam.get_view(); + auto hybm_h = hybm.get_view(); + auto nlevs = io_grid->get_num_vertical_levels(); + for (auto data : {SPAData_start, SPAData_end} ) { + auto spa_hyam = ekat::scalarize(data.hyam); + auto spa_hybm = ekat::scalarize(data.hybm); + auto spa_hyam_h = Kokkos::create_mirror_view(spa_hyam); + auto spa_hybm_h = Kokkos::create_mirror_view(spa_hybm); + for (int i=0; i(); - SPAData_out.AER_G_SW = get_field_out("aero_g_sw").get_view(); - SPAData_out.AER_SSA_SW = get_field_out("aero_ssa_sw").get_view(); - SPAData_out.AER_TAU_SW = get_field_out("aero_tau_sw").get_view(); - SPAData_out.AER_TAU_LW = get_field_out("aero_tau_lw").get_view(); - - // Retrieve the remap and data file locations from the parameter list: - EKAT_REQUIRE_MSG(m_params.isParameter("spa_remap_file"),"ERROR: spa_remap_file is missing from SPA parameter list."); - EKAT_REQUIRE_MSG(m_params.isParameter("spa_data_file"),"ERROR: spa_data_file is missing from SPA parameter list."); - m_spa_remap_file = m_params.get("spa_remap_file"); - - // Set the SPA remap weights. - // TODO: We may want to provide an option to calculate weights on-the-fly. - // If so, then the EKAT_REQUIRE_MSG above will need to be removed and - // we can have a default m_spa_data_file option that is online calculation. - using ci_string = ekat::CaseInsensitiveString; - ci_string no_filename = "none"; - if (m_spa_remap_file == no_filename) { - if (m_comm.am_i_root()) { - printf("WARNING: spa_remap_file has been set to 'NONE', assuming that SPA data and simulation are on the same grid - skipping horizontal interpolation\n"); - } - SPAFunc::set_remap_weights_one_to_one(m_min_global_dof,m_dofs_gids,SPAHorizInterp); - } else { - SPAFunc::get_remap_weights_from_file(m_spa_remap_file,m_min_global_dof,m_dofs_gids,SPAHorizInterp); - } - - // Initialize the size of the SPAData structures: add 2 to number of levels for padding - SPAData_start = SPAFunc::SPAInput(m_dofs_gids.size(), m_num_src_levs+2, m_nswbands, m_nlwbands); - SPAData_end = SPAFunc::SPAInput(m_dofs_gids.size(), m_num_src_levs+2, m_nswbands, m_nlwbands); - - // Update the local time state information and load the first set of SPA data for interpolation: - auto ts = timestamp(); - SPATimeState.inited = false; - SPATimeState.current_month = ts.get_month(); - SPAFunc::update_spa_timestate(m_spa_data_file,m_nswbands,m_nlwbands,ts,SPAHorizInterp,SPATimeState,SPAData_start,SPAData_end); - - // Set property checks for fields in this process + // Initialize SPAData_out with the views from the out fields + SPAData_out.CCN3 = get_field_out("nccn").get_view(); + SPAData_out.AER_G_SW = get_field_out("aero_g_sw").get_view(); + SPAData_out.AER_SSA_SW = get_field_out("aero_ssa_sw").get_view(); + SPAData_out.AER_TAU_SW = get_field_out("aero_tau_sw").get_view(); + SPAData_out.AER_TAU_LW = get_field_out("aero_tau_lw").get_view(); + + // Load the first month into spa_end. + // Note: At the first time step, the data will be moved into spa_beg, + // and spa_end will be reloaded from file with the new month. + const int curr_month = timestamp().get_month()-1; // 0-based + SPAFunc::update_spa_data_from_file(SPADataReader,SPAIOPDataReader,timestamp(),curr_month,*SPAHorizInterp,SPAData_end); + + // 6. Set property checks for fields in this process using Interval = FieldWithinIntervalCheck; const auto eps = std::numeric_limits::epsilon(); @@ -208,10 +250,10 @@ void SPA::run_impl (const double dt) /* Update the SPATimeState to reflect the current time, note the addition of dt */ SPATimeState.t_now = ts.frac_of_year_in_days(); /* Update time state and if the month has changed, update the data.*/ - SPAFunc::update_spa_timestate(m_spa_data_file,m_nswbands,m_nlwbands,ts,SPAHorizInterp,SPATimeState,SPAData_start,SPAData_end); + SPAFunc::update_spa_timestate(SPADataReader,SPAIOPDataReader,ts,*SPAHorizInterp,SPATimeState,SPAData_start,SPAData_end); // Call the main SPA routine to get interpolated aerosol forcings. - const auto& pmid_tgt = get_field_in("p_mid").get_view(); + const auto& pmid_tgt = get_field_in("p_mid").get_view(); SPAFunc::spa_main(SPATimeState, pmid_tgt, m_buffer.p_mid_src, SPAData_start,SPAData_end,m_buffer.spa_temp,SPAData_out); } diff --git a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp index b2ca826d65b5..b2d9f2b36388 100644 --- a/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp +++ b/components/eamxx/src/physics/spa/eamxx_spa_process_interface.hpp @@ -3,8 +3,9 @@ #include "physics/spa/spa_functions.hpp" #include "share/atm_process/atmosphere_process.hpp" -#include "ekat/ekat_parameter_list.hpp" -#include "ekat/util/ekat_lin_interp.hpp" +#include "share/io/scorpio_input.hpp" +#include "share/grid/remap/abstract_remapper.hpp" +#include #include @@ -21,20 +22,13 @@ namespace scream class SPA : public AtmosphereProcess { public: - using gid_type = AbstractGrid::gid_type; + using SPAFunc = spa::SPAFunctions; + using Spack = SPAFunc::Spack; + using KT = ekat::KokkosTypes; - using SPAFunc = spa::SPAFunctions; - using Spack = SPAFunc::Spack; - using Pack = ekat::Pack; - using KT = ekat::KokkosTypes; + using view_1d = typename SPAFunc::view_1d; + using view_2d = typename SPAFunc::view_2d; - using view_1d = typename SPAFunc::view_1d; - using view_2d = typename SPAFunc::view_2d; - using view_3d = typename SPAFunc::view_3d; - using view_1d_dof = typename SPAFunc::view_1d; - - template - using uview_1d = Unmanaged>; template using uview_2d = Unmanaged>; @@ -74,34 +68,30 @@ class SPA : public AtmosphereProcess void init_buffers(const ATMBufferManager &buffer_manager); // Keep track of field dimensions and the iteration count - int m_num_cols; + int m_num_cols; int m_num_levs; int m_num_src_levs; - int m_nk_pack; int m_nswbands = 14; int m_nlwbands = 16; - // DOF information - view_1d_dof m_dofs_gids; - int m_total_global_dofs; // Needed to make sure that remap data matches grid. - gid_type m_min_global_dof; - // Struct which contains temporary variables used during spa_main Buffer m_buffer; - // SPA specific files - std::string m_spa_remap_file; - std::string m_spa_data_file; + // IO structure to read in data for standard grids (keep it around to avoid re-creating PIO decomps) + std::shared_ptr SPADataReader; + // Similar to above, but stores info to read data for IOP grid + std::shared_ptr SPAIOPDataReader; // Structures to store the data used for interpolation + std::shared_ptr SPAHorizInterp; + SPAFunc::SPATimeState SPATimeState; - SPAFunc::SPAHorizInterp SPAHorizInterp; SPAFunc::SPAInput SPAData_start; SPAFunc::SPAInput SPAData_end; SPAFunc::SPAOutput SPAData_out; std::shared_ptr m_grid; -}; // class SPA +}; // class SPA } // namespace scream diff --git a/components/eamxx/src/physics/spa/spa_functions.hpp b/components/eamxx/src/physics/spa/spa_functions.hpp index 5a6efebe782e..631aae9eee78 100644 --- a/components/eamxx/src/physics/spa/spa_functions.hpp +++ b/components/eamxx/src/physics/spa/spa_functions.hpp @@ -1,15 +1,14 @@ #ifndef SPA_FUNCTIONS_HPP #define SPA_FUNCTIONS_HPP +#include "control/intensive_observation_period.hpp" #include "share/grid/abstract_grid.hpp" -#include "share/grid/remap/horizontal_remap_utility.hpp" -#include "share/scream_types.hpp" +#include "share/grid/remap/abstract_remapper.hpp" +#include "share/io/scorpio_input.hpp" #include "share/util/scream_time_stamp.hpp" +#include "share/scream_types.hpp" -#include "ekat/ekat_pack_kokkos.hpp" -#include "ekat/ekat_pack_utils.hpp" -#include "ekat/ekat_workspace.hpp" -#include "ekat/mpi/ekat_comm.hpp" +#include namespace scream { namespace spa { @@ -17,7 +16,6 @@ namespace spa { template struct SPAFunctions { - // // ------- Types -------- // @@ -25,22 +23,15 @@ struct SPAFunctions using Scalar = ScalarType; using Device = DeviceType; - template - using BigPack = ekat::Pack; - template - using SmallPack = ekat::Pack; - - using Pack = BigPack; - using Spack = SmallPack; + using Spack = ekat::Pack; using KT = KokkosTypes; using MemberType = typename KT::MemberType; - using WorkspaceManager = typename ekat::WorkspaceManager; - using Workspace = typename WorkspaceManager::Workspace; - using gid_type = AbstractGrid::gid_type; + using iop_ptr_type = std::shared_ptr; + template using view_1d = typename KT::template view_1d; template @@ -63,7 +54,6 @@ struct SPAFunctions struct SPATimeState { SPATimeState() = default; // Whether the timestate has been initialized. - bool inited = false; // The current month int current_month = -1; // Julian Date for the beginning of the month, as defined in @@ -137,31 +127,58 @@ struct SPAFunctions SPAData data; // All spa fields }; // SPAInput + struct IOPReader { + IOPReader (iop_ptr_type& iop_, + const std::string file_name_, + const std::vector& io_fields_, + const std::shared_ptr& io_grid_) + : iop(iop_), file_name(file_name_) + { + field_mgr = std::make_shared(io_grid_); + for (auto& f : io_fields_) { + field_mgr->add_field(f); + field_names.push_back(f.name()); + } + + // Set IO info for this grid and file in IOP object + iop->setup_io_info(file_name, io_grid_); + } + + void read_variables(const int time_index, const util::TimeStamp& ts) { + iop->read_fields_from_file_for_iop(file_name, field_names, ts, field_mgr, time_index); + } + + iop_ptr_type iop; + std::string file_name; + std::vector field_names; + std::shared_ptr field_mgr; + }; + // The output is really just SPAData, but for clarity it might // help to see a SPAOutput along a SPAInput in functions signatures using SPAOutput = SPAData; - struct SPAHorizInterp { - // This structure stores the information need by SPA to conduct horizontal - // interpolation from a set of source data to horizontal locations in the - // simulation grid. - // The source_grid_loc stores the column index in the source data, - // The target_grid_loc stores the column index in the target data that will be mapped to - // The weights stores the remapping weight to be applied to the source grid data for this location - // in the target data. - SPAHorizInterp() = default; - explicit SPAHorizInterp(const ekat::Comm& comm) - { - m_comm = comm; - } - // Horizontal Remap - HorizontalMap horiz_map; - // Comm group used for SPA - ekat::Comm m_comm; - - }; // SPAHorizInterp /* ------------------------------------------------------------------------------------------- */ // SPA routines + + static std::shared_ptr + create_horiz_remapper ( + const std::shared_ptr& model_grid, + const std::string& spa_data_file, + const std::string& map_file, + const bool use_iop = false); + + static std::shared_ptr + create_spa_data_reader ( + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file); + + static std::shared_ptr + create_spa_data_reader ( + iop_ptr_type& iop, + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file); + static void spa_main( const SPATimeState& time_state, const view_2d& p_tgt, @@ -171,34 +188,22 @@ struct SPAFunctions const SPAInput& data_tmp, // Temporary const SPAOutput& data_out); - static void get_remap_weights_from_file( - const std::string& remap_file_name, - const gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp); - - static void set_remap_weights_one_to_one( - gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp); - static void update_spa_data_from_file( - const std::string& spa_data_file_name, - const int time_index, - const int nswbands, - const int nlwbands, - SPAHorizInterp& spa_horiz_interp, - SPAInput& spa_data); + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + const int time_index, // zero-based + AbstractRemapper& spa_horiz_interp, + SPAInput& spa_input); static void update_spa_timestate( - const std::string& spa_data_file_name, - const int nswbands, - const int nlwbands, - const util::TimeStamp& ts, - SPAHorizInterp& spa_horiz_interp, - SPATimeState& time_state, - SPAInput& spa_beg, - SPAInput& spa_end); + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + AbstractRemapper& spa_horiz_interp, + SPATimeState& time_state, + SPAInput& spa_beg, + SPAInput& spa_end); // The following three are called during spa_main static void perform_time_interpolation ( diff --git a/components/eamxx/src/physics/spa/spa_functions_impl.hpp b/components/eamxx/src/physics/spa/spa_functions_impl.hpp index 665f4537083c..748983006a19 100644 --- a/components/eamxx/src/physics/spa/spa_functions_impl.hpp +++ b/components/eamxx/src/physics/spa/spa_functions_impl.hpp @@ -1,27 +1,27 @@ #ifndef SPA_FUNCTIONS_IMPL_HPP #define SPA_FUNCTIONS_IMPL_HPP -#include "share/scream_types.hpp" -#include "share/io/scream_scorpio_interface.hpp" -#include "share/io/scorpio_input.hpp" -#include "share/grid/point_grid.hpp" #include "physics/share/physics_constants.hpp" +#include "share/grid/remap/coarsening_remapper.hpp" +#include "share/grid/remap/refining_remapper_p2p.hpp" +#include "share/grid/remap/identity_remapper.hpp" +#include "share/io/scream_scorpio_interface.hpp" +#include "share/util/scream_timing.hpp" +#include "share/scream_types.hpp" -#include "ekat/kokkos/ekat_subview_utils.hpp" -#include "ekat/util/ekat_lin_interp.hpp" -#include "ekat/ekat_pack_utils.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" - -#include +#include +#include +#include +#include +#include -#include "share/util/scream_timing.hpp" /*----------------------------------------------------------------- * The main SPA routines used to convert SPA data into a format that * is usable by the rest of the atmosphere processes. * * SPA or Simple Prescribed Aerosols provides a way to prescribe * aerosols for an atmospheric simulation using pre-computed data. - * + * * The data is typically provided at a frequency of monthly, and * does not necessarily have to be on the same horizontal or vertical * domain as the atmospheric simulation. @@ -61,12 +61,119 @@ * temporally interpolated in the last step and the set of hybrid coordinates (hyam and hybm) * that are used in EAM to construct the physics pressure profiles. * The SPA data is then projected onto the simulation pressure profile (pmid) - * using EKAT linear interpolation. + * using EKAT linear interpolation. -----------------------------------------------------------------*/ namespace scream { namespace spa { +template +std::shared_ptr +SPAFunctions:: +create_horiz_remapper ( + const std::shared_ptr& model_grid, + const std::string& spa_data_file, + const std::string& map_file, + const bool use_iop) +{ + using namespace ShortFieldTagsNames; + + scorpio::register_file(spa_data_file,scorpio::Read); + const int nlevs_data = scorpio::get_dimlen(spa_data_file,"lev"); + const int ncols_data = scorpio::get_dimlen(spa_data_file,"ncol"); + const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); + const int nlwbands = scorpio::get_dimlen(spa_data_file,"lwband"); + scorpio::eam_pio_closefile(spa_data_file); + + // We could use model_grid directly if using same num levels, + // but since shallow clones are cheap, we may as well do it (less lines of code) + auto horiz_interp_tgt_grid = model_grid->clone("spa_horiz_interp_tgt_grid",true); + horiz_interp_tgt_grid->reset_num_vertical_lev(nlevs_data); + + const int ncols_model = model_grid->get_num_global_dofs(); + std::shared_ptr remapper; + if (ncols_data==ncols_model + or + use_iop /*IOP class defines it's own remapper for file data*/) { + remapper = std::make_shared(horiz_interp_tgt_grid,IdentityRemapper::SrcAliasTgt); + } else { + EKAT_REQUIRE_MSG (ncols_data<=ncols_model, + "Error! We do not allow to coarsen spa data to fit the model. We only allow\n" + " spa data to be at the same or coarser resolution as the model.\n"); + // We must have a valid map file + EKAT_REQUIRE_MSG (map_file!="", + "ERROR: Spa data is on a different grid than the model one,\n" + " but spa_remap_file is missing from SPA parameter list."); + + remapper = std::make_shared(horiz_interp_tgt_grid,map_file); + } + + remapper->registration_begins(); + + const auto tgt_grid = remapper->get_tgt_grid(); + + const auto layout_2d = tgt_grid->get_2d_scalar_layout(); + const auto layout_ccn3 = tgt_grid->get_3d_scalar_layout(true); + const auto layout_sw = tgt_grid->get_3d_vector_layout(true,SWBND,nswbands); + const auto layout_lw = tgt_grid->get_3d_vector_layout(true,LWBND,nlwbands); + const auto nondim = ekat::units::Units::nondimensional(); + + Field ps (FieldIdentifier("PS", layout_2d, nondim,tgt_grid->name())); + Field ccn3 (FieldIdentifier("CCN3", layout_ccn3,nondim,tgt_grid->name())); + Field aero_g_sw (FieldIdentifier("AER_G_SW", layout_sw, nondim,tgt_grid->name())); + Field aero_ssa_sw (FieldIdentifier("AER_SSA_SW",layout_sw, nondim,tgt_grid->name())); + Field aero_tau_sw (FieldIdentifier("AER_TAU_SW",layout_sw, nondim,tgt_grid->name())); + Field aero_tau_lw (FieldIdentifier("AER_TAU_LW",layout_lw, nondim,tgt_grid->name())); + ps.allocate_view(); + ccn3.allocate_view(); + aero_g_sw.allocate_view(); + aero_ssa_sw.allocate_view(); + aero_tau_sw.allocate_view(); + aero_tau_lw.allocate_view(); + + remapper->register_field_from_tgt (ps); + remapper->register_field_from_tgt (ccn3); + remapper->register_field_from_tgt (aero_g_sw); + remapper->register_field_from_tgt (aero_ssa_sw); + remapper->register_field_from_tgt (aero_tau_sw); + remapper->register_field_from_tgt (aero_tau_lw); + + remapper->registration_ends(); + + return remapper; +} + +template +std::shared_ptr +SPAFunctions:: +create_spa_data_reader ( + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file) +{ + std::vector io_fields; + for (int i=0; iget_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(spa_data_file,io_grid,io_fields,true); +} + +template +std::shared_ptr::IOPReader> +SPAFunctions:: +create_spa_data_reader ( + iop_ptr_type& iop, + const std::shared_ptr& horiz_remapper, + const std::string& spa_data_file) +{ + std::vector io_fields; + for (int i=0; iget_num_fields(); ++i) { + io_fields.push_back(horiz_remapper->get_src_field(i)); + } + const auto io_grid = horiz_remapper->get_src_grid(); + return std::make_shared(iop, spa_data_file, io_fields, io_grid); +} + /*-----------------------------------------------------------------*/ // The main SPA routine which handles projecting SPA data onto the // horizontal columns and vertical pressure profiles of the atmospheric @@ -82,9 +189,9 @@ namespace spa { // set of SPA data projected onto the pressure profile of the current atmosphere // state. This is the data that will be passed to other processes. // ncols_tgt, nlevs_tgt: The number of columns and levels in the simulation grid. -// (not to be confused with the number of columns and levels used for the SPA data, +// (not to be confused with the number of columns and levels used for the SPA data, // which can be different.) -// nswbands, nlwbands: The number of shortwave (sw) and longwave (lw) aerosol bands +// nswbands, nlwbands: The number of shortwave (sw) and longwave (lw) aerosol bands // for the data that will be passed to radiation. template void SPAFunctions @@ -268,7 +375,7 @@ perform_vertical_interpolation( KOKKOS_LAMBDA(typename LIV::MemberType const& team) { const int icol = team.league_rank(); - + // Setup vert_interp.setup(team, ekat::subview(p_src,icol), ekat::subview(p_tgt,icol)); @@ -295,62 +402,6 @@ perform_vertical_interpolation( Kokkos::fence(); } -/*-----------------------------------------------------------------*/ -// Function to set the remap and weights for a one-to-one mapping. -// This is used when the SPA data and the simulation grid are the -// same and no remapping is needed. -// Note: This function should be called only once during SPA::init -template -void SPAFunctions -::set_remap_weights_one_to_one( - gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp - ) -{ - // There may be cases where the SPA data is defined on the same grid as the simulation - // and thus no remapping is required. This simple routine establishes a 1-1 horizontal - // mapping - const int num_local_cols = dofs_gids.size(); - auto& spa_horiz_map = spa_horiz_interp.horiz_map; - spa_horiz_map = HorizontalMap(spa_horiz_interp.m_comm,"SPA 1-1 Remap",dofs_gids,min_dof); - view_1d src_dofs("",1); - view_1d src_wgts("",1); - auto dofs_gids_h = Kokkos::create_mirror_view(dofs_gids); - Kokkos::deep_copy(dofs_gids_h,dofs_gids); - for (int ii=0;ii -void SPAFunctions -::get_remap_weights_from_file( - const std::string& remap_file_name, - const gid_type min_dof, - const view_1d& dofs_gids, - SPAHorizInterp& spa_horiz_interp - ) -{ - start_timer("EAMxx::SPA::get_remap_weights_from_file"); - auto& spa_horiz_map = spa_horiz_interp.horiz_map; - spa_horiz_map = HorizontalMap(spa_horiz_interp.m_comm,"SPA File Remap", dofs_gids, min_dof); - spa_horiz_map.set_remap_segments_from_file(remap_file_name); - spa_horiz_map.set_unique_source_dofs(); - stop_timer("EAMxx::SPA::get_remap_weights_from_file"); - -} // END get_remap_weights_from_file /*-----------------------------------------------------------------*/ /* Note: In this routine the SPA source data is padded in the vertical * to facilitate the proper behavior at the boundaries when doing the @@ -370,7 +421,7 @@ ::get_remap_weights_from_file( * lhs of the hybrid coordinate system to ensure that the lhs of the * source pressure levels is 0.0 and the rhs is big enough to be larger than * the highest pressure in the target pressure levels. - * + * * The padding sets the lhs in a vertical column of data to 0.0. * This has the effect of ramping the top-of-model target data down to 0.0 * when the top target pressure level is smaller than the top source pressure, @@ -393,264 +444,143 @@ ::get_remap_weights_from_file( * pressure profile that is constructed in spa_main is * a) the current length, and * b) ensures that whatever the target pressure profile is, it's within the - * the range of the source data. + * the range of the source data. */ template void SPAFunctions ::update_spa_data_from_file( - const std::string& spa_data_file_name, - const int time_index, // zero-based - const int nswbands, - const int nlwbands, - SPAHorizInterp& spa_horiz_interp, - SPAInput& spa_data) + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + const int time_index, // zero-based + AbstractRemapper& spa_horiz_interp, + SPAInput& spa_input) { + using namespace ShortFieldTagsNames; + using ESU = ekat::ExeSpaceUtils; + using Member = typename KokkosTypes::MemberType; + start_timer("EAMxx::SPA::update_spa_data_from_file"); - // Ensure all ranks are operating independently when reading the file, so there's a copy on all ranks - auto comm = spa_horiz_interp.m_comm; - - // Use HorizontalMap to define the set of source column data we need to load - auto& spa_horiz_map = spa_horiz_interp.horiz_map; - auto unique_src_dofs = spa_horiz_map.get_unique_source_dofs(); - const int num_local_cols = spa_horiz_map.get_num_unique_dofs(); - scorpio::register_file(spa_data_file_name,scorpio::Read); - const int source_data_nlevs = scorpio::get_dimlen(spa_data_file_name,"lev"); - - // Construct local arrays to read data into - // Note, all of the views being created here are meant to hold the source resolution - // data that will need to be horizontally interpolated to the simulation grid using the remap - // data. For example, - // We will first define the data surface pressure PS_v and read that from file. - // then we will use the horizontal interpolation structure, spa_horiz_interp, to - // interpolate PS_v onto the simulation grid: PS_v -> spa_data.PS - // and so on for the other variables. + + // 1. Read from file start_timer("EAMxx::SPA::update_spa_data_from_file::read_data"); - std::vector fnames = {"hyam","hybm","PS","CCN3","AER_G_SW","AER_SSA_SW","AER_TAU_SW","AER_TAU_LW"}; - ekat::ParameterList spa_data_in_params; - spa_data_in_params.set("Field Names",fnames); - spa_data_in_params.set("Filename",spa_data_file_name); - spa_data_in_params.set("Skip_Grid_Checks",true); // We need to skip grid checks because multiple ranks may want the same column of source data. - - // Retrieve number of cols on spa_data_file. - scorpio::register_file(spa_data_file_name,scorpio::Read); - int num_global_cols = scorpio::get_dimlen(spa_data_file_name,"ncol"); - scorpio::eam_pio_closefile(spa_data_file_name); - - // Construct the grid needed for input: - auto grid = std::make_shared("grid",num_local_cols,num_global_cols,source_data_nlevs,comm); - Kokkos::deep_copy(grid->get_dofs_gids().template get_view(),unique_src_dofs); - grid->get_dofs_gids().sync_to_host(); - - // Check that padding matches source size: - EKAT_REQUIRE(source_data_nlevs+2 == spa_data.data.nlevs); - EKAT_REQUIRE_MSG(nswbands==scorpio::get_dimlen(spa_data_file_name,"swband"), - "ERROR update_spa_data_from_file: Number of SW bands in simulation doesn't match the SPA data file"); - EKAT_REQUIRE_MSG(nlwbands==scorpio::get_dimlen(spa_data_file_name,"lwband"), - "ERROR update_spa_data_from_file: Number of LW bands in simulation doesn't match the SPA data file"); - - // Constuct views to read source data in from file - typename view_1d::HostMirror hyam_v_h("hyam",source_data_nlevs); - typename view_1d::HostMirror hybm_v_h("hybm",source_data_nlevs); - view_1d PS_v("PS",num_local_cols); - view_2d CCN3_v("CCN3",num_local_cols,source_data_nlevs); - view_3d AER_G_SW_v("AER_G_SW",num_local_cols,nswbands,source_data_nlevs); - view_3d AER_SSA_SW_v("AER_SSA_SW",num_local_cols,nswbands,source_data_nlevs); - view_3d AER_TAU_SW_v("AER_TAU_SW",num_local_cols,nswbands,source_data_nlevs); - view_3d AER_TAU_LW_v("AER_TAU_LW",num_local_cols,nlwbands,source_data_nlevs); - - auto PS_v_h = Kokkos::create_mirror_view(PS_v); - auto CCN3_v_h = Kokkos::create_mirror_view(CCN3_v); - auto AER_G_SW_v_h = Kokkos::create_mirror_view(AER_G_SW_v); - auto AER_SSA_SW_v_h = Kokkos::create_mirror_view(AER_SSA_SW_v); - auto AER_TAU_SW_v_h = Kokkos::create_mirror_view(AER_TAU_SW_v); - auto AER_TAU_LW_v_h = Kokkos::create_mirror_view(AER_TAU_LW_v); - - // Set up input structure to read data from file. - using namespace ShortFieldTagsNames; - FieldLayout scalar1d_layout { {LEV}, {source_data_nlevs} }; - FieldLayout scalar2d_layout_mid { {COL}, {num_local_cols} }; - FieldLayout scalar3d_layout_mid { {COL,LEV}, {num_local_cols, source_data_nlevs} }; - FieldLayout scalar3d_swband_layout { {COL,SWBND, LEV}, {num_local_cols, nswbands, source_data_nlevs} }; - FieldLayout scalar3d_lwband_layout { {COL,LWBND, LEV}, {num_local_cols, nlwbands, source_data_nlevs} }; - std::map> host_views; - std::map layouts; - // Define each input variable we need - host_views["hyam"] = view_1d_host(hyam_v_h.data(),hyam_v_h.size()); - layouts.emplace("hyam", scalar1d_layout); - host_views["hybm"] = view_1d_host(hybm_v_h.data(),hybm_v_h.size()); - layouts.emplace("hybm", scalar1d_layout); - // - host_views["PS"] = view_1d_host(PS_v_h.data(),PS_v_h.size()); - layouts.emplace("PS", scalar2d_layout_mid); - // - host_views["CCN3"] = view_1d_host(CCN3_v_h.data(),CCN3_v_h.size()); - layouts.emplace("CCN3",scalar3d_layout_mid); - // - host_views["AER_G_SW"] = view_1d_host(AER_G_SW_v_h.data(),AER_G_SW_v_h.size()); - layouts.emplace("AER_G_SW",scalar3d_swband_layout); - // - host_views["AER_SSA_SW"] = view_1d_host(AER_SSA_SW_v_h.data(),AER_SSA_SW_v_h.size()); - layouts.emplace("AER_SSA_SW",scalar3d_swband_layout); - // - host_views["AER_TAU_SW"] = view_1d_host(AER_TAU_SW_v_h.data(),AER_TAU_SW_v_h.size()); - layouts.emplace("AER_TAU_SW",scalar3d_swband_layout); - // - host_views["AER_TAU_LW"] = view_1d_host(AER_TAU_LW_v_h.data(),AER_TAU_LW_v_h.size()); - layouts.emplace("AER_TAU_LW",scalar3d_lwband_layout); - // - - // Now that we have all the variables defined we can use the scorpio_input class to grab the data. - AtmosphereInput spa_data_input(spa_data_in_params,grid,host_views,layouts); - spa_data_input.read_variables(time_index); - spa_data_input.finalize(); - scorpio::eam_pio_closefile(spa_data_file_name); + if (iop_reader) { + iop_reader->read_variables(time_index, ts); + } else { + scorpio_reader->read_variables(time_index); + } stop_timer("EAMxx::SPA::update_spa_data_from_file::read_data"); - start_timer("EAMxx::SPA::update_spa_data_from_file::apply_remap"); - // Copy data from host back to the device views. - Kokkos::deep_copy(PS_v,PS_v_h); - Kokkos::deep_copy(CCN3_v , CCN3_v_h); - Kokkos::deep_copy(AER_G_SW_v , AER_G_SW_v_h); - Kokkos::deep_copy(AER_SSA_SW_v, AER_SSA_SW_v_h); - Kokkos::deep_copy(AER_TAU_SW_v, AER_TAU_SW_v_h); - Kokkos::deep_copy(AER_TAU_LW_v, AER_TAU_LW_v_h); - - // Apply the remap to this data - spa_horiz_map.apply_remap(PS_v,spa_data.PS); // Note PS is not padded, so remap can be applied right away - // For padded data we need create temporary arrays to store the direct remapped data, then we can add - // padding. - int tgt_ncol = spa_data.data.ncols; - int tgt_nlev = spa_data.data.nlevs-2; // Note, the spa data already accounts for padding in the nlevs, so we subtract 2 - view_2d CCN3_unpad("",tgt_ncol,tgt_nlev); - view_3d AER_G_SW_unpad("",tgt_ncol,nswbands,tgt_nlev); - view_3d AER_SSA_SW_unpad("",tgt_ncol,nswbands,tgt_nlev); - view_3d AER_TAU_SW_unpad("",tgt_ncol,nswbands,tgt_nlev); - view_3d AER_TAU_LW_unpad("",tgt_ncol,nlwbands,tgt_nlev); - // Apply remap to "unpadded" data - spa_horiz_map.apply_remap(CCN3_v,CCN3_unpad); - spa_horiz_map.apply_remap(AER_G_SW_v, AER_G_SW_unpad); - spa_horiz_map.apply_remap(AER_SSA_SW_v, AER_SSA_SW_unpad); - spa_horiz_map.apply_remap(AER_TAU_SW_v, AER_TAU_SW_unpad); - spa_horiz_map.apply_remap(AER_TAU_LW_v, AER_TAU_LW_unpad); - stop_timer("EAMxx::SPA::update_spa_data_from_file::apply_remap"); + + // 2. Run the horiz remapper (it is a do-nothing op if spa data is on same grid as model) + start_timer("EAMxx::SPA::update_spa_data_from_file::horiz_remap"); + spa_horiz_interp.remap(/*forward = */ true); + stop_timer("EAMxx::SPA::update_spa_data_from_file::horiz_remap"); + + // 3. Copy from the tgt field of the remapper into the spa_data, padding data if necessary start_timer("EAMxx::SPA::update_spa_data_from_file::copy_and_pad"); - // Copy unpadded data to SPA data structure, add padding. - // Note, all variables we map to are packed, while all the data we just loaded as - // input are in real N-D views. So we need to set the pack and index of the actual - // data ahead by one value. - // Note, we want to pad the actual source data such that - // Y[0] = 0.0, note this is handled by the deep copy above - // Y[k+1] = y[k], k = 0,source_data_nlevs (y is the data from file) - // Y[N+2] = y[N-1], N = source_data_nlevs - Kokkos::deep_copy(spa_data.data.CCN3,0.0); - Kokkos::deep_copy(spa_data.data.AER_G_SW,0.0); - Kokkos::deep_copy(spa_data.data.AER_SSA_SW,0.0); - Kokkos::deep_copy(spa_data.data.AER_TAU_SW,0.0); - Kokkos::deep_copy(spa_data.data.AER_TAU_LW,0.0); - // 2D vars - CCN3 - Kokkos::parallel_for("", tgt_ncol*tgt_nlev, KOKKOS_LAMBDA (const int& idx) { - int icol = idx / tgt_nlev; - int klev1 = idx % tgt_nlev; - int kpack = (klev1+1) / Spack::n; - int klev2 = (klev1+1) % Spack::n; - spa_data.data.CCN3(icol,kpack)[klev2] = CCN3_unpad(icol,klev1); - if (klev1 == tgt_nlev-1) { - int kpack = (tgt_nlev+1) / Spack::n; - int klev2 = (tgt_nlev+1) % Spack::n; - spa_data.data.CCN3(icol,kpack)[klev2] = CCN3_unpad(icol,klev1); - } - }); + // Recall, the fields are registered in the order: ps, ccn3, g_sw, ssa_sw, tau_sw, tau_lw + auto ps = spa_horiz_interp.get_tgt_field (0).get_view(); + auto ccn3 = spa_horiz_interp.get_tgt_field (1).get_view(); + auto aero_g_sw = spa_horiz_interp.get_tgt_field (2).get_view(); + auto aero_ssa_sw = spa_horiz_interp.get_tgt_field (3).get_view(); + auto aero_tau_sw = spa_horiz_interp.get_tgt_field (4).get_view(); + auto aero_tau_lw = spa_horiz_interp.get_tgt_field (5).get_view(); + + const auto& sw_layout = spa_horiz_interp.get_tgt_field (2).get_header().get_identifier().get_layout(); + const auto& lw_layout = spa_horiz_interp.get_tgt_field (5).get_header().get_identifier().get_layout(); + + const int ncols = sw_layout.dim(COL); + const int nlevs = sw_layout.dim(LEV); + const int nswbands = sw_layout.dim(SWBND); + const int nlwbands = lw_layout.dim(LWBND); + + Kokkos::deep_copy(spa_input.PS,ps); + Kokkos::fence(); + auto spa_data_ccn3 = ekat::scalarize(spa_input.data.CCN3); + auto spa_data_aero_g_sw = ekat::scalarize(spa_input.data.AER_G_SW); + auto spa_data_aero_ssa_sw = ekat::scalarize(spa_input.data.AER_SSA_SW); + auto spa_data_aero_tau_sw = ekat::scalarize(spa_input.data.AER_TAU_SW); + auto spa_data_aero_tau_lw = ekat::scalarize(spa_input.data.AER_TAU_LW); + + auto copy_and_pad = KOKKOS_LAMBDA (const Member& team) { + int icol = team.league_rank(); + + auto copy_col = [&](const int k) { + spa_data_ccn3(icol,k+1) = ccn3(icol,k); + for (int isw=0; isw void SPAFunctions ::update_spa_timestate( - const std::string& spa_data_file_name, - const int nswbands, - const int nlwbands, - const util::TimeStamp& ts, - SPAHorizInterp& spa_horiz_interp, - SPATimeState& time_state, - SPAInput& spa_beg, - SPAInput& spa_end) + std::shared_ptr& scorpio_reader, + std::shared_ptr& iop_reader, + const util::TimeStamp& ts, + AbstractRemapper& spa_horiz_interp, + SPATimeState& time_state, + SPAInput& spa_beg, + SPAInput& spa_end) { - // Now we check if we have to update the data that changes monthly // NOTE: This means that SPA assumes monthly data to update. Not // any other frequency. - const auto month = ts.get_month(); - if (month != time_state.current_month or !time_state.inited) { - + const auto month = ts.get_month() - 1; // Make it 0-based + if (month != time_state.current_month) { // Update the SPA time state information time_state.current_month = month; - time_state.t_beg_month = util::TimeStamp({ts.get_year(),month,1}, {0,0,0}).frac_of_year_in_days(); - time_state.days_this_month = util::days_in_month(ts.get_year(),month); + time_state.t_beg_month = util::TimeStamp({ts.get_year(),month+1,1}, {0,0,0}).frac_of_year_in_days(); + time_state.days_this_month = util::days_in_month(ts.get_year(),month+1); + + // Copy spa_end'data into spa_beg'data, and read in the new spa_end + std::swap(spa_beg,spa_end); + // Update the SPA forcing data for this month and next month - // Start by copying next months data to this months data structure. + // Start by copying next months data to this months data structure. // NOTE: If the timestep is bigger than monthly this could cause the wrong values // to be assigned. A timestep greater than a month is very unlikely so we // will proceed. - // NOTE: we use zero-based time indexing here. - update_spa_data_from_file(spa_data_file_name,time_state.current_month-1,nswbands,nlwbands,spa_horiz_interp,spa_beg); - int next_month = time_state.current_month==12 ? 1 : time_state.current_month+1; - update_spa_data_from_file(spa_data_file_name,next_month-1,nswbands,nlwbands,spa_horiz_interp,spa_end); - // If time state was not initialized it is now: - time_state.inited = true; + int next_month = (time_state.current_month + 1) % 12; + update_spa_data_from_file(scorpio_reader,iop_reader,ts,next_month,spa_horiz_interp,spa_end); } } // END updata_spa_timestate @@ -659,12 +589,12 @@ template KOKKOS_INLINE_FUNCTION auto SPAFunctions:: get_var_column (const SPAData& data, const int icol, const int ivar) - -> view_1d + -> view_1d { if (ivar==0) { return ekat::subview(data.CCN3,icol); } else { - // NOTE: if we ever get 2+ long-wave vars, you will have to do + // NOTE: if we ever get 2+ long-wave vars, you will have to do // something like // if (jvar -#include -#include -#include +#include namespace { @@ -21,7 +15,6 @@ template using view_1d = typename KokkosTypes::template view_1d; using SPAFunc = spa::SPAFunctions; -using Spack = SPAFunc::Spack; using gid_type = SPAFunc::gid_type; Real ps_func (const int t, const gid_type icol); @@ -31,44 +24,34 @@ Real aer_func (const int t, const gid_type icol, const int bnd, const int klev, TEST_CASE("spa_one_to_one_remap","spa") { // Set up the mpi communicator and init the pio subsystem - ekat::Comm spa_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - MPI_Fint fcomm = MPI_Comm_c2f(spa_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); std::string spa_data_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; - int max_time = 3; - int ncols = 20; - int nlevs = 4; - int nswbands = 2; - int nlwbands = 3; - - // Break the test set of columns into local degrees of freedom per mpi rank - auto comm_size = spa_comm.size(); - auto comm_rank = spa_comm.rank(); - - int my_ncols = ncols/comm_size + (comm_rank < ncols%comm_size ? 1 : 0); - view_1d dofs_gids("",my_ncols); - gid_type min_dof = 0; // We will set up the dof's to start from 0 - Kokkos::parallel_for("", my_ncols, KOKKOS_LAMBDA(const int& ii) { - dofs_gids(ii) = min_dof + static_cast(comm_rank + ii*comm_size); - }); - auto dofs_gids_h = Kokkos::create_mirror_view(dofs_gids); - Kokkos::deep_copy(dofs_gids_h,dofs_gids); - // Make sure that the total set of columns has been completely broken up. - int test_total_ncols = 0; - spa_comm.all_reduce(&my_ncols,&test_total_ncols,1,MPI_SUM); - REQUIRE(test_total_ncols == ncols); - - // Set up the set of SPA structures needed to run the test - SPAFunc::SPAHorizInterp spa_horiz_interp; - spa_horiz_interp.m_comm = spa_comm; - SPAFunc::set_remap_weights_one_to_one(min_dof,dofs_gids,spa_horiz_interp); - // Make sure one_to_one remap has the correct unique columns - auto spa_horiz_map = spa_horiz_interp.horiz_map; - REQUIRE(spa_horiz_map.get_num_unique_dofs()==my_ncols); + const int ncols_model = scorpio::get_dimlen(spa_data_file,"ncol"); + const int nlevs = scorpio::get_dimlen(spa_data_file,"lev"); + const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); + const int nlwbands = scorpio::get_dimlen(spa_data_file,"lwband"); + + auto grid_model = create_point_grid("model_grid",ncols_model,nlevs,comm); + + // Create horiz remapper + auto remapper = SPAFunc::create_horiz_remapper(grid_model,spa_data_file,"SHOULD_NOT_BE_NEEDED"); + const int ncols_data = remapper->get_src_grid()->get_num_global_dofs(); + + // This test should be tailored to have same number of columns as the spa data + REQUIRE (ncols_data==ncols_model); + REQUIRE (std::dynamic_pointer_cast(remapper)); + + // Create spa data reader + auto reader = SPAFunc::create_spa_data_reader(remapper,spa_data_file); + // TODO: We can add lat/lon to spa_data_file and test the impl for IOP + util::TimeStamp dummy_ts; + std::shared_ptr dummy_iop_reader; + // Recall, SPA data is padded, so we initialize with 2 more levels than the source data file. - SPAFunc::SPAInput spa_data(dofs_gids.size(), nlevs+2, nswbands, nlwbands); + SPAFunc::SPAInput spa_data(grid_model->get_num_local_dofs(), nlevs+2, nswbands, nlwbands); // Verify that the interpolated values match the algorithm for the data and the weights. // weights(i) = 1.0 @@ -79,42 +62,51 @@ TEST_CASE("spa_one_to_one_remap","spa") // aer_ssa_sw(t,i,b,k) = i // aer_tau_sw(t,i,b,k) = b // aer_tau_lw(t,i,b,k) = k - auto ps_h = Kokkos::create_mirror_view(spa_data.PS); - auto ccn3_h = Kokkos::create_mirror_view(spa_data.data.CCN3); - auto aer_g_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_G_SW); - auto aer_ssa_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_SSA_SW); - auto aer_tau_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_SW); - auto aer_tau_lw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_LW); + auto ps_d = spa_data.PS; + auto ccn3_d = ekat::scalarize(spa_data.data.CCN3); + auto aer_g_sw_d = ekat::scalarize(spa_data.data.AER_G_SW); + auto aer_ssa_sw_d = ekat::scalarize(spa_data.data.AER_SSA_SW); + auto aer_tau_sw_d = ekat::scalarize(spa_data.data.AER_TAU_SW); + auto aer_tau_lw_d = ekat::scalarize(spa_data.data.AER_TAU_LW); + + auto ps_h = Kokkos::create_mirror_view(ps_d); + auto ccn3_h = Kokkos::create_mirror_view(ccn3_d); + auto aer_g_sw_h = Kokkos::create_mirror_view(aer_g_sw_d); + auto aer_ssa_sw_h = Kokkos::create_mirror_view(aer_ssa_sw_d); + auto aer_tau_sw_h = Kokkos::create_mirror_view(aer_tau_sw_d); + auto aer_tau_lw_h = Kokkos::create_mirror_view(aer_tau_lw_d); + + auto dofs_gids_h = grid_model->get_dofs_gids().get_view(); + + const int max_time = 3; for (int time_index = 0;time_indexget_num_local_dofs(); ++idof) { + gid_type glob_i = dofs_gids_h(idof); + REQUIRE(ps_h(idof) == ps_func(time_index,glob_i)); + for (int kk=0; kk -#include -#include -#include +#include namespace { @@ -26,46 +20,36 @@ Real aer_func(const int t, const int bnd, const int klev, const int ncols, const TEST_CASE("spa_read_data","spa") { - // Set up the mpi communicator and init the pio subsystem - ekat::Comm spa_comm(MPI_COMM_WORLD); // MPI communicator group used for I/O set as ekat object. - MPI_Fint fcomm = MPI_Comm_c2f(spa_comm.mpi_comm()); // MPI communicator group used for I/O. In our simple test we use MPI_COMM_WORLD, however a subset could be used. - scorpio::eam_init_pio_subsystem(fcomm); // Gather the initial PIO subsystem data creater by component coupler - - // Establish the SPA function object using SPAFunc = spa::SPAFunctions; - using Spack = SPAFunc::Spack; - using gid_type = SPAFunc::gid_type; + constexpr Real tol = std::numeric_limits::epsilon()*1000; + // Set up the mpi communicator and init the pio subsystem + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + std::string spa_data_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; std::string spa_remap_file = SCREAM_DATA_DIR "/init/spa_data_for_testing.nc"; - int max_time = 3; - int ncols = 48; - int ncols_src = 20; - int nlevs = 4; - int nswbands = 2; - int nlwbands = 3; - // Break the test set of columns into local degrees of freedom per mpi rank - auto comm_size = spa_comm.size(); - auto comm_rank = spa_comm.rank(); - - int my_ncols = ncols/comm_size + (comm_rank < ncols%comm_size ? 1 : 0); - view_1d dofs_gids("",my_ncols); - gid_type min_dof = 1; // Start global-ids from 1 - Kokkos::parallel_for("", my_ncols, KOKKOS_LAMBDA(const int& ii) { - dofs_gids(ii) = min_dof + static_cast(comm_rank + ii*comm_size); - }); - // Make sure that the total set of columns has been completely broken up. - int test_total_ncols = 0; - spa_comm.all_reduce(&my_ncols,&test_total_ncols,1,MPI_SUM); - REQUIRE(test_total_ncols == ncols); - - // Set up the set of SPA structures needed to run the test - SPAFunc::SPAHorizInterp spa_horiz_interp; - spa_horiz_interp.m_comm = spa_comm; - SPAFunc::get_remap_weights_from_file(spa_remap_file,min_dof,dofs_gids,spa_horiz_interp); + + const int ncols_model = 48; + const int nlevs = scorpio::get_dimlen(spa_data_file,"lev"); + const int nswbands = scorpio::get_dimlen(spa_data_file,"swband"); + const int nlwbands = scorpio::get_dimlen(spa_data_file,"lwband"); + + auto grid_model = create_point_grid("model_grid",ncols_model,nlevs,comm); + + // Create horiz remapper + auto remapper = SPAFunc::create_horiz_remapper(grid_model,spa_data_file,spa_remap_file); + const int ncols_data = remapper->get_src_grid()->get_num_global_dofs(); + + // Create spa data reader + auto reader = SPAFunc::create_spa_data_reader(remapper,spa_data_file); + // TODO: We can add lat/lon to spa_data_file and test the impl for IOP + std::shared_ptr dummy_iop_reader; + util::TimeStamp dummy_iop_ts; + // Recall, SPA data is padded, so we initialize with 2 more levels than the source data file. - SPAFunc::SPAInput spa_data(dofs_gids.size(), nlevs+2, nswbands, nlwbands); + SPAFunc::SPAInput spa_data(grid_model->get_num_local_dofs(), nlevs+2, nswbands, nlwbands); // Verify that the interpolated values match the algorithm for the data and the weights. // weights(i) = 1 / (2**i), weights(-1) = 1 / (2**(ncols-1)) such that sum(weights) = 1., for i=0,1,2 @@ -76,45 +60,52 @@ TEST_CASE("spa_read_data","spa") // aer_ssa_sw(t,i,b,k) = i // aer_tau_sw(t,i,b,k) = b // aer_tau_lw(t,i,b,k) = k - auto ps_h = Kokkos::create_mirror_view(spa_data.PS); - auto ccn3_h = Kokkos::create_mirror_view(spa_data.data.CCN3); - auto aer_g_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_G_SW); - auto aer_ssa_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_SSA_SW); - auto aer_tau_sw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_SW); - auto aer_tau_lw_h = Kokkos::create_mirror_view(spa_data.data.AER_TAU_LW); - auto dofs_gids_h = Kokkos::create_mirror_view(dofs_gids); - Kokkos::deep_copy(dofs_gids_h,dofs_gids); + auto ps_d = spa_data.PS; + auto ccn3_d = ekat::scalarize(spa_data.data.CCN3); + auto aer_g_sw_d = ekat::scalarize(spa_data.data.AER_G_SW); + auto aer_ssa_sw_d = ekat::scalarize(spa_data.data.AER_SSA_SW); + auto aer_tau_sw_d = ekat::scalarize(spa_data.data.AER_TAU_SW); + auto aer_tau_lw_d = ekat::scalarize(spa_data.data.AER_TAU_LW); + + auto ps_h = Kokkos::create_mirror_view(ps_d); + auto ccn3_h = Kokkos::create_mirror_view(ccn3_d); + auto aer_g_sw_h = Kokkos::create_mirror_view(aer_g_sw_d); + auto aer_ssa_sw_h = Kokkos::create_mirror_view(aer_ssa_sw_d); + auto aer_tau_sw_h = Kokkos::create_mirror_view(aer_tau_sw_d); + auto aer_tau_lw_h = Kokkos::create_mirror_view(aer_tau_lw_d); + + const int max_time = 3; for (int time_index = 0;time_indexget_num_local_dofs(); ++idof) { + REQUIRE(std::abs(ps_h(idof) - ps_func(time_index,ncols_data)); + using iop_ptr = std::shared_ptr; + // Base constructor to set MPI communicator and params AtmosphereProcess (const ekat::Comm& comm, const ekat::ParameterList& params); @@ -276,6 +280,11 @@ class AtmosphereProcess : public ekat::enable_shared_from_thisset_iop(iop); + } + } + protected: // Adds fid to the list of required/computed fields of the group (as a whole). diff --git a/components/eamxx/src/share/field/field.cpp b/components/eamxx/src/share/field/field.cpp index 71a84fd6f7d9..c1d03f5905e4 100644 --- a/components/eamxx/src/share/field/field.cpp +++ b/components/eamxx/src/share/field/field.cpp @@ -122,7 +122,7 @@ get_component (const int i, const bool dynamic) { "Error! 'get_component' available only for vector fields.\n" " Layout of '" + fname + "': " + e2str(get_layout_type(layout.tags())) + "\n"); - const int idim = layout.get_vector_dim(); + const int idim = layout.get_vector_component_idx(); EKAT_REQUIRE_MSG (i>=0 && i FieldLayout::get_tensor_dims () const { + EKAT_REQUIRE_MSG (is_tensor_layout(), + "Error! 'get_tensor_dims' available only for tensor layouts.\n" + " Current layout: " + to_string(*this) + "\n" + " Layout type : " + e2str(get_layout_type(m_tags)) + "\n"); + + using namespace ShortFieldTagsNames; + std::vector cmp_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; + + std::vector idx; + auto it = m_tags.begin(); + do { + it = std::find_first_of (it,m_tags.cend(),cmp_tags.cbegin(),cmp_tags.cend()); + if (it!=m_tags.end()) { + idx.push_back(std::distance(m_tags.begin(),it)); + ++it; + } + } while (it!=m_tags.end()); + + EKAT_REQUIRE_MSG (idx.size()==2, + "Error! Could not find a two tensor tags in the layout.\n" + " - layout: " + to_string(*this) + "\n" + " - detected tags indices: " + ekat::join(idx,",") + "\n"); + + return idx; +} + +std::vector FieldLayout::get_tensor_tags () const { + auto idx = get_tensor_dims(); + return {m_tags[idx[0]], m_tags[idx[1]]}; } FieldLayout FieldLayout::strip_dim (const FieldTag tag) const { @@ -68,6 +117,14 @@ FieldLayout FieldLayout::strip_dim (const int idim) const { return FieldLayout (t,d); } +FieldLayout FieldLayout::clone_with_different_extent (const int idim, const int extent) const +{ + FieldLayout copy(m_tags,m_dims); + copy.set_dimension(idim,extent); + + return copy; +} + void FieldLayout::set_dimension (const int idim, const int dimension) { EKAT_REQUIRE_MSG(idim>=0 && idim=0, "Error! Dimensions must be non-negative."); @@ -95,6 +152,9 @@ LayoutType get_layout_type (const std::vector& field_tags) { // Start from undefined/invalid LayoutType result = LayoutType::Invalid; + // We don't care about TimeLevel + erase (tags,TL); + if (n_element==1 && ngp==2 && n_column==0) { // A Dynamics layout @@ -126,17 +186,17 @@ LayoutType get_layout_type (const std::vector& field_tags) { std::vector lev_tags = {LEV,ILEV}; return ekat::contains(lev_tags,t); }; - auto is_vec_tag = [](const FieldTag t) { - std::vector vec_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; - return ekat::contains(vec_tags,t); + auto is_cmp_tag = [](const FieldTag t) { + std::vector cmp_tags = {CMP,NGAS,SWBND,LWBND,SWGPT,ISCCPTAU,ISCCPPRS}; + return ekat::contains(cmp_tags,t); }; switch (size) { case 0: result = LayoutType::Scalar2D; break; case 1: - // The only tag left should be a vec tag, 'TL', or a lev tag - if (is_vec_tag(tags[0]) or tags[0]==TL) { + // The only tag left should be a cmp tag or a lev tag + if (is_cmp_tag(tags[0])) { result = LayoutType::Vector2D; } else if (is_lev_tag(tags[0])) { result = LayoutType::Scalar3D; @@ -144,18 +204,20 @@ LayoutType get_layout_type (const std::vector& field_tags) { break; case 2: // Possible supported scenarios: - // 1) - // 2) - if ( (is_vec_tag(tags[0]) or tags[0]==TL) and is_lev_tag(tags[1]) ) { + // 1) + // 3) + // where CMP,CMP1,CMP2 are any tag in cmp_tags + if ( is_cmp_tag(tags[0]) and is_lev_tag(tags[1]) ) { result = LayoutType::Vector3D; - } else if (tags[0]==TL && is_vec_tag(tags[1]) ) { + } else if (is_cmp_tag(tags[0]) and is_cmp_tag(tags[1])) { result = LayoutType::Tensor2D; } break; case 3: // The only supported scenario is: - // 1) - if ( tags[0]==TL && is_vec_tag(tags[1]) && is_lev_tag(tags[2]) ) { + // 1) + // where CMP1,CMP2 are any tag in cmp_tags + if (is_cmp_tag(tags[0]) and is_cmp_tag(tags[1]) and is_lev_tag(tags[2])) { result = LayoutType::Tensor3D; } } diff --git a/components/eamxx/src/share/field/field_layout.hpp b/components/eamxx/src/share/field/field_layout.hpp index d0ca9b76b565..d72f8649e9ef 100644 --- a/components/eamxx/src/share/field/field_layout.hpp +++ b/components/eamxx/src/share/field/field_layout.hpp @@ -13,7 +13,7 @@ namespace scream { -// The type of the layout, that is, the kind of field it represent. +// The type of the layout, that is, the kind of field it represents. enum class LayoutType { Invalid, Scalar0D, @@ -81,7 +81,7 @@ class FieldLayout { bool has_tags (const std::vector& tags) const; // The rank is the number of tags associated to this field. - int rank () const { return m_rank; } + int rank () const { return m_rank; } int dim (const FieldTag tag) const; int dim (const int idim) const; @@ -92,23 +92,35 @@ class FieldLayout { bool are_dimensions_set () const; - // Check if this layout is that of a vector field + // Check if this layout is that of a vector/tensor field bool is_vector_layout () const; + bool is_tensor_layout () const; - // If this is the layout of a vector field, get the idx of the vector dimension + // If this is the layout of a vector field, get the idx of the + // vector (CMP, Component) dimension // Note: throws if is_vector_layout()==false. + int get_vector_component_idx () const; + // get the dimension (extent) of the vector (CMP, Component) dimension + // calls get_vector_component_idx() int get_vector_dim () const; FieldTag get_vector_tag () const; + // If this is the layout of a tensor field, get the idx of the tensor dimensions + // Note: throws if is_tensor_layout()==false. + std::vector get_tensor_dims () const; + std::vector get_tensor_tags () const; + + // Returns a copy of this layout with a given dimension stripped FieldLayout strip_dim (const FieldTag tag) const; FieldLayout strip_dim (const int idim) const; + FieldLayout clone_with_different_extent (const int idim, const int extent) const; - // ----- Setters ----- // +protected: + // Only this class is allowed to change a layout. Customers can request + // a *slightly* different layout (via strip_dim or clone_with_different_extent) void set_dimension (const int idim, const int dimension); -protected: - int m_rank; std::vector m_tags; std::vector m_dims; @@ -121,6 +133,7 @@ std::string to_string (const FieldLayout& l); // ========================== IMPLEMENTATION ======================= // +// returns extent inline int FieldLayout::dim (const FieldTag t) const { auto it = ekat::find(m_tags,t); @@ -150,10 +163,10 @@ inline long long FieldLayout::size () const { return prod; } -inline FieldTag FieldLayout::tag (const int idim) const { +inline FieldTag FieldLayout::tag (const int idim) const { ekat::error::runtime_check(idim>=0 && idim& tags) const { bool b = true; @@ -181,4 +194,3 @@ inline bool operator== (const FieldLayout& fl1, const FieldLayout& fl2) { } // namespace scream #endif // SCREAM_FIELD_LAYOUT_HPP - diff --git a/components/eamxx/src/share/field/field_tag.hpp b/components/eamxx/src/share/field/field_tag.hpp index a96f781a465c..e685399a8bdc 100644 --- a/components/eamxx/src/share/field/field_tag.hpp +++ b/components/eamxx/src/share/field/field_tag.hpp @@ -40,7 +40,13 @@ enum class FieldTag { LongWaveBand, LongWaveGpoint, IsccpTau, - IsccpPrs + IsccpPrs, + // + MAM_NumModes, + MAM_NumRefIndexReal, + MAM_NumRefIndexImag, + MAM_NumCoefficients, + MAM_NumModesInFile }; // If using tags a lot, consider adding 'using namespace ShortFieldTagsNames' @@ -65,6 +71,13 @@ namespace ShortFieldTagsNames { constexpr auto LWGPT = FieldTag::LongWaveGpoint; constexpr auto ISCCPTAU = FieldTag::IsccpTau; constexpr auto ISCCPPRS = FieldTag::IsccpPrs; + constexpr auto NMODES = FieldTag::MAM_NumModes; + // + constexpr auto NREFINDEX_REAL = FieldTag::MAM_NumRefIndexReal; + constexpr auto NREFINDEX_IM = FieldTag::MAM_NumRefIndexImag; + + constexpr auto NCOEF_NUMBER = FieldTag::MAM_NumCoefficients; + constexpr auto MODE = FieldTag::MAM_NumModesInFile; } inline std::string e2str (const FieldTag ft) { @@ -117,6 +130,21 @@ inline std::string e2str (const FieldTag ft) { case FieldTag::IsccpPrs: name = "ISCCPPRS"; break; + case FieldTag::MAM_NumModes: + name = "num_modes"; + break; + case FieldTag::MAM_NumRefIndexReal: + name = "refindex_real"; + break; + case FieldTag::MAM_NumRefIndexImag: + name = "refindex_im"; + break; + case FieldTag::MAM_NumCoefficients: + name = "coef_number"; + break; + case FieldTag::MAM_NumModesInFile: + name = "mode"; + break; default: EKAT_ERROR_MSG("Error! Unrecognized field tag."); } diff --git a/components/eamxx/src/share/grid/abstract_grid.cpp b/components/eamxx/src/share/grid/abstract_grid.cpp index a345df2f7b19..ae304f18afd8 100644 --- a/components/eamxx/src/share/grid/abstract_grid.cpp +++ b/components/eamxx/src/share/grid/abstract_grid.cpp @@ -152,10 +152,9 @@ is_valid_layout (const FieldLayout& layout) const // Let's return true early to avoid segfautls below return true; } + const bool midpoints = layout.tags().back()==LEV; - const bool is_vec = layout.is_vector_layout(); - const int vec_dim = is_vec ? layout.dims()[layout.get_vector_dim()] : 0; - const auto vec_tag = is_vec ? layout.get_vector_tag() : INV; + const bool is3d = layout.tags().back()==LEV or layout.tags().back()==ILEV; switch (lt) { case LayoutType::Scalar1D: [[fallthrough]]; @@ -163,14 +162,29 @@ is_valid_layout (const FieldLayout& layout) const // 1d layouts need the right number of levels return layout.dims().back() == m_num_vert_levs or layout.dims().back() == (m_num_vert_levs+1); - case LayoutType::Scalar2D: - return layout==get_2d_scalar_layout(); - case LayoutType::Vector2D: - return layout==get_2d_vector_layout(vec_tag,vec_dim); + case LayoutType::Scalar2D: [[fallthrough]]; case LayoutType::Scalar3D: - return layout==get_3d_scalar_layout(midpoints); + return is3d ? layout==get_3d_scalar_layout(midpoints) + : layout==get_2d_scalar_layout(); + case LayoutType::Vector2D: [[fallthrough]]; case LayoutType::Vector3D: - return layout==get_3d_vector_layout(midpoints,vec_tag,vec_dim); + { + const auto vec_dim = layout.dims()[layout.get_vector_component_idx()]; + const auto vec_tag = layout.get_vector_tag(); + return is3d ? layout==get_3d_vector_layout(midpoints,vec_tag,vec_dim) + : layout==get_2d_vector_layout(vec_tag,vec_dim); + } + case LayoutType::Tensor2D: [[fallthrough]]; + case LayoutType::Tensor3D: + { + const auto ttags = layout.get_tensor_tags(); + std::vector tdims; + for (auto idx : layout.get_tensor_dims()) { + tdims.push_back(layout.dim(idx)); + } + return is3d ? layout==get_3d_tensor_layout(midpoints,ttags,tdims) + : layout==get_2d_tensor_layout(ttags,tdims); + } default: // Anything else is probably no return false; @@ -243,6 +257,17 @@ AbstractGrid::create_geometry_data (const FieldIdentifier& fid) return f; } +void +AbstractGrid::delete_geometry_data (const std::string& name) +{ + EKAT_REQUIRE_MSG (has_geometry_data(name), + "Error! Cannot delete geometry data, since it is does not exist.\n" + " - grid name: " + this->name() + "\n" + " - geo data name: " + name + "\n"); + + m_geo_fields.erase(name); +} + void AbstractGrid::set_geometry_data (const Field& f) { @@ -501,6 +526,9 @@ void AbstractGrid::copy_data (const AbstractGrid& src, const bool shallow) m_geo_fields[name] = src.m_geo_fields.at(name).clone(); } } + + m_global_max_dof_gid = src.m_global_max_dof_gid; + m_global_min_dof_gid = src.m_global_min_dof_gid; m_is_unique = src.m_is_unique; m_is_unique_computed = src.m_is_unique_computed; } diff --git a/components/eamxx/src/share/grid/abstract_grid.hpp b/components/eamxx/src/share/grid/abstract_grid.hpp index 46cf3a9446bf..10b1b2676540 100644 --- a/components/eamxx/src/share/grid/abstract_grid.hpp +++ b/components/eamxx/src/share/grid/abstract_grid.hpp @@ -74,8 +74,13 @@ class AbstractGrid : public ekat::enable_shared_from_this FieldLayout get_vertical_layout (const bool midpoints) const; virtual FieldLayout get_2d_scalar_layout () const = 0; virtual FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const = 0; + virtual FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const = 0; virtual FieldLayout get_3d_scalar_layout (const bool midpoints) const = 0; virtual FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const = 0; + virtual FieldLayout get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const = 0; int get_num_vertical_levels () const { return m_num_vert_levs; } @@ -120,6 +125,7 @@ class AbstractGrid : public ekat::enable_shared_from_this // Sets pre-existing field as geometry data. void set_geometry_data (const Field& f); + void delete_geometry_data (const std::string& name); bool has_geometry_data (const std::string& name) const { return m_geo_fields.find(name)!=m_geo_fields.end(); diff --git a/components/eamxx/src/share/grid/point_grid.cpp b/components/eamxx/src/share/grid/point_grid.cpp index fb3236a4a01c..c1db5c795e34 100644 --- a/components/eamxx/src/share/grid/point_grid.cpp +++ b/components/eamxx/src/share/grid/point_grid.cpp @@ -55,6 +55,20 @@ PointGrid::get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim return FieldLayout({COL,vector_tag},{get_num_local_dofs(),vector_dim}); } +FieldLayout +PointGrid::get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + std::vector tags = {COL}; + std::vector dims = {get_num_local_dofs()}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + return FieldLayout(tags,dims); +} + FieldLayout PointGrid::get_3d_scalar_layout (const bool midpoints) const { @@ -77,6 +91,26 @@ PointGrid::get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag return FieldLayout({COL,vector_tag,VL},{get_num_local_dofs(),vector_dim,nvl}); } +FieldLayout +PointGrid::get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); + auto VL = midpoints ? LEV : ILEV; + + std::vector tags = {COL}; + std::vector dims = {get_num_local_dofs()}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + tags.push_back(VL); + dims.push_back(nvl); + return FieldLayout(tags,dims); +} + std::shared_ptr PointGrid::clone (const std::string& clone_name, const bool shallow) const diff --git a/components/eamxx/src/share/grid/point_grid.hpp b/components/eamxx/src/share/grid/point_grid.hpp index 3a3d426d1cf9..a66bf5d7d764 100644 --- a/components/eamxx/src/share/grid/point_grid.hpp +++ b/components/eamxx/src/share/grid/point_grid.hpp @@ -45,8 +45,13 @@ class PointGrid : public AbstractGrid // E.g., for a 2d structured grid, this could be a set of 2 indices. FieldLayout get_2d_scalar_layout () const override; FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldTag get_partitioned_dim_tag () const override { return FieldTag::Column; diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp index 6d25787794fa..80df739aa4f4 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.cpp @@ -59,6 +59,34 @@ register_field (const field_type& src, const field_type& tgt) { bind_field(src,tgt); } +void AbstractRemapper:: +register_field_from_src (const field_type& src) { + const auto& src_fid = src.get_header().get_identifier(); + const auto& tgt_fid = create_tgt_fid(src_fid); + + Field tgt(tgt_fid); + const auto& src_ap = src.get_header().get_alloc_properties(); + auto& tgt_ap = tgt.get_header().get_alloc_properties(); + tgt_ap.request_allocation(src_ap.get_largest_pack_size()); + tgt.allocate_view(); + + register_field(src,tgt); +} + +void AbstractRemapper:: +register_field_from_tgt (const field_type& tgt) { + const auto& tgt_fid = tgt.get_header().get_identifier(); + const auto& src_fid = create_src_fid(tgt_fid); + + Field src(src_fid); + const auto& tgt_ap = tgt.get_header().get_alloc_properties(); + auto& src_ap = src.get_header().get_alloc_properties(); + src_ap.request_allocation(tgt_ap.get_largest_pack_size()); + src.allocate_view(); + + register_field(src,tgt); +} + void AbstractRemapper:: bind_field (const field_type& src, const field_type& tgt) { EKAT_REQUIRE_MSG(m_state!=RepoState::Clean, diff --git a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp index afbe63b04a84..af80889add1b 100644 --- a/components/eamxx/src/share/grid/remap/abstract_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/abstract_remapper.hpp @@ -56,6 +56,9 @@ class AbstractRemapper register_field(create_src_fid(tgt),tgt); } + virtual void register_field_from_src (const field_type& src); + virtual void register_field_from_tgt (const field_type& tgt); + // Call this to indicate that field registration is complete. void registration_ends (); diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp index c05ea30efdcc..c798cd9a31f3 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.cpp @@ -15,45 +15,53 @@ namespace scream CoarseningRemapper:: CoarseningRemapper (const grid_ptr_type& src_grid, const std::string& map_file, - const bool track_mask) + const bool track_mask, + const bool populate_tgt_grid_geo_data) : HorizInterpRemapperBase (src_grid,map_file,InterpType::Coarsen) , m_track_mask (track_mask) { using namespace ShortFieldTagsNames; - // Replicate the src grid geo data in the tgt grid. We use this remapper to do - // the remapping (if needed), and clean it up afterwards. - const auto& src_geo_data_names = src_grid->get_geometry_data_names(); - registration_begins(); - for (const auto& name : src_geo_data_names) { - const auto& src_data = src_grid->get_geometry_data(name); - const auto& src_data_fid = src_data.get_header().get_identifier(); - const auto& layout = src_data_fid.get_layout(); - if (layout.tags()[0]!=COL) { - // Not a field to be coarsened (perhaps a vertical coordinate field). - // Simply copy it in the tgt grid, but we still need to assign the new grid name. - FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); - auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); - tgt_data.deep_copy(src_data); - } else { - // This field needs to be remapped - auto tgt_data_fid = create_tgt_fid(src_data_fid); - auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); - register_field(src_data,tgt_data); + if (populate_tgt_grid_geo_data) { + // Replicate the src grid geo data in the tgt grid. We use this remapper to do + // the remapping (if needed), and clean it up afterwards. + const auto& src_geo_data_names = src_grid->get_geometry_data_names(); + registration_begins(); + for (const auto& name : src_geo_data_names) { + // Since different remappers may share the same data (if the map file is the same) + // the coarse grid may already have the geo data. + if (m_coarse_grid->has_geometry_data(name)) { + continue; + } + const auto& src_data = src_grid->get_geometry_data(name); + const auto& src_data_fid = src_data.get_header().get_identifier(); + const auto& layout = src_data_fid.get_layout(); + if (layout.tags()[0]!=COL) { + // Not a field to be coarsened (perhaps a vertical coordinate field). + // Simply copy it in the tgt grid, but we still need to assign the new grid name. + FieldIdentifier tgt_data_fid(src_data_fid.name(),src_data_fid.get_layout(),src_data_fid.get_units(),m_tgt_grid->name()); + auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); + tgt_data.deep_copy(src_data); + } else { + // This field needs to be remapped + auto tgt_data_fid = create_tgt_fid(src_data_fid); + auto tgt_data = m_coarse_grid->create_geometry_data(tgt_data_fid); + register_field(src_data,tgt_data); + } } - } - registration_ends(); - if (get_num_fields()>0) { - remap(true); - - // The remap phase only alters the fields on device. - // We need to sync them to host as well - for (int i=0; i0) { + remap(true); + + // The remap phase only alters the fields on device. + // We need to sync them to host as well + for (int i=0; i(); + bool mask1d = mask.rank()==1; + view_1d mask_1d; + view_2d mask_2d; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + if (mask.rank()==1) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); + } + const int dim1 = layout.dim(1); + const int dim2 = layout.dim(2); + const int dim3 = PackInfo::num_packs(layout.dim(3)); + auto policy = ESU::get_default_team_policy(ncols,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto icol = team.league_rank(); + if (mask1d) { + auto mask = mask_1d(icol); + if (mask>mask_threshold) { + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + auto x_sub = ekat::subview(x_view,icol,j,k); + x_sub(l) /= mask; + }); + } + } else { + auto m_sub = ekat::subview(mask_2d,icol); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + auto x_sub = ekat::subview(x_view,icol,j,k); + auto masked = m_sub(l) > mask_threshold; + + if (masked.any()) { + x_sub(l).set(masked,x_sub(l)/m_sub(l)); + } + x_sub(l).set(!masked,mask_val); + }); + } + }); + break; + } } } @@ -458,6 +517,46 @@ local_mat_vec (const Field& x, const Field& y, const Field& mask) const }); break; } + case 4: + { + auto x_view = x.get_view(); + auto y_view = y.get_view< Pack****>(); + // Note, the mask is still assumed to be defined on COLxLEV so still only 2D for case 3. + view_1d mask_1d; + view_2d mask_2d; + bool mask1d = mask.rank()==1; + // If the mask comes from FieldAtLevel, it's only defined on columns (rank=1) + // If the mask comes from vert interpolation remapper, it is defined on ncols x nlevs (rank=2) + if (mask1d) { + mask_1d = mask.get_view(); + } else { + mask_2d = mask.get_view(); + } + const int dim1 = src_layout.dim(1); + const int dim2 = src_layout.dim(2); + const int dim3 = PackInfo::num_packs(src_layout.dim(3)); + auto policy = ESU::get_default_team_policy(nrows,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto row = team.league_rank(); + + const auto beg = row_offsets(row); + const auto end = row_offsets(row+1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + y_view(row,j,k,l) = weights(beg)*x_view(col_lids(beg),j,k,l) * + (mask1d ? mask_1d (col_lids(beg)) : mask_2d(col_lids(beg),l)); + for (int icol=beg+1; icol(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + auto policy = ESU::get_default_team_policy(num_send_gids,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team){ + const int i = team.league_rank(); + const int lid = lids_pids(i,0); + const int pid = lids_pids(i,1); + const int lidpos = i - pid_lid_start(pid); + const int offset = f_pid_offsets(pid); + + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx) { + const int idim = (idx / dim3) / dim2; + const int jdim = (idx / dim3) % dim2; + const int ilev = idx % dim3; + buf(offset + lidpos*dim1*dim2*dim3 + idim*dim2*dim3 + jdim*dim3 + ilev) = v(lid,idim,jdim,ilev); + }); + }); + } break; default: EKAT_ERROR_MSG ("Unexpected field rank in CoarseningRemapper::pack.\n" @@ -659,6 +782,34 @@ void CoarseningRemapper::recv_and_unpack () }); } break; + case 4: + { + auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + auto policy = ESU::get_default_team_policy(num_tgt_dofs,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team){ + const int lid = team.league_rank(); + const int recv_beg = recv_lids_beg(lid); + const int recv_end = recv_lids_end(lid); + for (int irecv=recv_beg; irecv>& pid2gids_send) const void CoarseningRemapper::setup_mpi_data_structures () { using namespace ShortFieldTagsNames; + using gid_type = AbstractGrid::gid_type; const auto mpi_comm = m_comm.mpi_comm(); const auto mpi_real = ekat::get_mpi_type(); diff --git a/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp b/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp index 6c8088ead14b..8146fff8acf6 100644 --- a/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/coarsening_remapper.hpp @@ -44,7 +44,8 @@ class CoarseningRemapper : public HorizInterpRemapperBase CoarseningRemapper (const grid_ptr_type& src_grid, const std::string& map_file, - const bool track_mask = false); + const bool track_mask = false, + const bool populate_tgt_grid_geo_data = true); ~CoarseningRemapper (); diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp index cb65bdda88d6..5fa1e46ba646 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.cpp @@ -16,6 +16,7 @@ HorizInterpRemapperBase (const grid_ptr_type& fine_grid, const std::string& map_file, const InterpType type) : m_fine_grid(fine_grid) + , m_map_file (map_file) , m_type (type) , m_comm (fine_grid->get_comm()) { @@ -25,27 +26,66 @@ HorizInterpRemapperBase (const grid_ptr_type& fine_grid, " - fine grid name: " + fine_grid->name() + "\n" " - fine_grid_type: " + e2str(fine_grid->type()) + "\n"); EKAT_REQUIRE_MSG (fine_grid->is_unique(), - "Error! CoarseningRemapper requires a unique source grid.\n"); + "Error! HorizInterpRemapperBase requires a unique fine grid.\n"); // This is a special remapper. We only go in one direction m_bwd_allowed = false; - // Read the map file, loading the triplets this rank needs for the crs matrix - // in the map file that this rank has to read - auto my_triplets = get_my_triplets (map_file); + // Get the remap data (if not already present, it will be built) + auto& data = s_remapper_data[m_map_file]; + if (data.num_customers==0) { + data.build(m_map_file,m_fine_grid,m_comm,m_type); + } + ++data.num_customers; + + m_row_offsets = data.row_offsets; + m_col_lids = data.col_lids; + m_weights = data.weights; + + // The grids really only matter for the horiz part. We may have 2+ remappers with + // fine grids that only differ in terms of number of levs. Such remappers cannot + // store the same coarse grid. So we soft-clone the grid, and reset the number of levels + auto coarse_grid = data.coarse_grid->clone(data.coarse_grid->name(),true); + auto ov_coarse_grid = data.ov_coarse_grid->clone(data.ov_coarse_grid->name(),true); - // Create coarse/ov_coarse grids - create_coarse_grids (my_triplets); + // Reset num levs, and remove any geo data that depends on levs + using namespace ShortFieldTagsNames; + for (std::shared_ptr grid : {coarse_grid,ov_coarse_grid}) { + grid->reset_num_vertical_lev(fine_grid->get_num_vertical_levels()); + for (const auto& name : grid->get_geometry_data_names()) { + const auto& f = grid->get_geometry_data(name); + const auto& fl = f.get_header().get_identifier().get_layout(); + if (fl.has_tag(LEV) or fl.has_tag(ILEV)) { + grid->delete_geometry_data(name); + } + } + } + m_coarse_grid = coarse_grid; + m_ov_coarse_grid = ov_coarse_grid; - // Set src/tgt grid, based on interpolation type if (m_type==InterpType::Refine) { - set_grids (m_coarse_grid,m_fine_grid); + set_grids(m_coarse_grid,m_fine_grid); } else { - set_grids (m_fine_grid,m_coarse_grid); + set_grids(m_fine_grid,m_coarse_grid); + } +} + +HorizInterpRemapperBase:: +~HorizInterpRemapperBase () +{ + auto it = s_remapper_data.find(m_map_file); + if (it==s_remapper_data.end()) { + // This would be very suspicious. But since the error is "benign", + // and since we want to avoid throwing inside a destructor, just issue a warning. + std::cerr << "WARNING! Remapper data for this map file was already deleted!\n" + " - map file: " << m_map_file << "\n"; + return; } - // Create crs matrix - create_crs_matrix_structures (my_triplets); + --it->second.num_customers; + if (it->second.num_customers==0) { + s_remapper_data.erase(it); + } } FieldLayout HorizInterpRemapperBase:: @@ -58,28 +98,7 @@ create_src_layout (const FieldLayout& tgt_layout) const "[HorizInterpRemapperBase] Error! Input target layout is not valid for this remapper.\n" " - input layout: " + to_string(tgt_layout)); - using namespace ShortFieldTagsNames; - const auto lt = get_layout_type(tgt_layout.tags()); - const bool midpoints = tgt_layout.has_tag(LEV); - const int vec_dim = tgt_layout.is_vector_layout() ? tgt_layout.dim(CMP) : -1; - auto src = FieldLayout::invalid(); - switch (lt) { - case LayoutType::Scalar2D: - src = m_src_grid->get_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - src = m_src_grid->get_2d_vector_layout(CMP,vec_dim); - break; - case LayoutType::Scalar3D: - src = m_src_grid->get_3d_scalar_layout(midpoints); - break; - case LayoutType::Vector3D: - src = m_src_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); - break; - default: - EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); - } - return src; + return create_layout (tgt_layout, m_src_grid); } FieldLayout HorizInterpRemapperBase:: @@ -92,28 +111,55 @@ create_tgt_layout (const FieldLayout& src_layout) const "[HorizInterpRemapperBase] Error! Input source layout is not valid for this remapper.\n" " - input layout: " + to_string(src_layout)); + return create_layout (src_layout, m_tgt_grid); +} + +FieldLayout HorizInterpRemapperBase:: +create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid) const +{ using namespace ShortFieldTagsNames; - const auto lt = get_layout_type(src_layout.tags()); - auto tgt = FieldLayout::invalid(); - const bool midpoints = src_layout.has_tag(LEV); - const int vec_dim = src_layout.is_vector_layout() ? src_layout.dim(CMP) : -1; - switch (lt) { - case LayoutType::Scalar2D: - tgt = m_tgt_grid->get_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - tgt = m_tgt_grid->get_2d_vector_layout(CMP,vec_dim); - break; + const auto type = get_layout_type(fl_in.tags()); + auto fl_out = FieldLayout::invalid(); + const bool midpoints = fl_in.has_tag(LEV); + const bool is3d = fl_in.has_tag(LEV) or fl_in.has_tag(ILEV); + switch (type) { + case LayoutType::Scalar2D: [[ fallthrough ]]; case LayoutType::Scalar3D: - tgt = m_tgt_grid->get_3d_scalar_layout(midpoints); + fl_out = is3d + ? grid->get_3d_scalar_layout(midpoints) + : grid->get_2d_scalar_layout(); break; + case LayoutType::Vector2D: [[ fallthrough ]]; case LayoutType::Vector3D: - tgt = m_tgt_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); + { + auto vtag = fl_in.get_vector_tag(); + auto vdim = fl_in.dim(vtag); + fl_out = is3d + ? grid->get_3d_vector_layout(midpoints,vtag,vdim) + : grid->get_2d_vector_layout(vtag,vdim); break; + } + + case LayoutType::Tensor2D: [[ fallthrough ]]; + case LayoutType::Tensor3D: + { + auto ttags = fl_in.get_tensor_tags(); + std::vector tdims; + for (auto idx : fl_in.get_tensor_dims()) { + tdims.push_back(fl_in.dim(idx)); + } + fl_out = is3d + ? grid->get_3d_tensor_layout(midpoints,ttags,tdims) + : grid->get_2d_tensor_layout(ttags,tdims); + break; + } + default: - EKAT_ERROR_MSG ("Layout not supported by CoarseningRemapper: " + e2str(lt) + "\n"); + EKAT_ERROR_MSG ("Layout not supported by HorizInterpRemapperBase:\n" + " - layout: " + to_string(fl_in) + "\n"); } - return tgt; + return fl_out; } void HorizInterpRemapperBase::do_registration_ends () @@ -159,222 +205,6 @@ do_bind_field (const int ifield, const field_type& src, const field_type& tgt) } } -auto HorizInterpRemapperBase:: -get_my_triplets (const std::string& map_file) const - -> std::vector -{ - using gid_type = AbstractGrid::gid_type; - using namespace ShortFieldTagsNames; - - // 1. Load the map file chunking it evenly across all ranks - scorpio::register_file(map_file,scorpio::FileMode::Read); - - // 1.1 Create a "helper" grid, with as many dofs as the number - // of triplets in the map file, and divided linearly across ranks - const int ngweights = scorpio::get_dimlen(map_file,"n_s"); - int nlweights = ngweights / m_comm.size(); - if (m_comm.rank() < (ngweights % m_comm.size())) { - nlweights += 1; - } - - gid_type offset = nlweights; - m_comm.scan(&offset,1,MPI_SUM); - offset -= nlweights; // scan is inclusive, but we need exclusive - - // Create a unique decomp tag, which ensures all refining remappers have - // their own decomposition - static int tag_counter = 0; - const std::string int_decomp_tag = "RR::gmtg,int,grid-idx=" + std::to_string(tag_counter++); - const std::string real_decomp_tag = "RR::gmtg,real,grid-idx=" + std::to_string(tag_counter++); - - // 1.2 Read a chunk of triplets col indices - std::vector cols(nlweights); - std::vector rows(nlweights); - std::vector S(nlweights); - - scorpio::register_variable(map_file, "col", "col", {"n_s"}, "int", int_decomp_tag); - scorpio::register_variable(map_file, "row", "row", {"n_s"}, "int", int_decomp_tag); - scorpio::register_variable(map_file, "S", "S", {"n_s"}, "real", real_decomp_tag); - - std::vector dofs_offsets(nlweights); - std::iota(dofs_offsets.begin(),dofs_offsets.end(),offset); - scorpio::set_dof(map_file,"col",nlweights,dofs_offsets.data()); - scorpio::set_dof(map_file,"row",nlweights,dofs_offsets.data()); - scorpio::set_dof(map_file,"S" ,nlweights,dofs_offsets.data()); - scorpio::set_decomp(map_file); - - scorpio::grid_read_data_array(map_file,"col",-1,cols.data(),cols.size()); - scorpio::grid_read_data_array(map_file,"row",-1,rows.data(),rows.size()); - scorpio::grid_read_data_array(map_file,"S" ,-1,S.data(),S.size()); - - scorpio::eam_pio_closefile(map_file); - - // 1.3 Dofs in grid are likely 0-based, while row/col ids in map file - // are likely 1-based. To match dofs, we need to offset the row/cols - // ids we just read in. - int map_file_min_row = std::numeric_limits::max(); - int map_file_min_col = std::numeric_limits::max(); - for (int id=0; idget_global_min_dof_gid(); - } else { - col_offset -= m_fine_grid->get_global_min_dof_gid(); - } - for (auto& id : rows) { - id -= row_offset; - } - for (auto& id : cols) { - id -= col_offset; - } - - // Create a grid based on the row gids I read in (may be duplicated across ranks) - std::vector unique_gids; - const auto& gids = m_type==InterpType::Refine ? rows : cols; - for (auto gid : gids) { - if (not ekat::contains(unique_gids,gid)) { - unique_gids.push_back(gid); - } - } - auto io_grid = std::make_shared ("helper",unique_gids.size(),0,m_comm); - auto io_grid_gids_h = io_grid->get_dofs_gids().get_view(); - int k = 0; - for (auto gid : unique_gids) { - io_grid_gids_h(k++) = gid; - } - io_grid->get_dofs_gids().sync_to_dev(); - - // Create Triplets to export, sorted by gid - std::map> io_triplets; - auto io_grid_gid2lid = io_grid->get_gid2lid_map(); - for (int i=0; i(); - auto mpi_real_t = ekat::get_mpi_type(); - int lengths[3] = {1,1,1}; - MPI_Aint displacements[3] = {0, offsetof(Triplet,col), offsetof(Triplet,w)}; - MPI_Datatype types[3] = {mpi_gid_t,mpi_gid_t,mpi_real_t}; - MPI_Datatype mpi_triplet_t; - MPI_Type_create_struct (3,lengths,displacements,types,&mpi_triplet_t); - MPI_Type_commit(&mpi_triplet_t); - - // Create import-export - GridImportExport imp_exp (m_fine_grid,io_grid); - std::map> my_triplets_map; - imp_exp.gather(mpi_triplet_t,io_triplets,my_triplets_map); - MPI_Type_free(&mpi_triplet_t); - - std::vector my_triplets; - for (auto& it : my_triplets_map) { - my_triplets.reserve(my_triplets.size()+it.second.size()); - std::move(it.second.begin(),it.second.end(),std::back_inserter(my_triplets)); - } - - return my_triplets; -} - -void HorizInterpRemapperBase:: -create_coarse_grids (const std::vector& triplets) -{ - const int nlevs = m_fine_grid->get_num_vertical_levels(); - - // Gather overlapped coarse grid gids (rows or cols, depending on m_type) - std::map ov_gid2lid; - bool pickRow = m_type==InterpType::Coarsen; - for (const auto& t : triplets) { - ov_gid2lid.emplace(pickRow ? t.row : t.col,ov_gid2lid.size()); - } - int num_ov_gids = ov_gid2lid.size(); - - // Use a temp and then assing, b/c grid_ptr_type is a pointer to const, - // so you can't modify gids using that pointer - auto ov_coarse_grid = std::make_shared("ov_coarse_grid",num_ov_gids,nlevs,m_comm); - auto ov_coarse_gids_h = ov_coarse_grid->get_dofs_gids().get_view(); - for (const auto& it : ov_gid2lid) { - ov_coarse_gids_h[it.second] = it.first; - } - ov_coarse_grid->get_dofs_gids().sync_to_dev(); - m_ov_coarse_grid = ov_coarse_grid; - - // Create the unique coarse grid - auto coarse_gids = m_ov_coarse_grid->get_unique_gids(); - int num_gids = coarse_gids.size(); - m_coarse_grid = std::make_shared("coarse_grid",num_gids,nlevs,m_comm); - auto coarse_gids_h = m_coarse_grid->get_dofs_gids().get_view(); - std::copy(coarse_gids.begin(),coarse_gids.end(),coarse_gids_h.data()); - m_coarse_grid->get_dofs_gids().sync_to_dev(); -} - -void HorizInterpRemapperBase:: -create_crs_matrix_structures (std::vector& triplets) -{ - // Get row/col data depending on interp type - bool refine = m_type==InterpType::Refine; - auto row_grid = refine ? m_fine_grid : m_ov_coarse_grid; - auto col_grid = refine ? m_ov_coarse_grid : m_fine_grid; - const int num_rows = row_grid->get_num_local_dofs(); - - auto col_gid2lid = col_grid->get_gid2lid_map(); - auto row_gid2lid = row_grid->get_gid2lid_map(); - - // Sort triplets so that row GIDs appear in the same order as - // in the row grid. If two row GIDs are the same, use same logic - // with col - auto compare = [&] (const Triplet& lhs, const Triplet& rhs) { - auto lhs_lrow = row_gid2lid.at(lhs.row); - auto rhs_lrow = row_gid2lid.at(rhs.row); - auto lhs_lcol = col_gid2lid.at(lhs.col); - auto rhs_lcol = col_gid2lid.at(rhs.col); - return lhs_lrow("",num_rows+1); - m_col_lids = view_1d("",nnz); - m_weights = view_1d("",nnz); - - auto row_offsets_h = Kokkos::create_mirror_view(m_row_offsets); - auto col_lids_h = Kokkos::create_mirror_view(m_col_lids); - auto weights_h = Kokkos::create_mirror_view(m_weights); - - // Fill col ids and weights - for (int i=0; i row_counts(num_rows); - for (int i=0; i(); + auto y_view = y.get_view< Pack****>(); + const int dim1 = src_layout.dim(1); + const int dim2 = src_layout.dim(2); + const int dim3 = PackInfo::num_packs(src_layout.dim(3)); + auto policy = ESU::get_default_team_policy(nrows,dim1*dim2*dim3); + Kokkos::parallel_for(policy, + KOKKOS_LAMBDA(const MemberType& team) { + const auto row = team.league_rank(); + + const auto beg = row_offsets(row); + const auto end = row_offsets(row+1); + Kokkos::parallel_for(Kokkos::TeamVectorRange(team,dim1*dim2*dim3), + [&](const int idx){ + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + y_view(row,j,k,l) = weights(beg)*x_view(col_lids(beg),j,k,l); + for (int icol=beg+1; icol HorizInterpRemapperBase::s_remapper_data; + // ETI, so derived classes can call this method template void HorizInterpRemapperBase:: diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp index 9a91220d9a5d..c604ea085c04 100644 --- a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_base.hpp @@ -2,6 +2,7 @@ #define SCREAM_HORIZ_INTERP_REMAPPER_BASE_HPP #include "share/grid/remap/abstract_remapper.hpp" +#include "share/grid/remap/horiz_interp_remapper_data.hpp" namespace scream { @@ -16,18 +17,12 @@ namespace scream class HorizInterpRemapperBase : public AbstractRemapper { -protected: - enum class InterpType { - Refine, - Coarsen - }; - public: HorizInterpRemapperBase (const grid_ptr_type& fine_grid, const std::string& map_file, const InterpType type); - virtual ~HorizInterpRemapperBase () = default; + ~HorizInterpRemapperBase (); FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override; FieldLayout create_tgt_layout (const FieldLayout& src_layout) const override; @@ -44,6 +39,9 @@ class HorizInterpRemapperBase : public AbstractRemapper protected: + FieldLayout create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid) const; + const identifier_type& do_get_src_field_id (const int ifield) const override { return m_src_fields[ifield].get_header().get_identifier(); } @@ -66,32 +64,11 @@ class HorizInterpRemapperBase : public AbstractRemapper EKAT_ERROR_MSG ("HorizInterpRemapperBase only supports fwd remapping.\n"); } - using gid_type = AbstractGrid::gid_type; using KT = KokkosTypes; template using view_1d = typename KT::template view_1d; - struct Triplet { - // Note: unfortunately, C++17 does not support emplace-ing POD - // types as aggregates unless a ctor is declared. C++20 does though. - Triplet () = default; - Triplet(const gid_type rr, const gid_type cc, const Real ww) - : row(rr), col(cc), w(ww) {} - gid_type row; - gid_type col; - Real w; - }; - - std::vector - get_my_triplets (const std::string& map_file) const; - - void create_coarse_grids (const std::vector& triplets); - - // Not a const ref, since we'll sort the triplets according to - // how row gids appear in the coarse grid - void create_crs_matrix_structures (std::vector& triplets); - void create_ov_fields (); void clean_up (); @@ -129,9 +106,15 @@ class HorizInterpRemapperBase : public AbstractRemapper view_1d m_col_lids; view_1d m_weights; + // Keep track of this, since we need to tell the remap data repo + // we are releasing the data for our map file. + std::string m_map_file; + InterpType m_type; ekat::Comm m_comm; + + static std::map s_remapper_data; }; } // namespace scream diff --git a/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp new file mode 100644 index 000000000000..d44fc6aa2f0f --- /dev/null +++ b/components/eamxx/src/share/grid/remap/horiz_interp_remapper_data.cpp @@ -0,0 +1,271 @@ +#include "horiz_interp_remapper_data.hpp" + +#include "share/grid/point_grid.hpp" +#include "share/grid/grid_import_export.hpp" +#include "share/io/scream_scorpio_interface.hpp" + +#include + +namespace scream { + +// --------------- HorizRemapperData ---------------- // + +void HorizRemapperData:: +build (const std::string& map_file, + const std::shared_ptr& fine_grid_in, + const ekat::Comm& comm_in, + const InterpType type_in) +{ + comm = comm_in; + fine_grid = fine_grid_in; + type = type_in; + + // Gather sparse matrix triplets needed by this rank + auto my_triplets = get_my_triplets (map_file); + + // Create coarse/ov_coarse grids + create_coarse_grids (my_triplets); + + // Create crs matrix + create_crs_matrix_structures (my_triplets); +} + +auto HorizRemapperData:: +get_my_triplets (const std::string& map_file) const + -> std::vector +{ + using gid_type = AbstractGrid::gid_type; + using namespace ShortFieldTagsNames; + + // 1. Load the map file chunking it evenly across all ranks + scorpio::register_file(map_file,scorpio::FileMode::Read); + + // 1.1 Create a "helper" grid, with as many dofs as the number + // of triplets in the map file, and divided linearly across ranks + const int ngweights = scorpio::get_dimlen(map_file,"n_s"); + int nlweights = ngweights / comm.size(); + if (comm.rank() < (ngweights % comm.size())) { + nlweights += 1; + } + + gid_type offset = nlweights; + comm.scan(&offset,1,MPI_SUM); + offset -= nlweights; // scan is inclusive, but we need exclusive + + // Create a unique decomp tag, which ensures all refining remappers have + // their own decomposition + static int tag_counter = 0; + const std::string int_decomp_tag = "RR::gmtg,int,grid-idx=" + std::to_string(tag_counter++); + const std::string real_decomp_tag = "RR::gmtg,real,grid-idx=" + std::to_string(tag_counter++); + + // 1.2 Read a chunk of triplets col indices + std::vector cols(nlweights); + std::vector rows(nlweights); + std::vector S(nlweights); + + scorpio::register_variable(map_file, "col", "col", {"n_s"}, "int", int_decomp_tag); + scorpio::register_variable(map_file, "row", "row", {"n_s"}, "int", int_decomp_tag); + scorpio::register_variable(map_file, "S", "S", {"n_s"}, "real", real_decomp_tag); + + std::vector dofs_offsets(nlweights); + std::iota(dofs_offsets.begin(),dofs_offsets.end(),offset); + scorpio::set_dof(map_file,"col",nlweights,dofs_offsets.data()); + scorpio::set_dof(map_file,"row",nlweights,dofs_offsets.data()); + scorpio::set_dof(map_file,"S" ,nlweights,dofs_offsets.data()); + scorpio::set_decomp(map_file); + + // Figure out if we are reading the right map, that is: + // - n_a or n_b matches the fine grid ncols + // - the map "direction" (fine->coarse or coarse->fine) matches m_type + const int n_a = scorpio::get_dimlen(map_file,"n_a"); + const int n_b = scorpio::get_dimlen(map_file,"n_b"); + const int ncols_fine = fine_grid->get_num_global_dofs(); + EKAT_REQUIRE_MSG (n_a==ncols_fine or n_b==ncols_fine, + "Error! The input map seems incompatible with the remapper fine grid.\n" + " - map file: " + map_file + "\n" + " - map file n_a: " + std::to_string(n_a) + "\n" + " - map file n_b: " + std::to_string(n_b) + "\n" + " - fine grid ncols: " + std::to_string(ncols_fine) + "\n"); + const bool map_is_coarsening = n_a==ncols_fine; + EKAT_REQUIRE_MSG (map_is_coarsening==(type==InterpType::Coarsen), + "Error! The input map seems incompatible with the remapper type.\n" + " - map file: " + map_file + "\n" + " - map file n_a: " + std::to_string(n_a) + "\n" + " - map file n_b: " + std::to_string(n_b) + "\n" + " - fine grid ncols: " + std::to_string(ncols_fine) + "\n" + " - remapper type: " + std::string(type==InterpType::Refine ? "refine" : "coarsen") + "\n"); + + scorpio::grid_read_data_array(map_file,"col",-1,cols.data(),nlweights); + scorpio::grid_read_data_array(map_file,"row",-1,rows.data(),nlweights); + scorpio::grid_read_data_array(map_file,"S" ,-1,S.data(),nlweights); + + scorpio::eam_pio_closefile(map_file); + + // 1.3 Dofs in grid are likely 0-based, while row/col ids in map file + // are likely 1-based. To match dofs, we need to offset the row/cols + // ids we just read in. + int map_file_min_row = std::numeric_limits::max(); + int map_file_min_col = std::numeric_limits::max(); + for (int id=0; idget_global_min_dof_gid(); + } else { + col_offset -= fine_grid->get_global_min_dof_gid(); + } + for (auto& id : rows) { + id -= row_offset; + } + for (auto& id : cols) { + id -= col_offset; + } + + // Create a grid based on the row gids I read in (may be duplicated across ranks) + std::vector unique_gids; + const auto& gids = type==InterpType::Refine ? rows : cols; + for (auto gid : gids) { + if (not ekat::contains(unique_gids,gid)) { + unique_gids.push_back(gid); + } + } + auto io_grid = std::make_shared ("helper",unique_gids.size(),0,comm); + auto io_grid_gids_h = io_grid->get_dofs_gids().get_view(); + int k = 0; + for (auto gid : unique_gids) { + io_grid_gids_h(k++) = gid; + } + io_grid->get_dofs_gids().sync_to_dev(); + + // Create Triplets to export, sorted by gid + std::map> io_triplets; + auto io_grid_gid2lid = io_grid->get_gid2lid_map(); + for (int i=0; i(); + auto mpi_real_t = ekat::get_mpi_type(); + int lengths[3] = {1,1,1}; + MPI_Aint displacements[3] = {0, offsetof(Triplet,col), offsetof(Triplet,w)}; + MPI_Datatype types[3] = {mpi_gid_t,mpi_gid_t,mpi_real_t}; + MPI_Datatype mpi_triplet_t; + MPI_Type_create_struct (3,lengths,displacements,types,&mpi_triplet_t); + MPI_Type_commit(&mpi_triplet_t); + + // Create import-export + GridImportExport imp_exp (fine_grid,io_grid); + std::map> my_triplets_map; + imp_exp.gather(mpi_triplet_t,io_triplets,my_triplets_map); + MPI_Type_free(&mpi_triplet_t); + + std::vector my_triplets; + for (auto& it : my_triplets_map) { + my_triplets.reserve(my_triplets.size()+it.second.size()); + std::move(it.second.begin(),it.second.end(),std::back_inserter(my_triplets)); + } + + return my_triplets; +} + +void HorizRemapperData:: +create_coarse_grids (const std::vector& triplets) +{ + // Gather overlapped coarse grid gids (rows or cols, depending on type) + std::map ov_gid2lid; + bool pickRow = type==InterpType::Coarsen; + for (const auto& t : triplets) { + ov_gid2lid.emplace(pickRow ? t.row : t.col,ov_gid2lid.size()); + } + int num_ov_gids = ov_gid2lid.size(); + + // Use a temp and then assing, b/c grid_ptr_type is a pointer to const, + // so you can't modify gids using that pointer + ov_coarse_grid = std::make_shared("ov_coarse_grid",num_ov_gids,0,comm); + auto ov_coarse_gids_h = ov_coarse_grid->get_dofs_gids().get_view(); + for (const auto& it : ov_gid2lid) { + ov_coarse_gids_h[it.second] = it.first; + } + auto beg = ov_coarse_gids_h.data(); + auto end = beg+ov_coarse_gids_h.size(); + std::sort(beg,end); + + ov_coarse_grid->get_dofs_gids().sync_to_dev(); + + // Create the unique coarse grid + auto coarse_gids = ov_coarse_grid->get_unique_gids(); + int num_gids = coarse_gids.size(); + coarse_grid = std::make_shared("coarse_grid",num_gids,0,comm); + auto coarse_gids_h = coarse_grid->get_dofs_gids().get_view(); + std::copy(coarse_gids.begin(),coarse_gids.end(),coarse_gids_h.data()); + coarse_grid->get_dofs_gids().sync_to_dev(); +} + +void HorizRemapperData:: +create_crs_matrix_structures (std::vector& triplets) +{ + // Get row/col data depending on interp type + bool refine = type==InterpType::Refine; + auto row_grid = refine ? fine_grid : ov_coarse_grid; + auto col_grid = refine ? ov_coarse_grid : fine_grid; + const int num_rows = row_grid->get_num_local_dofs(); + + auto col_gid2lid = col_grid->get_gid2lid_map(); + auto row_gid2lid = row_grid->get_gid2lid_map(); + + // Sort triplets so that row GIDs appear in the same order as + // in the row grid. If two row GIDs are the same, use same logic + // with col + auto compare = [&] (const Triplet& lhs, const Triplet& rhs) { + auto lhs_lrow = row_gid2lid.at(lhs.row); + auto rhs_lrow = row_gid2lid.at(rhs.row); + auto lhs_lcol = col_gid2lid.at(lhs.col); + auto rhs_lcol = col_gid2lid.at(rhs.col); + return lhs_lrow("",num_rows+1); + col_lids = view_1d("",nnz); + weights = view_1d("",nnz); + + auto row_offsets_h = Kokkos::create_mirror_view(row_offsets); + auto col_lids_h = Kokkos::create_mirror_view(col_lids); + auto weights_h = Kokkos::create_mirror_view(weights); + + // Fill col ids and weights + for (int i=0; i row_counts(num_rows); + for (int i=0; i + +#include +#include +#include + +namespace scream { + +enum class InterpType { + Refine, + Coarsen +}; + +// A small struct to hold horiz remap data, which can +// be shared across multiple horiz remappers +struct HorizRemapperData { + using KT = KokkosTypes; + template + using view_1d = typename KT::template view_1d; + + void build (const std::string& map_file, + const std::shared_ptr& fine_grid, + const ekat::Comm& comm, + const InterpType type); + + // The coarse grid data + std::shared_ptr coarse_grid; + std::shared_ptr ov_coarse_grid; + + // The CRS matrix data for online interpolation + view_1d row_offsets; + view_1d col_lids; + view_1d weights; + + int num_customers = 0; +private: + using gid_type = AbstractGrid::gid_type; + + InterpType type; + std::shared_ptr fine_grid; + ekat::Comm comm; + + struct Triplet { + // Note: unfortunately, C++17 does not support emplace-ing POD + // types as aggregates unless a ctor is declared. C++20 does though. + Triplet () = default; + Triplet(const gid_type rr, const gid_type cc, const Real ww) + : row(rr), col(cc), w(ww) {} + gid_type row; + gid_type col; + Real w; + }; + + std::vector + get_my_triplets (const std::string& map_file) const; + + void create_coarse_grids (const std::vector& triplets); + + // Not a const ref, since we'll sort the triplets according to + // how row gids appear in the coarse grid + void create_crs_matrix_structures (std::vector& triplets); +}; + +} // namespace scream + +#endif // EAMXX_HORIZ_INTERP_REMAP_DATA_HPP diff --git a/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp b/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp deleted file mode 100644 index c9950f126cce..000000000000 --- a/components/eamxx/src/share/grid/remap/horizontal_remap_utility.cpp +++ /dev/null @@ -1,546 +0,0 @@ -#include "share/grid/remap/horizontal_remap_utility.hpp" -#include "share/util/scream_timing.hpp" - -namespace scream { - -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMap::HorizontalMap(const ekat::Comm& comm) - : m_comm (comm) -{ - m_dofs_set = false; -} -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMap::HorizontalMap(const ekat::Comm& comm, const std::string& map_name) - : m_name (map_name) - , m_comm (comm) -{ - m_dofs_set = false; -} -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMap::HorizontalMap(const ekat::Comm& comm, const std::string& map_name, const view_1d& dofs_gids, const gid_type min_dof) - : m_name (map_name) - , m_comm (comm) -{ - set_dof_gids(dofs_gids,min_dof); -} -/*-----------------------------------------------------------------------------------------------*/ -/*-----------------------------------------------------------------------------------------------*/ -// This function sets the remap segments for this map given a remap file that was created offline. -// Note: This function assumes that the remap file follows the convention: -// col - This variable in the file represents the set of source dofs that map to a specific target. -// row - This variable represents the corresponding list of target columns mapped to -// S - This variable stores the corresponding weights for each col -> row pair -// n_s - Is the integer number of col -> row mappings. -// for reference there may also be -// n_a - Is the size of the source grid. -// n_b - Is the size of the target grid. -// Following these conventions we assume that -// col - will be used to populate a segment's "source_dofs" -// row - will be used to populate a segment's "m_dof" -// S - will be used to populate a segment's "weights" -void HorizontalMap::set_remap_segments_from_file(const std::string& remap_filename) -{ - // Ensure each horiz remap file gets a unique decomp tag - static std::map file2idx; - if (file2idx.find(remap_filename)==file2idx.end()) { - file2idx[remap_filename] = file2idx.size(); - } - - start_timer("EAMxx::HorizontalMap::set_remap_segments_from_file"); - // Open remap file and determine the amount of data to be read - scorpio::register_file(remap_filename,scorpio::Read); - const auto remap_size = scorpio::get_dimlen(remap_filename,"n_s"); // Note, here we assume a standard format of col, row, S - // Step 1: Read in the "row" data from the file to figure out which mpi ranks care about which - // chunk of the remap data. This step reduces the memory footprint of reading in the - // map data, which can be rather large. - // Distribute responsibility for reading remap data over all ranks - const int my_rank = m_comm.rank(); - const int num_ranks = m_comm.size(); - // my_chunk will represent the chunk of data this rank will read from file. - int my_chunk = remap_size/num_ranks; - int remainder = remap_size - (my_chunk*num_ranks); - if (remainder != 0) { - my_chunk += my_rank chunks_glob(num_ranks); - m_comm.all_gather(&my_chunk,chunks_glob.data(),1); - int my_start = 0; - for (int ii=0; ii tgt_col("row",my_chunk); - auto tgt_col_h = Kokkos::create_mirror_view(tgt_col); - std::vector vec_of_dims = {"n_s"}; - std::string i_decomp = "HR::srsff,phase1,dt=int,n_s=" + std::to_string(my_chunk) + ",file-idx=" + std::to_string(file2idx[remap_filename]); - scorpio::register_variable(remap_filename, "row", "row", vec_of_dims, "int", i_decomp); - std::vector var_dof(my_chunk); - std::iota(var_dof.begin(),var_dof.end(),my_start); - scorpio::set_dof(remap_filename,"row",var_dof.size(),var_dof.data()); - scorpio::set_decomp(remap_filename); - scorpio::grid_read_data_array(remap_filename,"row",0,tgt_col_h.data(),tgt_col_h.size()); - scorpio::eam_pio_closefile(remap_filename); - // Step 2: Now that we have the data distributed among all ranks we organize the data - // into sets of target column, start location in data and length of data. - // At the same time, determine the min_dof for remap column indices. - std::vector chunk_dof, chunk_start, chunk_len; - chunk_dof.push_back(tgt_col_h(0)); - chunk_start.push_back(my_start); - chunk_len.push_back(1); - int remap_min_dof = tgt_col_h(0); - for (int ii=1; ii num_chunks_per_rank(num_ranks), chunk_displacement(num_ranks); - int total_num_chunks; - int global_remap_min_dof; - m_comm.all_gather(&num_chunks, num_chunks_per_rank.data(),1); - m_comm.all_reduce(&remap_min_dof,&global_remap_min_dof,1,MPI_MIN); - chunk_displacement[0] = 0; - total_num_chunks = num_chunks_per_rank[0]; - for (int ii=1; ii buff_dof(total_num_chunks), buff_sta(total_num_chunks), buff_len(total_num_chunks); - MPI_Allgatherv(chunk_dof.data(), chunk_dof.size(),MPI_INT,buff_dof.data(),num_chunks_per_rank.data(),chunk_displacement.data(),MPI_INT,m_comm.mpi_comm()); - MPI_Allgatherv(chunk_start.data(),chunk_dof.size(),MPI_INT,buff_sta.data(),num_chunks_per_rank.data(),chunk_displacement.data(),MPI_INT,m_comm.mpi_comm()); - MPI_Allgatherv(chunk_len.data(), chunk_dof.size(),MPI_INT,buff_len.data(),num_chunks_per_rank.data(),chunk_displacement.data(),MPI_INT,m_comm.mpi_comm()); - // Step 3: Now that all of the ranks are aware of all of the "sets" of source -> target mappings we - // construct and add segments for just the DOF's this rank cares about. - std::vector seg_dof, seg_start, seg_length; - var_dof.clear(); - auto dofs_gids_h = Kokkos::create_mirror_view(m_dofs_gids); - Kokkos::deep_copy(dofs_gids_h,m_dofs_gids); - for (int ii=0; ii var_tmp(buff_len[ii]); - std::iota(var_tmp.begin(),var_tmp.end(),buff_sta[ii]); - seg_dof.push_back(buff_dof[ii]); - seg_start.push_back(var_dof.size()); - seg_length.push_back(buff_len[ii]); - var_dof.insert(var_dof.end(),var_tmp.begin(),var_tmp.end()); - } - } - } - // Now that we know which parts of the remap file this rank cares about we can construct segments - view_1d col("col",var_dof.size()); - view_1d S("S",var_dof.size()); - auto col_h = Kokkos::create_mirror_view(col); - auto S_h = Kokkos::create_mirror_view(S); - vec_of_dims = {"n_s"}; - i_decomp = "HR::srsff,phase2,dt=int,n_s=" + std::to_string(var_dof.size()) + ",file-idx=" + std::to_string(file2idx[remap_filename]); - std::string r_decomp = "HR::srsff,phase2,dt=real,n_s=" + std::to_string(var_dof.size()) + ",file-idx=" + std::to_string(file2idx[remap_filename]); - scorpio::register_file(remap_filename,scorpio::Read); - scorpio::register_variable(remap_filename, "col", "col", vec_of_dims, "int", i_decomp); - scorpio::register_variable(remap_filename, "S", "S", vec_of_dims, "real", r_decomp); - scorpio::set_dof(remap_filename,"col",var_dof.size(),var_dof.data()); - scorpio::set_dof(remap_filename,"S",var_dof.size(),var_dof.data()); - scorpio::set_decomp(remap_filename); - scorpio::grid_read_data_array(remap_filename,"col",0,col_h.data(),col_h.size()); - scorpio::grid_read_data_array(remap_filename,"S",0,S_h.data(),S_h.size()); - scorpio::eam_pio_closefile(remap_filename); - Kokkos::deep_copy(col,col_h); - Kokkos::deep_copy(S,S_h); - // Construct segments based on data just read from file - for (size_t ii=0; ii source_dofs("",seglength); - view_1d weights("",seglength); - Kokkos::parallel_for("", seglength, KOKKOS_LAMBDA (const int& jj) { - int idx = segstart + jj; - source_dofs(jj) = col(idx)-global_remap_min_dof; // Offset to zero based dofs - weights(jj) = S(idx); - }); - HorizontalMapSegment seg(seg_dof[ii]-global_remap_min_dof,seglength,source_dofs,weights); - add_remap_segment(seg); - } - stop_timer("EAMxx::HorizontalMap::set_remap_segments_from_file"); -} -/*-----------------------------------------------------------------------------------------------*/ -// This function is used to set the internal set of degrees of freedom (dof) this map is responsible for. -// We use the global dofs, offset by the minimum global dof to make everything zero based. Note, when -// gathering remap parameters from a file, depending on the algorithm that made the file the dof -// indices may be 1-based or 0-based. By offsetting everything to 0-based we avoid potential bugs. -void HorizontalMap::set_dof_gids(const view_1d& dofs_gids, const gid_type min_dof) -{ - start_timer("EAMxx::HorizontalMap::set_dof_gids"); - EKAT_REQUIRE(dofs_gids.size()>0); - m_dofs_gids = view_1d("",dofs_gids.size()); - const auto l_dofs_gids = m_dofs_gids; - m_num_dofs = m_dofs_gids.extent(0); - Kokkos::parallel_for("", m_num_dofs, KOKKOS_LAMBDA (const int& ii) { - l_dofs_gids(ii) = dofs_gids(ii)-min_dof; - }); - m_dofs_set = true; - stop_timer("EAMxx::HorizontalMap::set_dof_gids"); -} -/*-----------------------------------------------------------------------------------------------*/ -// This function adds a remap segment to a HorizontalMap, note, we want each segment to represent a full -// remapping. This function also checks if a segment already exists for the degree of freedom -// in question. If it does then instead of add the segment to the end, this function finds that -// segment and combines them into a new comprehensive segment. -void HorizontalMap::add_remap_segment(const HorizontalMapSegment& seg) -{ - // First determine if a segment already exists in this map for the seg_dof. - gid_type seg_dof = seg.get_dof(); - int match_loc = -999; - for (int iseg=0; iseg unique_dofs; - // Check all segments and add unique dofs. Done on HOST so we can use a vector, only done once - // per map so it's alright to not be optimized for performance. - int min_gid = INT_MAX; - int max_gid = INT_MIN; - for (int iseg=0; iseg("",m_num_unique_dofs); - auto temp_h = Kokkos::create_mirror_view(m_unique_dofs); - for (int ii=0; ii& source_data, const view_1d& remapped_data) { - start_timer("EAMxx::HorizontalMap::apply_remap_1d"); - if (m_num_dofs==0) { return; } // This HorizontalMap has nothing to do for this rank. - auto remapped_data_h = Kokkos::create_mirror_view(remapped_data); - auto source_data_h = Kokkos::create_mirror_view(source_data); - Kokkos::deep_copy(source_data_h,source_data); - Kokkos::deep_copy(remapped_data_h,0.0); - for (int iseg=0; iseg& source_data, const view_2d& remapped_data) { - start_timer("EAMxx::HorizontalMap::apply_remap_2d"); - if (m_num_dofs==0) { return; } // This HorizontalMap has nothing to do for this rank. - int num_levs = source_data.extent(1); - auto remapped_data_h = Kokkos::create_mirror_view(remapped_data); - auto source_data_h = Kokkos::create_mirror_view(source_data); - Kokkos::deep_copy(source_data_h,source_data); - Kokkos::deep_copy(remapped_data_h,0.0); - for (int iseg=0; iseg& source_data, const view_3d& remapped_data) { - start_timer("EAMxx::HorizontalMap::apply_remap_3d"); - if (m_num_dofs==0) { return; } // This HorizontalMap has nothing to do for this rank. - int num_levs = source_data.extent(2); - int num_bands = source_data.extent(1); - auto remapped_data_h = Kokkos::create_mirror_view(remapped_data); - auto source_data_h = Kokkos::create_mirror_view(source_data); - Kokkos::deep_copy(source_data_h,source_data); - Kokkos::deep_copy(remapped_data_h,0.0); - for (int iseg=0; iseg("",m_length); - m_source_idx = view_1d("",m_length); - m_weights = view_1d("",m_length); - m_source_idx_h = Kokkos::create_mirror_view(m_source_idx); - m_weights_h = Kokkos::create_mirror_view(m_weights); -} -/*-----------------------------------------------------------------------------------------------*/ -HorizontalMapSegment::HorizontalMapSegment(const gid_type dof_gid, const int length, const view_1d& source_dofs, const view_1d& weights) - : m_dof (dof_gid) - , m_length (length) -{ - m_source_dofs = view_1d("",m_length); - m_weights = view_1d("",m_length); - m_source_idx = view_1d("",m_length); - Kokkos::deep_copy(m_source_dofs,source_dofs); - Kokkos::deep_copy(m_weights,weights); - m_source_idx_h = Kokkos::create_mirror_view(m_source_idx); - m_weights_h = Kokkos::create_mirror_view(m_weights); -} -/*-----------------------------------------------------------------------------------------------*/ -void HorizontalMapSegment::sync_to_host() -{ - Kokkos::deep_copy(m_source_idx_h,m_source_idx); - Kokkos::deep_copy(m_weights_h,m_weights); -} -/*-----------------------------------------------------------------------------------------------*/ -bool HorizontalMapSegment::check() const -{ - // Basic check for bounds for arrays - EKAT_REQUIRE_MSG(m_source_dofs.extent_int(0)==m_length,"Error remap segment for DOF: " + std::to_string(m_dof) + ", source_dofs view not the correct length"); - EKAT_REQUIRE_MSG(m_weights.extent_int(0)==m_length,"Error remap segment for DOF: " + std::to_string(m_dof) + ", weightss view not the correct length"); - EKAT_REQUIRE_MSG(m_source_idx.extent_int(0)==m_length,"Error remap segment for DOF: " + std::to_string(m_dof) + ", source_idx view not the correct length"); - // Check that the segment weight adds up to 1.0 - Real wgt = 0.0; - const auto weights = m_weights; - Kokkos::parallel_reduce("", m_length, KOKKOS_LAMBDA (const int& ii, Real& lsum) { - lsum += weights(ii); - },wgt); - Real tol = std::numeric_limits::epsilon() * 100.0; - if (std::abs(wgt-1.0)>=tol) { - printf("ERROR: HorizontalMap: checking remap segment for DOF = %d, total weight = %e.\n",m_dof,wgt); - return false; - } - - // If we made it this far, things all passed - return true; -} -/*-----------------------------------------------------------------------------------------------*/ -void HorizontalMapSegment::print() const -{ - printf("\n--------------------\n"); - printf("Printing information for segment with DOF = %d, DOF_idx for local decomp = %d\n",m_dof,m_dof_idx); - printf(" length = %d\n",m_length); - - auto source_dofs_h = Kokkos::create_mirror_view(m_source_dofs); - auto source_idx_h = Kokkos::create_mirror_view(m_source_idx); - auto weights_h = Kokkos::create_mirror_view(m_weights); - Kokkos::deep_copy(source_dofs_h,m_source_dofs); - Kokkos::deep_copy(source_idx_h ,m_source_idx ); - Kokkos::deep_copy(weights_h ,m_weights ); - Real total_wgt = 0.0; - printf("%10s: %10s, %10s, %s\n","ii","source dof","source idx","weight"); - for (int ii=0; ii - -namespace scream { - -/*===============================================================================================*/ -/* HorizontalMapSegment - * Lightweight structure to represent a single remapping of source data to a single target column. - * Y_target = sum_(n=1)^N ( w_n * Y_source_n ) - * See description of HorizontalMap structure for more details on the mapping. - * - * This structure is used to organize the overall horizontal remapping stored in the HorizontalMap structure. - * Each segment represents a single target column in the HorizontalMap. - * -------------------------------------- - * A.S. Donahue (LLNL): 2022-09-07 - *===============================================================================================*/ -struct HorizontalMapSegment { - - using gid_type = AbstractGrid::gid_type; - using KT = KokkosTypes; - - template - using view_1d = typename KT::template view_1d; - - template - using view_1d_host = typename KT::template view_1d::HostMirror; - - // Constructors/Destructor - HorizontalMapSegment() {}; - HorizontalMapSegment(const gid_type dof_gid, const int length); - HorizontalMapSegment(const gid_type dof_gid, const int length, const view_1d& source_dofs, const view_1d& weights); - - // Helper Functions - bool check() const; // Check if this segment is valid - void print() const; // Useful for debugging, print the segment mapping info - - // Setter Functions - void set_dof_idx(const int idx) { m_dof_idx = idx; } - void sync_to_host(); - - // Getter Functions - gid_type get_dof() const { return m_dof; } - int get_dof_idx() const { return m_dof_idx; } - int get_length() const { return m_length; } - view_1d get_source_dofs() const { return m_source_dofs; } - view_1d get_source_idx() const { return m_source_idx; } - view_1d get_weights() const { return m_weights; } - view_1d_host get_source_idx_on_host() const { return m_source_idx_h; } - view_1d_host get_weights_on_host() const { return m_weights_h; } - - // TODO: Not sure why, but this can't be set as private, otherwise `set_dof_idx` doesn't work. - int m_dof_idx = -999; // The degree of freedom w.r.t. to the local index for this map -private: - - // Remap views - view_1d m_source_dofs; - view_1d m_source_idx; - view_1d m_weights; - view_1d_host m_source_idx_h; - view_1d_host m_weights_h; - - // Segment ID - gid_type m_dof; // The global degree of freedom this segment maps to - int m_length; -}; // HorizontalMapSegment - -/*===============================================================================================*/ -/* HorizontalMap - * Structure which can be used to setup and control a horizontal remapping. This structure - * follows the basic premise that there are a set of source columns >=1 that will map to a - * single target column with a specific set of weights. Mapping follows the expression: - * Y_target = sum_(n=1)^N ( w_n * Y_source_n ) - * where, - * Y_target: Is the remapped value on the target column. - * w_n: Is the n'th weight - * Y_source_n: Is the n'th column in the source data - * N: Is the total number of source columns mapping to the target column (N>=1) - * - * This structure follows the format used by the component coupler. - * -------------------------------------- - * A.S. Donahue (LLNL): 2022-09-07 - *===============================================================================================*/ - -class HorizontalMap { - // Note: The name used for mapping in the component coupler is HorizontalMap. We could adopt a different - // name if desired. - using gid_type = AbstractGrid::gid_type; - using KT = KokkosTypes; - - template - using view_1d = typename KT::template view_1d; - - template - using view_2d = typename KT::template view_2d; - - template - using view_3d = typename KT::template view_3d; - - -public: - // Constructors/Destructor - ~HorizontalMap() = default; - HorizontalMap() {}; - explicit HorizontalMap(const ekat::Comm& comm); - HorizontalMap(const ekat::Comm& comm, const std::string& map_name); - HorizontalMap(const ekat::Comm& comm, const std::string& map_name, const view_1d& dofs_gids, const gid_type min_dof); - - // Main remap functions - void apply_remap(const view_1d& source_data, const view_1d& remapped_data); - void apply_remap(const view_2d& source_data, const view_2d& remapped_data); - void apply_remap(const view_3d& source_data, const view_3d& remapped_data); - - // Helper functions - void check() const; // A check to make sure the map is valid - void print() const; // Useful for debugging - - // Builder functions - used to build the HorizontalMap - void set_dof_gids(const view_1d& dofs_gids, const gid_type min_dof); - void set_unique_source_dofs(); - void add_remap_segment(const HorizontalMapSegment& seg); - void set_remap_segments_from_file(const std::string& remap_filename); - - // Getter functions - view_1d get_unique_source_dofs() const { return m_unique_dofs; } - int get_num_unique_dofs() const { return m_num_unique_dofs; } - int get_num_of_dofs() const { return m_num_dofs; } - std::vector get_map_segments() const { return m_map_segments; } - int get_num_of_segments() const { return m_num_segments; } - -private: - - // Global degrees of freedom information on target grid - view_1d m_dofs_gids; - int m_num_dofs = 0; - // Global degrees of freedom information on source grid - view_1d m_unique_dofs; - int m_num_unique_dofs; - bool m_unique_set = false; - // HorizontalMap data - std::string m_name = ""; - ekat::Comm m_comm; - bool m_dofs_set = false; - std::vector m_map_segments; - int m_num_segments = 0; - -}; // struct HorizontalMap - -/*===============================================================================================*/ - -} //namespace scream - -#endif // EAMXX_HORIZONTAL_REMAP_UTILITY_HPP diff --git a/components/eamxx/src/share/grid/remap/identity_remapper.hpp b/components/eamxx/src/share/grid/remap/identity_remapper.hpp index 6b6659c3f60e..047bd90ef835 100644 --- a/components/eamxx/src/share/grid/remap/identity_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/identity_remapper.hpp @@ -12,29 +12,40 @@ namespace scream * This remapper effectively does nothing, since its source and target * grids are the same. There is no *real* need for this routine, * but it makes it easier to 'generically' and 'agnostically' create - * remappers. If one hits the case of src_grid=tgt_grid, he/she can - * simply create an identity remapper. This remapper is guaranteed - * to do absolutely nothing (except for possibly some correctness - * checks, mostly through the base class interface) when the remap - * method is called. It *does* still store ids for source and target - * fields, mostly to allow queries via the base class interface. - * However, no field is actually stored (no views, that is), since - * there is no need to actually access the data. + * remappers. If one hits the case of src_grid=tgt_grid, they can + * simply create an identity remapper. When remap methods are called, + * this remapper will + * - call Field::deep_copy if m_aliasing = NoAliasing + * - do nothing if m_aliasing != NoAliasing */ class IdentityRemapper : public AbstractRemapper { public: - using base_type = AbstractRemapper; - - IdentityRemapper (const grid_ptr_type grid) - : base_type(grid,grid) + using base_type = AbstractRemapper; + + // If + enum Aliasing { + SrcAliasTgt, + TgtAliasSrc, + NoAliasing + }; + + IdentityRemapper (const grid_ptr_type grid, + const Aliasing aliasing = NoAliasing) + : base_type (grid,grid) { - // Nothing to do here + set_aliasing(aliasing); } ~IdentityRemapper () = default; + void set_aliasing (const Aliasing aliasing) { + EKAT_REQUIRE_MSG (get_state()!=RepoState::Closed, + "Error! Aliasing in IdentityRemapper must be set *before* registration ends.\n"); + m_aliasing = aliasing; + } + FieldLayout create_src_layout (const FieldLayout& tgt_layout) const override { EKAT_REQUIRE_MSG (is_valid_tgt_layout(tgt_layout), "[IdentityRemapper] Error! Input target layout is not valid for this remapper.\n" @@ -52,6 +63,24 @@ class IdentityRemapper : public AbstractRemapper return src_layout; } + void register_field_from_src (const field_type& src) { + EKAT_REQUIRE_MSG (m_aliasing!=SrcAliasTgt, + "Error! Makes no sense to register from src and ask that src alias tgt.\n"); + if (m_aliasing==TgtAliasSrc) { + register_field(src,src); + } else { + AbstractRemapper::register_field_from_src(src); + } + } + void register_field_from_tgt (const field_type& tgt) { + EKAT_REQUIRE_MSG (m_aliasing!=TgtAliasSrc, + "Error! Makes no sense to register from tgt and ask that tgt alias src.\n"); + if (m_aliasing==SrcAliasTgt) { + register_field(tgt,tgt); + } else { + AbstractRemapper::register_field_from_tgt(tgt); + } + } protected: const identifier_type& do_get_src_field_id (const int ifield) const override { @@ -81,17 +110,40 @@ class IdentityRemapper : public AbstractRemapper m_fields[ifield].second = tgt; } void do_registration_ends () override { - // Do nothing + // If src is an alias of tgt (or viceversa), make the pair of fields store the same field + if (m_aliasing==Aliasing::SrcAliasTgt) { + for (auto& it : m_fields) { + it.first = it.second; + } + } else if (m_aliasing==Aliasing::TgtAliasSrc) { + for (auto& it : m_fields) { + it.second = it.first; + } + } } void do_remap_fwd () override { - // Do nothing + // If src is an alias of tgt (or viceversa), no need to run the remapper. + // Otherwise, we can simply run Field::deep_copy. + if (m_aliasing==Aliasing::NoAliasing) { + for (auto& it : m_fields) { + it.second.deep_copy(it.first); + } + } } void do_remap_bwd () override { - // Do nothing + // If src is an alias of tgt (or viceversa), no need to run the remapper. + // Otherwise, we can simply run Field::deep_copy. + if (m_aliasing==Aliasing::NoAliasing) { + for (auto& it : m_fields) { + it.first.deep_copy(it.second); + } + } } std::vector> m_fields; + + Aliasing m_aliasing; }; } // namespace scream diff --git a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp index b1c738fe9256..d4f878f77a85 100644 --- a/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp +++ b/components/eamxx/src/share/grid/remap/refining_remapper_p2p.cpp @@ -242,6 +242,35 @@ void RefiningRemapperP2P::pack_and_send () Kokkos::parallel_for(policy,pack); break; } + case 4: + { + const auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + const int f_col_size = dim1*dim2*dim3; + auto policy = ESU::get_default_team_policy(num_exports,dim1*dim2*dim3); + auto pack = KOKKOS_LAMBDA(const TeamMember& team) { + const int iexp = team.league_rank(); + const int icol = export_lids(iexp); + const int pid = export_pids(iexp); + auto pid_offset = pids_send_offsets(pid); + auto pos_within_pid = iexp - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_send(pid)*f_col_sizes_scan_sum + + pos_within_pid*f_col_size; + auto col_pack = [&](const int& idx) { + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + send_buf(offset+idx) = v(icol,j,k,l); + }; + auto tvr = Kokkos::TeamVectorRange(team,f_col_size); + Kokkos::parallel_for(tvr,col_pack); + }; + Kokkos::parallel_for(policy,pack); + break; + } default: EKAT_ERROR_MSG ("Unexpected field rank in RefiningRemapperP2P::pack.\n" " - MPI rank : " + std::to_string(m_comm.rank()) + "\n" @@ -357,6 +386,35 @@ void RefiningRemapperP2P::recv_and_unpack () Kokkos::parallel_for(policy,unpack); break; } + case 4: + { + auto v = f.get_view(); + const int dim1 = fl.dim(1); + const int dim2 = fl.dim(2); + const int dim3 = fl.dim(3); + const int f_col_size = dim1*dim2*dim3; + auto policy = ESU::get_default_team_policy(num_imports,dim1*dim2*dim3); + auto unpack = KOKKOS_LAMBDA (const TeamMember& team) { + const int idx = team.league_rank(); + const int pid = import_pids(idx); + const int icol = import_lids(idx); + const auto pid_offset = pids_recv_offsets(pid); + const auto pos_within_pid = idx - pid_offset; + auto offset = pid_offset*total_col_size + + ncols_recv(pid)*f_col_sizes_scan_sum + + pos_within_pid*f_col_size; + auto col_unpack = [&](const int& idx) { + const int j = (idx / dim3) / dim2; + const int k = (idx / dim3) % dim2; + const int l = idx % dim3; + v(icol,j,k,l) = recv_buf(offset+idx); + }; + auto tvr = Kokkos::TeamVectorRange(team,f_col_size); + Kokkos::parallel_for(tvr,col_unpack); + }; + Kokkos::parallel_for(policy,unpack); + break; + } default: EKAT_ERROR_MSG ("Unexpected field rank in RefiningRemapperP2P::unpack.\n" " - MPI rank : " + std::to_string(m_comm.rank()) + "\n" diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp index beda1f5d7850..9abefed9eed7 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.cpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.cpp @@ -5,6 +5,7 @@ #include "share/io/scorpio_input.hpp" #include "share/field/field_tag.hpp" #include "share/field/field_identifier.hpp" +#include "share/util/scream_universal_constants.hpp" #include "ekat/util/ekat_units.hpp" #include @@ -19,21 +20,21 @@ VerticalRemapper:: VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, const Field& lev_prof, - const Field& ilev_prof, - const Real mask_val) - : VerticalRemapper(src_grid,map_file,lev_prof,ilev_prof) + const Field& ilev_prof) + : VerticalRemapper(src_grid,map_file,lev_prof,ilev_prof,constants::DefaultFillValue::value) { - m_mask_val = mask_val; + // Nothing to do here } VerticalRemapper:: VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, const Field& lev_prof, - const Field& ilev_prof) + const Field& ilev_prof, + const Real mask_val) : AbstractRemapper() , m_comm (src_grid->get_comm()) - , m_mask_val(std::numeric_limits::max()/10.0) + , m_mask_val(mask_val) { using namespace ShortFieldTagsNames; @@ -67,6 +68,8 @@ VerticalRemapper (const grid_ptr_type& src_grid, // Gather the pressure level data for vertical remapping set_pressure_levels(map_file); + // Add tgt pressure levels to the tgt grid + tgt_grid->set_geometry_data(m_remap_pres); scorpio::eam_pio_closefile(map_file); } @@ -79,28 +82,9 @@ create_src_layout (const FieldLayout& tgt_layout) const "[VerticalRemapper] Error! Input target layout is not valid for this remapper.\n" " - input layout: " + to_string(tgt_layout)); - const auto lt = get_layout_type(tgt_layout.tags()); - auto src = FieldLayout::invalid(); - const bool midpoints = tgt_layout.has_tag(LEV); - const int vec_dim = tgt_layout.is_vector_layout() ? tgt_layout.dim(CMP) : -1; - switch (lt) { - case LayoutType::Scalar2D: - src = m_src_grid->get_2d_scalar_layout(); - break; - case LayoutType::Vector2D: - src = m_src_grid->get_2d_vector_layout(CMP,vec_dim); - break; - case LayoutType::Scalar3D: - src = m_src_grid->get_3d_scalar_layout(midpoints); - break; - case LayoutType::Vector3D: - src = m_src_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); - break; - default: - EKAT_ERROR_MSG ("Layout not supported by VerticalRemapper: " + e2str(lt) + "\n"); - } - return src; + return create_layout(tgt_layout,m_src_grid); } + FieldLayout VerticalRemapper:: create_tgt_layout (const FieldLayout& src_layout) const { @@ -110,27 +94,51 @@ create_tgt_layout (const FieldLayout& src_layout) const "[VerticalRemapper] Error! Input source layout is not valid for this remapper.\n" " - input layout: " + to_string(src_layout)); - const auto lt = get_layout_type(src_layout.tags()); - auto tgt = FieldLayout::invalid(); - const bool midpoints = true; //src_layout.has_tag(LEV); - const int vec_dim = src_layout.is_vector_layout() ? src_layout.dim(CMP) : -1; + return create_layout(src_layout,m_tgt_grid); +} + +FieldLayout VerticalRemapper:: +create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid_out) const +{ + using namespace ShortFieldTagsNames; + + // NOTE: for the vert remapper, it doesn't really make sense to distinguish + // between midpoints and interfaces: we're simply asking for a quantity + // at a given set of pressure levels. So we choose to have fl_out + // to *always* have LEV as vertical tag. + const auto lt = get_layout_type(fl_in.tags()); + auto fl_out = FieldLayout::invalid(); switch (lt) { - case LayoutType::Scalar2D: - tgt = m_tgt_grid->get_2d_scalar_layout(); + case LayoutType::Scalar0D: [[ fallthrough ]]; + case LayoutType::Vector0D: [[ fallthrough ]]; + case LayoutType::Scalar2D: [[ fallthrough ]]; + case LayoutType::Vector2D: [[ fallthrough ]]; + case LayoutType::Tensor2D: + // These layouts do not have vertical dim tags, so no change + fl_out = fl_in; break; - case LayoutType::Vector2D: - tgt = m_tgt_grid->get_2d_vector_layout(CMP,vec_dim); + case LayoutType::Scalar1D: + fl_out = grid_out->get_vertical_layout(true); break; case LayoutType::Scalar3D: - tgt = m_tgt_grid->get_3d_scalar_layout(midpoints); + fl_out = grid_out->get_3d_scalar_layout(true); break; case LayoutType::Vector3D: - tgt = m_tgt_grid->get_3d_vector_layout(midpoints,CMP,vec_dim); + { + const auto vec_tag = fl_in.get_vector_tag(); + const auto vec_dim = fl_in.dim(vec_tag); + fl_out = grid_out->get_3d_vector_layout(true,vec_tag,vec_dim); break; + } default: - EKAT_ERROR_MSG ("Layout not supported by VerticalRemapper: " + e2str(lt) + "\n"); + // NOTE: this also include Tensor3D. We don't really have any atm proc + // that needs to handle a tensor3d quantity, so no need to add it + EKAT_ERROR_MSG ( + "Layout not supported by VerticalRemapper.\n" + " - input layout: " + to_string(fl_in) + "\n"); } - return tgt; + return fl_out; } void VerticalRemapper:: @@ -146,7 +154,7 @@ set_pressure_levels(const std::string& map_file) std::vector tags = {LEV}; std::vector dims = {m_num_remap_levs}; FieldLayout layout(tags,dims); - FieldIdentifier fid("p_remap",layout,ekat::units::Pa,m_tgt_grid->name()); + FieldIdentifier fid("p_levs",layout,ekat::units::Pa,m_tgt_grid->name()); m_remap_pres = Field(fid); m_remap_pres.get_header().get_alloc_properties().request_allocation(mPack::n); m_remap_pres.allocate_view(); @@ -312,7 +320,6 @@ void VerticalRemapper::do_remap_fwd () { using namespace ShortFieldTagsNames; // Loop over each field - constexpr auto can_pack = SCREAM_PACK_SIZE>1; const auto& tgt_pres_ap = m_remap_pres.get_header().get_alloc_properties(); for (int i=0; i>() && - tgt_ap.is_compatible>() && - src_pres_ap.is_compatible>() && - tgt_pres_ap.is_compatible>()) { + if (src_ap.is_compatible>() && + tgt_ap.is_compatible>() && + src_pres_ap.is_compatible>() && + tgt_pres_ap.is_compatible>()) { apply_vertical_interpolation(f_src,f_tgt); } else { apply_vertical_interpolation<1>(f_src,f_tgt); @@ -358,10 +365,10 @@ void VerticalRemapper::do_remap_fwd () const auto& src_ap = f_src.get_header().get_alloc_properties(); const auto& tgt_ap = f_tgt.get_header().get_alloc_properties(); const auto& src_pres_ap = src_tag == LEV ? m_src_mid.get_header().get_alloc_properties() : m_src_int.get_header().get_alloc_properties(); - if (can_pack && src_ap.is_compatible>() && - tgt_ap.is_compatible>() && - src_pres_ap.is_compatible>() && - tgt_pres_ap.is_compatible>()) { + if (src_ap.is_compatible>() && + tgt_ap.is_compatible>() && + src_pres_ap.is_compatible>() && + tgt_pres_ap.is_compatible>()) { apply_vertical_interpolation(f_src,f_tgt,true); } else { apply_vertical_interpolation<1>(f_src,f_tgt,true); @@ -378,44 +385,42 @@ template void VerticalRemapper:: apply_vertical_interpolation(const Field& f_src, const Field& f_tgt, const bool mask_interp) const { - - using Pack = ekat::Pack; - using namespace ShortFieldTagsNames; - using namespace scream::vinterp; - const auto& layout = f_src.get_header().get_identifier().get_layout(); - const auto rank = f_src.rank(); - const auto src_tag = layout.tags().back(); - const auto src_num_levs = layout.dims().back(); - // ARG mask_interp checks if this is a vertical interpolation of the mask array that tracks masked 0.0 or not 1.0 - Real mask_val = mask_interp ? 0.0 : m_mask_val; - - Field src_lev_f; - if (src_tag == ILEV) { - src_lev_f = m_src_int; - } else { - src_lev_f = m_src_mid; + using Pack = ekat::Pack; + using namespace ShortFieldTagsNames; + using namespace scream::vinterp; + const auto& layout = f_src.get_header().get_identifier().get_layout(); + const auto rank = f_src.rank(); + const auto src_tag = layout.tags().back(); + const auto src_num_levs = layout.dims().back(); + // ARG mask_interp checks if this is a vertical interpolation of the mask array that tracks masked 0.0 or not 1.0 + Real mask_val = mask_interp ? 0.0 : m_mask_val; + + Field src_lev_f; + if (src_tag == ILEV) { + src_lev_f = m_src_int; + } else { + src_lev_f = m_src_mid; + } + auto src_lev = src_lev_f.get_view(); + auto remap_pres_view = m_remap_pres.get_view(); + switch(rank) { + case 2: + { + auto src_view = f_src.get_view(); + auto tgt_view = f_tgt.get_view< Pack**>(); + perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); + break; } - auto src_lev = src_lev_f.get_view(); - auto remap_pres_view = m_remap_pres.get_view(); - switch(rank) { - case 2: - { - auto src_view = f_src.get_view(); - auto tgt_view = f_tgt.get_view< Pack**>(); - perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); - break; - } - case 3: - { - auto src_view = f_src.get_view(); - auto tgt_view = f_tgt.get_view< Pack***>(); - perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); - break; - } - default: - EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by VerticalRemapper.\n"); + case 3: + { + auto src_view = f_src.get_view(); + auto tgt_view = f_tgt.get_view< Pack***>(); + perform_vertical_interpolation(src_lev,remap_pres_view,src_view,tgt_view,src_num_levs,m_num_remap_levs,mask_val); + break; } - + default: + EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by VerticalRemapper.\n"); + } } } // namespace scream diff --git a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp index 880d305b0638..f04baa933cc8 100644 --- a/components/eamxx/src/share/grid/remap/vertical_remapper.hpp +++ b/components/eamxx/src/share/grid/remap/vertical_remapper.hpp @@ -23,6 +23,7 @@ class VerticalRemapper : public AbstractRemapper const Field& ilev_prof, const Real mask_val); + // Calls the above one, with mask_val=max_float/10 VerticalRemapper (const grid_ptr_type& src_grid, const std::string& map_file, const Field& lev_prof, @@ -67,9 +68,21 @@ class VerticalRemapper : public AbstractRemapper src_col_size == tgt_col_size; } + // NOTE: for the vert remapper, it doesn't really make sense to distinguish + // between midpoints and interfaces: we're simply asking for a quantity + // at a given set of pressure levels. So we choose to NOT allow a tgt + // layout with ILEV tag. + bool is_valid_tgt_layout (const layout_type& layout) const override { + using namespace ShortFieldTagsNames; + return not layout.has_tag(ILEV) + and AbstractRemapper::is_valid_tgt_layout(layout); + } protected: + FieldLayout create_layout (const FieldLayout& fl_in, + const grid_ptr_type& grid_out) const; + void register_vertical_source_field(const Field& src); const identifier_type& do_get_src_field_id (const int ifield) const override { diff --git a/components/eamxx/src/share/grid/se_grid.cpp b/components/eamxx/src/share/grid/se_grid.cpp index 1e5556de6cad..1b941125e1cb 100644 --- a/components/eamxx/src/share/grid/se_grid.cpp +++ b/components/eamxx/src/share/grid/se_grid.cpp @@ -47,6 +47,24 @@ SEGrid::get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) c return FieldLayout({EL,vector_tag,GP,GP},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp}); } +FieldLayout +SEGrid::get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + std::vector tags = {EL}; + std::vector dims = {m_num_local_elem}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + tags.push_back(GP); + tags.push_back(GP); + dims.push_back(m_num_gp); + dims.push_back(m_num_gp); + return FieldLayout(tags,dims); +} + FieldLayout SEGrid::get_3d_scalar_layout (const bool midpoints) const { @@ -69,6 +87,30 @@ SEGrid::get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, c return FieldLayout({EL,vector_tag,GP,GP,VL},{m_num_local_elem,vector_dim,m_num_gp,m_num_gp,nvl}); } +FieldLayout +SEGrid::get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const +{ + using namespace ShortFieldTagsNames; + + int nvl = this->get_num_vertical_levels() + (midpoints ? 0 : 1); + auto VL = midpoints ? LEV : ILEV; + + std::vector tags = {EL}; + std::vector dims = {m_num_local_elem}; + + tags.insert(tags.end(),cmp_tags.begin(),cmp_tags.end()); + dims.insert(dims.end(),cmp_dims.begin(),cmp_dims.end()); + tags.push_back(GP); + tags.push_back(GP); + tags.push_back(VL); + dims.push_back(m_num_gp); + dims.push_back(m_num_gp); + dims.push_back(nvl); + return FieldLayout(tags,dims); +} + Field SEGrid::get_cg_dofs_gids () { EKAT_REQUIRE_MSG (m_cg_dofs_gids.is_allocated(), diff --git a/components/eamxx/src/share/grid/se_grid.hpp b/components/eamxx/src/share/grid/se_grid.hpp index f31167b79a76..90af8de066da 100644 --- a/components/eamxx/src/share/grid/se_grid.hpp +++ b/components/eamxx/src/share/grid/se_grid.hpp @@ -22,8 +22,13 @@ class SEGrid : public AbstractGrid // Native layout of a dof. This is the natural way to index a dof in the grid. FieldLayout get_2d_scalar_layout () const override; FieldLayout get_2d_vector_layout (const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_2d_tensor_layout (const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldLayout get_3d_scalar_layout (const bool midpoints) const override; FieldLayout get_3d_vector_layout (const bool midpoints, const FieldTag vector_tag, const int vector_dim) const override; + FieldLayout get_3d_tensor_layout (const bool midpoints, + const std::vector& cmp_tags, + const std::vector& cmp_dims) const override; FieldTag get_partitioned_dim_tag () const override { return FieldTag::Element; diff --git a/components/eamxx/src/share/io/scorpio_input.cpp b/components/eamxx/src/share/io/scorpio_input.cpp index 29a62763cec4..c4f023cc396f 100644 --- a/components/eamxx/src/share/io/scorpio_input.cpp +++ b/components/eamxx/src/share/io/scorpio_input.cpp @@ -12,7 +12,7 @@ namespace scream AtmosphereInput:: AtmosphereInput (const ekat::ParameterList& params, - const std::shared_ptr& field_mgr) + const std::shared_ptr& field_mgr) { init(params,field_mgr); } @@ -29,11 +29,13 @@ AtmosphereInput (const ekat::ParameterList& params, AtmosphereInput:: AtmosphereInput (const std::string& filename, const std::shared_ptr& grid, - const std::vector& fields) + const std::vector& fields, + const bool skip_grid_checks) { // Create param list and field manager on the fly ekat::ParameterList params; params.set("Filename",filename); + params.set("Skip_Grid_Checks",skip_grid_checks); auto& names = params.get>("Field Names",{}); auto fm = std::make_shared(grid); @@ -204,7 +206,12 @@ void AtmosphereInput::read_variables (const int time_index) { auto func_start = std::chrono::steady_clock::now(); if (m_atm_logger) { - m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file:\n\t " + m_filename + " ...\n"); + m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file"); + m_atm_logger->info(" file name: " + m_filename); + m_atm_logger->info(" var names: " + ekat::join(m_fields_names,", ")); + if (time_index!=-1) { + m_atm_logger->info(" time idx : " + std::to_string(time_index)); + } } EKAT_REQUIRE_MSG (m_inited_with_views || m_inited_with_fields, "Error! Scorpio structures not inited yet. Did you forget to call 'init(..)'?\n"); @@ -321,7 +328,7 @@ void AtmosphereInput::read_variables (const int time_index) auto func_finish = std::chrono::steady_clock::now(); if (m_atm_logger) { auto duration = std::chrono::duration_cast(func_finish - func_start)/1000.0; - m_atm_logger->info("[EAMxx::scorpio_input] Reading variables from file:\n\t " + m_filename + " ... done! (Elapsed time = " + std::to_string(duration.count()) +" seconds)\n"); + m_atm_logger->info(" Done! Elapsed time: " + std::to_string(duration.count()) +" seconds"); } } @@ -442,6 +449,15 @@ void AtmosphereInput::set_degrees_of_freedom() std::vector AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) { + using namespace ShortFieldTagsNames; + + // Precompute this *before* the early return, since it involves collectives. + // If one rank owns zero cols, and returns prematurely, the others will be left waiting. + AbstractGrid::gid_type min_gid; + if (layout.has_tag(COL) or layout.has_tag(EL)) { + min_gid = m_io_grid->get_global_min_dof_gid(); + } + // It may be that this MPI ranks owns no chunk of the field if (layout.size()==0) { return {}; @@ -461,17 +477,13 @@ AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) // of the MPI-local array w.r.t. the global array. So long as the offsets are in // the same order as the corresponding entry in the data to be read/written, we're good. auto dofs_h = m_io_grid->get_dofs_gids().get_view(); - if (layout.has_tag(ShortFieldTagsNames::COL)) { + if (layout.has_tag(COL)) { const int num_cols = m_io_grid->get_num_local_dofs(); // Note: col_size might be *larger* than the number of vertical levels, or even smaller. // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int icol=0; icolget_2d_scalar_layout(); const int num_my_elems = layout2d.dim(0); const int ngp = layout2d.dim(1); @@ -492,10 +504,6 @@ AtmosphereInput::get_var_dof_offsets(const FieldLayout& layout) // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int ie=0,icol=0; ie& layouts); AtmosphereInput (const std::string& filename, const std::shared_ptr& grid, - const std::vector& fields); + const std::vector& fields, + const bool skip_grid_checks = false); ~AtmosphereInput (); diff --git a/components/eamxx/src/share/io/scorpio_output.cpp b/components/eamxx/src/share/io/scorpio_output.cpp index 08b4b83410d5..3943e59d5da1 100644 --- a/components/eamxx/src/share/io/scorpio_output.cpp +++ b/components/eamxx/src/share/io/scorpio_output.cpp @@ -41,9 +41,9 @@ void combine (const Real& new_val, Real& curr_val, const OutputAvgType avg_type) } // This one covers cases where a variable might be masked. KOKKOS_INLINE_FUNCTION -void combine_and_fill (const Real& new_val, Real& curr_val, Real& avg_coeff, const OutputAvgType avg_type, const Real fill_value) +void combine_and_fill (const Real& new_val, Real& curr_val, const OutputAvgType avg_type, const Real fill_value) { - const bool new_fill = (avg_coeff == 0.0); + const bool new_fill = new_val == fill_value; const bool curr_fill = curr_val == fill_value; if (curr_fill && new_fill) { // Then the value is already set to be filled and the new value doesn't change things. @@ -119,20 +119,6 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, { using vos_t = std::vector; - if (params.isParameter("fill_value")) { - m_fill_value = static_cast(params.get("fill_value")); - // If the fill_value is specified there is a good chance the user expects the average count to track filling. - m_track_avg_cnt = true; - } - if (params.isParameter("track_fill")) { - // Note, we do this after checking for fill_value to give users that opportunity to turn off fill tracking, even - // if they specify a specific fill value. - m_track_avg_cnt = params.get("track_fill"); - } - if (params.isParameter("fill_threshold")) { - m_avg_coeff_threshold = params.get("fill_threshold"); - } - // Figure out what kind of averaging is requested auto avg_type = params.get("Averaging Type"); m_avg_type = str2avg(avg_type); @@ -202,6 +188,29 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, // Register any diagnostics needed by this output stream set_diagnostics(); + // Avg count only makes sense if we have + // - non-instant output + // - we have one between: + // - vertically remapped output + // - field_at_XhPa diagnostic + // We already set m_track_avg_cnt to true if field_at_XhPa is found in set_diagnostics. + // Hence, here we only check if vert remap is active + + if (params.isParameter("track_avg_cnt")) { + // This is to be used for unit testing only, so that we can test avg cnt even + // if there is no vert remap and no field_at_XhPa diagnostic in the stream + m_track_avg_cnt = params.get("track_avg_cnt"); + } + if (use_vertical_remap_from_file) { + m_track_avg_cnt = true; + } + if (params.isParameter("fill_value")) { + m_fill_value = static_cast(params.get("fill_value")); + } + if (params.isParameter("fill_threshold")) { + m_avg_coeff_threshold = params.get("fill_threshold"); + } + // Helper lambda, to copy io string attributes. This will be used if any // remapper is created, to ensure atts set by atm_procs are not lost auto transfer_io_str_atts = [&] (const Field& src, Field& tgt) { @@ -216,8 +225,6 @@ AtmosphereOutput (const ekat::Comm& comm, const ekat::ParameterList& params, // Setup remappers - if needed if (use_vertical_remap_from_file) { - // When vertically remapping there is a chance that filled values will be present, so be sure to track these - m_track_avg_cnt = true; // We build a remapper, to remap fields from the fm grid to the io grid auto vert_remap_file = params.get("vertical_remap_file"); auto f_lev = get_field("p_mid","sim"); @@ -369,7 +376,8 @@ run (const std::string& filename, Real duration_write = 0.0; // Record of time spent writing output if (is_write_step) { if (m_atm_logger) { - m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file:\n\t " + filename + " ...\n"); + m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file"); + m_atm_logger->info(" file name: " + filename); } } @@ -418,37 +426,36 @@ run (const std::string& filename, // temporary views for each layout that are either 0 or 1 depending on if the // value is filled or unfilled. // We then use these values to update the overall average count views for that layout. - if (m_track_avg_cnt && m_add_time_dim) { + if (m_track_avg_cnt) { // Note, we assume that all fields that share a layout are also masked/filled in the same - // way. If, we need to handle a case where only a subset of output variables are expected to + // way. If we need to handle a case where only a subset of output variables are expected to // be masked/filled then the recommendation is to request those variables in a separate output // stream. - // We cycle through all fields and mark points that are filled/masked in the local views. First - // initialize them to 1 representing unfilled. - for (const auto& name : m_avg_cnt_names) { - auto& dev_view = m_local_tmp_avg_cnt_views_1d.at(name); - Kokkos::deep_copy(dev_view,1.0); - } - // Now we cycle through all the fields + // We cycle through all fields and we + // 1. Find the avg_cnt view for this field. + // 2. If we already processed the avg_cnt view, go to next field, and start from 1 again. + // 3. Add 1 to all entries of avg_cnt where field!=fill_value + std::set avg_updated; for (const auto& name : m_fields_names) { - auto field = get_field(name,"io"); - auto lookup = m_field_to_avg_cnt_map.at(name); - auto dev_view = m_local_tmp_avg_cnt_views_1d.at(lookup); - update_avg_cnt_view(field,dev_view); - } - // Finally, we update the overall avg_cnt_views - for (const auto& name : m_avg_cnt_names) { - auto track_view = m_dev_views_1d.at(name); - auto local_view = m_local_tmp_avg_cnt_views_1d.at(name); - const auto layout = m_layouts.at(name); - KT::RangePolicy policy(0,layout.size()); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - track_view(i) += local_view(i); - }); + auto avg_cnt_name = m_field_to_avg_cnt_map.at(name); + if (avg_updated.count(avg_cnt_name)==1) { + // We updated this avg_cnt by checking another field + continue; + } + auto field = get_field(name,"io"); + update_avg_cnt_view(field,m_dev_views_1d.at(avg_cnt_name)); + + // Make sure we don't double update this avg cnt + avg_updated.insert(avg_cnt_name); } } // Take care of updating and possibly writing fields. + // These are needed inside kernels, so crate local copies + auto do_avg_cnt = m_track_avg_cnt; + auto avg_type = m_avg_type; + auto fill_value = m_fill_value; + auto avg_coeff_threshold = m_avg_coeff_threshold; for (auto const& name : m_fields_names) { // Get all the info for this field. auto field = get_field(name,"io"); @@ -481,26 +488,6 @@ run (const std::string& filename, KT::RangePolicy policy(0,layout.size()); const auto extents = layout.extents(); - // Averaging count data - // If we are not tracking the average count then we don't need to build the - // views for the average count, so we leave them as essentially empty. - auto avg_cnt_dims = dims; - auto avg_cnt_data = data; - if (m_track_avg_cnt && m_add_time_dim) { - const auto lookup = m_field_to_avg_cnt_map.at(name); - avg_cnt_data = m_local_tmp_avg_cnt_views_1d.at(lookup).data(); - } else { - for (auto& dim : avg_cnt_dims) { - dim = 1; - } - avg_cnt_data = nullptr; - } - - auto avg_type = m_avg_type; - auto track_avg_cnt = m_track_avg_cnt; - auto add_time_dim = m_add_time_dim; - auto fill_value = m_fill_value; - auto avg_coeff_threshold = m_avg_coeff_threshold; // If the dev_view_1d is aliasing the field device view (must be Instant output), // then there's no point in copying from the field's view to dev_view if (not is_aliasing_field_view) { @@ -511,12 +498,11 @@ run (const std::string& filename, // handling a few more scenarios auto new_view_1d = field.get_strided_view(); auto avg_view_1d = view_Nd_dev<1>(data,dims[0]); - auto avg_coeff_1d = view_Nd_dev<1>(avg_cnt_data,avg_cnt_dims[0]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_1d(i), avg_view_1d(i),avg_coeff_1d(i),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_1d(i),avg_view_1d(i),avg_type,fill_value); } else { - combine(new_view_1d(i), avg_view_1d(i),avg_type); + combine(new_view_1d(i),avg_view_1d(i),avg_type); } }); break; @@ -525,12 +511,11 @@ run (const std::string& filename, { auto new_view_2d = field.get_view(); auto avg_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); - auto avg_coeff_2d = view_Nd_dev<2>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j; unflatten_idx(idx,extents,i,j); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_2d(i,j), avg_view_2d(i,j),avg_coeff_2d(i,j),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_2d(i,j),avg_view_2d(i,j),avg_type,fill_value); } else { combine(new_view_2d(i,j), avg_view_2d(i,j),avg_type); } @@ -541,12 +526,11 @@ run (const std::string& filename, { auto new_view_3d = field.get_view(); auto avg_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); - auto avg_coeff_3d = view_Nd_dev<3>(avg_cnt_data,dims[0],avg_cnt_dims[1],avg_cnt_dims[2]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k; unflatten_idx(idx,extents,i,j,k); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_3d(i,j,k), avg_view_3d(i,j,k),avg_coeff_3d(i,j,k),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_3d(i,j,k),avg_view_3d(i,j,k),avg_type,fill_value); } else { combine(new_view_3d(i,j,k), avg_view_3d(i,j,k),avg_type); } @@ -557,12 +541,11 @@ run (const std::string& filename, { auto new_view_4d = field.get_view(); auto avg_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); - auto avg_coeff_4d = view_Nd_dev<4>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l; unflatten_idx(idx,extents,i,j,k,l); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_coeff_4d(i,j,k,l),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_type,fill_value); } else { combine(new_view_4d(i,j,k,l), avg_view_4d(i,j,k,l),avg_type); } @@ -573,12 +556,11 @@ run (const std::string& filename, { auto new_view_5d = field.get_view(); auto avg_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); - auto avg_coeff_5d = view_Nd_dev<5>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3],avg_cnt_dims[4]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l,m; unflatten_idx(idx,extents,i,j,k,l,m); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_coeff_5d(i,j,k,l,m),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_type,fill_value); } else { combine(new_view_5d(i,j,k,l,m), avg_view_5d(i,j,k,l,m),avg_type); } @@ -589,12 +571,11 @@ run (const std::string& filename, { auto new_view_6d = field.get_view(); auto avg_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); - auto avg_coeff_6d = view_Nd_dev<6>(avg_cnt_data,avg_cnt_dims[0],avg_cnt_dims[1],avg_cnt_dims[2],avg_cnt_dims[3],avg_cnt_dims[4],avg_cnt_dims[5]); Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { int i,j,k,l,m,n; unflatten_idx(idx,extents,i,j,k,l,m,n); - if (track_avg_cnt && add_time_dim) { - combine_and_fill(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n), avg_coeff_6d(i,j,k,l,m,n),avg_type,fill_value); + if (do_avg_cnt) { + combine_and_fill(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n), avg_type,fill_value); } else { combine(new_view_6d(i,j,k,l,m,n), avg_view_6d(i,j,k,l,m,n),avg_type); } @@ -608,7 +589,7 @@ run (const std::string& filename, if (is_write_step) { if (output_step and avg_type==OutputAvgType::Average) { - if (m_track_avg_cnt && m_add_time_dim) { + if (do_avg_cnt) { const auto avg_cnt_lookup = m_field_to_avg_cnt_map.at(name); const auto avg_cnt_view = m_dev_views_1d.at(avg_cnt_lookup); const auto avg_nsteps = avg_cnt_view.data(); @@ -654,7 +635,7 @@ run (const std::string& filename, } if (is_write_step) { if (m_atm_logger) { - m_atm_logger->info("[EAMxx::scorpio_output] Writing variables to file:\n\t " + filename + " ...done! (Elapsed time = " + std::to_string(duration_write/1000.0) +" seconds)\n"); + m_atm_logger->info(" Done! Elapsed time: " + std::to_string(duration_write/1000.0) +" seconds"); } } } // run @@ -837,18 +818,20 @@ void AtmosphereOutput::register_views() m_host_views_1d.emplace(name,Kokkos::create_mirror(m_dev_views_1d[name])); } - // Now create and store a dev view to track the averaging count for this layout (if we are tracking) - // We don't need to track average counts for files that are not tracking the time dim - set_avg_cnt_tracking(name,"",layout); + if (m_track_avg_cnt) { + // Now create and store a dev view to track the averaging count for this layout (if we are tracking) + // We don't need to track average counts for files that are not tracking the time dim + set_avg_cnt_tracking(name,layout); + } } // Initialize the local views reset_dev_views(); } /* ---------------------------------------------------------- */ -void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const std::string& avg_cnt_suffix, const FieldLayout& layout) +void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const FieldLayout& layout) { - // Make sure this field "name" hasn't already been regsitered with avg_cnt tracking. + // Make sure this field "name" hasn't already been registered with avg_cnt tracking. // Note, we check this because some diagnostics need to have their own tracking which // is created at the 'create_diagnostics' function. if (m_field_to_avg_cnt_map.count(name)>0) { @@ -866,9 +849,10 @@ void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const std:: // Now create and store a dev view to track the averaging count for this layout (if we are tracking) // We don't need to track average counts for files that are not tracking the time dim + const auto& avg_cnt_suffix = m_field_to_avg_cnt_suffix[name]; const auto size = layout.size(); const auto tags = layout.tags(); - if (m_add_time_dim && m_track_avg_cnt) { + if (m_track_avg_cnt) { std::string avg_cnt_name = "avg_count" + avg_cnt_suffix; for (int ii=0; iiget_dim_name(layout.tag(ii)); @@ -879,7 +863,6 @@ void AtmosphereOutput::set_avg_cnt_tracking(const std::string& name, const std:: } m_field_to_avg_cnt_map.emplace(name,avg_cnt_name); m_dev_views_1d.emplace(avg_cnt_name,view_1d_dev("",size)); // Note, emplace will only add a new key if one isn't already there - m_local_tmp_avg_cnt_views_1d.emplace(avg_cnt_name,view_1d_dev("",size)); // Note, emplace will only add a new key if one isn't already there m_host_views_1d.emplace(avg_cnt_name,Kokkos::create_mirror(m_dev_views_1d[avg_cnt_name])); m_layouts.emplace(avg_cnt_name,layout); } @@ -890,7 +873,7 @@ reset_dev_views() { // Reset the local device views depending on the averaging type // Init dev view with an "identity" for avg_type - const Real fill_for_average = (m_track_avg_cnt && m_add_time_dim) ? m_fill_value : 0.0; + const Real fill_for_average = m_track_avg_cnt ? m_fill_value : 0.0; for (auto const& name : m_fields_names) { switch (m_avg_type) { case OutputAvgType::Instant: @@ -1019,7 +1002,7 @@ register_variables(const std::string& filename, } // If tracking average count variables then add the name of the tracking variable for this variable - if (m_track_avg_cnt && m_add_time_dim) { + if (m_track_avg_cnt) { const auto lookup = m_field_to_avg_cnt_map.at(name); set_variable_metadata(filename,name,"averaging_count_tracker",lookup); } @@ -1033,7 +1016,7 @@ register_variables(const std::string& filename, } } // Now register the average count variables - if (m_track_avg_cnt && m_add_time_dim) { + if (m_track_avg_cnt) { for (const auto& name : m_avg_cnt_names) { const auto layout = m_layouts.at(name); auto io_decomp_tag = set_decomp_tag(layout); @@ -1047,6 +1030,15 @@ register_variables(const std::string& filename, std::vector AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) { + using namespace ShortFieldTagsNames; + + // Precompute this *before* the early return, since it involves collectives. + // If one rank owns zero cols, and returns prematurely, the others will be left waiting. + AbstractGrid::gid_type min_gid; + if (layout.has_tag(COL) or layout.has_tag(EL)) { + min_gid = m_io_grid->get_global_min_dof_gid(); + } + // It may be that this MPI rank owns no chunk of the field if (layout.size()==0) { return {}; @@ -1071,17 +1063,13 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) // NOTE: In the case of regional output this rank may have 0 columns to write, thus, var_dof // should be empty, we check for this special case and return an empty var_dof. auto dofs_h = m_io_grid->get_dofs_gids().get_view(); - if (layout.has_tag(ShortFieldTagsNames::COL)) { + if (layout.has_tag(COL)) { const int num_cols = m_io_grid->get_num_local_dofs(); // Note: col_size might be *larger* than the number of vertical levels, or even smaller. // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int icol=0; icolget_2d_scalar_layout(); const int num_my_elems = layout2d.dim(0); const int ngp = layout2d.dim(1); @@ -1102,10 +1090,6 @@ AtmosphereOutput::get_var_dof_offsets(const FieldLayout& layout) // E.g., (ncols,2,nlevs), or (ncols,2) respectively. scorpio::offset_t col_size = layout.size() / num_cols; - // Precompute this *before* the loop, since it involves expensive collectives. - // Besides, the loop might have different length on different ranks, so - // computing it inside might cause deadlocks. - auto min_gid = m_io_grid->get_global_min_dof_gid(); for (int ie=0,icol=0; ieinitialize(util::TimeStamp(),RunType::Initial); // If specified, set avg_cnt tracking for this diagnostic. - if (m_add_time_dim && m_track_avg_cnt) { + if (m_track_avg_cnt) { const auto diag_field = diag->get_diagnostic(); const auto name = diag_field.name(); - const auto layout = diag_field.get_header().get_identifier().get_layout(); - set_avg_cnt_tracking(name,diag_avg_cnt_name,layout); + m_field_to_avg_cnt_suffix.emplace(name,diag_avg_cnt_name); } return diag; @@ -1368,105 +1371,98 @@ AtmosphereOutput::create_diagnostic (const std::string& diag_field_name) { // Helper function to mark filled points in a specific layout void AtmosphereOutput:: update_avg_cnt_view(const Field& field, view_1d_dev& dev_view) { - // If the dev_view_1d is aliasing the field device view (must be Instant output), - // then there's no point in copying from the field's view to dev_view const auto& name = field.name(); const auto& layout = m_layouts.at(name); const auto& dims = layout.dims(); - const auto rank = layout.rank(); auto data = dev_view.data(); - const bool is_diagnostic = (m_diagnostics.find(name) != m_diagnostics.end()); - const bool is_aliasing_field_view = - m_avg_type==OutputAvgType::Instant && - field.get_header().get_alloc_properties().get_padding()==0 && - field.get_header().get_parent().expired() && - not is_diagnostic; const auto fill_value = m_fill_value; - if (not is_aliasing_field_view) { - KT::RangePolicy policy(0,layout.size()); - const auto extents = layout.extents(); - switch (rank) { - case 1: - { - // For rank-1 views, we use strided layout, since it helps us - // handling a few more scenarios - auto src_view_1d = field.get_strided_view(); - auto tgt_view_1d = view_Nd_dev<1>(data,dims[0]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { - if (src_view_1d(i)==fill_value) { - tgt_view_1d(i) = 0.0; - } - }); - break; - } - case 2: - { - auto src_view_2d = field.get_view(); - auto tgt_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j; - unflatten_idx(idx,extents,i,j); - if (src_view_2d(i,j)==fill_value) { - tgt_view_2d(i,j) = 0.0; - } - }); - break; - } - case 3: - { - auto src_view_3d = field.get_view(); - auto tgt_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k; - unflatten_idx(idx,extents,i,j,k); - if (src_view_3d(i,j,k)==fill_value) { - tgt_view_3d(i,j,k) = 0.0; - } - }); - break; - } - case 4: - { - auto src_view_4d = field.get_view(); - auto tgt_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k,l; - unflatten_idx(idx,extents,i,j,k,l); - if (src_view_4d(i,j,k,l)==fill_value) { - tgt_view_4d(i,j,k,l) = 0.0; - } - }); - break; - } - case 5: - { - auto src_view_5d = field.get_view(); - auto tgt_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k,l,m; - unflatten_idx(idx,extents,i,j,k,l,m); - if (src_view_5d(i,j,k,l,m)==fill_value) { - tgt_view_5d(i,j,k,l,m) = 0.0; - } - }); - break; - } - case 6: - { - auto src_view_6d = field.get_view(); - auto tgt_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); - Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { - int i,j,k,l,m,n; - unflatten_idx(idx,extents,i,j,k,l,m,n); - if (src_view_6d(i,j,k,l,m,n)==fill_value) { - tgt_view_6d(i,j,k,l,m,n) = 0.0; - } - }); - break; - } - default: - EKAT_ERROR_MSG ("Error! Field rank (" + std::to_string(rank) + ") not supported by AtmosphereOutput.\n"); + + KT::RangePolicy policy(0,layout.size()); + const auto extents = layout.extents(); + switch (layout.rank()) { + case 1: + { + // For rank-1 views, we use strided layout, since it helps us + // handling a few more scenarios + auto src_view_1d = field.get_strided_view(); + auto tgt_view_1d = view_Nd_dev<1>(data,dims[0]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int i) { + if (src_view_1d(i)!=fill_value) { + tgt_view_1d(i) += 1; + } + }); + break; } + case 2: + { + auto src_view_2d = field.get_view(); + auto tgt_view_2d = view_Nd_dev<2>(data,dims[0],dims[1]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j; + unflatten_idx(idx,extents,i,j); + if (src_view_2d(i,j)!=fill_value) { + tgt_view_2d(i,j) += 1; + } + }); + break; + } + case 3: + { + auto src_view_3d = field.get_view(); + auto tgt_view_3d = view_Nd_dev<3>(data,dims[0],dims[1],dims[2]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k; + unflatten_idx(idx,extents,i,j,k); + if (src_view_3d(i,j,k)!=fill_value) { + tgt_view_3d(i,j,k) += 1; + } + }); + break; + } + case 4: + { + auto src_view_4d = field.get_view(); + auto tgt_view_4d = view_Nd_dev<4>(data,dims[0],dims[1],dims[2],dims[3]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l; + unflatten_idx(idx,extents,i,j,k,l); + if (src_view_4d(i,j,k,l)!=fill_value) { + tgt_view_4d(i,j,k,l) += 1; + } + }); + break; + } + case 5: + { + auto src_view_5d = field.get_view(); + auto tgt_view_5d = view_Nd_dev<5>(data,dims[0],dims[1],dims[2],dims[3],dims[4]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l,m; + unflatten_idx(idx,extents,i,j,k,l,m); + if (src_view_5d(i,j,k,l,m)!=fill_value) { + tgt_view_5d(i,j,k,l,m) += 1; + } + }); + break; + } + case 6: + { + auto src_view_6d = field.get_view(); + auto tgt_view_6d = view_Nd_dev<6>(data,dims[0],dims[1],dims[2],dims[3],dims[4],dims[5]); + Kokkos::parallel_for(policy, KOKKOS_LAMBDA(int idx) { + int i,j,k,l,m,n; + unflatten_idx(idx,extents,i,j,k,l,m,n); + if (src_view_6d(i,j,k,l,m,n)!=fill_value) { + tgt_view_6d(i,j,k,l,m,n) += 1; + } + }); + break; + } + default: + EKAT_ERROR_MSG ( + "Error! Field rank not not supported by AtmosphereOutput.\n" + " - field name: " + field.name() + "\n" + " - field layout: " + to_string(layout) + "\n"); } } diff --git a/components/eamxx/src/share/io/scorpio_output.hpp b/components/eamxx/src/share/io/scorpio_output.hpp index 25a3da0d170c..13877425f614 100644 --- a/components/eamxx/src/share/io/scorpio_output.hpp +++ b/components/eamxx/src/share/io/scorpio_output.hpp @@ -175,7 +175,7 @@ class AtmosphereOutput create_diagnostic (const std::string& diag_name); // Tracking the averaging of any filled values: - void set_avg_cnt_tracking(const std::string& name, const std::string& avg_cnt_suffix, const FieldLayout& layout); + void set_avg_cnt_tracking(const std::string& name, const FieldLayout& layout); // --- Internal variables --- // ekat::Comm m_comm; @@ -198,8 +198,8 @@ class AtmosphereOutput std::vector m_fields_names; std::vector m_avg_cnt_names; std::map m_field_to_avg_cnt_map; + std::map m_field_to_avg_cnt_suffix; std::map m_layouts; - std::map m_dofs; std::map> m_dims; std::map> m_diagnostics; std::map> m_diag_depends_on_diags; @@ -215,8 +215,6 @@ class AtmosphereOutput // Local views of each field to be used for "averaging" output and writing to file. std::map m_host_views_1d; std::map m_dev_views_1d; - std::map m_local_tmp_avg_cnt_views_1d; - std::map m_avg_coeff_views_1d; bool m_add_time_dim; bool m_track_avg_cnt = false; diff --git a/components/eamxx/src/share/io/scream_io_control.hpp b/components/eamxx/src/share/io/scream_io_control.hpp new file mode 100644 index 000000000000..63cef4111991 --- /dev/null +++ b/components/eamxx/src/share/io/scream_io_control.hpp @@ -0,0 +1,78 @@ +#ifndef SCREAM_IO_CONTROL_HPP +#define SCREAM_IO_CONTROL_HPP + +#include "share/util/scream_time_stamp.hpp" + +#include + +#include + +namespace scream +{ + +// Mini struct to hold IO frequency info +struct IOControl { + + // If frequency_units is not "none" or "never", frequency *must* be set to a positive number + int frequency = -1; + std::string frequency_units = "none"; + + int nsamples_since_last_write = 0; // Needed when updating output data, such as with the OAT::Average flag + + util::TimeStamp next_write_ts; + util::TimeStamp last_write_ts; + + bool output_enabled () const { + return frequency_units!="none" && frequency_units!="never"; + } + + bool is_write_step (const util::TimeStamp& ts) const { + if (not output_enabled()) return false; + return frequency_units=="nsteps" ? ts.get_num_steps()==next_write_ts.get_num_steps() + : ts==next_write_ts; + } + + // Computes next_write_ts from frequency and last_write_ts + void compute_next_write_ts () { + EKAT_REQUIRE_MSG (last_write_ts.is_valid(), + "Error! Cannot compute next_write_ts, since last_write_ts was never set.\n"); + if (frequency_units=="nsteps") { + // This avoids having an invalid date/time in the above check next time this fcn runs + next_write_ts = last_write_ts; + next_write_ts.set_num_steps(last_write_ts.get_num_steps()+frequency); + } else if (frequency_units=="nsecs") { + next_write_ts = last_write_ts; + next_write_ts += frequency; + } else if (frequency_units=="nmins") { + next_write_ts = last_write_ts; + next_write_ts += frequency*60; + } else if (frequency_units=="nhours") { + next_write_ts = last_write_ts; + next_write_ts += frequency*3600; + } else if (frequency_units=="ndays") { + next_write_ts = last_write_ts; + next_write_ts += frequency*86400; + } else if (frequency_units=="nmonths" or frequency_units=="nyears") { + auto date = last_write_ts.get_date(); + if (frequency_units=="nmonths") { + int temp = date[1] + frequency - 1; + date[1] = temp % 12 + 1; + date[0] += temp / 12; + } else { + date[0] += frequency; + } + + // Fix day, in case we moved to a month/year where current days. E.g., if last_write + // was on Mar 31st, and units='nmonths', next write is on Apr 30th. HOWEVER, this + // means we will *always* write on the 30th of each month after then, since we have + // no memory of the fact that we were writing on the 31st before. + date[2] = std::min(date[2],util::days_in_month(date[0],date[1])); + next_write_ts = util::TimeStamp(date,last_write_ts.get_time()); + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported frequency unit '" + frequency_units + "'\n"); + } + } +}; + +} // namespace scream +#endif // SCREAM_IO_CONTROL_HPP diff --git a/components/eamxx/src/share/io/scream_io_file_specs.hpp b/components/eamxx/src/share/io/scream_io_file_specs.hpp new file mode 100644 index 000000000000..0e9ab2e042af --- /dev/null +++ b/components/eamxx/src/share/io/scream_io_file_specs.hpp @@ -0,0 +1,115 @@ +#ifndef SCREAM_IO_FILE_SPECS_HPP +#define SCREAM_IO_FILE_SPECS_HPP + +#include "share/io/scream_io_utils.hpp" +#include "share/util/scream_time_stamp.hpp" + +#include + +#include +#include + +namespace scream +{ + +// How the file capacity is specified +enum StorageType { + NumSnaps, // Fixed number of snaps per file + Monthly, // Each file contains output for one month + Yearly // Each file contains output for one year +}; + +inline std::string e2str (const StorageType st) { + switch (st) { + case NumSnaps: return "num_snapshots"; + case Yearly: return "one_year"; + case Monthly: return "one_month"; + default: return "unknown"; + } +} + +struct StorageSpecs { + + StorageType type = NumSnaps; + + // Current index ***in terms of this->type*** + // If type==NumSnaps, curr_idx=num_snapshots_in_file, + // otherwise it is the month/year index stored in this file + int curr_idx = -1; + + // A snapshot fits if + // - type=NumSnaps: the number of stored snaps is less than the max allowed per file. + // - otherwise: the snapshot month/year index match the one currently stored in the file + // or the file has no snapshot stored yet + bool snapshot_fits (const util::TimeStamp& t) { + const auto& idx = type==Monthly ? t.get_month() : t.get_year(); + switch (type) { + case Yearly: + case Monthly: + return curr_idx==-1 or curr_idx==idx; + case NumSnaps: + return num_snapshots_in_file::max(); +}; + +// Mini struct to hold some specs of an IO file +// To keep nc files small, we limit the number of snapshots in each nc file +// When the number of snapshots in a file reaches m_out_max_steps, it's time +// to close the out file, and open a new one. +struct IOFileSpecs { + + StorageSpecs storage = {}; + + bool is_open = false; + std::string filename; + + // If positive, flush the output file every these many snapshots + int flush_frequency = std::numeric_limits::max(); + + // bool file_is_full () const { return num_snapshots_in_file>=max_snapshots_in_file; } + bool file_needs_flush () const { + return storage.num_snapshots_in_file%flush_frequency==0; + } + + // Whether it is a model output, model restart, or history restart file + FileType ftype = FileType::Unset; + + bool is_restart_file () const { + return ftype==FileType::ModelRestart or ftype==FileType::HistoryRestart; + } + + std::string suffix () const { + if (ftype==FileType::HistoryRestart) + return ".rhist"; + else if (ftype==FileType::ModelRestart) + return ".r"; + else + return ""; + } + + void close () { + is_open = false; + storage.num_snapshots_in_file = 0; + storage.curr_idx = -1; + } +}; + +} // namespace scream +#endif // SCREAM_IO_FILE_SPECS_HPP diff --git a/components/eamxx/src/share/io/scream_io_utils.hpp b/components/eamxx/src/share/io/scream_io_utils.hpp index e4d803703128..8cbcd39a1a2c 100644 --- a/components/eamxx/src/share/io/scream_io_utils.hpp +++ b/components/eamxx/src/share/io/scream_io_utils.hpp @@ -3,14 +3,31 @@ #include "share/util/scream_time_stamp.hpp" -#include "ekat/util/ekat_string_utils.hpp" -#include "ekat/mpi/ekat_comm.hpp" +#include +#include #include namespace scream { +enum class FileType { + ModelOutput, + ModelRestart, + HistoryRestart, + Unset +}; + +inline std::string e2str(const FileType avg) { + using FT = FileType; + switch (avg) { + case FT::ModelOutput: return "model-output"; + case FT::ModelRestart: return "model-restart"; + case FT::HistoryRestart: return "history-restart"; + default: return "UNSET"; + } +} + enum class OutputAvgType { Instant, Max, @@ -42,95 +59,6 @@ inline OutputAvgType str2avg (const std::string& s) { return OAT::Invalid; } -// Mini struct to hold IO frequency info -struct IOControl { - // If units is not "none" or "never", freq *must* be set to a positive number - int frequency = -1; - int nsamples_since_last_write = 0; // Needed when updating output data, such as with the OAT::Average flag - util::TimeStamp timestamp_of_last_write; - std::string frequency_units = "none"; - - bool output_enabled () const { - return frequency_units!="none" && frequency_units!="never"; - } - - bool is_write_step (const util::TimeStamp& ts) { - // Mini-routine to determine if it is time to write output to file. - // The current allowable options are nsteps, nsecs, nmins, nhours, ndays, nmonths, nyears - // We query the frequency_units string value to determine which option it is. - bool ret = false; - if (frequency_units != "never" && frequency_units != "none") { - auto ts_diff = (ts-timestamp_of_last_write); - if (frequency_units == "nsteps") { - // Just use the num_steps from timestamps - return ((ts.get_num_steps()-timestamp_of_last_write.get_num_steps()) % frequency == 0); - // We will need to use timestamp information - } else if (frequency_units == "nsecs") { - ret = ((ts_diff > 0) && (ts_diff % frequency == 0)); - } else if (frequency_units == "nmins") { - ret = (ts_diff >= 60) && (ts_diff % (frequency*60) == 0); - } else if (frequency_units == "nhours") { - ret = (ts_diff >= 3600) && (ts_diff % (frequency*3600) == 0); - } else if (frequency_units == "ndays") { - ret = (ts_diff >= 86400) && (ts_diff % (frequency*86400) == 0); - } else if (frequency_units == "nmonths" || frequency_units == "nyears") { - // For months and years we need to be careful, can't just divide ts_diff by a set value. - // First we make sure that if we are the same day of the month and at the same time of day. - // TODO: Potential bug, if the day of last write >=29 there is a chance that we won't write - // in some subset of months, think Feb (28 days) and all of the months with only 30 days. - if (ts.get_day() == timestamp_of_last_write.get_day() && - ts.sec_of_day() == timestamp_of_last_write.sec_of_day()) { - auto diff = 0; - // Determine how many years have passed - diff += (ts.get_year() - timestamp_of_last_write.get_year()); - if (frequency_units == "nyears") { - ret = (diff>0) && (diff % frequency == 0); - } - // Determine number of months that have passed - diff *= 12; - diff += (ts.get_month() - timestamp_of_last_write.get_month()); - if (frequency_units == "nmonths") { - ret = (diff>0) && (diff % frequency == 0); - } - } - } else { - EKAT_REQUIRE_MSG(false,"Invalid frequency unit of [" + frequency_units + "] for output stream. Please check that all outputs have frequency_units of\n" - "none, never, nsteps, nsecs, nmins, nhours, ndays, nmonths, nyears"); - } - } - return ret; - } // End function is_write_step -}; - -// Mini struct to hold some specs of an IO file -// To keep nc files small, we limit the number of snapshots in each nc file -// When the number of snapshots in a file reaches m_out_max_steps, it's time -// to close the out file, and open a new one. -struct IOFileSpecs { - bool is_open = false; - std::string filename; - int num_snapshots_in_file = 0; - int max_snapshots_in_file; - - // If positive, flush the output file every these many snapshots - int flush_frequency = -1; - - bool file_is_full () const { return num_snapshots_in_file>=max_snapshots_in_file; } - bool file_needs_flush () const { return flush_frequency>0 and num_snapshots_in_file%flush_frequency==0; } - - // Adding number of MPI ranks to the filenamea is useful in testing, since we can run - // multiple instances of the same test in parallel (with different number of ranks), - // without the risk of them overwriting each other output. - // For production runs, this is not desirable. - bool filename_with_mpiranks = false; - - bool save_grid_data = true; - - // Whether this struct refers to a history restart file - bool hist_restart_file = false; - -}; - std::string find_filename_in_rpointer ( const std::string& casename, const bool model_restart, diff --git a/components/eamxx/src/share/io/scream_output_manager.cpp b/components/eamxx/src/share/io/scream_output_manager.cpp index 263cb9e8803f..e9f8f5887025 100644 --- a/components/eamxx/src/share/io/scream_output_manager.cpp +++ b/components/eamxx/src/share/io/scream_output_manager.cpp @@ -55,36 +55,9 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, m_is_restarted_run = (case_t0("use_case_as_start_reference",!m_is_model_restart_output); - m_output_control.frequency_units = out_control_pl.get("frequency_units"); - // In case output is disabled, no point in doing anything else - if (m_output_control.frequency_units=="none" || m_output_control.frequency_units=="never") { - m_output_disabled = true; - return; - } - m_output_control.frequency = out_control_pl.get("Frequency"); - EKAT_REQUIRE_MSG (m_output_control.frequency>0, - "Error! Invalid frequency (" + std::to_string(m_output_control.frequency) + ") in Output Control. Please, use positive number.\n"); - - m_output_control.timestamp_of_last_write = start_ref ? m_case_t0 : m_run_t0; - - // File specs - constexpr auto large_int = 1000000; - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); - m_output_file_specs.filename_with_mpiranks = out_control_pl.get("MPI Ranks in Filename",false); - m_output_file_specs.save_grid_data = out_control_pl.get("save_grid_data",!m_is_model_restart_output); - // Here, store if PG2 fields will be present in output streams. // Will be useful if multiple grids are defined (see below). bool pg2_grid_in_io_streams = false; @@ -136,7 +109,7 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, // For normal output, setup the geometry data streams, which we used to write the // geo data in the output file when we create it. - if (m_output_file_specs.save_grid_data) { + if (m_save_grid_data) { std::map> grids; for (const auto& it : m_output_streams) { grids[it->get_io_grid()->name()] = it->get_io_grid(); @@ -177,65 +150,38 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, } } - if (m_params.isSublist("Checkpoint Control")) { - // Output control - // TODO: It would be great if there was an option where, if Checkpoint Control was not a sublist, we - // could query the restart control information and just use that. - auto& pl = m_params.sublist("Checkpoint Control"); - m_checkpoint_control.frequency_units = pl.get("frequency_units"); - - if (m_checkpoint_control.output_enabled()) { - m_checkpoint_control.timestamp_of_last_write = run_t0; - m_checkpoint_control.frequency = pl.get("Frequency"); - EKAT_REQUIRE_MSG (m_output_control.frequency>0, - "Error! Invalid frequency (" + std::to_string(m_checkpoint_control.frequency) + ") in Checkpoint Control. Please, use positive number.\n"); - - // File specs - m_checkpoint_file_specs.max_snapshots_in_file = 1; - m_checkpoint_file_specs.flush_frequency = 1; - m_checkpoint_file_specs.filename_with_mpiranks = pl.get("MPI Ranks in Filename",false); - m_checkpoint_file_specs.save_grid_data = false; - m_checkpoint_file_specs.hist_restart_file = true; - } - } - - // If this is normal output (not the model restart output) and the output specs - // require it, we need to restart the output history. - // E.g., we might save 30-day avg value for field F, but due to job size - // break the run into three 10-day runs. We then need to save the state of - // our averaging in a "restart" file (e.g., the current avg). + // If this is model output (not model restart) we need to restart the output history. + // For instant output, this just entails restarting timestamps and counters, while + // for average output we also need to restore the accumulation state of the output fields // Note: the user might decide *not* to restart the output, so give the option // of disabling the restart. Also, the user might want to change the // filename_prefix, so allow to specify a different filename_prefix for the restart file. - if (m_is_restarted_run) { + if (m_is_restarted_run and not m_is_model_restart_output) { // Allow to skip history restart, or to specify a filename_prefix for the restart file // that is different from the filename_prefix of the current output. auto& restart_pl = m_params.sublist("Restart"); bool perform_history_restart = restart_pl.get("Perform Restart",true); auto hist_restart_filename_prefix = restart_pl.get("filename_prefix",m_filename_prefix); - if (m_is_model_restart_output) { - // For model restart output, the restart time (which is the start time of this run) is precisely - // when the last write happened, so we can quickly init the output control. - m_output_control.timestamp_of_last_write = m_run_t0; - m_output_control.nsamples_since_last_write = 0; - } else if (perform_history_restart) { + if (perform_history_restart) { using namespace scorpio; auto rhist_file = find_filename_in_rpointer(hist_restart_filename_prefix,false,m_io_comm,m_run_t0); + scorpio::register_file(rhist_file,scorpio::Read); // From restart file, get the time of last write, as well as the current size of the avg sample - m_output_control.timestamp_of_last_write = read_timestamp(rhist_file,"last_write"); + m_output_control.last_write_ts = read_timestamp(rhist_file,"last_write",true); + m_output_control.compute_next_write_ts(); m_output_control.nsamples_since_last_write = get_attribute(rhist_file,"num_snapshots_since_last_write"); if (m_avg_type!=OutputAvgType::Instant) { m_time_bnds.resize(2); - m_time_bnds[0] = m_output_control.timestamp_of_last_write.days_from(m_case_t0); + m_time_bnds[0] = m_output_control.last_write_ts.days_from(m_case_t0); } // If the type/freq of output needs restart data, we need to restart the streams - const bool output_every_step = m_output_control.frequency_units=="nsteps" && - m_output_control.frequency==1; - const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant && not output_every_step; + const bool output_every_step = m_output_control.frequency_units=="nsteps" && + m_output_control.frequency==1; + const bool has_checkpoint_data = m_avg_type!=OutputAvgType::Instant && not output_every_step; if (has_checkpoint_data && m_output_control.nsamples_since_last_write>0) { for (auto stream : m_output_streams) { stream->restart(rhist_file); @@ -259,14 +205,26 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, "Error! Cannot change avg type when performing history restart.\n" " - old avg type: " << old_avg_type + "\n" " - new avg type: " << e2str(m_avg_type) << "\n"); - auto old_max_snaps = scorpio::get_attribute(rhist_file,"max_snapshots_per_file"); - EKAT_REQUIRE_MSG (old_max_snaps == m_output_file_specs.max_snapshots_in_file, - "Error! Cannot change max snapshots per file when performing history restart.\n" - " - old max snaps: " << old_max_snaps << "\n" - " - new max snaps: " << m_output_file_specs.max_snapshots_in_file << "\n" - "If you *really* want to change the file capacity, you need to force using a new file, setting\n" + + + auto old_storage_type = scorpio::get_attribute(rhist_file,"file_max_storage_type"); + EKAT_REQUIRE_MSG (old_storage_type == e2str(m_output_file_specs.storage.type), + "Error! Cannot change file storage type when performing history restart.\n" + " - old file_max_storage_type: " << old_storage_type << "\n" + " - new file_max_storage_type: " << e2str(m_output_file_specs.storage.type) << "\n" + "If you *really* want to change the file storage type, you need to force using a new file, setting\n" " Restart:\n" " force_new_file: true\n"); + if (old_storage_type=="num_snapshot") { + auto old_max_snaps = scorpio::get_attribute(rhist_file,"max_snapshots_per_file"); + EKAT_REQUIRE_MSG (old_max_snaps == m_output_file_specs.storage.max_snapshots_in_file, + "Error! Cannot change max snapshots per file when performing history restart.\n" + " - old max snaps: " << old_max_snaps << "\n" + " - new max snaps: " << m_output_file_specs.storage.max_snapshots_in_file << "\n" + "If you *really* want to change the file capacity, you need to force using a new file, setting\n" + " Restart:\n" + " force_new_file: true\n"); + } std::string fp_precision = m_params.get("Floating Point Precision"); auto old_fp_precision = scorpio::get_attribute(rhist_file,"fp_precision"); EKAT_REQUIRE_MSG (old_fp_precision == fp_precision, @@ -279,26 +237,20 @@ setup (const ekat::Comm& io_comm, const ekat::ParameterList& params, const auto& last_output_filename = get_attribute(rhist_file,"last_output_filename"); m_resume_output_file = last_output_filename!="" and not restart_pl.get("force_new_file",false); if (m_resume_output_file) { + scorpio::register_file(last_output_filename,scorpio::Read); int num_snaps = scorpio::get_dimlen(last_output_filename,"time"); - - // End of checks. Close the file. scorpio::eam_pio_closefile(last_output_filename); - // If last output was full, we can no longer try to resume the file - if (num_snaps("skip_t0_output",false)) // This will be true for ERS/ERP tests + { + // In order to trigger a t0 write, we need to have next_write_ts matching run_t0 + m_output_control.next_write_ts = m_run_t0; this->run(m_run_t0); } @@ -328,7 +286,7 @@ add_global (const std::string& name, const ekat::any& global) { void OutputManager::run(const util::TimeStamp& timestamp) { // In case output is disabled, no point in doing anything else - if (m_output_disabled) { + if (not m_output_control.output_enabled()) { return; } @@ -340,6 +298,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) std::string timer_root = m_is_model_restart_output ? "EAMxx::IO::restart" : "EAMxx::IO::standard"; start_timer(timer_root); + // Check if we need to open a new file ++m_output_control.nsamples_since_last_write; ++m_checkpoint_control.nsamples_since_last_write; @@ -365,17 +324,32 @@ void OutputManager::run(const util::TimeStamp& timestamp) // Create and setup output/checkpoint file(s), if necessary start_timer(timer_root+"::get_new_file"); - auto setup_output_file = [&](IOControl& control, IOFileSpecs& filespecs, - bool add_to_rpointer, const std::string& file_type) { + auto setup_output_file = [&](IOControl& control, IOFileSpecs& filespecs) { + // Check if the new snapshot fits, if not, close the file + // NOTE: if output is average/max/min AND we save one file per month/year, + // we don't want to check if *this* timestamp fits in the file, since + // it may be in the next month/year even though most of the time averaging + // window is in the right year. E.g., if the avg is over the whole month, + // you would end up saving Jan average in the Feb file, since the end + // of the window is Feb 1st 00:00:00. So instead, we use the *start* + // of the avg window. If the avg is such that it spans 2 months (e.g., + // a 7-day avg), then where we put the avg is arbitrary, so our choice + // is still fine. + util::TimeStamp snapshot_start; + if (m_avg_type==OutputAvgType::Instant or filespecs.storage.type==NumSnaps) { + snapshot_start = timestamp; + } else { + snapshot_start = m_case_t0; + snapshot_start += m_time_bnds[0]; + } + if (not filespecs.storage.snapshot_fits(snapshot_start)) { + eam_pio_closefile(filespecs.filename); + filespecs.close(); + } + // Check if we need to open a new file if (not filespecs.is_open) { - // If this is normal output, with some sort of average, then the timestamp should be - // the one of the last write, since that's when the current avg window started. - // For Instant output (includes model restart output) and history restart, use the current timestamp - auto file_ts = m_avg_type==OutputAvgType::Instant or filespecs.hist_restart_file - ? timestamp : control.timestamp_of_last_write; - - filespecs.filename = compute_filename (control,filespecs,file_ts); + filespecs.filename = compute_filename (control,filespecs,timestamp); // Register all dims/vars, write geometry data (e.g. lat/lon/hyam/hybm) setup_file(filespecs,control); } @@ -383,7 +357,7 @@ void OutputManager::run(const util::TimeStamp& timestamp) // If we are going to write an output checkpoint file, or a model restart file, // we need to append to the filename ".rhist" or ".r" respectively, and add // the filename to the rpointer.atm file. - if (add_to_rpointer && m_io_comm.am_i_root()) { + if (m_io_comm.am_i_root() and filespecs.is_restart_file()) { std::ofstream rpointer; if (m_is_model_restart_output) { rpointer.open("rpointer.atm"); // Open rpointer and nuke its content @@ -406,19 +380,19 @@ void OutputManager::run(const util::TimeStamp& timestamp) } if (m_atm_logger) { - m_atm_logger->info("[EAMxx::output_manager] - Writing " + file_type + ":"); + m_atm_logger->info("[EAMxx::output_manager] - Writing " + e2str(filespecs.ftype) + ":"); m_atm_logger->info("[EAMxx::output_manager] FILE: " + filespecs.filename); } }; if (is_output_step) { - setup_output_file(m_output_control,m_output_file_specs,m_is_model_restart_output,m_is_model_restart_output ? "model restart" : "model output"); + setup_output_file(m_output_control,m_output_file_specs); // Update time (must be done _before_ writing fields) pio_update_time(m_output_file_specs.filename,timestamp.days_from(m_case_t0)); } if (is_checkpoint_step) { - setup_output_file(m_checkpoint_control,m_checkpoint_file_specs,true,"history restart"); + setup_output_file(m_checkpoint_control,m_checkpoint_file_specs); if (is_full_checkpoint_step) { // Update time (must be done _before_ writing fields) @@ -455,13 +429,19 @@ void OutputManager::run(const util::TimeStamp& timestamp) if (m_atm_logger) { m_atm_logger->debug("[OutputManager]: writing globals...\n"); } + + // Since we wrote to file we need to reset the timestamps + control.last_write_ts = timestamp; + control.compute_next_write_ts(); + control.nsamples_since_last_write = 0; + if (m_is_model_restart_output) { // Only write nsteps on model restart set_attribute(filespecs.filename,"nsteps",timestamp.get_num_steps()); } else { - if (filespecs.hist_restart_file) { + if (filespecs.ftype==FileType::HistoryRestart) { // Update the date of last write and sample size - scorpio::write_timestamp (filespecs.filename,"last_write",m_output_control.timestamp_of_last_write); + scorpio::write_timestamp (filespecs.filename,"last_write",m_output_control.last_write_ts,true); scorpio::set_attribute (filespecs.filename,"last_output_filename",m_output_file_specs.filename); scorpio::set_attribute (filespecs.filename,"num_snapshots_since_last_write",m_output_control.nsamples_since_last_write); } @@ -470,7 +450,10 @@ void OutputManager::run(const util::TimeStamp& timestamp) set_attribute(filespecs.filename,"averaging_type",e2str(m_avg_type)); set_attribute(filespecs.filename,"averaging_frequency_units",m_output_control.frequency_units); set_attribute(filespecs.filename,"averaging_frequency",m_output_control.frequency); - set_attribute(filespecs.filename,"max_snapshots_per_file",m_output_file_specs.max_snapshots_in_file); + set_attribute(filespecs.filename,"file_max_storage_type",e2str(m_output_file_specs.storage.type)); + if (m_output_file_specs.storage.type==NumSnaps) { + set_attribute(filespecs.filename,"max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); + } const auto& fp_precision = m_params.get("Floating Point Precision"); set_attribute(filespecs.filename,"fp_precision",fp_precision); } @@ -483,29 +466,21 @@ void OutputManager::run(const util::TimeStamp& timestamp) } // We're adding one snapshot to the file - ++filespecs.num_snapshots_in_file; + filespecs.storage.update_storage(timestamp); if (m_time_bnds.size()>0) { scorpio::grid_write_data_array(filespecs.filename, "time_bnds", m_time_bnds.data(), 2); } - // Since we wrote to file we need to reset the nsamples_since_last_write, the timestamp ... - control.nsamples_since_last_write = 0; - control.timestamp_of_last_write = timestamp; - - // Check if we need to close the output file - if (filespecs.file_is_full()) { - eam_pio_closefile(filespecs.filename); - filespecs.num_snapshots_in_file = 0; - filespecs.is_open = false; - } else if (filespecs.file_needs_flush()) { + // Check if we need to flush the output file + if (filespecs.file_needs_flush()) { eam_flush_file (filespecs.filename); } }; start_timer(timer_root+"::update_snapshot_tally"); // Important! Process output file first, and hist restart (if any) second. - // That's b/c write_global_data will update m_output_control.timestamp_of_last_write, + // That's b/c write_global_data will update m_output_control.last_write_ts, // which is later written as global data in the hist restart file if (is_output_step) { write_global_data(m_output_control,m_output_file_specs); @@ -551,25 +526,30 @@ compute_filename (const IOControl& control, const IOFileSpecs& file_specs, const util::TimeStamp& timestamp) const { - std::string suffix = - file_specs.hist_restart_file ? ".rhist" - : (m_is_model_restart_output ? ".r" : ""); - auto filename = m_filename_prefix + suffix; + auto filename = m_filename_prefix + file_specs.suffix(); // Always add avg type and frequency info filename += "." + e2str(m_avg_type); filename += "." + control.frequency_units+ "_x" + std::to_string(control.frequency); // Optionally, add number of mpi ranks (useful mostly in unit tests, to run multiple MPI configs in parallel) - if (file_specs.filename_with_mpiranks) { + if (m_params.get("MPI Ranks in Filename")) { filename += ".np" + std::to_string(m_io_comm.size()); } // Always add a time stamp - if (m_avg_type==OutputAvgType::Instant || file_specs.hist_restart_file) { - filename += "." + timestamp.to_string(); - } else { - filename += "." + control.timestamp_of_last_write.to_string(); + auto ts = (m_avg_type==OutputAvgType::Instant || file_specs.ftype==FileType::HistoryRestart) + ? timestamp : control.last_write_ts; + + switch (file_specs.storage.type) { + case NumSnaps: + filename += "." + ts.to_string(); break; + case Yearly: + filename += "." + std::to_string(ts.get_year()); break; + case Monthly: + filename += "." + std::to_string(ts.get_year()) + "-" + std::to_string(ts.get_month()); break; + default: + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); } return filename + ".nc"; @@ -585,21 +565,10 @@ set_params (const ekat::ParameterList& params, if (m_is_model_restart_output) { // We build some restart parameters internally - auto avg_type = m_params.get("Averaging Type","INSTANT"); - m_avg_type = str2avg(avg_type); - EKAT_REQUIRE_MSG (m_avg_type==OutputAvgType::Instant, - "Error! For restart output, the averaging type must be 'Instant'.\n" - " Note: you don't have to specify this parameter for restart output.\n"); - - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",1); - EKAT_REQUIRE_MSG (m_output_file_specs.max_snapshots_in_file==1, - "Error! For restart output, max snapshots per file must be 1.\n" - " Note: you don't have to specify this parameter for restart output.\n"); - - m_output_file_specs.flush_frequency = m_params.get("flush_frequency",1); - EKAT_REQUIRE_MSG (m_output_file_specs.flush_frequency==1, - "Error! For restart output, file flush frequency must be 1.\n" - " Note: you don't have to specify this parameter for restart output.\n"); + m_avg_type = OutputAvgType::Instant; + m_output_file_specs.storage.type = NumSnaps; + m_output_file_specs.storage.max_snapshots_in_file = 1; + m_output_file_specs.flush_frequency = 1; auto& fields_pl = m_params.sublist("Fields"); for (const auto& it : field_mgrs) { @@ -617,7 +586,9 @@ set_params (const ekat::ParameterList& params, fields_pl.sublist(it.first).set("Field Names",fnames); } m_filename_prefix = m_params.get("filename_prefix"); - // Match precision of Fields + + // Hard code some parameters in case we access them later + m_params.set("MPI Ranks in Filename",false); m_params.set("Floating Point Precision","real"); } else { auto avg_type = m_params.get("Averaging Type"); @@ -626,10 +597,24 @@ set_params (const ekat::ParameterList& params, "Error! Unsupported averaging type '" + avg_type + "'.\n" " Valid options: Instant, Max, Min, Average. Case insensitive.\n"); - constexpr auto large_int = 1000000; - m_output_file_specs.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); + const auto& storage_type = m_params.get("file_max_storage_type","num_snapshots"); + auto& storage = m_output_file_specs.storage; + constexpr auto large_int = std::numeric_limits::max(); + if (storage_type=="num_snapshots") { + storage.type = NumSnaps; + storage.max_snapshots_in_file = m_params.get("Max Snapshots Per File",large_int); + EKAT_REQUIRE_MSG (storage.max_snapshots_in_file>0, + "Error! Value for 'Max Snapshots Per File' should be positive.\n" + " To request 'unlimited' storage, leave parameter unset.\n"); + } else if (storage_type=="one_year") { + storage.type = Yearly; + } else if (storage_type=="one_month") { + storage.type = Monthly; + } else { + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); + } m_filename_prefix = m_params.get("filename_prefix"); - m_output_file_specs.flush_frequency = m_params.get("flush_frequency",m_output_file_specs.max_snapshots_in_file); + m_output_file_specs.flush_frequency = m_params.get("flush_frequency",large_int); // Allow user to ask for higher precision for normal model output, // but default to single to save on storage @@ -639,8 +624,64 @@ set_params (const ekat::ParameterList& params, "Error! Invalid/unsupported value for 'Floating Point Precision'.\n" " - input value: " + prec + "\n" " - supported values: float, single, double, real\n"); + + // If not set, hard code to false for CIME cases, and true for standalone, + // since standalone may be running multiple versions of the same test at once + if (not m_params.isParameter("MPI Ranks in Filename")) { + m_params.set("MPI Ranks in Filename",is_scream_standalone()); + } + } + + // Output control + EKAT_REQUIRE_MSG(m_params.isSublist("output_control"), + "Error! The output control YAML file for " + m_filename_prefix + " is missing the sublist 'output_control'"); + auto& out_control_pl = m_params.sublist("output_control"); + m_output_control.frequency_units = out_control_pl.get("frequency_units"); + + // In case output is disabled, no point in doing anything else + if (m_output_control.frequency_units=="none" || m_output_control.frequency_units=="never") { + return; + } + m_output_control.frequency = out_control_pl.get("Frequency"); + EKAT_REQUIRE_MSG (m_output_control.frequency>0, + "Error! Invalid frequency (" + std::to_string(m_output_control.frequency) + ") in Output Control. Please, use positive number.\n"); + + // Determine which timestamp to use a reference for output frequency. Two options: + // 1. use_case_as_start_reference: TRUE - implies we want to calculate frequency from the beginning of the whole simulation, even if this is a restarted run. + // 2. use_case_as_start_reference: FALSE - implies we want to base the frequency of output on when this particular simulation started. + // Note, (2) is needed for restarts since the restart frequency in CIME assumes a reference of when this run began. + // NOTE: m_is_restarted_run and not m_is_model_restart_output, these timestamps will be corrected once we open the hist restart file + const bool use_case_as_ref = out_control_pl.get("use_case_as_start_reference",!m_is_model_restart_output); + const bool perform_history_restart = m_params.sublist("Restart").get("Perform Restart",true); + const auto& start_ref = use_case_as_ref and perform_history_restart ? m_case_t0 : m_run_t0; + m_output_control.last_write_ts = start_ref; + m_output_control.compute_next_write_ts(); + + // File specs + m_save_grid_data = out_control_pl.get("save_grid_data",!m_is_model_restart_output); + m_output_file_specs.ftype = m_is_model_restart_output ? FileType::ModelRestart : FileType::ModelOutput; + + if (m_params.isSublist("Checkpoint Control")) { + auto& pl = m_params.sublist("Checkpoint Control"); + m_checkpoint_control.frequency_units = pl.get("frequency_units"); + + if (m_checkpoint_control.output_enabled()) { + m_checkpoint_control.frequency = pl.get("Frequency"); + EKAT_REQUIRE_MSG (m_output_control.frequency>0, + "Error! Invalid frequency (" + std::to_string(m_checkpoint_control.frequency) + ") in Checkpoint Control. Please, use positive number.\n"); + + m_checkpoint_control.last_write_ts = m_run_t0; + m_checkpoint_control.compute_next_write_ts(); + + // File specs + m_checkpoint_file_specs.storage.type = NumSnaps; + m_checkpoint_file_specs.storage.max_snapshots_in_file = 1; + m_checkpoint_file_specs.flush_frequency = 1; + m_checkpoint_file_specs.ftype = FileType::HistoryRestart; + } } } + /*===============================================================================================*/ void OutputManager:: setup_file ( IOFileSpecs& filespecs, @@ -693,7 +734,10 @@ setup_file ( IOFileSpecs& filespecs, set_attribute(filename,"averaging_type",e2str(m_avg_type)); set_attribute(filename,"averaging_frequency_units",m_output_control.frequency_units); set_attribute(filename,"averaging_frequency",m_output_control.frequency); - set_attribute(filename,"max_snapshots_per_file",m_output_file_specs.max_snapshots_in_file); + set_attribute(filespecs.filename,"file_max_storage_type",e2str(m_output_file_specs.storage.type)); + if (m_output_file_specs.storage.type==NumSnaps) { + set_attribute(filename,"max_snapshots_per_file",m_output_file_specs.storage.max_snapshots_in_file); + } set_attribute(filename,"fp_precision",fp_precision); set_file_header(filespecs); } @@ -713,7 +757,7 @@ setup_file ( IOFileSpecs& filespecs, // If grid data is needed, also register geo data fields. Skip if file is resumed, // since grid data was written in the previous run - if (filespecs.save_grid_data and not m_resume_output_file) { + if (m_save_grid_data and not filespecs.is_restart_file() and not m_resume_output_file) { for (auto& it : m_geo_data_streams) { it->setup_output_file(filename,fp_precision,mode); } @@ -724,7 +768,7 @@ setup_file ( IOFileSpecs& filespecs, // the dims/vars are already in the file (we don't allow adding dims/vars) eam_pio_enddef (filename); - if (filespecs.save_grid_data and not m_resume_output_file) { + if (m_save_grid_data and not filespecs.is_restart_file() and not m_resume_output_file) { // Immediately run the geo data streams for (const auto& it : m_geo_data_streams) { it->run(filename,true,false,0); @@ -766,13 +810,7 @@ void OutputManager::set_file_header(const IOFileSpecs& file_specs) set_attribute(filename,"realm","atmos"); set_attribute(filename,"history",ts_str); set_attribute(filename,"Conventions","CF-1.8"); - if (m_is_model_restart_output) { - set_attribute(filename,"product","model-restart"); - } else if (file_specs.hist_restart_file) { - set_attribute(filename,"product","history-restart"); - } else { - set_attribute(filename,"product","model-output"); - } + set_attribute(filename,"product",e2str(file_specs.ftype)); } /*===============================================================================================*/ void OutputManager:: @@ -790,13 +828,28 @@ push_to_logger() m_atm_logger->info(" Filename prefix: " + m_filename_prefix); m_atm_logger->info(" Run t0: " + m_run_t0.to_string()); m_atm_logger->info(" Case t0: " + m_case_t0.to_string()); - m_atm_logger->info(" Reference t0: " + m_output_control.timestamp_of_last_write.to_string()); + m_atm_logger->info(" Reference t0: " + m_output_control.last_write_ts.to_string()); m_atm_logger->info(" Is Restart File ?: " + bool_to_string(m_is_model_restart_output)); m_atm_logger->info(" Is Restarted Run ?: " + bool_to_string(m_is_restarted_run)); m_atm_logger->info(" Averaging Type: " + e2str(m_avg_type)); m_atm_logger->info(" Output Frequency: " + std::to_string(m_output_control.frequency) + " " + m_output_control.frequency_units); - m_atm_logger->info(" Max snaps in file: " + std::to_string(m_output_file_specs.max_snapshots_in_file)); // TODO: add "not set" if the value is -1 - m_atm_logger->info(" Includes Grid Data ?: " + bool_to_string(m_output_file_specs.save_grid_data)); + switch (m_output_file_specs.storage.type) { + case NumSnaps: + { + auto ms = m_output_file_specs.storage.max_snapshots_in_file; + m_atm_logger->info(" File Capacity: " + (ms>0 ? std::to_string(ms) + "snapshots" : "UNLIMITED")); + break; + } + case Monthly: + m_atm_logger->info(" File Capacity: one month per file"); + break; + case Yearly: + m_atm_logger->info(" File Capacity: one year per file"); + break; + default: + EKAT_ERROR_MSG ("Error! Unrecognized/unsupported file storage type.\n"); + } + m_atm_logger->info(" Includes Grid Data ?: " + bool_to_string(m_save_grid_data)); // List each GRID - TODO // List all FIELDS - TODO } diff --git a/components/eamxx/src/share/io/scream_output_manager.hpp b/components/eamxx/src/share/io/scream_output_manager.hpp index e546922b4d6e..b707c61ee4c0 100644 --- a/components/eamxx/src/share/io/scream_output_manager.hpp +++ b/components/eamxx/src/share/io/scream_output_manager.hpp @@ -4,6 +4,8 @@ #include "share/io/scorpio_output.hpp" #include "share/io/scream_scorpio_interface.hpp" #include "share/io/scream_io_utils.hpp" +#include "share/io/scream_io_file_specs.hpp" +#include "share/io/scream_io_control.hpp" #include "share/field/field_manager.hpp" #include "share/grid/grids_manager.hpp" @@ -171,9 +173,6 @@ class OutputManager // Whether a restarted run can resume filling previous run output file (if not full) bool m_resume_output_file = false; - // If the user specifies freq units "none" or "never", output is disabled - bool m_output_disabled = false; - // The initial time stamp of the simulation and run. For initial runs, they coincide, // but for restarted runs, run_t0>case_t0, with the former being the time at which the // restart happens, and the latter being the start time of the *original* run. @@ -182,6 +181,9 @@ class OutputManager // The logger to be used throughout the ATM to log message std::shared_ptr m_atm_logger; + + // If true, we save grid data in output file + bool m_save_grid_data; }; } // namespace scream diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.F90 b/components/eamxx/src/share/io/scream_scorpio_interface.F90 index 6f8b83886547..50401f7f3051 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.F90 +++ b/components/eamxx/src/share/io/scream_scorpio_interface.F90 @@ -408,7 +408,7 @@ subroutine register_variable(filename,shortname,longname,units, & prev => curr curr => prev%next end do - + allocate(prev%next) curr => prev%next allocate(curr%var) @@ -562,7 +562,7 @@ function get_variable_metadata_float(filename, varname, metaname) result(metaval character(len=256), intent(in) :: varname character(len=256), intent(in) :: metaname real(kind=c_float) :: metaval - + ! Local variables type(pio_atm_file_t),pointer :: pio_file @@ -998,7 +998,7 @@ subroutine eam_pio_flush_file(fname) call lookup_pio_atm_file(trim(fname),pio_atm_file,found) if (found) then - if ( is_write(pio_atm_file%purpose) ) then + if ( is_write(pio_atm_file%purpose) .or. is_append(pio_atm_file%purpose) ) then call PIO_syncfile(pio_atm_file%pioFileDesc) else call errorHandle("PIO ERROR: unable to flush file: "//trim(fname)//", is not open in write mode",-999) @@ -1674,7 +1674,6 @@ subroutine grid_read_darray_double(filename, varname, buf, buf_size, time_index) call lookup_pio_atm_file(trim(filename),pio_atm_file,found) call get_var(pio_atm_file,varname,var) - ! Set the timesnap we are reading if (time_index .gt. 0) then ! The user has set a valid time index to read from diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.cpp b/components/eamxx/src/share/io/scream_scorpio_interface.cpp index 676b72f60aee..a74ceecd3930 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.cpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.cpp @@ -190,7 +190,7 @@ bool has_variable (const std::string& filename, const std::string& varname) return false; } EKAT_REQUIRE_MSG (err==PIO_NOERR, - "Error! Something went wrong while retrieving dimension id.\n" + "Error! Something went wrong while retrieving variable id.\n" " - filename : " + filename + "\n" " - varname : " + varname + "\n" " - pio error: " + std::to_string(err) + "\n"); @@ -200,6 +200,54 @@ bool has_variable (const std::string& filename, const std::string& varname) return true; } + +bool has_attribute (const std::string& filename, const std::string& attname) +{ + return has_attribute(filename,"GLOBAL",attname); +} + +bool has_attribute (const std::string& filename, const std::string& varname, const std::string& attname) +{ + int ncid, varid, attid, err; + + bool was_open = is_file_open_c2f(filename.c_str(),-1); + if (not was_open) { + register_file(filename,Read); + } + + // Get file id + ncid = get_file_ncid_c2f (filename.c_str()); + + // Get var id + if (varname=="GLOBAL") { + varid = PIO_GLOBAL; + } else { + err = PIOc_inq_varid(ncid,varname.c_str(),&varid); + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while retrieving variable id.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + } + + // Get att id + err = PIOc_inq_attid(ncid,varid,attname.c_str(),&attid); + if (err==PIO_ENOTATT) { + return false; + } + EKAT_REQUIRE_MSG (err==PIO_NOERR, + "Error! Something went wrong while retrieving attribute id.\n" + " - filename : " + filename + "\n" + " - varname : " + varname + "\n" + " - attname : " + attname + "\n" + " - pio error: " + std::to_string(err) + "\n"); + + if (not was_open) { + eam_pio_closefile(filename); + } + + return true; +} /* ----------------------------------------------------------------- */ void set_dof(const std::string& filename, const std::string& varname, const Int dof_len, const std::int64_t* x_dof) { @@ -231,6 +279,7 @@ void register_dimension(const std::string &filename, const std::string& shortnam EKAT_REQUIRE_MSG (err==PIO_NOERR, "Error! Something went wrong querying for the unlimited dimension id.\n" " - filename: " + filename + "\n" + " - dimension : " + shortname + "\n" " - pio error: " + std::to_string(err) + "\n"); if (length==0) { EKAT_REQUIRE_MSG ( unlimid==dimid, @@ -292,7 +341,8 @@ void register_variable(const std::string &filename, const std::string& shortname if (mode==Write) { EKAT_REQUIRE_MSG ( units!="" and nc_dtype!="", "Error! Missing valid units and/or nc_dtype arguments for file open in Write mode.\n" - " - filename: " + filename + "\n"); + " - filename: " + filename + "\n" + " - varname : " + shortname + "\n"); } else { EKAT_REQUIRE_MSG ( has_var, "Error! Variable not found in file open in " + mode_str + " mode.\n" @@ -357,6 +407,7 @@ void register_variable(const std::string &filename, const std::string& shortname EKAT_REQUIRE_MSG(var_dimensions==dims_from_file, "Error! Input variable dimensions do not match the ones from the file.\n" " - filename : " + filename + "\n" + " - varname : " + shortname + "\n" " - input dims: (" + ekat::join(var_dimensions,",") + ")\n" " - file dims : (" + ekat::join(dims_from_file,",") + ")\n"); @@ -626,14 +677,24 @@ void grid_write_data_array(const std::string &filename, const std::strin grid_write_data_array_c2f_double(filename.c_str(),varname.c_str(),hbuf,buf_size); } /* ----------------------------------------------------------------- */ -void write_timestamp (const std::string& filename, const std::string& ts_name, const util::TimeStamp& ts) +void write_timestamp (const std::string& filename, const std::string& ts_name, + const util::TimeStamp& ts, const bool write_nsteps) { set_attribute(filename,ts_name,ts.to_string()); + if (write_nsteps) { + set_attribute(filename,ts_name+"_nsteps",ts.get_num_steps()); + } } /* ----------------------------------------------------------------- */ -util::TimeStamp read_timestamp (const std::string& filename, const std::string& ts_name) +util::TimeStamp read_timestamp (const std::string& filename, + const std::string& ts_name, + const bool read_nsteps) { - return util::str_to_time_stamp(get_attribute(filename,ts_name)); + auto ts = util::str_to_time_stamp(get_attribute(filename,ts_name)); + if (read_nsteps and has_attribute(filename,ts_name+"_nsteps")) { + ts.set_num_steps(get_attribute(filename,ts_name+"_nsteps")); + } + return ts; } /* ----------------------------------------------------------------- */ } // namespace scorpio diff --git a/components/eamxx/src/share/io/scream_scorpio_interface.hpp b/components/eamxx/src/share/io/scream_scorpio_interface.hpp index 4289265d07aa..b64ca3ef25fd 100644 --- a/components/eamxx/src/share/io/scream_scorpio_interface.hpp +++ b/components/eamxx/src/share/io/scream_scorpio_interface.hpp @@ -38,6 +38,8 @@ namespace scorpio { int get_dimlen(const std::string& filename, const std::string& dimname); bool has_dim(const std::string& filename, const std::string& dimname); bool has_variable (const std::string& filename, const std::string& varname); + bool has_attribute (const std::string& filename, const std::string& attname); + bool has_attribute (const std::string& filename, const std::string& varname, const std::string& attname); void set_decomp(const std::string& filename); /* Sets the degrees-of-freedom for a particular variable in a particular file. Called once for each variable, for each file. */ void set_dof(const std::string &filename, const std::string &varname, const Int dof_len, const offset_t* x_dof); @@ -93,8 +95,11 @@ namespace scorpio { } // Shortcut to write/read to/from YYYYMMDD/HHMMSS attributes in the NC file - void write_timestamp (const std::string& filename, const std::string& ts_name, const util::TimeStamp& ts); - util::TimeStamp read_timestamp (const std::string& filename, const std::string& ts_name); + void write_timestamp (const std::string& filename, const std::string& ts_name, + const util::TimeStamp& ts, const bool write_nsteps = false); + util::TimeStamp read_timestamp (const std::string& filename, + const std::string& ts_name, + const bool read_nsteps = false); extern "C" { /* Query whether the pio subsystem is inited or not */ diff --git a/components/eamxx/src/share/io/tests/CMakeLists.txt b/components/eamxx/src/share/io/tests/CMakeLists.txt index 21d293dbb15f..94bcee1790b0 100644 --- a/components/eamxx/src/share/io/tests/CMakeLists.txt +++ b/components/eamxx/src/share/io/tests/CMakeLists.txt @@ -17,6 +17,12 @@ CreateUnitTest(io_basic "io_basic.cpp" MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) +## Test output where we write one file per month +CreateUnitTest(io_monthly "io_monthly.cpp" + LIBS scream_io LABELS io + MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} +) + ## Test basic output (no packs, no diags, all avg types, all freq units) CreateUnitTest(io_filled "io_filled.cpp" LIBS scream_io LABELS io @@ -36,8 +42,7 @@ CreateUnitTest(io_diags "io_diags.cpp" ) # Test output on SE grid -configure_file(io_test_se_grid.yaml io_test_se_grid.yaml) -CreateUnitTest(io_test_se_grid "io_se_grid.cpp" +CreateUnitTest(io_se_grid "io_se_grid.cpp" LIBS scream_io LABELS io MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} ) diff --git a/components/eamxx/src/share/io/tests/io_basic.cpp b/components/eamxx/src/share/io/tests/io_basic.cpp index 4b7ee47bc5f5..378a956f8932 100644 --- a/components/eamxx/src/share/io/tests/io_basic.cpp +++ b/components/eamxx/src/share/io/tests/io_basic.cpp @@ -152,7 +152,6 @@ void write (const std::string& avg_type, const std::string& freq_units, auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",freq_units); ctrl_pl.set("Frequency",freq); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager @@ -172,8 +171,8 @@ void write (const std::string& avg_type, const std::string& freq_units, t += dt; // Add 1 to all fields entries - for (const auto& n : fnames) { - auto f = fm->get_field(n); + for (const auto& name : fnames) { + auto f = fm->get_field(name); add(f,1.0); } @@ -231,6 +230,7 @@ void read (const std::string& avg_type, const std::string& freq_units, // The last one comes from // (a+1 + a+2 +..+a+freq)/freq = // a + sum(i)/freq = a + (freq(freq+1)/2)/freq + // = a + (freq+1)/2 double delta = (freq+1)/2.0; for (int n=0; n Averaging type: " + avg + " ", 40); write(avg,units,freq,seed,comm); - read(avg,units,freq,seed,comm); + read (avg,units,freq,seed,comm); print(" PASS\n"); } } diff --git a/components/eamxx/src/share/io/tests/io_diags.cpp b/components/eamxx/src/share/io/tests/io_diags.cpp index 90abb4c57775..b2fe7f994db6 100644 --- a/components/eamxx/src/share/io/tests/io_diags.cpp +++ b/components/eamxx/src/share/io/tests/io_diags.cpp @@ -186,7 +186,6 @@ void write (const int seed, const ekat::Comm& comm) auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",1); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager diff --git a/components/eamxx/src/share/io/tests/io_filled.cpp b/components/eamxx/src/share/io/tests/io_filled.cpp index 93e14589c44c..ba5eb7d871d8 100644 --- a/components/eamxx/src/share/io/tests/io_filled.cpp +++ b/components/eamxx/src/share/io/tests/io_filled.cpp @@ -133,12 +133,11 @@ void write (const std::string& avg_type, const std::string& freq_units, om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", avg_type); om_pl.set("fill_value",FillValue); - om_pl.set("track_fill",true); om_pl.set("fill_threshold",fill_threshold); + om_pl.set("track_avg_cnt",true); auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",freq_units); ctrl_pl.set("Frequency",freq); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager @@ -222,24 +221,24 @@ void read (const std::string& avg_type, const std::string& freq_units, auto f0 = fm0->get_field(fn).clone(); auto f = fm->get_field(fn); if (avg_type=="MIN") { - Real test_val = ((n+1)*freq%2==0) ? n*freq+1 : n*freq+2; + Real test_val = ((n+1)*freq%2==0) ? n*freq+1 : n*freq+2; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); } else if (avg_type=="MAX") { - Real test_val = ((n+1)*freq%2==0) ? (n+1)*freq : (n+1)*freq-1; + Real test_val = ((n+1)*freq%2==0) ? (n+1)*freq : (n+1)*freq-1; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); } else if (avg_type=="INSTANT") { - Real test_val = (n*freq%2==0) ? n*freq : FillValue; + Real test_val = (n*freq%2==0) ? n*freq : FillValue; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); } else { // Is avg_type = AVERAGE - // Note, for AVERAGE type output with filling we need to check that the - // number of contributing fill steps surpasses the fill_threshold, if not - // then we know that the snap will reflect the fill value. - Real test_val; - Real M = freq/2 + (n%2==0 ? 0.0 : 1.0); - Real a = n*freq + (n%2==0 ? 0.0 : -1.0); + // Note, for AVERAGE type output with filling we need to check that the + // number of contributing fill steps surpasses the fill_threshold, if not + // then we know that the snap will reflect the fill value. + Real test_val; + Real M = freq/2 + (n%2==0 ? 0.0 : 1.0); + Real a = n*freq + (n%2==0 ? 0.0 : -1.0); test_val = (M/freq > fill_threshold) ? a + (M+1.0) : FillValue; set(f0,test_val); REQUIRE (views_are_equal(f,f0)); diff --git a/components/eamxx/src/share/io/tests/io_monthly.cpp b/components/eamxx/src/share/io/tests/io_monthly.cpp new file mode 100644 index 000000000000..4cf18fff1c71 --- /dev/null +++ b/components/eamxx/src/share/io/tests/io_monthly.cpp @@ -0,0 +1,230 @@ +#include + +#include "share/io/scream_output_manager.hpp" +#include "share/io/scorpio_input.hpp" + +#include "share/grid/mesh_free_grids_manager.hpp" + +#include "share/field/field_utils.hpp" +#include "share/field/field.hpp" +#include "share/field/field_manager.hpp" + +#include "share/util/scream_universal_constants.hpp" +#include "share/util/scream_setup_random_test.hpp" +#include "share/util/scream_time_stamp.hpp" +#include "share/scream_types.hpp" + +#include "ekat/util/ekat_units.hpp" +#include "ekat/ekat_parameter_list.hpp" +#include "ekat/ekat_assert.hpp" +#include "ekat/mpi/ekat_comm.hpp" +#include "ekat/util/ekat_test_utils.hpp" + +#include +#include + +namespace scream { + +void add (const Field& f, const double v) { + auto data = f.get_internal_view_data(); + auto nscalars = f.get_header().get_alloc_properties().get_num_scalars(); + for (int i=0; i +get_gm (const ekat::Comm& comm) +{ + // For 2+ ranks tests, this will check IO works correctly + // even if one rank owns 0 dofs + const int ngcols = std::max(comm.size()-1,1); + const int nlevs = 4; + auto gm = create_mesh_free_grids_manager(comm,0,0,nlevs,ngcols); + gm->build_grids(); + return gm; +} + +std::shared_ptr +get_fm (const std::shared_ptr& grid, + const util::TimeStamp& t0, const int seed) +{ + using FL = FieldLayout; + using FID = FieldIdentifier; + using namespace ShortFieldTagsNames; + + // Random number generation stuff + // NOTES + // - Use integers, so we can check answers without risk of + // non bfb diffs due to different order of sums. + // - Uniform_int_distribution returns an int, and the randomize + // util checks that return type matches the Field data type. + // So wrap the int pdf in a lambda, that does the cast. + std::mt19937_64 engine(seed); + auto my_pdf = [&](std::mt19937_64& engine) -> Real { + std::uniform_int_distribution pdf (0,100); + Real v = pdf(engine); + return v; + }; + + const int nlcols = grid->get_num_local_dofs(); + const int nlevs = grid->get_num_vertical_levels(); + + std::vector layouts = + { + FL({COL }, {nlcols }), + FL({COL, LEV}, {nlcols, nlevs}), + FL({COL,CMP,ILEV}, {nlcols,2,nlevs+1}) + }; + + auto fm = std::make_shared(grid); + + const auto units = ekat::units::Units::nondimensional(); + int count=0; + for (const auto& fl : layouts) { + FID fid("f_"+std::to_string(count),fl,units,grid->name()); + Field f(fid); + f.allocate_view(); + randomize (f,engine,my_pdf); + f.get_header().get_tracking().update_time_stamp(t0); + fm->add_field(f); + ++count; + } + + return fm; +} + +// Returns fields after initialization +void write (const int seed, const ekat::Comm& comm) +{ + // Create grid + auto gm = get_gm(comm); + auto grid = gm->get_grid("Point Grid"); + + // Time advance parameters + auto t0 = get_t0(); + const int dt = 86400*30; // 30 days + + // Create some fields + auto fm = get_fm(grid,t0,seed); + std::vector fnames; + for (auto it : *fm) { + fnames.push_back(it.second->name()); + } + + // Create output params + ekat::ParameterList om_pl; + om_pl.set("MPI Ranks in Filename",true); + om_pl.set("filename_prefix",std::string("io_monthly")); + om_pl.set("Field Names",fnames); + om_pl.set("Averaging Type", std::string("Instant")); + om_pl.set("file_max_storage_type",std::string("one_month")); + om_pl.set("Floating Point Precision",std::string("single")); + auto& ctrl_pl = om_pl.sublist("output_control"); + ctrl_pl.set("frequency_units",std::string("nsteps")); + ctrl_pl.set("Frequency",1); + ctrl_pl.set("save_grid_data",false); + + // Create Output manager + OutputManager om; + om.setup(comm,om_pl,fm,gm,t0,t0,false); + + // Time loop: do 11 steps, since we already did Jan output at t0 + const int nsteps = 11; + auto t = t0; + for (int n=0; nget_field(name); + add(f,1); + } + + // Run output manager + om.run (t); + } + + // Close file and cleanup + om.finalize(); +} + +void read (const int seed, const ekat::Comm& comm) +{ + // Time quantities + auto t0 = get_t0(); + int dt = 86400*30; + + // Get gm + auto gm = get_gm (comm); + auto grid = gm->get_grid("Point Grid"); + + // Get initial fields. Use wrong seed for fm, so fields are not + // inited with right data (avoid getting right answer without reading). + auto fm0 = get_fm(grid,t0,seed); + auto fm = get_fm(grid,t0,-seed-1); + std::vector fnames; + for (auto it : *fm) { + fnames.push_back(it.second->name()); + } + + // Get filename from timestamp + std::string casename = "io_monthly"; + auto get_filename = [&](const util::TimeStamp& t) { + std::string fname = casename + + ".INSTANT.nsteps_x1" + + ".np" + std::to_string(comm.size()) + + "." + std::to_string(t.get_year()) + + "-" + std::to_string(t.get_month()) + + ".nc"; + return fname; + }; + + // Create reader pl + ekat::ParameterList reader_pl; + reader_pl.set("Field Names",fnames); + + for (int n=0; n<12; ++n) { + auto t = t0 + n*dt; + auto filename = get_filename(t); + + // There should be just one time snapshot per file + REQUIRE(scorpio::get_dimlen(filename,"time")==1); + + reader_pl.set("Filename",filename); + AtmosphereInput reader(reader_pl,fm); + reader.read_variables(); + + for (const auto& fn : fnames) { + auto f0 = fm0->get_field(fn).clone(); + auto f = fm->get_field(fn); + add(f0,n); + REQUIRE (views_are_equal(f,f0)); + } + } +} + +TEST_CASE ("io_monthly") { + ekat::Comm comm(MPI_COMM_WORLD); + scorpio::eam_init_pio_subsystem(comm); + + auto seed = get_random_test_seed(&comm); + + if (comm.am_i_root()) { + std::cout << " -> Testing output with one file per month ...\n"; + } + write(seed,comm); + read (seed,comm); + if (comm.am_i_root()) { + std::cout << " -> Testing output with one file per month ... PASS\n"; + } + scorpio::eam_pio_finalize(); +} + +} // anonymous namespace diff --git a/components/eamxx/src/share/io/tests/io_packed.cpp b/components/eamxx/src/share/io/tests/io_packed.cpp index f4b467c619c1..585fc7b26f69 100644 --- a/components/eamxx/src/share/io/tests/io_packed.cpp +++ b/components/eamxx/src/share/io/tests/io_packed.cpp @@ -120,7 +120,6 @@ void write (const int freq, const int seed, const int ps, const ekat::Comm& comm auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",freq); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); // Create Output manager diff --git a/components/eamxx/src/share/io/tests/io_remap_test.cpp b/components/eamxx/src/share/io/tests/io_remap_test.cpp index 91fa95b3fc29..bfed937636e2 100644 --- a/components/eamxx/src/share/io/tests/io_remap_test.cpp +++ b/components/eamxx/src/share/io/tests/io_remap_test.cpp @@ -243,6 +243,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_source.setup(io_comm,source_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_source.run(t0); + om_source.finalize(); print (" -> source data ... done\n",io_comm); print (" -> vertical remap ... \n",io_comm); @@ -250,6 +251,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_vert.setup(io_comm,vert_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_vert.run(t0); + om_vert.finalize(); print (" -> vertical remap ... done\n",io_comm); print (" -> horizontal remap ... \n",io_comm); @@ -257,6 +259,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_horiz.setup(io_comm,horiz_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_horiz.run(t0); + om_horiz.finalize(); print (" -> horizontal remap ... done\n",io_comm); print (" -> vertical-horizontal remap ... \n",io_comm); @@ -264,6 +267,7 @@ TEST_CASE("io_remap_test","io_remap_test") om_vert_horiz.setup(io_comm,vert_horiz_remap_control,field_manager,gm,t0,t0,false); io_comm.barrier(); om_vert_horiz.run(t0); + om_vert_horiz.finalize(); print (" -> vertical-horizontal remap ... done\n",io_comm); print (" -> Create output ... done\n",io_comm); @@ -640,7 +644,7 @@ std::shared_ptr get_test_fm(std::shared_ptr gr auto f_Vi = fm->get_field(fid_Vi); // Set some string to be written to file as attribute to the variables - for (const std::string& fname : {"Y_flat","Y_mid","Y_int","V_mid","V_int"}) { + for (const std::string fname : {"Y_flat","Y_mid","Y_int","V_mid","V_int"}) { auto& f = fm->get_field(fname); auto& str_atts = f.get_header().get_extra_data("io: string attributes"); str_atts["test"] = fname; @@ -669,14 +673,14 @@ std::shared_ptr get_test_fm(std::shared_ptr gr ekat::ParameterList set_output_params(const std::string& name, const std::string& remap_filename, const int p_ref, const bool vert_remap, const bool horiz_remap) { using vos_type = std::vector; - ekat::ParameterList output_yaml; - - output_yaml.set("filename_prefix",name); - output_yaml.set("Averaging Type","Instant"); - output_yaml.set("Max Snapshots Per File",1); - output_yaml.set("Floating Point Precision","real"); - auto& oc = output_yaml.sublist("output_control"); - oc.set("MPI Ranks in Filename",true); + ekat::ParameterList params; + + params.set("filename_prefix",name); + params.set("Averaging Type","Instant"); + params.set("Max Snapshots Per File",1); + params.set("Floating Point Precision","real"); + params.set("MPI Ranks in Filename",true); + auto& oc = params.sublist("output_control"); oc.set("Frequency",1); oc.set("frequency_units","nsteps"); @@ -689,16 +693,16 @@ ekat::ParameterList set_output_params(const std::string& name, const std::string fields_out.push_back("p_mid"); fields_out.push_back("p_int"); } - output_yaml.set("Field Names",fields_out); + params.set("Field Names",fields_out); if (vert_remap) { - output_yaml.set("vertical_remap_file",remap_filename); // TODO, make this work for general np=? + params.set("vertical_remap_file",remap_filename); // TODO, make this work for general np=? } if (horiz_remap) { - output_yaml.set("horiz_remap_file",remap_filename); // TODO, make this work for general np=? + params.set("horiz_remap_file",remap_filename); // TODO, make this work for general np=? } - return output_yaml; + return params; } /*==========================================================================================================*/ ekat::ParameterList set_input_params(const std::string& name, ekat::Comm& comm, const std::string& tstamp, const int p_ref) diff --git a/components/eamxx/src/share/io/tests/io_se_grid.cpp b/components/eamxx/src/share/io/tests/io_se_grid.cpp index 16872a7491f8..edf5f5dbbcd9 100644 --- a/components/eamxx/src/share/io/tests/io_se_grid.cpp +++ b/components/eamxx/src/share/io/tests/io_se_grid.cpp @@ -39,6 +39,7 @@ ekat::ParameterList get_in_params(const ekat::Comm& comm, TEST_CASE("se_grid_io") { + using strvec_t = std::vector; ekat::Comm io_comm(MPI_COMM_WORLD); int num_my_elems = 2; @@ -58,8 +59,15 @@ TEST_CASE("se_grid_io") auto fm0 = get_test_fm(grid,t0,true); ekat::ParameterList params; - ekat::parse_yaml_file("io_test_se_grid.yaml",params); + params.set("filename_prefix","io_se_grid"); + params.set("Averaging Type","Instant"); + params.set("Max Snapshots Per File",1); + params.set("Field Names",{"field_1","field_2","field_3","field_packed"}); params.set("Floating Point Precision","real"); + params.set("MPI Ranks in Filename",true); + auto& ctl_pl = params.sublist("output_control"); + ctl_pl.set("Frequency",1); + ctl_pl.set("frequency_units","nsteps"); OutputManager om; om.setup(io_comm,params,fm0,gm,t0,t0,false); @@ -164,7 +172,7 @@ ekat::ParameterList get_in_params(const ekat::Comm& comm, using vos_type = std::vector; ekat::ParameterList in_params("Input Parameters"); - std::string filename = "io_test_se_grid.INSTANT.nsteps_x1.np" + std::string filename = "io_se_grid.INSTANT.nsteps_x1.np" + std::to_string(comm.size()) + "." + t0.to_string() + ".nc"; diff --git a/components/eamxx/src/share/io/tests/io_test_se_grid.yaml b/components/eamxx/src/share/io/tests/io_test_se_grid.yaml deleted file mode 100644 index f4dd56e5cde6..000000000000 --- a/components/eamxx/src/share/io/tests/io_test_se_grid.yaml +++ /dev/null @@ -1,11 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: io_test_se_grid -Averaging Type: Instant -Max Snapshots Per File: 1 -Field Names: [field_1, field_2, field_3, field_packed] -output_control: - MPI Ranks in Filename: true - Frequency: 1 - frequency_units: nsteps -... diff --git a/components/eamxx/src/share/io/tests/io_utils.cpp b/components/eamxx/src/share/io/tests/io_utils.cpp index 6064a29567a0..77779307a720 100644 --- a/components/eamxx/src/share/io/tests/io_utils.cpp +++ b/components/eamxx/src/share/io/tests/io_utils.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -40,22 +41,21 @@ TEST_CASE ("io_control") { IOControl control; control.frequency = 2; - control.timestamp_of_last_write = t0; + control.last_write_ts = t0; SECTION ("none") { control.frequency_units = "none"; REQUIRE (not control.output_enabled()); - REQUIRE (not control.is_write_step(t0)); } SECTION ("never") { control.frequency_units = "never"; REQUIRE (not control.output_enabled()); - REQUIRE (not control.is_write_step(t0)); } SECTION ("nsteps") { control.frequency_units = "nsteps"; + control.compute_next_write_ts(); auto t1 = t0 + 1; auto t2 = t1 + 1; REQUIRE (control.output_enabled()); @@ -65,6 +65,7 @@ TEST_CASE ("io_control") { SECTION ("nsecs") { control.frequency_units = "nsecs"; + control.compute_next_write_ts(); auto t1 = t0 + 1; auto t2 = t1 + 1; REQUIRE (control.output_enabled()); @@ -74,6 +75,7 @@ TEST_CASE ("io_control") { SECTION ("nmins") { control.frequency_units = "nmins"; + control.compute_next_write_ts(); auto t1 = t0 + 60; auto t2 = t1 + 60; REQUIRE (control.output_enabled()); @@ -83,6 +85,7 @@ TEST_CASE ("io_control") { SECTION ("nhours") { control.frequency_units = "nhours"; + control.compute_next_write_ts(); auto t1 = t0 + 3600; auto t2 = t1 + 3600; REQUIRE (control.output_enabled()); @@ -92,6 +95,7 @@ TEST_CASE ("io_control") { SECTION ("ndays") { control.frequency_units = "ndays"; + control.compute_next_write_ts(); auto t1 = t0 + 86400; auto t2 = t1 + 86400; REQUIRE (control.output_enabled()); @@ -101,6 +105,7 @@ TEST_CASE ("io_control") { SECTION ("nmonths") { control.frequency_units = "nmonths"; + control.compute_next_write_ts(); util::TimeStamp t1({2023,10,7},{12,0,0}); util::TimeStamp t2({2023,11,7},{12,0,0}); util::TimeStamp t3({2023,11,7},{13,0,0}); @@ -112,6 +117,7 @@ TEST_CASE ("io_control") { SECTION ("nyears") { control.frequency_units = "nyears"; + control.compute_next_write_ts(); util::TimeStamp t1({2024,9,7},{12,0,0}); util::TimeStamp t2({2025,9,7},{12,0,0}); util::TimeStamp t3({2025,9,7},{13,0,0}); diff --git a/components/eamxx/src/share/io/tests/output_restart.cpp b/components/eamxx/src/share/io/tests/output_restart.cpp index 118b9a84836a..62676450efee 100644 --- a/components/eamxx/src/share/io/tests/output_restart.cpp +++ b/components/eamxx/src/share/io/tests/output_restart.cpp @@ -79,14 +79,13 @@ TEST_CASE("output_restart","io") output_params.set("Floating Point Precision","real"); output_params.set>("Field Names",{"field_1", "field_2", "field_3", "field_4","field_5"}); output_params.set("fill_value",FillValue); - output_params.sublist("output_control").set("MPI Ranks in Filename","true"); + output_params.set("MPI Ranks in Filename","true"); + output_params.set("flush_frequency",1); output_params.sublist("output_control").set("frequency_units","nsteps"); output_params.sublist("output_control").set("Frequency",10); - output_params.sublist("Checkpoint Control").set("MPI Ranks in Filename","true"); output_params.sublist("Checkpoint Control").set("Frequency",5); // This skips a test that only matters for AD runs output_params.sublist("Checkpoint Control").set("is_unit_testing","true"); - output_params.sublist("Restart").set("MPI Ranks in Filename","true"); // Creates and runs an OM from output_params and given inputs auto run = [&](std::shared_ptr fm, @@ -118,7 +117,7 @@ TEST_CASE("output_restart","io") } }; // Run test for different avg type choices - for (const std::string& avg_type : {"INSTANT","AVERAGE"}) { + for (const std::string avg_type : {"INSTANT","AVERAGE"}) { { // In normal runs, the OM for the model restart takes care of nuking rpointer.atm, // and re-creating a new one. Here, we don't have that, so we must nuke it manually diff --git a/components/eamxx/src/share/scream_config.hpp b/components/eamxx/src/share/scream_config.hpp index fea495c5ae3f..391f0afdfbde 100644 --- a/components/eamxx/src/share/scream_config.hpp +++ b/components/eamxx/src/share/scream_config.hpp @@ -22,6 +22,15 @@ std::string scream_config_string(); bool use_leap_year (); void set_use_leap_year (const bool use_leap); +// Allow downstream code to avoid macros +bool constexpr is_scream_standalone () { +#ifdef SCREAM_CIME_BUILD + return false; +#else + return true; +#endif +} + } // namespace scream #endif // SCREAM_CONFIG_HPP diff --git a/components/eamxx/src/share/tests/CMakeLists.txt b/components/eamxx/src/share/tests/CMakeLists.txt index 0b2363fad2f6..bfdc0bba2b7e 100644 --- a/components/eamxx/src/share/tests/CMakeLists.txt +++ b/components/eamxx/src/share/tests/CMakeLists.txt @@ -1,5 +1,5 @@ # NOTE: tests inside this if statement won't be built in a baselines-only build -if (NOT ${SCREAM_BASELINES_ONLY}) +if (NOT SCREAM_ONLY_GENERATE_BASELINES) include(ScreamUtils) # Test vertical interpolation @@ -63,11 +63,4 @@ if (NOT ${SCREAM_BASELINES_ONLY}) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/atm_process_tests_named_procs.yaml ${CMAKE_CURRENT_BINARY_DIR}/atm_process_tests_named_procs.yaml COPYONLY) CreateUnitTest(atm_proc "atm_process_tests.cpp") - - # Test horizontal remapping utility - CreateUnitTest(horizontal_remap "horizontal_remap_test.cpp" - LIBS scream_io - LABELS horiz_remap - MPI_RANKS 1 ${SCREAM_TEST_MAX_RANKS} - ) endif() diff --git a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp index e56b43aab7ef..13832c490441 100644 --- a/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp +++ b/components/eamxx/src/share/tests/coarsening_remapper_tests.cpp @@ -105,6 +105,8 @@ build_src_grid(const ekat::Comm& comm, const int ngdofs, Engine& engine) } constexpr int vec_dim = 2; +constexpr int tens_dim1 = 3; +constexpr int tens_dim2 = 4; Field create_field (const std::string& name, const LayoutType lt, const AbstractGrid& grid, const bool midpoints) { const auto u = ekat::units::Units::nondimensional(); @@ -116,6 +118,8 @@ Field create_field (const std::string& name, const LayoutType lt, const Abstract f = Field(FieldIdentifier(name,grid.get_2d_scalar_layout(),u,gn)); break; case LayoutType::Vector2D: f = Field(FieldIdentifier(name,grid.get_2d_vector_layout(CMP,vec_dim),u,gn)); break; + case LayoutType::Tensor2D: + f = Field(FieldIdentifier(name,grid.get_2d_tensor_layout({CMP,CMP},{tens_dim1,tens_dim2}),u,gn)); break; case LayoutType::Scalar3D: f = Field(FieldIdentifier(name,grid.get_3d_scalar_layout(midpoints),u,gn)); f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); @@ -124,6 +128,10 @@ Field create_field (const std::string& name, const LayoutType lt, const Abstract f = Field(FieldIdentifier(name,grid.get_3d_vector_layout(midpoints,CMP,vec_dim),u,gn)); f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); break; + case LayoutType::Tensor3D: + f = Field(FieldIdentifier(name,grid.get_3d_tensor_layout(midpoints,{CMP,CMP},{tens_dim1,tens_dim2}),u,gn)); + f.get_header().get_alloc_properties().request_allocation(SCREAM_PACK_SIZE); + break; default: EKAT_ERROR_MSG ("Invalid layout type for this unit test.\n"); } @@ -192,6 +200,13 @@ Field all_gather_field_impl (const Field& f, const ekat::Comm& comm) { data = data_vec.data(); } break; + case 4: + if (pid==comm.rank()) { + data = ekat::subview(f.get_view(),icol).data(); + } else { + data = data_vec.data(); + } + break; default: EKAT_ERROR_MSG ( "Unexpected rank in RefiningRemapperRMA unit test.\n" @@ -306,20 +321,26 @@ TEST_CASE("coarsening_remap") auto src_s2d = create_field("s2d", LayoutType::Scalar2D, *src_grid, false, engine); auto src_v2d = create_field("v2d", LayoutType::Vector2D, *src_grid, false, engine); + auto src_t2d = create_field("t2d", LayoutType::Tensor2D, *src_grid, false, engine); auto src_s3d_m = create_field("s3d_m",LayoutType::Scalar3D, *src_grid, true, engine); auto src_s3d_i = create_field("s3d_i",LayoutType::Scalar3D, *src_grid, false, engine); auto src_v3d_m = create_field("v3d_m",LayoutType::Vector3D, *src_grid, true, engine); auto src_v3d_i = create_field("v3d_i",LayoutType::Vector3D, *src_grid, false, engine); + auto src_t3d_m = create_field("t3d_m",LayoutType::Tensor3D, *src_grid, true, engine); + auto src_t3d_i = create_field("t3d_i",LayoutType::Tensor3D, *src_grid, false, engine); auto tgt_s2d = create_field("s2d", LayoutType::Scalar2D, *tgt_grid, false); auto tgt_v2d = create_field("v2d", LayoutType::Vector2D, *tgt_grid, false); + auto tgt_t2d = create_field("t2d", LayoutType::Tensor2D, *tgt_grid, false); auto tgt_s3d_m = create_field("s3d_m",LayoutType::Scalar3D, *tgt_grid, true ); auto tgt_s3d_i = create_field("s3d_i",LayoutType::Scalar3D, *tgt_grid, false); auto tgt_v3d_m = create_field("v3d_m",LayoutType::Vector3D, *tgt_grid, true ); auto tgt_v3d_i = create_field("v3d_i",LayoutType::Vector3D, *tgt_grid, false); + auto tgt_t3d_m = create_field("t3d_m",LayoutType::Tensor3D, *tgt_grid, true ); + auto tgt_t3d_i = create_field("t3d_i",LayoutType::Tensor3D, *tgt_grid, false); - std::vector src_f = {src_s2d,src_v2d,src_s3d_m,src_s3d_i,src_v3d_m,src_v3d_i}; - std::vector tgt_f = {tgt_s2d,tgt_v2d,tgt_s3d_m,tgt_s3d_i,tgt_v3d_m,tgt_v3d_i}; + std::vector src_f = {src_s2d,src_v2d,src_t2d,src_s3d_m,src_s3d_i,src_v3d_m,src_v3d_i,src_t3d_m,src_t3d_i}; + std::vector tgt_f = {tgt_s2d,tgt_v2d,tgt_t2d,tgt_s3d_m,tgt_s3d_i,tgt_v3d_m,tgt_v3d_i,tgt_t3d_m,tgt_t3d_i}; // -------------------------------------- // // Register fields in the remapper // @@ -357,7 +378,7 @@ TEST_CASE("coarsening_remap") const auto& l = gsrc.get_header().get_identifier().get_layout(); const auto ls = to_string(l); - std::string dots (25-ls.size(),'.'); + std::string dots (30-ls.size(),'.'); auto msg = " -> Checking field with layout " + to_string(l) + " " + dots; root_print (msg + "\n",comm); bool ok = true; @@ -396,6 +417,26 @@ TEST_CASE("coarsening_remap") } } } break; + case LayoutType::Tensor2D: + { + const auto v_src = gsrc.get_view(); + const auto v_tgt = gtgt.get_view(); + for (int idof=0; idof(); @@ -436,6 +477,29 @@ TEST_CASE("coarsening_remap") } } } break; + case LayoutType::Tensor3D: + { + const auto v_src = gsrc.get_view(); + const auto v_tgt = gtgt.get_view(); + auto f_nlevs = gsrc.get_header().get_identifier().get_layout().dims().back(); + for (int idof=0; idof; + using IVec = std::vector; + + FieldLayout fl1 ({COL},{1}); + FieldLayout fl2 ({COL,CMP},{1,1}); + FieldLayout fl3 ({COL,SWBND,LWBND},{1,1,1}); + FieldLayout fl4 ({COL,LEV},{1,1}); + FieldLayout fl5 ({COL,CMP,LEV},{1,1,1}); + FieldLayout fl6 ({COL,ISCCPTAU,ISCCPPRS,ILEV},{1,1,1,1}); + + REQUIRE (get_layout_type(fl1.tags())==LayoutType::Scalar2D); + REQUIRE (get_layout_type(fl2.tags())==LayoutType::Vector2D); + REQUIRE (get_layout_type(fl3.tags())==LayoutType::Tensor2D); + REQUIRE (get_layout_type(fl4.tags())==LayoutType::Scalar3D); + REQUIRE (get_layout_type(fl5.tags())==LayoutType::Vector3D); + REQUIRE (get_layout_type(fl6.tags())==LayoutType::Tensor3D); + + REQUIRE (not fl1.is_vector_layout()); + REQUIRE ( fl2.is_vector_layout()); + REQUIRE (not fl3.is_vector_layout()); + REQUIRE (not fl4.is_vector_layout()); + REQUIRE ( fl5.is_vector_layout()); + REQUIRE (not fl6.is_vector_layout()); + + REQUIRE (not fl1.is_tensor_layout()); + REQUIRE (not fl2.is_tensor_layout()); + REQUIRE ( fl3.is_tensor_layout()); + REQUIRE (not fl4.is_tensor_layout()); + REQUIRE (not fl5.is_tensor_layout()); + REQUIRE ( fl6.is_tensor_layout()); + + REQUIRE (fl2.get_vector_tag()==CMP); + REQUIRE (fl5.get_vector_tag()==CMP); + REQUIRE (fl2.get_vector_component_idx()==1); + REQUIRE (fl5.get_vector_component_idx()==1); + REQUIRE (fl2.get_vector_dim()==1); + REQUIRE (fl5.get_vector_dim()==1); + + REQUIRE (fl3.get_tensor_tags()==TVec{SWBND,LWBND}); + REQUIRE (fl6.get_tensor_tags()==TVec{ISCCPTAU,ISCCPPRS}); + REQUIRE (fl3.get_tensor_dims()==IVec{1,2}); + REQUIRE (fl6.get_tensor_dims()==IVec{1,2}); +} + TEST_CASE("field_identifier", "") { using namespace scream; using namespace ekat::units; diff --git a/components/eamxx/src/share/tests/horizontal_remap_test.cpp b/components/eamxx/src/share/tests/horizontal_remap_test.cpp deleted file mode 100644 index f9bdca47dae4..000000000000 --- a/components/eamxx/src/share/tests/horizontal_remap_test.cpp +++ /dev/null @@ -1,525 +0,0 @@ -#include - -#include "share/grid/remap/horizontal_remap_utility.hpp" - -#include "share/io/scorpio_input.hpp" -#include "share/io/scream_output_manager.hpp" - -#include "share/grid/mesh_free_grids_manager.hpp" -#include "share/grid/point_grid.hpp" - -#include "share/field/field_manager.hpp" - -#include "share/util/scream_setup_random_test.hpp" - -#include "ekat/util/ekat_units.hpp" -#include "ekat/util/ekat_test_utils.hpp" -#include "ekat/ekat_parse_yaml_file.hpp" -namespace { - -using namespace scream; - -using gid_type = AbstractGrid::gid_type; -using Pack = ekat::Pack; - -using KT = KokkosTypes; - -template -using view_1d = typename KT::template view_1d; - -template -using view_2d = typename KT::template view_2d; - -template -using view_3d = typename KT::template view_3d; - -template -using view_1d_host = typename KT::template view_1d::HostMirror; - -std::shared_ptr get_test_gm(const ekat::Comm& comm, const Int num_gcols, const Int num_levs); - -void run(std::mt19937_64& engine, const ekat::Comm& comm, const gid_type src_min_dof) -{ -/* Testing for the HorizontalMap structure and using it to apply a remap to data. - * This test has three main parts: - * 1. Construction of the remapping data, construction of the source data to be - * remapped, and the establishment of a baseline of remapped data. - * 2. The construction of a HorizontalMap where the remap data is gathered directly from - * the views derived in part 1. Apply the remap and compare against baseline. - * 3. After writing the remapping data and source data to file, we construct a - * HorizontalMap reampper with data gathered from the file. We then apply the remap - * and compare against the baseline. - * - * By breaking the test into these parts we are able to make sure that each test - * generate random data, i.e. we are not getting lucky with static numbers. We are - * also able to test the two main implementations of the horizontal remap structure; - * the generation of weights using a map constructed online, and the use of remapping - * weights constructed offline and save to file. - * - * INPUT ARGS - * - engine: This is just the random seed provided by the EAMxx testing setup - * - src_min_dof: This is the degree-of-freedom index for the first column. This - * simulates that some remapping algorithms may treat the first column - * with index=0 and others with index=1. Making this an input allows - * the test to check both cases. - */ - - using IPDF = std::uniform_int_distribution; - using RPDF = std::uniform_real_distribution; - constexpr Real tol = std::numeric_limits::epsilon()*10; - -//---------------------- Test configuration - int num_src_cols = 20; // Number of columns from source data - int num_tgt_cols = 10; // Number of columns on target grid - int num_levels = 2*SCREAM_PACK_SIZE+1; // Dummy variable needed for initialization of a grid. - - // Construct a target grid for remapped data - auto gm_tgt = get_test_gm(comm, num_tgt_cols, num_levels); - auto grid_tgt = gm_tgt->get_grid("Point Grid"); // Retrieve the actual grid from the grids manager. - auto num_loc_tgt_cols = grid_tgt->get_num_local_dofs(); // Number of columns (dofs) on this rank. - auto tgt_min_dof = grid_tgt->get_global_min_dof_gid(); // The starting index of the EAMxx grid. - auto dofs_gids = grid_tgt->get_dofs_gids().get_view(); // A view of the dofs on this rank with their global id. - auto dofs_gids_h = grid_tgt->get_dofs_gids().get_view(); - -//---------------------- -// Step 1: Generate a random remapping (source col, target col, weight) and random source data. -// Store these in local 1D views and create a remap file. -// Note, randomization is done entirely on ROOT so that we only have one version of the remap -// data. -// -// OUT - y_baseline, num_src_to_tgt_cols, n_s, map_src_cols, map_tgt_cols, map_wgts - - // When we create random source data we again only do this on root so that we - // only have one set of data. - std::vector vec_src_data(num_src_cols); - if (comm.am_i_root()) { - RPDF pdf_src_data(-1,1); // Randomizer for generating source data to be mapped from. - for (int ii=0; ii source_remap_data("",num_src_cols); - auto source_remap_data_h = Kokkos::create_mirror_view(source_remap_data); - for (int ii=0; iitgt pairs. - std::vector vec_src_col, vec_tgt_col; - std::vector vec_wgt; - std::map y_baseline; - std::map num_src_to_tgt_cols; // A record of how many source columns map to a specific target. Will be used later for testing segments. - IPDF pdf_map_src_num(2,num_src_cols); // Randomizer for number of source columns mapping to a target column - IPDF pdf_map_src(0,num_src_cols-1); // Randomizer for which source columns map to a target column - RPDF pdf_map_wgt(0,1); // Randomizer for mapping weights - // Cycle through all tgt_cols and set up a remap - for (int tcol=0; tcol temp_wgt; - std::vector temp_src; - Real wgt_sum = 0.0; - Real y_sum = 0.0; - for (int ii=0; ii map_src_cols_h(vec_src_col.data(),vec_wgt.size()); - view_1d_host map_tgt_cols_h(vec_tgt_col.data(),vec_wgt.size()); - view_1d_host map_wgts_h(vec_wgt.data(),vec_wgt.size()); - auto map_src_cols = Kokkos::create_mirror_view(map_src_cols_h); - auto map_tgt_cols = Kokkos::create_mirror_view(map_tgt_cols_h); - auto map_wgts = Kokkos::create_mirror_view(map_wgts_h ); - Kokkos::deep_copy(map_src_cols, map_src_cols_h); - Kokkos::deep_copy(map_tgt_cols, map_tgt_cols_h); - Kokkos::deep_copy(map_wgts , map_wgts_h ); - -//---------------------- - // Step 2: Use HorizontalMap with remap values and source data, and compare against baseline. - std::map y_from_views; - HorizontalMap remap_from_views(comm,"From Views", dofs_gids, tgt_min_dof); - for (int iseg=0; iseg source_dofs("",seg_len); - view_1d weights("",seg_len); - auto source_dofs_h = Kokkos::create_mirror_view(source_dofs); - auto weights_h = Kokkos::create_mirror_view(weights); - int idx = 0; - for (int ii=0; ii x_data_from_views("",num_unique_dofs_from_views); - Kokkos::parallel_for("", num_unique_dofs_from_views, KOKKOS_LAMBDA (const int& ii) { - const int idx = unique_dofs_from_views(ii); - x_data_from_views(ii) = source_remap_data(idx); - }); - // Apply the remap - view_1d y_data_from_views("",num_loc_tgt_cols); - remap_from_views.apply_remap(x_data_from_views,y_data_from_views); - // The remapped values should match the y_baseline - auto y_data_from_views_h = Kokkos::create_mirror_view(y_data_from_views); - Kokkos::deep_copy(y_data_from_views_h,y_data_from_views); - for (int ii=0; ii vec_of_remap_dims = {"n_s"}; - std::vector vec_of_data_dims = {"ncol"}; - std::string int_type = "int"; - scorpio::register_variable(filename,"col","col","unitless",vec_of_remap_dims,"int","int",remap_decomp_tag_i); - scorpio::register_variable(filename,"row","row","unitless",vec_of_remap_dims,"int","int",remap_decomp_tag_i); - scorpio::register_variable(filename,"S","S", "unitless",vec_of_remap_dims,"real","real",remap_decomp_tag_r); - scorpio::register_variable(filename,"src_data","src_data", "m",vec_of_data_dims,"real","real",data_decomp_tag_r); - // - DOFs - std::vector var_dof(n_s_local); - std::iota(var_dof.begin(),var_dof.end(),n_s_offset); - scorpio::set_dof(filename,"col",var_dof.size(),var_dof.data()); - scorpio::set_dof(filename,"row",var_dof.size(),var_dof.data()); - scorpio::set_dof(filename,"S",var_dof.size(),var_dof.data()); - var_dof.resize(unique_dofs_from_views.size()); - auto unique_dofs_from_views_h = Kokkos::create_mirror_view(unique_dofs_from_views); - Kokkos::deep_copy(unique_dofs_from_views_h,unique_dofs_from_views); - for (size_t ii=0; ii x_data_from_file("",var_dof.size()); - auto x_data_from_file_h = Kokkos::create_mirror_view(x_data_from_file); - scorpio::grid_read_data_array(filename,"src_data",0,x_data_from_file.data(),x_data_from_file.size()); - scorpio::eam_pio_closefile(filename); - Kokkos::deep_copy(x_data_from_file_h,x_data_from_file); - // Apply remap using the data - view_1d y_data_from_file("",num_loc_tgt_cols); - remap_from_file.apply_remap(x_data_from_file,y_data_from_file); - // The remapped values should match the y_baseline - auto y_data_from_file_h = Kokkos::create_mirror_view(y_data_from_file); - Kokkos::deep_copy(y_data_from_file_h,y_data_from_file); - for (int ii=0; ii x_2d_data("",unique_dofs_from_file.size(),num_levels); - view_2d y_2d_data("",num_loc_tgt_cols,num_levels); - view_3d x_3d_data("",unique_dofs_from_file.size(),num_bands,num_levels); - view_3d y_3d_data("",num_loc_tgt_cols,num_bands,num_levels); - auto x_2d_data_h = Kokkos::create_mirror_view(x_2d_data); - auto x_3d_data_h = Kokkos::create_mirror_view(x_3d_data); - for (size_t ii=0;ii Testing horizontal remapping for minimum source dof = 0..."); - } - run(engine, comm, 0); - if (comm.am_i_root()) { - printf("ok!\n"); - } - - if (comm.am_i_root()) { - printf(" -> Testing horizontal remapping for minimum source dof = 1..."); - } - run(engine, comm, 1); - if (comm.am_i_root()) { - printf("ok!\n"); - } - -} // TEST_CASE -//=============================================================================== - -TEST_CASE("horizontal_remap_units", "") { - - using namespace scream; - using IPDF = std::uniform_int_distribution; - using RPDF = std::uniform_real_distribution; - auto engine = scream::setup_random_test(); - ekat::Comm comm (MPI_COMM_WORLD); - - // Test the remap_segment structure - { - IPDF pdf_seg_len(2,100); - RPDF pdf_seg_wgt(0,1); - Int len = pdf_seg_len(engine); - HorizontalMapSegment test_segment(0,len); - auto seg_weights = test_segment.get_weights(); - ekat::genRandArray(seg_weights,engine,pdf_seg_wgt); - Real wgt_normalize = 0; - auto weights_h = Kokkos::create_mirror_view(seg_weights); - Kokkos::deep_copy(weights_h,seg_weights); - for (int ii=0; ii dof_gids("",num_dofs); - Kokkos::parallel_for("", num_dofs, KOKKOS_LAMBDA (const int& ii) { - dof_gids(ii) = ii; - }); - test_map.set_dof_gids(dof_gids,0); - // Create a remap segment - Int len = pdf_seg_len(engine); - HorizontalMapSegment test_seg(1,len); - auto test_weights = test_seg.get_weights(); - ekat::genRandArray(test_weights,engine,pdf_seg_wgt); - Real wgt_normalize = 0; - auto weights_h = Kokkos::create_mirror_view(test_weights); - Kokkos::deep_copy(weights_h,test_weights); - for (int ii=0; ii dof_gids("",num_dofs); - Kokkos::parallel_for("", num_dofs, KOKKOS_LAMBDA (const int& ii) { - dof_gids(ii) = ii; - }); - test_map.set_dof_gids(dof_gids,0); - // All of the segments combined should overlap by 2 on either side. The set of unique - // columns for all segments should be 0,1,...,seg_start+seg_len-1. So we check that. - test_map.set_unique_source_dofs(); - const auto& unique_dofs = test_map.get_unique_source_dofs(); - REQUIRE(unique_dofs.extent_int(0) == dof_end); - Kokkos::parallel_for("", dof_end, KOKKOS_LAMBDA (const int& ii) { - EKAT_KERNEL_REQUIRE(unique_dofs(ii) == ii); - }); - } - -} // TEST_CASE horizontal remap -/*===================================================================================================*/ -std::shared_ptr get_test_gm(const ekat::Comm& comm, const Int num_gcols, const Int num_levs) -{ -/* Simple routine to construct and return a grids manager given number of columns and levels */ - auto gm = create_mesh_free_grids_manager(comm,0,0,num_levs,num_gcols); - gm->build_grids(); - return gm; -} // end get_test_gm -/*==========================================================================================================*/ - -} // end namespace diff --git a/components/eamxx/src/share/tests/utils_tests.cpp b/components/eamxx/src/share/tests/utils_tests.cpp index e8a9c4f5c4ce..b3b18bf206b5 100644 --- a/components/eamxx/src/share/tests/utils_tests.cpp +++ b/components/eamxx/src/share/tests/utils_tests.cpp @@ -179,6 +179,16 @@ TEST_CASE ("time_stamp") { auto ts6 = ts1 + spd*1000; REQUIRE ( (ts6-ts1)==spd*1000 ); } + SECTION ("large_updates") { + TS base ({1850,1,1},{0,0,0},1); + for (int i=1; i<=500; ++i) { + TS curr ({1850+i,1,1},{0,0,0},0); + auto diff = curr.seconds_from(base); + TS time = base; + time += diff; + REQUIRE (time==curr); + } + } } TEST_CASE ("array_utils") { diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp index a3d132bf4f82..3515bcdaac91 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.cpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.cpp @@ -161,6 +161,7 @@ void TimeInterpolation::initialize_data_from_files() input_params.set("Field Names",m_field_names); input_params.set("Filename",triplet_curr.filename); m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); + m_file_data_atm_input.set_logger(m_logger); // Assign the mask value gathered from the FillValue found in the source file. // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? float var_fill_value; @@ -319,6 +320,7 @@ void TimeInterpolation::read_data() input_params.set("Field Names",m_field_names); input_params.set("Filename",triplet_curr.filename); m_file_data_atm_input = AtmosphereInput(input_params,m_fm_time1); + m_file_data_atm_input.set_logger(m_logger); // Also determine the FillValue, if used // TODO: Should we make it possible to check if FillValue is in the metadata and only assign mask_value if it is? float var_fill_value; @@ -328,6 +330,11 @@ void TimeInterpolation::read_data() field.get_header().set_extra_data("mask_value",var_fill_value); } } + + if (m_logger) { + m_logger->info(m_header); + m_logger->info("[EAMxx:time_interpolation] Reading data at time " + triplet_curr.timestamp.to_string()); + } m_file_data_atm_input.read_variables(triplet_curr.time_idx); m_time1 = triplet_curr.timestamp; } diff --git a/components/eamxx/src/share/util/eamxx_time_interpolation.hpp b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp index 89d4c917d276..870dad330135 100644 --- a/components/eamxx/src/share/util/eamxx_time_interpolation.hpp +++ b/components/eamxx/src/share/util/eamxx_time_interpolation.hpp @@ -45,6 +45,13 @@ class TimeInterpolation { // Informational void print(); + // Option to add a logger + void set_logger(const std::shared_ptr& logger, + const std::string& header) { + m_logger = logger; + m_header = header; + } + protected: // Internal structure to store data source triplets (when using data from file) @@ -85,7 +92,8 @@ class TimeInterpolation { AtmosphereInput m_file_data_atm_input; bool m_is_data_from_file=false; - + std::shared_ptr m_logger; + std::string m_header; }; // class TimeInterpolation } // namespace util diff --git a/components/eamxx/src/share/util/scream_time_stamp.cpp b/components/eamxx/src/share/util/scream_time_stamp.cpp index fd0ef3785f10..d2ef8243e258 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.cpp +++ b/components/eamxx/src/share/util/scream_time_stamp.cpp @@ -135,12 +135,6 @@ double TimeStamp::frac_of_year_in_days () const { return doy; } -void TimeStamp::set_num_steps (const int num_steps) { - EKAT_REQUIRE_MSG (m_num_steps==0, - "Error! Cannot reset m_num_steps once the count started.\n"); - m_num_steps = num_steps; -} - TimeStamp& TimeStamp::operator+=(const double seconds) { // Sanity checks // Note: (x-int(x)) only works for x small enough that can be stored in an int, @@ -161,30 +155,32 @@ TimeStamp& TimeStamp::operator+=(const double seconds) { auto& yy = m_date[0]; ++m_num_steps; - sec += seconds; + + constexpr auto spd = constants::seconds_per_day; + auto add_days = std::floor(seconds / spd); + auto add_secs = seconds - add_days*spd; + + sec += add_secs; + dd += add_days; // Carry over int carry; carry = sec / 60; - if (carry==0) { - return *this; - } - - sec = sec % 60; - min += carry; - carry = min / 60; - if (carry==0) { - return *this; - } - min = min % 60; - hour += carry; - carry = hour / 24; - - if (carry==0) { - return *this; + if (carry!=0) { + sec = sec % 60; + min += carry; + carry = min / 60; + if (carry!=0) { + min = min % 60; + hour += carry; + carry = hour / 24; + + if (carry!=0) { + hour = hour % 24; + dd += carry; + } + } } - hour = hour % 24; - dd += carry; while (dd>days_in_month(yy,mm)) { dd -= days_in_month(yy,mm); diff --git a/components/eamxx/src/share/util/scream_time_stamp.hpp b/components/eamxx/src/share/util/scream_time_stamp.hpp index fdfec5bb0ebf..259686eacf65 100644 --- a/components/eamxx/src/share/util/scream_time_stamp.hpp +++ b/components/eamxx/src/share/util/scream_time_stamp.hpp @@ -4,6 +4,7 @@ #include #include #include +#include namespace scream { namespace util { @@ -46,9 +47,8 @@ class TimeStamp { // === Update method(s) === // - // Set the counter for the number of steps. Must be called while m_num_steps==0, - // for safety reasons (do not alter num steps while the count started). - void set_num_steps (const int num_steps); + // Set the counter for the number of steps. + void set_num_steps (const int num_steps) { m_num_steps = num_steps; } TimeStamp& operator= (const TimeStamp&) = default; @@ -63,7 +63,7 @@ class TimeStamp { std::vector m_date; // [year, month, day] std::vector m_time; // [hour, min, sec] - int m_num_steps = 0; // Number of steps since simulation started + int m_num_steps = std::numeric_limits::lowest(); // Number of steps since simulation started }; // Overload operators for TimeStamp diff --git a/components/eamxx/src/share/util/scream_universal_constants.hpp b/components/eamxx/src/share/util/scream_universal_constants.hpp index c133cd93cd39..9937bbf595fe 100644 --- a/components/eamxx/src/share/util/scream_universal_constants.hpp +++ b/components/eamxx/src/share/util/scream_universal_constants.hpp @@ -2,6 +2,7 @@ #define SCREAM_UNIVERSAL_CONSTANTS_HPP #include +#include namespace scream { @@ -14,9 +15,9 @@ constexpr int days_per_nonleap_year = 365; // TODO: When we switch to supporting C++17 we can use a simple `inline constexpr` rather than a struct template struct DefaultFillValue { - static const bool is_float = std::is_floating_point::value; - static const bool is_int = std::is_integral::value; - T value = is_int ? std::numeric_limits::max() / 2 : + static constexpr bool is_float = std::is_floating_point::value; + static constexpr bool is_int = std::is_integral::value; + static constexpr T value = is_int ? std::numeric_limits::max() / 2 : is_float ? std::numeric_limits::max() / 1e5 : std::numeric_limits::max(); }; diff --git a/components/eamxx/src/share/util/scream_utils.cpp b/components/eamxx/src/share/util/scream_utils.cpp index 91a412dfd8a4..2083f4d62b64 100644 --- a/components/eamxx/src/share/util/scream_utils.cpp +++ b/components/eamxx/src/share/util/scream_utils.cpp @@ -1,4 +1,5 @@ #include "share/util/scream_utils.hpp" +#include #if defined(SCREAM_ENABLE_STATM) #include @@ -43,4 +44,33 @@ long long get_mem_usage (const MemoryUnits u) { return mem; } +std::vector filename_glob(const std::vector& patterns) { + std::vector all_files; + for (const auto& pattern : patterns) { + auto files = globloc(pattern); + all_files.insert(all_files.end(), files.begin(), files.end()); + } + return all_files; +} + +std::vector globloc(const std::string& pattern) { + // glob struct resides on the stack + glob_t glob_result; + memset(&glob_result, 0, sizeof(glob_result)); + + int return_value = ::glob(pattern.c_str(), GLOB_TILDE, NULL, &glob_result); + if (return_value != 0) { + globfree(&glob_result); + EKAT_REQUIRE_MSG(return_value == 0, "glob() failed with return value " + std::to_string(return_value)); + } + + std::vector filenames; + for (size_t i = 0; i < glob_result.gl_pathc; ++i) { + filenames.push_back(std::string(glob_result.gl_pathv[i])); + } + + globfree(&glob_result); + return filenames; +} + } // namespace scream diff --git a/components/eamxx/src/share/util/scream_utils.hpp b/components/eamxx/src/share/util/scream_utils.hpp index 4dddebf75aa5..6e4f08ffd22e 100644 --- a/components/eamxx/src/share/util/scream_utils.hpp +++ b/components/eamxx/src/share/util/scream_utils.hpp @@ -347,6 +347,12 @@ check_mpi_call (int err, const std::string& context) { " - context: " + context + "\n"); } +// Find the full filename list from patterns +std::vector filename_glob(const std::vector& patterns); + +// Use globloc for each filename pattern +std::vector globloc(const std::string& pattern); + } // namespace scream #endif // SCREAM_UTILS_HPP diff --git a/components/eamxx/tests/CMakeLists.txt b/components/eamxx/tests/CMakeLists.txt index ab168e67e79f..7a7a9e821493 100644 --- a/components/eamxx/tests/CMakeLists.txt +++ b/components/eamxx/tests/CMakeLists.txt @@ -1,5 +1,55 @@ include (ScreamUtils) +# This function compares an output file created by an AD-driven test with its +# corresponding one in the baselines dir. The inputs (all REQUIRED) are +# +# - TEST_BASE_NAME: the base name of the test that generated the file; will be used also +# as the prefix for the baseline_cmp test name +# - GEN_TEST_NRANKS: the number of ranks used in the test that generated the file +# - OUT_FILE: the name of the output nc file +# - FIXTURES_BASE_NAME: base name of the FIXTURES_SETUP property of the test that generated the file. +# The string _np${GEN_TEST_NRANKS}_omp1 will be attached to this + +function (CreateBaselineTest TEST_BASE_NAME GEN_TEST_NRANKS OUT_FILE FIXTURES_BASE_NAME) + # Get names of src and tgt files + set (SRC_FILE ${CMAKE_CURRENT_BINARY_DIR}/${OUT_FILE}) + set (TGT_FILE ${SCREAM_BASELINES_DIR}/data/${OUT_FILE}) + + # Add comparison test using the CprncTest.cmake scritp shipped by EAMxx + add_test ( + NAME ${TEST_BASE_NAME}_baseline_cmp + COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + set_tests_properties( + ${TEST_BASE_NAME}_baseline_cmp + PROPERTIES + LABELS baseline_cmp + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${TEST_RANK_END}_omp1) + + # Add the test that generated the baseline to the baseline_gen label, so scripts/test-all-scream + # can run it when it has to generate baselines + if (TEST ${TEST_BASE_NAME}_np${GEN_TEST_NRANKS}) + set (GEN_TEST_FULL_NAME ${TEST_BASE_NAME}_np${GEN_TEST_NRANKS}) + elseif(TEST ${TEST_BASE_NAME}) + set (GEN_TEST_FULL_NAME ${TEST_BASE_NAME}) + else() + string (CONCAT msg + "Could not find the test that generated the output file\n" + " TEST_BASE_NAME: ${TEST_BASE_NAME}\n" + " OUT_FILE : ${OUT_FILE}\n") + message ("${msg}") + message (FATAL_ERROR "Aborting...") + endif() + set_tests_properties (${GEN_TEST_FULL_NAME} PROPERTIES LABELS baseline_gen) + + # test-all-scream will read this file to get the list of baseline nc files to + # copy into the baseline dir + file (APPEND ${SCREAM_TEST_OUTPUT_DIR}/baseline_list + "${SRC_FILE}\n" + ) + +endfunction() + # Some tests for checking that certain testing infrastructures work add_subdirectory(generic) @@ -10,13 +60,14 @@ if (NOT DEFINED ENV{SCREAM_FAKE_ONLY}) set(TEST_RANK_END ${SCREAM_TEST_MAX_RANKS}) # Initial condition files used in the tests - set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") - set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") - set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") + set(EAMxx_tests_IC_FILE_72lev "screami_unit_tests_ne2np4L72_20220822.nc") + set(EAMxx_tests_IC_FILE_128lev "screami_unit_tests_ne2np4L128_20220822.nc") + set(EAMxx_tests_TOPO_FILE "USGS-gtopo30_ne2np4pg2_x6t_20230331.nc") + set(EAMxx_tests_IC_FILE_MAM4xx_72lev "scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc") # Testing individual atm processes - add_subdirectory(uncoupled) + add_subdirectory(single-process) # Testing multiple atm processes coupled together - add_subdirectory(coupled) + add_subdirectory(multi-process) endif() diff --git a/components/eamxx/tests/coupled/CMakeLists.txt b/components/eamxx/tests/coupled/CMakeLists.txt deleted file mode 100644 index b71893963fdb..000000000000 --- a/components/eamxx/tests/coupled/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT SCREAM_BASELINES_ONLY) - add_subdirectory(physics_only) - if (NOT "${SCREAM_DYNAMICS_DYCORE}" STREQUAL "NONE") - add_subdirectory(dynamics_physics) - endif() -endif() diff --git a/components/eamxx/tests/generic/fail_check/CMakeLists.txt b/components/eamxx/tests/generic/fail_check/CMakeLists.txt index 7c23f4fd52d5..8bbd96ccb67b 100644 --- a/components/eamxx/tests/generic/fail_check/CMakeLists.txt +++ b/components/eamxx/tests/generic/fail_check/CMakeLists.txt @@ -8,29 +8,25 @@ include(ScreamUtils) # - a REQUIRE clause that fails makes the test fail # - Our Create unit test logic does work for catching failures CreateUnitTest (fail "fail.cpp" - PROPERTIES WILL_FAIL TRUE - LABELS "fail") + WILL_FAIL LABELS "fail") if (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK) # Ensure that Kokkos OOB are caught CreateUnitTest (kokkos_fail "kokkos_fail.cpp" - PROPERTIES WILL_FAIL TRUE - LABELS "fail") + WILL_FAIL LABELS "fail") endif() if (EKAT_ENABLE_VALGRIND) # Ensure that valgrind errors are caught EkatCreateUnitTest (valg_fail "valg_fail.cpp" - PROPERTIES WILL_FAIL TRUE - LABELS "fail") + WILL_FAIL LABELS "fail") endif() # Ensure that FPE *do* throw when we expect them to CreateUnitTestExec (scream_fpe_check "fpe_check.cpp") if (SCREAM_FPE) CreateUnitTestFromExec (scream_fpe_check scream_fpe_check - PROPERTIES WILL_FAIL TRUE - LABELS "check") + WILL_FAIL LABELS "check") else() CreateUnitTestFromExec (scream_fpe_check scream_fpe_check LABELS "check") diff --git a/components/eamxx/tests/meta-tests/CMakeLists.txt b/components/eamxx/tests/meta-tests/CMakeLists.txt index d5cf8f1b41e2..b583d5ef825e 100644 --- a/components/eamxx/tests/meta-tests/CMakeLists.txt +++ b/components/eamxx/tests/meta-tests/CMakeLists.txt @@ -11,24 +11,30 @@ include(ScreamUtils) # the test executable). # Test to ensure that a build failure is detected by our testing scripts -EkatCreateUnitTest(build_fail build_fail.cpp +EkatCreateUnitTest(build_fail build_fail.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION LABELS "fail") # Test to ensure that a test failure is detected by our testing scripts -EkatCreateUnitTest(test_fail test_fail.cpp +EkatCreateUnitTest(test_fail test_fail.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION LABELS "fail") # Tests to ensure the testing infrastructure is correctly spreading # concurrent tests across the available resources if (SCREAM_TEST_MAX_TOTAL_THREADS GREATER_EQUAL 16) - EkatCreateUnitTestExec(resource_spread resource_spread.cpp) + EkatCreateUnitTestExec(resource_spread resource_spread.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) - CreateUnitTestFromExec(resource_spread_thread resource_spread + # When scripts-tests builds this folder, they are not building Ekat (or even Kokkos). + # So we must add openmp to the compiler/linker flags manually + # NOTE: scripts-tests is already checking that this is an "OpenMP machine", so this is safe + find_package(OpenMP REQUIRED COMPONENTS CXX) + target_link_libraries(resource_spread OpenMP::OpenMP_CXX MPI::MPI_C) + + EkatCreateUnitTestFromExec(resource_spread_thread resource_spread PRINT_OMP_AFFINITY THREADS 1 4 1 MPI_RANKS 4) - CreateUnitTestFromExec(resource_spread_rank resource_spread + EkatCreateUnitTestFromExec(resource_spread_rank resource_spread PRINT_OMP_AFFINITY THREADS 4 MPI_RANKS 1 4 1) diff --git a/components/eamxx/tests/meta-tests/build_fail.cpp b/components/eamxx/tests/meta-tests/build_fail.cpp index 23f46a51e802..8ed7844235a6 100644 --- a/components/eamxx/tests/meta-tests/build_fail.cpp +++ b/components/eamxx/tests/meta-tests/build_fail.cpp @@ -1,16 +1,10 @@ -#include - #include -namespace scream { - #ifdef SCREAM_FORCE_BUILD_FAIL #error "Forcing failure to test test-all-scream" #endif -TEST_CASE("pass", "[fake_infra_test]") +int main(int,char**) { - REQUIRE(true); + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/resource_spread.cpp b/components/eamxx/tests/meta-tests/resource_spread.cpp index c3905cf25701..9b59297b84dd 100644 --- a/components/eamxx/tests/meta-tests/resource_spread.cpp +++ b/components/eamxx/tests/meta-tests/resource_spread.cpp @@ -1,19 +1,17 @@ -#include "ekat/kokkos/ekat_kokkos_types.hpp" - -#include - -#include #include #include -namespace scream { +#include -TEST_CASE("rank_and_thread_spread", "[fake_infra_test]") +int main(int argc, char** argv) { + MPI_Init(&argc,&argv); + // Nothing needs to be done here except sleeping to give a chance // for tests to run concurrently. const auto seconds_to_sleep = 5; std::this_thread::sleep_for(std::chrono::seconds(seconds_to_sleep)); -} -} // empty namespace + MPI_Finalize(); + return 0; +} diff --git a/components/eamxx/tests/meta-tests/test_fail.cpp b/components/eamxx/tests/meta-tests/test_fail.cpp index 7acfe34adc40..4e8acdb584a8 100644 --- a/components/eamxx/tests/meta-tests/test_fail.cpp +++ b/components/eamxx/tests/meta-tests/test_fail.cpp @@ -1,16 +1,10 @@ -#include - #include -namespace scream { - -TEST_CASE("pass", "[fake_infra_test]") +int main(int, char**) { #ifdef SCREAM_FORCE_RUN_FAIL - REQUIRE(false); + return 1; #else - REQUIRE(true); + return 0; #endif } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt b/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt index 3c015db4f93e..18f94c286b10 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt +++ b/components/eamxx/tests/meta-tests/test_level_check/CMakeLists.txt @@ -1,8 +1,8 @@ include(ScreamUtils) -EkatCreateUnitTestExec(at_unit at_unit.cpp) -EkatCreateUnitTestExec(nightly_unit nightly_unit.cpp) -EkatCreateUnitTestExec(experimental_unit experimental_unit.cpp) +EkatCreateUnitTestExec(at_unit at_unit.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) +EkatCreateUnitTestExec(nightly_unit nightly_unit.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) +EkatCreateUnitTestExec(experimental_unit experimental_unit.cpp EXCLUDE_MAIN_CPP EXCLUDE_TEST_SESSION) CreateUnitTestFromExec(at_unit at_unit) CreateUnitTestFromExec(nightly_unit nightly_unit MINIMUM_TEST_LEVEL ${SCREAM_TEST_LEVEL_NIGHTLY}) diff --git a/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp b/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp index 5c2ef4bbbec0..280c9d343650 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp +++ b/components/eamxx/tests/meta-tests/test_level_check/at_unit.cpp @@ -1,12 +1,7 @@ -#include - #include -namespace scream { - -TEST_CASE("at_unit") +int main(int,char**) { std::cout << "AT unit test running" << std::endl; + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp b/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp index 7598eb628ae0..012060d405b5 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp +++ b/components/eamxx/tests/meta-tests/test_level_check/experimental_unit.cpp @@ -1,12 +1,7 @@ -#include - #include -namespace scream { - -TEST_CASE("experimental_unit") +int main(int,char**) { std::cout << "EXPERIMENTAL unit test running" << std::endl; + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp b/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp index e04ffccff98c..22c9fbcb7ce3 100644 --- a/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp +++ b/components/eamxx/tests/meta-tests/test_level_check/nightly_unit.cpp @@ -1,12 +1,7 @@ -#include - #include -namespace scream { - -TEST_CASE("nightly_unit") +int main(int,char**) { std::cout << "NIGHTLY unit test running" << std::endl; + return 0; } - -} // empty namespace diff --git a/components/eamxx/tests/multi-process/CMakeLists.txt b/components/eamxx/tests/multi-process/CMakeLists.txt new file mode 100644 index 000000000000..827a6167e13c --- /dev/null +++ b/components/eamxx/tests/multi-process/CMakeLists.txt @@ -0,0 +1,4 @@ +add_subdirectory(physics_only) +if (NOT "${SCREAM_DYNAMICS_DYCORE}" STREQUAL "NONE") + add_subdirectory(dynamics_physics) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt similarity index 57% rename from components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt index 5127f42969a2..5b263cbd66e5 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/CMakeLists.txt @@ -2,13 +2,17 @@ if (SCREAM_DOUBLE_PRECISION) if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") add_subdirectory(homme_shoc_cld_p3_rrtmgp) + add_subdirectory(homme_shoc_cld_p3_rrtmgp_pg2) add_subdirectory(model_restart) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_128levels) - add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_pg2) add_subdirectory(homme_shoc_cld_spa_p3_rrtmgp_dp) if (SCREAM_ENABLE_MAM) - add_subdirectory(homme_mam4xx_pg2) + # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the + # corresponding test here needs to be reworked with valid aerosol + # initial conditions. + #add_subdirectory(homme_mam4xx_pg2) + add_subdirectory(mam/homme_shoc_cld_p3_mam_optics_rrtmgp) endif() endif() endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt similarity index 84% rename from components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt index bb24e93b28d5..9463333ce39c 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/CMakeLists.txt @@ -1,5 +1,8 @@ include (ScreamUtils) +# This test needs to be reworked for microphysics -- currently it's still using +# input for nucleation. + set (TEST_BASE_NAME homme_mam4xx_pg2) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) @@ -82,3 +85,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics physics mam4xx META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/input.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/input.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/output.yaml similarity index 96% rename from components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/output.yaml index 23beda243aea..c2e1a6bf17de 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_mam4xx_pg2/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_mam4xx_pg2/output.yaml @@ -40,5 +40,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt similarity index 87% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt index d4618934c7c4..22acf3d88a4f 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/CMakeLists.txt @@ -79,3 +79,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics physics shoc cld p3 rrtmgp META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml similarity index 95% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml index 5eef64e73f5f..316b43ffdf73 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/input.yaml @@ -15,15 +15,10 @@ time_stepping: initial_conditions: Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} - get_topo_from_file: true surf_evap: 0.0 surf_sens_flux: 0.0 precip_liq_surf_mass: 0.0 precip_ice_surf_mass: 0.0 - aero_g_sw: 0.0 - aero_ssa_sw: 0.0 - aero_tau_sw: 0.0 - aero_tau_lw: 0.0 atmosphere_processes: atm_procs_list: [homme,physics] diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml index 824d648f2b2e..4b7cc02e6f8f 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp/output.yaml @@ -82,5 +82,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt similarity index 82% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt index 46d0513ae3f4..c3fa3c7bc2a9 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/CMakeLists.txt @@ -1,6 +1,6 @@ include (ScreamUtils) -set (TEST_BASE_NAME homme_shoc_cld_spa_p3_rrtmgp_pg2) +set (TEST_BASE_NAME homme_shoc_cld_p3_rrtmgp_pg2) set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) # Get or create the dynamics lib @@ -10,7 +10,7 @@ CreateDynamicsLib("theta-l_kokkos" 4 72 10) # Create the test CreateADUnitTest(${TEST_BASE_NAME} LABELS dynamics tms shoc cld p3 rrtmgp physics pg2 - LIBS cld_fraction nudging tms shoc spa p3 scream_rrtmgp ${dynLibName} diagnostics + LIBS cld_fraction nudging tms shoc p3 scream_rrtmgp ${dynLibName} diagnostics MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) @@ -67,8 +67,6 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc scream/init/${EAMxx_tests_IC_FILE_72lev} cam/topo/${EAMxx_tests_TOPO_FILE} ) @@ -83,6 +81,13 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS dynamics physics tms shoc cld p3 rrtmgp spa pg2 + LABELS dynamics physics tms shoc cld p3 rrtmgp pg2 META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml similarity index 86% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml index b778dbc2bbd8..8e463b8be24c 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/input.yaml @@ -15,6 +15,9 @@ initial_conditions: surf_sens_flux: 0.0 precip_liq_surf_mass: 0.0 precip_ice_surf_mass: 0.0 + surf_lw_flux_up: 400.0 + eddy_diff_mom: 0.02 + sgs_buoy_flux: -0.001 tke: 0.0 qm: 0.0 bm: 0.0 @@ -30,18 +33,16 @@ atmosphere_processes: homme: Moisture: moist physics: - atm_procs_list: [mac_aero_mic,rrtmgp] + atm_procs_list: [mac_mic,rrtmgp] schedule_type: Sequential Type: Group - mac_aero_mic: - atm_procs_list: [tms,shoc,CldFraction,spa,p3] + mac_mic: + atm_procs_list: [tms,shoc,CldFraction,p3] Type: Group schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} - spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc p3: + do_prescribed_ccn: false max_total_ni: 740.0e3 shoc: lambda_low: 0.001 @@ -63,6 +64,7 @@ atmosphere_processes: rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + do_aerosol_rad: false grids_manager: Type: Homme diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml similarity index 93% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml index ca226588db27..d07cdf5a4d0b 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_pg2/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_p3_rrtmgp_pg2/output.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -filename_prefix: homme_shoc_cld_spa_p3_rrtmgp_pg2_output +filename_prefix: homme_shoc_cld_p3_rrtmgp_pg2_output Averaging Type: Instant Max Snapshots Per File: 1 Fields: @@ -25,12 +25,6 @@ Fields: # CLD - cldfrac_ice - cldfrac_tot - # SPA - - aero_g_sw - - aero_ssa_sw - - aero_tau_lw - - aero_tau_sw - - nccn # P3 - bm - nc @@ -125,5 +119,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt similarity index 86% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt index 03617b46bb0e..6308e2794112 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/CMakeLists.txt @@ -68,8 +68,7 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES scream/init/spa_init_ne2np4.nc - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc scream/init/${EAMxx_tests_IC_FILE_72lev} cam/topo/${EAMxx_tests_TOPO_FILE} ) @@ -87,3 +86,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics physics shoc cld p3 rrtmgp spa META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml similarity index 94% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml index fc2d6cab6c25..d2f51e9e1ccb 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/input.yaml @@ -31,8 +31,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml index 9728316223a9..21489de893ce 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp/output.yaml @@ -104,5 +104,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt similarity index 88% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt index 1daeade05d61..fe5282265857 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/CMakeLists.txt @@ -67,7 +67,7 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/init/spa_file_unified_and_complete_ne4_20220428.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc scream/init/${EAMxx_tests_IC_FILE_128lev} cam/topo/${EAMxx_tests_TOPO_FILE} ) @@ -98,3 +98,10 @@ foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) LABELS "dynamics;shoc;cld;p3;rrtmgp;physics" FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) endforeach() + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml similarity index 94% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml index 7c51e54dad0a..b92889cdcaa0 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/input.yaml @@ -32,8 +32,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml similarity index 85% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml index 9e5249efe800..75c1d29a05ea 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_128levels/output.yaml @@ -68,9 +68,17 @@ Fields: - sfc_flux_sw_net - EAMxx_T_mid_tend - EAMxx_qv_tend + # Sliced Output + - T_mid_at_2m_above_surface + - T_mid_at_1000m_above_sealevel + - T_mid_at_500hPa + - T_mid_at_50000Pa + - T_mid_at_500mb + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_lev_10 output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt similarity index 84% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt index 8a6bf81d4039..ef2494304f58 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/CMakeLists.txt @@ -8,7 +8,7 @@ set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) CreateDynamicsLib("theta-l_kokkos" 4 128 10) # Create the test -set (TEST_LABELS "dynamics;driver;shoc;cld;p3;rrtmgp;physics;dp") +set (TEST_LABELS "dynamics;driver;shoc;cld;spa;p3;rrtmgp;physics;dp") CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_dp "homme_shoc_cld_spa_p3_rrtmgp_dp.cpp" LABELS ${TEST_LABELS} LIBS cld_fraction shoc spa p3 scream_rrtmgp ${dynLibName} scream_control diagnostics @@ -17,7 +17,7 @@ CreateUnitTest(homme_shoc_cld_spa_p3_rrtmgp_dp "homme_shoc_cld_spa_p3_rrtmgp_dp. ) # Set AD configurable options -set (ATM_TIME_STEP 50) +set (ATM_TIME_STEP 100) SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h set (RUN_T0 1999-07-10-00000) @@ -70,9 +70,6 @@ configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta-dp.nl # Ensure test input files are present in the data dir set (TEST_INPUT_FILES - scream/init/spa_init_ne2np4.nc - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc scream/init/${EAMxx_tests_IC_FILE_128lev} cam/topo/${EAMxx_tests_TOPO_FILE} cam/scam/iop/DYCOMSrf01_iopfile_4scam.nc @@ -88,6 +85,13 @@ CompareNCFilesFamilyMpi ( TEST_BASE_NAME ${TEST_BASE_NAME} FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS dynamics physics shoc cld p3 rrtmgp spa dp + LABELS dynamics physics shoc cld spa p3 rrtmgp dp META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp index 5a8dc0fb29bf..235b7f22a16b 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/homme_shoc_cld_spa_p3_rrtmgp_dp.cpp @@ -56,7 +56,6 @@ TEST_CASE("scream_homme_physics", "scream_homme_physics") { ad.init_time_stamps (t0, t0); ad.create_atm_processes (); ad.create_grids (); - ad.setup_intensive_observation_period (); ad.create_fields (); // Setup surface coupler import to be NaNs for fields IOP should overwrite @@ -82,7 +81,6 @@ TEST_CASE("scream_homme_physics", "scream_homme_physics") { ad.setup_surface_coupling_data_manager(SurfaceCouplingTransferType::Import, 4, 4, ncols, import_data.data(), import_names[0], import_cpl_indices.data(), import_vec_comps.data(), import_constant_multiple.data(), do_import_during_init.data()); - ad.initialize_fields (); ad.initialize_output_managers (); ad.initialize_atm_procs (); diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml similarity index 91% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml index c545245de90c..950ef7e4db38 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/input.yaml @@ -2,13 +2,15 @@ --- driver_options: atmosphere_dag_verbosity_level: 5 - enable_intensive_observation_period: true + enable_iop: true -intensive_observation_period_options: +iop_options: doubly_periodic_mode: true iop_file: ${IOP_DATA_DIR}/DYCOMSrf01_iopfile_4scam.nc target_latitude: 31.5 target_longitude: 238.5 + iop_dosubsidence: true + iop_srf_prop: true time_stepping: time_step: ${ATM_TIME_STEP} @@ -25,6 +27,9 @@ initial_conditions: perturbed_fields: [T_mid] perturbation_limit: 0.001 perturbation_minimum_pressure: 900.0 # in millibar + surf_lw_flux_up: 400.0 + eddy_diff_mom: 0.02 + sgs_buoy_flux: -0.001 atmosphere_processes: atm_procs_list: [sc_import,homme,physics] @@ -41,8 +46,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: @@ -67,6 +71,7 @@ atmosphere_processes: rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + do_aerosol_rad: false grids_manager: Type: Homme diff --git a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml index 1c7f06085f81..ddec1a3b9fa6 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/homme_shoc_cld_spa_p3_rrtmgp_dp/output.yaml @@ -104,6 +104,5 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/CMakeLists.txt new file mode 100644 index 000000000000..024aa833df39 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/CMakeLists.txt @@ -0,0 +1,109 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME homme_shoc_cld_p3_mam_optics_rrtmgp) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Get or create the dynamics lib +# HOMME_TARGET NP PLEV QSIZE_D +CreateDynamicsLib("theta-l_kokkos" 4 72 41) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS cld_fraction ${dynLibName} shoc p3 scream_rrtmgp mam + LABELS dynamics shoc cld p3 rrtmgp physics mam4_optics + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 4 48) # 1h 2h 24h +set (RUN_T0 2021-10-12-45000) + +# Determine num subcycles needed to keep shoc dt<=300s +set (SHOC_MAX_DT 300) +math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_MAX_DT}") + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Set homme's test options, so that we can configure the namelist correctly +# Discretization/algorithm settings +set (HOMME_TEST_NE 2) +set (HOMME_TEST_LIM 9) +set (HOMME_TEST_REMAP_FACTOR 3) +set (HOMME_TEST_TRACERS_FACTOR 1) +set (HOMME_TEST_TIME_STEP 300) +set (HOMME_THETA_FORM 1) +set (HOMME_TTYPE 5) +set (HOMME_SE_FTYPE 0) +set (HOMME_TEST_TRANSPORT_ALG 0) +set (HOMME_TEST_CUBED_SPHERE_MAP 0) + +# Hyperviscosity settings +set (HOMME_TEST_HVSCALING 0) +set (HOMME_TEST_HVS 1) +set (HOMME_TEST_HVS_TOM 0) +set (HOMME_TEST_HVS_Q 1) + +set (HOMME_TEST_NU 7e15) +set (HOMME_TEST_NUDIV 1e15) +set (HOMME_TEST_NUTOP 2.5e5) + +# Testcase settings +set (HOMME_TEST_MOISTURE notdry) +set (HOMME_THETA_HY_MODE true) + +# Vert coord settings +set (HOMME_TEST_VCOORD_INT_FILE acme-72i.ascii) +set (HOMME_TEST_VCOORD_MID_FILE acme-72m.ascii) + +# Configure the namelist into the test directory +configure_file(${SCREAM_SRC_DIR}/dynamics/homme/tests/theta.nl + ${CMAKE_CURRENT_BINARY_DIR}/namelist.nl) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS dynamics physics shoc cld p3 rrtmgp mam4_optics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml new file mode 100644 index 000000000000..e6b942364886 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/input.yaml @@ -0,0 +1,90 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + mass_column_conservation_error_tolerance: 1e-3 + energy_column_conservation_error_tolerance: 1e-4 + column_conservation_checks_fail_handling_type: Warning + property_check_data_fields: [phis] + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +initial_conditions: + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_liq_surf_mass: 0.0 + precip_ice_surf_mass: 0.0 + pbl_height : 1.0 + phis : 1.0 + +atmosphere_processes: + atm_procs_list: [homme,physics] + schedule_type: Sequential + homme: + Moisture: moist + physics: + atm_procs_list: [mac_mic,mam4_optics,rrtmgp] + schedule_type: Sequential + Type: Group + mac_mic: + atm_procs_list: [shoc,CldFraction,p3] + schedule_type: Sequential + Type: Group + number_of_subcycles: ${MAC_MIC_SUBCYCLES} + p3: + do_prescribed_ccn: false + enable_column_conservation_checks: true + max_total_ni: 740.0e3 + shoc: + enable_column_conservation_checks: true + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + enable_column_conservation_checks: true + +grids_manager: + Type: Homme + physics_grid_type: GLL + dynamics_namelist_file_name: namelist.nl + vertical_coordinate_filename: IC_FILE + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml new file mode 100644 index 000000000000..559f5158bb60 --- /dev/null +++ b/components/eamxx/tests/multi-process/dynamics_physics/mam/homme_shoc_cld_p3_mam_optics_rrtmgp/output.yaml @@ -0,0 +1,121 @@ +%YAML 1.1 +--- +filename_prefix: homme_shoc_cld_p3_mam_optics_rrtmgp +Averaging Type: Instant +Max Snapshots Per File: 1 +Fields: + Physics GLL: + Field Names: + # HOMME + - ps + - pseudo_density + - omega + - p_int + - p_mid + - pseudo_density_dry + - p_dry_int + - p_dry_mid + # SHOC + - cldfrac_liq + - eddy_diff_mom + - sgs_buoy_flux + - tke + - inv_qc_relvar + - pbl_height + # CLD + - cldfrac_ice + - cldfrac_tot + # P3 + - bm + - nc + - ni + - nr + - qi + - qm + - qr + - T_prev_micro_step + - qv_prev_micro_step + - eff_radius_qc + - eff_radius_qi + - eff_radius_qr + - micro_liq_ice_exchange + - micro_vap_ice_exchange + - micro_vap_liq_exchange + - precip_ice_surf_mass + - precip_liq_surf_mass + - rainfrac + # SHOC + HOMME + - horiz_winds + # SHOC + P3 + - qc + - qv + # SHOC + P3 + RRTMGP + HOMME + - T_mid + #mam_optics + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + # Diagnostics + - T_mid_at_lev_2 + - T_mid_at_model_top + - T_mid_at_model_bot + - T_mid_at_500mb + - T_mid_at_500hPa + - T_mid_at_50000Pa + - PotentialTemperature + - AtmosphereDensity + - Exner + - VirtualTemperature + - z_int + - geopotential_int_at_lev_2 + - z_mid_at_500mb + - geopotential_mid + - dz + - DryStaticEnergy + - SeaLevelPressure + - LiqWaterPath + - IceWaterPath + - VapWaterPath + - RainWaterPath + - RimeWaterPath + - ShortwaveCloudForcing + - LongwaveCloudForcing + - RelativeHumidity + - ZonalVapFlux + - MeridionalVapFlux + - PotentialTemperature_at_model_top + - PotentialTemperature_at_500mb + # GLL output for homme states. These + # represent all current possible homme + # states available. + Dynamics: + Field Names: + - v_dyn + - vtheta_dp_dyn + - dp3d_dyn + - phi_int_dyn + - ps_dyn + - phis_dyn + - omega_dyn + - Qdp_dyn + IO Grid Name: Physics GLL + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/CMakeLists.txt b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/CMakeLists.txt rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/CMakeLists.txt diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/input_baseline.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/input_baseline.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_baseline.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/input_initial.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/input_initial.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_initial.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/input_restarted.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml similarity index 100% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/input_restarted.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/input_restarted.yaml diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_output.yaml similarity index 98% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/model_output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_output.yaml index 284b86f3a06f..671a10e57701 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_output.yaml @@ -80,7 +80,6 @@ Fields: - vtheta_dp_dyn - dp3d_dyn output_control: - MPI Ranks in Filename: true Frequency: 1 frequency_units: nmins ... diff --git a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_restart_output.yaml b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_restart_output.yaml similarity index 96% rename from components/eamxx/tests/coupled/dynamics_physics/model_restart/model_restart_output.yaml rename to components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_restart_output.yaml index 13ca88309659..d4026069b167 100644 --- a/components/eamxx/tests/coupled/dynamics_physics/model_restart/model_restart_output.yaml +++ b/components/eamxx/tests/multi-process/dynamics_physics/model_restart/model_restart_output.yaml @@ -80,9 +80,7 @@ Fields: - vtheta_dp_dyn - dp3d_dyn output_control: - MPI Ranks in Filename: true Frequency: 1 frequency_units: nmins Checkpoint Control: - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/physics_only/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt similarity index 77% rename from components/eamxx/tests/coupled/physics_only/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/CMakeLists.txt index ed15610160ed..61b224f71ae6 100644 --- a/components/eamxx/tests/coupled/physics_only/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/CMakeLists.txt @@ -2,6 +2,9 @@ if (SCREAM_DOUBLE_PRECISION) add_subdirectory(shoc_cld_p3_rrtmgp) add_subdirectory(shoc_cld_spa_p3_rrtmgp) + if (SCREAM_ENABLE_MAM) + add_subdirectory(mam/optics_rrtmgp) + endif() endif() add_subdirectory (atm_proc_subcycling) diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt similarity index 92% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt index 8b16178ccd83..71fe2a5f43c2 100644 --- a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/CMakeLists.txt @@ -45,8 +45,8 @@ CreateUnitTestFromExec (shoc_p3_monolithic shoc_p3 include (BuildCprnc) BuildCprnc() -set (SRC_FILE "shoc_p3_monolithic.INSTANT.nsteps_x3.${RUN_T0}.nc") -set (TGT_FILE "shoc_p3_subcycled.INSTANT.nsteps_x1.${RUN_T0}.nc") +set (SRC_FILE "shoc_p3_monolithic.INSTANT.nsteps_x3.np1.${RUN_T0}.nc") +set (TGT_FILE "shoc_p3_subcycled.INSTANT.nsteps_x1.np1.${RUN_T0}.nc") set (TEST_NAME check_subcycling) add_test (NAME ${TEST_NAME} COMMAND cmake -P ${CMAKE_BINARY_DIR}/bin/CprncTest.cmake ${SRC_FILE} ${TGT_FILE} @@ -57,7 +57,7 @@ set_tests_properties(${TEST_NAME} PROPERTIES # Check calculation of shoc tendencies when the parent group is subcycled set (script ${SCREAM_BASE_DIR}/scripts/check-tendencies) -set (fname shoc_p3_tend_subcycled.INSTANT.nsteps_x1.${RUN_T0}.nc) +set (fname shoc_p3_tend_subcycled.INSTANT.nsteps_x1.np1.${RUN_T0}.nc) add_test (NAME ${TEST_NAME}_tend_check COMMAND ${script} -f ${fname} -v tke -t shoc_tke_tend WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/input.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml similarity index 100% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/input.yaml rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/input.yaml diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output.yaml similarity index 94% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output.yaml rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output.yaml index 2074321ad09a..2e9d18f64317 100644 --- a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output.yaml @@ -35,5 +35,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output_tend.yaml b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output_tend.yaml similarity index 86% rename from components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output_tend.yaml rename to components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output_tend.yaml index 1b51444a0a3f..5103e182d60e 100644 --- a/components/eamxx/tests/coupled/physics_only/atm_proc_subcycling/output_tend.yaml +++ b/components/eamxx/tests/multi-process/physics_only/atm_proc_subcycling/output_tend.yaml @@ -9,5 +9,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/CMakeLists.txt new file mode 100644 index 000000000000..14e7a94e5c22 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/CMakeLists.txt @@ -0,0 +1,57 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME mam_optics_rrtmgp) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LIBS shoc scream_rrtmgp mam + LABELS rrtmgp physics diagnostics mam4_optics + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +set (ATM_TIME_STEP 1800) +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) # 1h 4h 24h +set (RUN_T0 2021-10-12-45000) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +## Copy (and configure) yaml files needed by tests +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS rrtmgp mam4_optics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/input.yaml new file mode 100644 index 000000000000..347054e5d9fd --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/input.yaml @@ -0,0 +1,55 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_optics, rrtmgp] + schedule_type: Sequential + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + rrtmgp: + column_chunk_size: 123 + active_gases: ["h2o", "co2", "o3", "n2o", "co" , "ch4", "o2", "n2"] + orbital_year: 1990 + rrtmgp_coefficients_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-sw-g112-210809.nc + rrtmgp_coefficients_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-data-lw-g128-210809.nc + rrtmgp_cloud_optics_file_sw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-sw.nc + rrtmgp_cloud_optics_file_lw: ${SCREAM_DATA_DIR}/init/rrtmgp-cloud-optics-coeffs-lw.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml new file mode 100644 index 000000000000..24e1dfbb3e7a --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/mam/optics_rrtmgp/output.yaml @@ -0,0 +1,30 @@ +%YAML 1.1 +--- +filename_prefix: mam_optics_rrtmgp_output +Averaging Type: Instant +Max Snapshots Per File: 1 +Field Names: + # MAM_OPTICS + - aero_g_sw + - aero_ssa_sw + - aero_tau_lw + - aero_tau_sw + # RRTMGP + - sfc_alb_dif_nir + - sfc_alb_dif_vis + - sfc_alb_dir_nir + - sfc_alb_dir_vis + - LW_flux_dn + - LW_flux_up + - SW_flux_dn + - SW_flux_dn_dir + - SW_flux_up + - rad_heating_pdel + - sfc_flux_lw_dn + - sfc_flux_sw_net + +output_control: + Frequency: ${NUM_STEPS} + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/CMakeLists.txt diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml similarity index 100% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/input.yaml diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/output.yaml similarity index 96% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/output.yaml index 868970c7d5c9..a8b22c16ba09 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_p3_rrtmgp/output.yaml @@ -54,5 +54,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt similarity index 92% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt index 2cf18454eef3..799010ad10be 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/CMakeLists.txt @@ -23,9 +23,7 @@ math (EXPR MAC_MIC_SUBCYCLES "(${ATM_TIME_STEP} + ${SHOC_MAX_DT} - 1) / ${SHOC_M # Ensure test input files are present in the data dir set (TEST_INPUT_FILES scream/init/${EAMxx_tests_IC_FILE_72lev} - scream/init/spa_init_ne2np4.nc - scream/maps/map_ne4np4_to_ne2np4_mono.nc - scream/init/spa_file_unified_and_complete_ne4_20220428.nc + scream/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc cam/topo/${EAMxx_tests_TOPO_FILE} ) foreach (file IN ITEMS ${TEST_INPUT_FILES}) diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml similarity index 94% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml index aae97f2190e7..542625798f53 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/input.yaml @@ -17,8 +17,7 @@ atmosphere_processes: schedule_type: Sequential number_of_subcycles: ${MAC_MIC_SUBCYCLES} spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc p3: max_total_ni: 740.0e3 shoc: diff --git a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml similarity index 97% rename from components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml index be396798eb49..1d2d91de183c 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_cld_spa_p3_rrtmgp/output.yaml @@ -63,5 +63,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/CMakeLists.txt b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt similarity index 77% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/CMakeLists.txt rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt index 74283073c62c..973e58b7da53 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/CMakeLists.txt +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/CMakeLists.txt @@ -19,6 +19,7 @@ CreateUnitTest(create_vert_remap_and_weights "create_vert_remap_and_weights.cpp" # Run a test to setup nudging source data: set (NUM_STEPS 5) set (POSTFIX source_data) +set (ADD_RANKS false) set (ATM_TIME_STEP 300) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_source_data.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_source_data.yaml) @@ -35,6 +36,7 @@ CreateUnitTestFromExec (shoc_p3_source shoc_p3_nudging set (NUM_STEPS 5) set (ATM_TIME_STEP 300) set (POSTFIX nudged) +set (ADD_RANKS true) set (VERT_TYPE TIME_DEPENDENT_3D_PROFILE) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging.yaml ${CMAKE_CURRENT_BINARY_DIR}/input_nudging.yaml) @@ -57,3 +59,16 @@ CreateUnitTestFromExec (shoc_p3_nudged_remapped shoc_p3_nudging EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_remapped.yaml" FIXTURES_REQUIRED shoc_p3_source_data) +# Run a test with nudging using data read in glob pattern and skip vertical interpolation: +set (NUM_STEPS 5) +set (ATM_TIME_STEP 300) +set (POSTFIX nudged_glob_novert) +set (VERT_TYPE TIME_DEPENDENT_3D_PROFILE) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_nudging_glob_novert.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_nudging_glob_novert.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_nudged_glob_novert.yaml) +CreateUnitTestFromExec (shoc_p3_nudging_glob_novert shoc_p3_nudging + EXE_ARGS "--use-colour no --ekat-test-params ifile=input_nudging_glob_novert.yaml" + FIXTURES_REQUIRED shoc_p3_source_data) + diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp similarity index 98% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp index b9bdfc9fdb07..702d83f16d37 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/create_vert_remap_and_weights.cpp @@ -80,5 +80,6 @@ TEST_CASE("create_vert_remap_and_weights","create_vert_remap_and_weights") { create_vert_remap(); create_nudging_weights_ncfile(1, 218, 72, "nudging_weights.nc"); + create_nudging_weights_ncfile(1, 218, 128, "nudging_weights_L128.nc"); } } // end namespace diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml similarity index 93% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml index 2c9b58b678cc..2b33d15f9f75 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_nudging.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging.yaml @@ -14,7 +14,7 @@ atmosphere_processes: p3: max_total_ni: 720.0e3 nudging: - nudging_filename: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] + nudging_filenames_patterns: [shoc_p3_source_data_${POSTFIX}.INSTANT.nsteps_x${NUM_STEPS}.${RUN_T0}.nc] nudging_fields: ["T_mid", "qv"] nudging_timescale: 1000 use_nudging_weights: true diff --git a/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml new file mode 100644 index 000000000000..35d75015e2a1 --- /dev/null +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_nudging_glob_novert.yaml @@ -0,0 +1,61 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + schedule_type: Sequential + atm_procs_list: [shoc,p3,nudging] + p3: + max_total_ni: 720.0e3 + nudging: + nudging_filenames_patterns: [./shoc_p3_source_data_nudged.INSTANT.nsteps_x*.nc] + nudging_fields: ["T_mid", "qv"] + nudging_timescale: 1000 + use_nudging_weights: true + nudging_weights_file: nudging_weights_L128.nc + skip_vert_interpolation: true + source_pressure_type: ${VERT_TYPE} + source_pressure_file: vertical_remap.nc ## Only used in the case of STATIC_1D_VERTICAL_PROFILE + shoc: + lambda_low: 0.001 + lambda_high: 0.04 + lambda_slope: 2.65 + lambda_thresh: 0.02 + thl2tune: 1.0 + qw2tune: 1.0 + qwthl2tune: 1.0 + w2tune: 1.0 + length_fac: 0.5 + c_diag_3rd_mom: 7.0 + Ckh: 0.1 + Ckm: 0.1 + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + aliases: [Physics] + type: point_grid + number_of_global_columns: 218 + number_of_vertical_levels: 128 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_128lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + surf_evap: 0.0 + surf_sens_flux: 0.0 + precip_ice_surf_mass: 0.0 + precip_liq_surf_mass: 0.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: [output_${POSTFIX}.yaml] +... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_source_data.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml similarity index 100% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/input_source_data.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/input_source_data.yaml diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml similarity index 94% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml index d4779131bcbb..e740c5cdf1f1 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output.yaml @@ -3,6 +3,7 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged Averaging Type: Instant Max Snapshots Per File: 2 +MPI Ranks in Filename: ${ADD_RANKS} Field Names: - p_mid - T_mid @@ -35,5 +36,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml similarity index 87% rename from components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml rename to components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml index 4dde5dae2d5a..c75a6a862f71 100644 --- a/components/eamxx/tests/coupled/physics_only/shoc_p3_nudging/output_remapped.yaml +++ b/components/eamxx/tests/multi-process/physics_only/shoc_p3_nudging/output_remapped.yaml @@ -4,6 +4,7 @@ filename_prefix: shoc_p3_${POSTFIX}_nudged_remapped Averaging Type: Instant Max Snapshots Per File: 2 vertical_remap_file: vertical_remap.nc +MPI Ranks in Filename: ${ADD_RANKS} Field Names: - T_mid - qv @@ -11,5 +12,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: false ... diff --git a/components/eamxx/tests/postrun/check_hashes_ers.py b/components/eamxx/tests/postrun/check_hashes_ers.py deleted file mode 100755 index be1da31ffff0..000000000000 --- a/components/eamxx/tests/postrun/check_hashes_ers.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 - -import os, sys, re, glob, subprocess - -def readall(fn): - with open(fn,'r') as f: - txt = f.read() - return txt - -def greptxt(pattern, txt): - return re.findall('(?:' + pattern + ').*', txt, flags=re.MULTILINE) - -def grep(pattern, fn): - txt = readall(fn) - return greptxt(pattern, txt) - -def get_log_glob_from_atm_modelio(case_dir): - filename = case_dir + os.path.sep + 'CaseDocs' + os.path.sep + 'atm_modelio.nml' - ln = grep('diro = ', filename)[0] - run_dir = ln.split()[2].split('"')[1] - ln = grep('logfile = ', filename)[0] - atm_log_fn = ln.split()[2].split('"')[1] - id = atm_log_fn.split('.')[2] - return run_dir + os.path.sep + 'e3sm.log.' + id + '*' - -def get_hash_lines(fn): - rlns = subprocess.run(['zgrep', 'exxhash', fn], capture_output=True) - rlns = rlns.stdout.decode().split('\n') - lns = [] - if len(rlns) == 0: return lns - for rln in rlns: - pos = rln.find('exxhash') - lns.append(rln[pos:]) - return lns - -def parse_time(hash_ln): - return hash_ln.split()[1:3] - -def all_equal(t1, t2): - if len(t1) != len(t2): return False - for i in range(len(t1)): - if t1[i] != t2[i]: return False - return True - -def find_first_index_at_time(lns, time): - for i, ln in enumerate(lns): - t = parse_time(ln) - if all_equal(time, t): return i - return None - -def diff(l1, l2): - diffs = [] - for i in range(len(l1)): - if l1[i] != l2[i]: - diffs.append((l1[i], l2[i])) - return diffs - -def main(case_dir): - # Look for the two e3sm.log files. - glob_pat = get_log_glob_from_atm_modelio(case_dir) - e3sm_fns = glob.glob(glob_pat) - if len(e3sm_fns) == 0: - print('Could not find e3sm.log files with glob string {}'.format(glob_pat)) - return False - e3sm_fns.sort() - if len(e3sm_fns) == 1: - # This is the first run. Exit and wait for the second - # run. (POSTRUN_SCRIPT is called after each of the two runs.) - print('Exiting on first run.') - return True - print('Diffing base {} and restart {}'.format(e3sm_fns[0], e3sm_fns[1])) - - # Because of the prefixed 1: and 2: on some systems, we can't just use - # zdiff. - lns = [] - for f in e3sm_fns: - lns.append(get_hash_lines(f)) - time = parse_time(lns[1][0]) - time_idx = find_first_index_at_time(lns[0], time) - if time_idx is None: - print('Could not find a start time.') - return False - lns[0] = lns[0][time_idx:] - if len(lns[0]) != len(lns[1]): - print('Number of hash lines starting at restart time do not agree.') - return False - diffs = diff(lns[0], lns[1]) - - # Flushed prints to e3sm.log can sometimes conflict with other - # output. Permit up to 'thr' diffs so we don't fail due to badly printed - # lines. This isn't a big loss in checking because an ERS_Ln22 second run - # writes > 1000 hash lines, and a true loss of BFBness is nearly certain to - # propagate to a large number of subsequent hashes. - thr = 5 - if len(lns[0]) < 100: thr = 0 - - ok = True - if len(diffs) > thr: - print('DIFF') - print(diffs[-10:]) - ok = False - else: - print('OK') - - return ok - -case_dir = sys.argv[1] -sys.exit(0 if main(case_dir) else 1) diff --git a/components/eamxx/tests/single-process/CMakeLists.txt b/components/eamxx/tests/single-process/CMakeLists.txt new file mode 100644 index 000000000000..d417a9920f46 --- /dev/null +++ b/components/eamxx/tests/single-process/CMakeLists.txt @@ -0,0 +1,26 @@ +if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") + add_subdirectory(homme) +endif() +add_subdirectory(p3) +add_subdirectory(shoc) +add_subdirectory(cld_fraction) +add_subdirectory(spa) +add_subdirectory(surface_coupling) +if (SCREAM_ENABLE_ML_CORRECTION ) + add_subdirectory(ml_correction) +endif() +if (SCREAM_DOUBLE_PRECISION) + add_subdirectory(rrtmgp) + add_subdirectory(cosp) +else() + message(STATUS "RRTMGP and COSP only supported for double precision builds; skipping") +endif() +if (SCREAM_ENABLE_MAM) + # Once the mam4xx aerosol microphysics AtmosphereProcess is running, the + # corresponding test here needs to be reworked with valid aerosol + # initial conditions. + add_subdirectory(mam/optics) +endif() +if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) + add_subdirectory(zm) +endif() diff --git a/components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt b/components/eamxx/tests/single-process/cld_fraction/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/cld_fraction/CMakeLists.txt rename to components/eamxx/tests/single-process/cld_fraction/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/cld_fraction/input.yaml b/components/eamxx/tests/single-process/cld_fraction/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/cld_fraction/input.yaml rename to components/eamxx/tests/single-process/cld_fraction/input.yaml diff --git a/components/eamxx/tests/uncoupled/cosp/CMakeLists.txt b/components/eamxx/tests/single-process/cosp/CMakeLists.txt similarity index 50% rename from components/eamxx/tests/uncoupled/cosp/CMakeLists.txt rename to components/eamxx/tests/single-process/cosp/CMakeLists.txt index 769c6593aca8..6de9f09b2cd7 100644 --- a/components/eamxx/tests/uncoupled/cosp/CMakeLists.txt +++ b/components/eamxx/tests/single-process/cosp/CMakeLists.txt @@ -1,10 +1,14 @@ include (ScreamUtils) +set (TEST_BASE_NAME cosp_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + # Create the test -CreateADUnitTest(cosp_standalone +CreateADUnitTest(${TEST_BASE_NAME} LABELS cosp physics LIBS eamxx_cosp MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} ) # Set AD configurable options @@ -20,3 +24,11 @@ configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: for other tests we do np1-vs-npX bfb tests, which is why one is enough. + # COSP, is not bfb w.r.t. num ranks though. Still, we only check 1 output files against baselines + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/cosp/input.yaml b/components/eamxx/tests/single-process/cosp/input.yaml similarity index 96% rename from components/eamxx/tests/uncoupled/cosp/input.yaml rename to components/eamxx/tests/single-process/cosp/input.yaml index 3336b0694db5..1447bcc654f1 100644 --- a/components/eamxx/tests/uncoupled/cosp/input.yaml +++ b/components/eamxx/tests/single-process/cosp/input.yaml @@ -28,7 +28,7 @@ initial_conditions: topography_filename: ${TOPO_DATA_DIR}/USGS-gtopo30_ne4np4pg2_16x_converted.c20200527.nc dtau067: 1.0 dtau105: 1.0 - cldfrac_tot_for_analysis: 0.5 + cldfrac_rad: 0.5 eff_radius_qc: 0.0 eff_radius_qi: 0.0 sunlit: 1.0 diff --git a/components/eamxx/tests/uncoupled/cosp/output.yaml b/components/eamxx/tests/single-process/cosp/output.yaml similarity index 86% rename from components/eamxx/tests/uncoupled/cosp/output.yaml rename to components/eamxx/tests/single-process/cosp/output.yaml index 59cc18aeddf9..7e942489c728 100644 --- a/components/eamxx/tests/uncoupled/cosp/output.yaml +++ b/components/eamxx/tests/single-process/cosp/output.yaml @@ -10,5 +10,4 @@ Fields: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/homme/CMakeLists.txt b/components/eamxx/tests/single-process/homme/CMakeLists.txt similarity index 86% rename from components/eamxx/tests/uncoupled/homme/CMakeLists.txt rename to components/eamxx/tests/single-process/homme/CMakeLists.txt index 13dcdaea21b5..f1c28036782b 100644 --- a/components/eamxx/tests/uncoupled/homme/CMakeLists.txt +++ b/components/eamxx/tests/single-process/homme/CMakeLists.txt @@ -77,3 +77,10 @@ CompareNCFilesFamilyMpi ( LABELS dynamics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/homme/input.yaml b/components/eamxx/tests/single-process/homme/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/homme/input.yaml rename to components/eamxx/tests/single-process/homme/input.yaml diff --git a/components/eamxx/tests/uncoupled/homme/output.yaml b/components/eamxx/tests/single-process/homme/output.yaml similarity index 92% rename from components/eamxx/tests/uncoupled/homme/output.yaml rename to components/eamxx/tests/single-process/homme/output.yaml index d34812a6a834..4b88510d770e 100644 --- a/components/eamxx/tests/uncoupled/homme/output.yaml +++ b/components/eamxx/tests/single-process/homme/output.yaml @@ -20,5 +20,4 @@ Fields: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt new file mode 100644 index 000000000000..a3e6f64ec7d0 --- /dev/null +++ b/components/eamxx/tests/single-process/mam/optics/CMakeLists.txt @@ -0,0 +1,66 @@ +include (ScreamUtils) + +set (TEST_BASE_NAME mam4_optics_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Create the test +CreateADUnitTest(${TEST_BASE_NAME} + LABELS mam4_optics physics + LIBS mam + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME} +) + +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 12 24 36) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev}) +GetInputFile(cam/topo/${EAMxx_tests_TOPO_FILE}) + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) + +# Ensure test input files are present in the data dir +set (TEST_INPUT_FILES + scream/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + scream/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + scream/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + scream/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + scream/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + scream/mam4xx/physprops/ssam_rrtmg_c20240206.nc + scream/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + scream/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + scream/mam4xx/physprops/poly_rrtmg_c20240206.nc + scream/init/scream_unit_tests_aerosol_optics_ne2np4L72_20220822.nc +) + +foreach (file IN ITEMS ${TEST_INPUT_FILES}) + GetInputFile(${file}) +endforeach() + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME} + FILE_META_NAME ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS mam4_optics physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 +) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x2.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() \ No newline at end of file diff --git a/components/eamxx/tests/single-process/mam/optics/input.yaml b/components/eamxx/tests/single-process/mam/optics/input.yaml new file mode 100644 index 000000000000..1aa6f3f7134f --- /dev/null +++ b/components/eamxx/tests/single-process/mam/optics/input.yaml @@ -0,0 +1,47 @@ +%YAML 1.1 +--- +driver_options: + atmosphere_dag_verbosity_level: 5 + +time_stepping: + time_step: ${ATM_TIME_STEP} + run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX + number_of_steps: ${NUM_STEPS} + +atmosphere_processes: + atm_procs_list: [mam4_optics] + mam4_optics: + mam4_mode1_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode1_rrtmg_aeronetdust_c20240206.nc + mam4_mode2_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode2_rrtmg_c20240206.nc + mam4_mode3_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode3_rrtmg_aeronetdust_c20240206.nc + mam4_mode4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/mam4_mode4_rrtmg_c20240206.nc + mam4_water_refindex_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/water_refindex_rrtmg_c20240206.nc + mam4_soa_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocphi_rrtmg_c20240206.nc + mam4_dust_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/dust_aeronet_rrtmg_c20240206.nc + mam4_nacl_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ssam_rrtmg_c20240206.nc + mam4_so4_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/sulfate_rrtmg_c20240206.nc + mam4_pom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/ocpho_rrtmg_c20240206.nc + mam4_bc_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/bcpho_rrtmg_c20240206.nc + mam4_mom_physical_properties_file : ${SCREAM_DATA_DIR}/mam4xx/physprops/poly_rrtmg_c20240206.nc + +grids_manager: + Type: Mesh Free + geo_data_source: IC_FILE + grids_names: [Physics GLL] + Physics GLL: + type: point_grid + aliases: [Physics] + number_of_global_columns: 218 + number_of_vertical_levels: 72 + +initial_conditions: + # The name of the file containing the initial conditions for this test. + Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_MAM4xx_72lev} + topography_filename: ${TOPO_DATA_DIR}/${EAMxx_tests_TOPO_FILE} + pbl_height : 1.0 + phis : 1.0 + +# The parameters for I/O control +Scorpio: + output_yaml_files: ["output.yaml"] +... diff --git a/components/eamxx/tests/single-process/mam/optics/output.yaml b/components/eamxx/tests/single-process/mam/optics/output.yaml new file mode 100644 index 000000000000..8ea3d237933d --- /dev/null +++ b/components/eamxx/tests/single-process/mam/optics/output.yaml @@ -0,0 +1,17 @@ +%YAML 1.1 +--- +filename_prefix: mam4_optics_standalone_output +Averaging Type: Instant +Fields: + Physics: + Field Names: + - aero_g_sw + - aero_ssa_sw + - aero_tau_sw + - aero_tau_lw + +output_control: + Frequency: 2 + frequency_units: nsteps + MPI Ranks in Filename: true +... diff --git a/components/eamxx/tests/uncoupled/ml_correction/CMakeLists.txt b/components/eamxx/tests/single-process/ml_correction/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/CMakeLists.txt rename to components/eamxx/tests/single-process/ml_correction/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/ml_correction/input.yaml b/components/eamxx/tests/single-process/ml_correction/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/input.yaml rename to components/eamxx/tests/single-process/ml_correction/input.yaml diff --git a/components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp b/components/eamxx/tests/single-process/ml_correction/ml_correction_standalone.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/ml_correction_standalone.cpp rename to components/eamxx/tests/single-process/ml_correction/ml_correction_standalone.cpp diff --git a/components/eamxx/tests/uncoupled/ml_correction/test_correction.py b/components/eamxx/tests/single-process/ml_correction/test_correction.py similarity index 100% rename from components/eamxx/tests/uncoupled/ml_correction/test_correction.py rename to components/eamxx/tests/single-process/ml_correction/test_correction.py diff --git a/components/eamxx/tests/uncoupled/p3/CMakeLists.txt b/components/eamxx/tests/single-process/p3/CMakeLists.txt similarity index 83% rename from components/eamxx/tests/uncoupled/p3/CMakeLists.txt rename to components/eamxx/tests/single-process/p3/CMakeLists.txt index a155ebc4dc74..969ed3a9da71 100644 --- a/components/eamxx/tests/uncoupled/p3/CMakeLists.txt +++ b/components/eamxx/tests/single-process/p3/CMakeLists.txt @@ -48,3 +48,10 @@ foreach (NRANKS RANGE ${TEST_RANK_START} ${TEST_RANK_END}) LABELS "p3;physics" FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${NRANKS}_omp1) endforeach() + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/p3/input.yaml b/components/eamxx/tests/single-process/p3/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/p3/input.yaml rename to components/eamxx/tests/single-process/p3/input.yaml diff --git a/components/eamxx/tests/uncoupled/p3/output.yaml b/components/eamxx/tests/single-process/p3/output.yaml similarity index 91% rename from components/eamxx/tests/uncoupled/p3/output.yaml rename to components/eamxx/tests/single-process/p3/output.yaml index a83da3a3848c..e52e02c4a19b 100644 --- a/components/eamxx/tests/uncoupled/p3/output.yaml +++ b/components/eamxx/tests/single-process/p3/output.yaml @@ -4,7 +4,7 @@ filename_prefix: p3_standalone_output Averaging Type: Instant Field Names: - T_mid - - T_mid_at_2m + - T_mid_at_2m_above_sealevel - T_prev_micro_step - qv - qc @@ -30,5 +30,4 @@ Field Names: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt new file mode 100644 index 000000000000..792d3946a4c2 --- /dev/null +++ b/components/eamxx/tests/single-process/rrtmgp/CMakeLists.txt @@ -0,0 +1,92 @@ +INCLUDE (ScreamUtils) + +set (TEST_BASE_NAME rrtmgp_standalone) +set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) + +# Ensure test input files are present in the data dir +GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) + +if (SCREAM_ENABLE_BASELINE_TESTS AND NOT SCREAM_ONLY_GENERATE_BASELINES) + # Unit test to compare against raw rrtmgp output + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_unit.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_unit.yaml) + CreateUnitTest(${TEST_BASE_NAME}_unit rrtmgp_standalone_unit.cpp + LABELS rrtmgp physics driver + LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics rrtmgp_test_utils + EXE_ARGS "--ekat-test-params rrtmgp_inputfile=${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc,rrtmgp_baseline=${SCREAM_BASELINES_DIR}/data/rrtmgp-allsky-baseline.nc" + ) +endif() + +## Create rrtmgp stand alone executable +CreateUnitTestExec(${TEST_BASE_NAME} "rrtmgp_standalone.cpp" + LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics +) + +# The RRTMGP stand-alone test that runs multi-step +# Set AD configurable options +SetVarDependingOnTestSize(NUM_STEPS 2 5 48) +set (ATM_TIME_STEP 1800) +set (RUN_T0 2021-10-12-45000) + +# Test non-chunked version (sweep multiple ranks) +set (SUFFIX "_not_chunked") +set (COL_CHUNK_SIZE 1000) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_not_chunked.yaml) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_not_chunked.yaml) +CreateUnitTestFromExec( + ${TEST_BASE_NAME}_not_chunked ${TEST_BASE_NAME} + LABELS rrtmgp physics driver + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + EXE_ARGS "--ekat-test-params inputfile=input_not_chunked.yaml" + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_not_chunked +) + +# Compare output files produced by npX tests, to ensure they are bfb +include (CompareNCFiles) + +CompareNCFilesFamilyMpi ( + TEST_BASE_NAME ${TEST_BASE_NAME}_not_chunked + FILE_META_NAME ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc + MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} + LABELS rrtmgp physics + META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_not_chunked_npMPIRANKS_omp1 +) + +## Test chunked version (only for ${TEST_RANK_END}) and compare against non-chunked +set (SUFFIX "_chunked") +math (EXPR COL_PER_RANK "218 / ${TEST_RANK_END}") +math (EXPR COL_CHUNK_SIZE "${COL_PER_RANK} / 2") +if (COL_CHUNK_SIZE LESS 1) + message (FATAL_ERROR "Error! Chunk size for rrtmgp unit test is less than 1.") +endif() + +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml + ${CMAKE_CURRENT_BINARY_DIR}/input_chunked.yaml) +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml + ${CMAKE_CURRENT_BINARY_DIR}/output_chunked.yaml) +CreateUnitTestFromExec( + ${TEST_BASE_NAME}_chunked ${TEST_BASE_NAME} + LABELS rrtmgp physics driver + MPI_RANKS ${TEST_RANK_END} + EXE_ARGS "--ekat-test-params inputfile=input_chunked.yaml" + FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_chunked + PROPERTIES PASS_REGULAR_EXPRESSION "(beg.end: 0, ${COL_CHUNK_SIZE})" +) + +CompareNCFiles( + TEST_NAME ${TEST_BASE_NAME}_chunked_vs_not_chunked + SRC_FILE ${TEST_BASE_NAME}_output_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc + TGT_FILE ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc + LABELS rrtmgp physics + FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_chunked_np${TEST_RANK_END}_omp1 + ${FIXTURES_BASE_NAME}_not_chunked_np${TEST_RANK_END}_omp1) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX, + # and that chunked is bfb with not_chunked + set (OUT_FILE ${TEST_BASE_NAME}_output_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME}_chunked ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}_chunked) +endif() diff --git a/components/eamxx/tests/uncoupled/rrtmgp/input.yaml b/components/eamxx/tests/single-process/rrtmgp/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/input.yaml rename to components/eamxx/tests/single-process/rrtmgp/input.yaml diff --git a/components/eamxx/tests/uncoupled/rrtmgp/input_unit.yaml b/components/eamxx/tests/single-process/rrtmgp/input_unit.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/input_unit.yaml rename to components/eamxx/tests/single-process/rrtmgp/input_unit.yaml diff --git a/components/eamxx/tests/uncoupled/rrtmgp/output.yaml b/components/eamxx/tests/single-process/rrtmgp/output.yaml similarity index 95% rename from components/eamxx/tests/uncoupled/rrtmgp/output.yaml rename to components/eamxx/tests/single-process/rrtmgp/output.yaml index 0baa5da841d2..b067d1412bc6 100644 --- a/components/eamxx/tests/uncoupled/rrtmgp/output.yaml +++ b/components/eamxx/tests/single-process/rrtmgp/output.yaml @@ -30,5 +30,4 @@ Field Names: output_control: Frequency: ${NUM_STEPS} frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone.cpp rename to components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone.cpp diff --git a/components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp b/components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/rrtmgp/rrtmgp_standalone_unit.cpp rename to components/eamxx/tests/single-process/rrtmgp/rrtmgp_standalone_unit.cpp diff --git a/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt b/components/eamxx/tests/single-process/shoc/CMakeLists.txt similarity index 93% rename from components/eamxx/tests/uncoupled/shoc/CMakeLists.txt rename to components/eamxx/tests/single-process/shoc/CMakeLists.txt index bebc54448ca0..bae7f719856d 100644 --- a/components/eamxx/tests/uncoupled/shoc/CMakeLists.txt +++ b/components/eamxx/tests/single-process/shoc/CMakeLists.txt @@ -76,6 +76,13 @@ foreach (RANK RANGE ${TEST_RANK_START} ${TEST_RANK_END}) FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_np${RANK}_omp1) endforeach() +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() + ################################ # MUST FAIL tests # ################################ diff --git a/components/eamxx/tests/uncoupled/shoc/input.yaml b/components/eamxx/tests/single-process/shoc/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/shoc/input.yaml rename to components/eamxx/tests/single-process/shoc/input.yaml diff --git a/components/eamxx/tests/uncoupled/shoc/output.yaml b/components/eamxx/tests/single-process/shoc/output.yaml similarity index 95% rename from components/eamxx/tests/uncoupled/shoc/output.yaml rename to components/eamxx/tests/single-process/shoc/output.yaml index 82ccde097b1d..1ad0d7d990c5 100644 --- a/components/eamxx/tests/uncoupled/shoc/output.yaml +++ b/components/eamxx/tests/single-process/shoc/output.yaml @@ -32,5 +32,4 @@ Fields: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/spa/CMakeLists.txt b/components/eamxx/tests/single-process/spa/CMakeLists.txt similarity index 79% rename from components/eamxx/tests/uncoupled/spa/CMakeLists.txt rename to components/eamxx/tests/single-process/spa/CMakeLists.txt index 559f48af9e53..8501f48224cb 100644 --- a/components/eamxx/tests/uncoupled/spa/CMakeLists.txt +++ b/components/eamxx/tests/single-process/spa/CMakeLists.txt @@ -42,3 +42,10 @@ CompareNCFilesFamilyMpi ( LABELS spa physics META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_npMPIRANKS_omp1 ) + +if (SCREAM_ENABLE_BASELINE_TESTS) + # Compare one of the output files with the baselines. + # Note: one is enough, since we already check that np1 is BFB with npX + set (OUT_FILE ${TEST_BASE_NAME}_output.INSTANT.nsteps_x1.np${TEST_RANK_END}.${RUN_T0}.nc) + CreateBaselineTest(${TEST_BASE_NAME} ${TEST_RANK_END} ${OUT_FILE} ${FIXTURES_BASE_NAME}) +endif() diff --git a/components/eamxx/tests/uncoupled/spa/input.yaml b/components/eamxx/tests/single-process/spa/input.yaml similarity index 88% rename from components/eamxx/tests/uncoupled/spa/input.yaml rename to components/eamxx/tests/single-process/spa/input.yaml index ffd8b4f50b8a..6682710df0c7 100644 --- a/components/eamxx/tests/uncoupled/spa/input.yaml +++ b/components/eamxx/tests/single-process/spa/input.yaml @@ -11,8 +11,7 @@ time_stepping: atmosphere_processes: atm_procs_list: [spa] spa: - spa_remap_file: ${SCREAM_DATA_DIR}/maps/map_ne4np4_to_ne2np4_mono.nc - spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne4_20220428.nc + spa_data_file: ${SCREAM_DATA_DIR}/init/spa_file_unified_and_complete_ne2np4L72_20231222.nc grids_manager: Type: Mesh Free diff --git a/components/eamxx/tests/uncoupled/spa/output.yaml b/components/eamxx/tests/single-process/spa/output.yaml similarity index 89% rename from components/eamxx/tests/uncoupled/spa/output.yaml rename to components/eamxx/tests/single-process/spa/output.yaml index 408d676c64df..89c6ef76bda4 100644 --- a/components/eamxx/tests/uncoupled/spa/output.yaml +++ b/components/eamxx/tests/single-process/spa/output.yaml @@ -12,5 +12,4 @@ Field Names: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/surface_coupling/CMakeLists.txt b/components/eamxx/tests/single-process/surface_coupling/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/surface_coupling/CMakeLists.txt rename to components/eamxx/tests/single-process/surface_coupling/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/surface_coupling/input.yaml b/components/eamxx/tests/single-process/surface_coupling/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/surface_coupling/input.yaml rename to components/eamxx/tests/single-process/surface_coupling/input.yaml diff --git a/components/eamxx/tests/uncoupled/surface_coupling/output.yaml b/components/eamxx/tests/single-process/surface_coupling/output.yaml similarity index 93% rename from components/eamxx/tests/uncoupled/surface_coupling/output.yaml rename to components/eamxx/tests/single-process/surface_coupling/output.yaml index de4da9b32b31..432079db9685 100644 --- a/components/eamxx/tests/uncoupled/surface_coupling/output.yaml +++ b/components/eamxx/tests/single-process/surface_coupling/output.yaml @@ -22,5 +22,4 @@ Field Names: output_control: Frequency: 1 frequency_units: nsteps - MPI Ranks in Filename: true ... diff --git a/components/eamxx/tests/uncoupled/surface_coupling/surface_coupling.cpp b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp similarity index 99% rename from components/eamxx/tests/uncoupled/surface_coupling/surface_coupling.cpp rename to components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp index 106618b827ed..d34059f8559f 100644 --- a/components/eamxx/tests/uncoupled/surface_coupling/surface_coupling.cpp +++ b/components/eamxx/tests/single-process/surface_coupling/surface_coupling.cpp @@ -90,7 +90,6 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons // Create output manager to handle the data scorpio::eam_init_pio_subsystem(comm); ekat::ParameterList om_pl; - om_pl.set("MPI Ranks in Filename",true); om_pl.set("filename_prefix",std::string("surface_coupling_forcing")); om_pl.set("Field Names",fnames); om_pl.set("Averaging Type", std::string("INSTANT")); @@ -99,7 +98,6 @@ std::vector create_from_file_test_data(const ekat::Comm& comm, cons auto& ctrl_pl = om_pl.sublist("output_control"); ctrl_pl.set("frequency_units",std::string("nsteps")); ctrl_pl.set("Frequency",1); - ctrl_pl.set("MPI Ranks in Filename",true); ctrl_pl.set("save_grid_data",false); OutputManager4Test om; om.setup(comm,om_pl,fm,gm,t0,false); diff --git a/components/eamxx/tests/uncoupled/zm/CMakeLists.txt b/components/eamxx/tests/single-process/zm/CMakeLists.txt similarity index 100% rename from components/eamxx/tests/uncoupled/zm/CMakeLists.txt rename to components/eamxx/tests/single-process/zm/CMakeLists.txt diff --git a/components/eamxx/tests/uncoupled/zm/input.yaml b/components/eamxx/tests/single-process/zm/input.yaml similarity index 100% rename from components/eamxx/tests/uncoupled/zm/input.yaml rename to components/eamxx/tests/single-process/zm/input.yaml diff --git a/components/eamxx/tests/uncoupled/zm/zm_standalone.cpp b/components/eamxx/tests/single-process/zm/zm_standalone.cpp similarity index 100% rename from components/eamxx/tests/uncoupled/zm/zm_standalone.cpp rename to components/eamxx/tests/single-process/zm/zm_standalone.cpp diff --git a/components/eamxx/tests/uncoupled/CMakeLists.txt b/components/eamxx/tests/uncoupled/CMakeLists.txt deleted file mode 100644 index dd0fd493e29a..000000000000 --- a/components/eamxx/tests/uncoupled/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# NOTE: if you have baseline-type tests, add the subdirectory OUTSIDE the following if statement -if (NOT SCREAM_BASELINES_ONLY) - if ("${SCREAM_DYNAMICS_DYCORE}" STREQUAL "HOMME") - add_subdirectory(homme) - endif() - add_subdirectory(p3) - add_subdirectory(shoc) - add_subdirectory(cld_fraction) - add_subdirectory(spa) - add_subdirectory(surface_coupling) - if (SCREAM_ENABLE_ML_CORRECTION ) - add_subdirectory(ml_correction) - endif() - if (SCREAM_DOUBLE_PRECISION) - add_subdirectory(rrtmgp) - add_subdirectory(cosp) - else() - message(STATUS "RRTMGP and COSP only supported for double precision builds; skipping") - endif() - if (SCREAM_ENABLE_MAM) - add_subdirectory(mam4) - endif() - if (SCREAM_TEST_LEVEL GREATER_EQUAL SCREAM_TEST_LEVEL_EXPERIMENTAL) - add_subdirectory(zm) - endif() -endif() diff --git a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt b/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt deleted file mode 100644 index 66475af4cdd3..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -include (ScreamUtils) - -# Create the test -CreateADUnitTest(mam4_nucleation_standalone - LABELS mam4 physics - LIBS mam - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} -) - -# Set AD configurable options -set (ATM_TIME_STEP 1) -SetVarDependingOnTestSize(NUM_STEPS 12 24 36) -set (RUN_T0 2021-10-12-45000) - -## Copy (and configure) yaml files needed by tests -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input.yaml) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output.yaml) - -# Ensure test input files are present in the data dir -GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) diff --git a/components/eamxx/tests/uncoupled/mam4/input.yaml b/components/eamxx/tests/uncoupled/mam4/input.yaml deleted file mode 100644 index 80571b614f0d..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/input.yaml +++ /dev/null @@ -1,38 +0,0 @@ -%YAML 1.1 ---- -driver_options: - atmosphere_dag_verbosity_level: 5 - -time_stepping: - time_step: ${ATM_TIME_STEP} - run_t0: ${RUN_T0} # YYYY-MM-DD-XXXXX - number_of_steps: ${NUM_STEPS} - -atmosphere_processes: - atm_procs_list: [mam4_micro] - mam4_micro: - compute_tendencies: [q_aitken_so4, n_aitken, q_h2so4] - -grids_manager: - Type: Mesh Free - grids_names: [Physics] - Physics: - type: point_grid - number_of_global_columns: 218 - number_of_vertical_levels: 72 - -initial_conditions: - # The name of the file containing the initial conditions for this test. - Filename: ${SCREAM_DATA_DIR}/init/${EAMxx_tests_IC_FILE_72lev} - q_aitken_so4: 0.0 - n_aitken: 0.0 - q_h2so4: 1.9186478479542893e-011 # 0.65e-10 is from namelist, but this is what gets to nucleation - pbl_height: 1100.0 - T_mid: 273.0 - p_mid: 1.e5 - qv: 0.0018908932854425809 # computed from relative humidity = 0.5 using Hardy formulae - -# The parameters for I/O control -Scorpio: - output_yaml_files: ["output.yaml"] -... diff --git a/components/eamxx/tests/uncoupled/mam4/output.yaml b/components/eamxx/tests/uncoupled/mam4/output.yaml deleted file mode 100644 index 4e6ad2ac9128..000000000000 --- a/components/eamxx/tests/uncoupled/mam4/output.yaml +++ /dev/null @@ -1,15 +0,0 @@ -%YAML 1.1 ---- -filename_prefix: mam4_nucleation_standalone_output -Averaging Type: Instant -Field Names: - - T_mid - - p_mid - - q_aitken_so4 - - n_aitken - - q_h2so4 -output_control: - Frequency: 1 - frequency_units: nsteps - MPI Ranks in Filename: true -... diff --git a/components/eamxx/tests/uncoupled/rrtmgp/CMakeLists.txt b/components/eamxx/tests/uncoupled/rrtmgp/CMakeLists.txt deleted file mode 100644 index 90a6eed8dcf7..000000000000 --- a/components/eamxx/tests/uncoupled/rrtmgp/CMakeLists.txt +++ /dev/null @@ -1,88 +0,0 @@ -INCLUDE (ScreamUtils) - -# Test atmosphere processes -if (NOT SCREAM_BASELINES_ONLY) - # Ensure test input files are present in the data dir - GetInputFile(scream/init/${EAMxx_tests_IC_FILE_72lev}) - - set (TEST_BASE_NAME rrtmgp_standalone) - set (FIXTURES_BASE_NAME ${TEST_BASE_NAME}_generate_output_nc_files) - - # Unit test to compare against raw rrtmgp output - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/input_unit.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input_unit.yaml) - CreateUnitTest(${TEST_BASE_NAME}_unit rrtmgp_standalone_unit.cpp - LABELS rrtmgp physics driver - LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics rrtmgp_test_utils - EXE_ARGS "--ekat-test-params rrtmgp_inputfile=${SCREAM_DATA_DIR}/init/rrtmgp-allsky.nc,rrtmgp_baseline=${SCREAM_TEST_DATA_DIR}/rrtmgp-allsky-baseline.nc" - ) - # This test needs the allsky baselines file - add_dependencies (rrtmgp_standalone_unit rrtmgp_allsky_baseline.nc) - - ## Create rrtmgp stand alone executable - CreateUnitTestExec(${TEST_BASE_NAME} "rrtmgp_standalone.cpp" - LIBS scream_rrtmgp rrtmgp scream_control yakl diagnostics - ) - - # The RRTMGP stand-alone test that runs multi-step - # Set AD configurable options - SetVarDependingOnTestSize(NUM_STEPS 2 5 48) - set (ATM_TIME_STEP 1800) - set (RUN_T0 2021-10-12-45000) - - # Test non-chunked version (sweep multiple ranks) - set (SUFFIX "_not_chunked") - set (COL_CHUNK_SIZE 1000) - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output_not_chunked.yaml) - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input_not_chunked.yaml) - CreateUnitTestFromExec( - ${TEST_BASE_NAME}_not_chunked ${TEST_BASE_NAME} - LABELS rrtmgp physics driver - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_not_chunked.yaml" - FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_not_chunked - ) - - # Compare output files produced by npX tests, to ensure they are bfb - include (CompareNCFiles) - - CompareNCFilesFamilyMpi ( - TEST_BASE_NAME ${TEST_BASE_NAME}_not_chunked - FILE_META_NAME ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.npMPIRANKS.${RUN_T0}.nc - MPI_RANKS ${TEST_RANK_START} ${TEST_RANK_END} - LABELS rrtmgp physics - META_FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_not_chunked_npMPIRANKS_omp1 - ) - - - ## Test chunked version (only for ${TEST_RANK_END}) and compare against non-chunked - set (SUFFIX "_chunked") - math (EXPR COL_PER_RANK "218 / ${TEST_RANK_END}") - math (EXPR COL_CHUNK_SIZE "${COL_PER_RANK} / 2") - if (COL_CHUNK_SIZE LESS 1) - message (FATAL_ERROR "Error! Chunk size for rrtmgp unit test is less than 1.") - endif() - - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/input.yaml - ${CMAKE_CURRENT_BINARY_DIR}/input_chunked.yaml) - configure_file (${CMAKE_CURRENT_SOURCE_DIR}/output.yaml - ${CMAKE_CURRENT_BINARY_DIR}/output_chunked.yaml) - CreateUnitTestFromExec( - ${TEST_BASE_NAME}_chunked ${TEST_BASE_NAME} - LABELS rrtmgp physics driver - MPI_RANKS ${TEST_RANK_END} - EXE_ARGS "--ekat-test-params inputfile=input_chunked.yaml" - FIXTURES_SETUP_INDIVIDUAL ${FIXTURES_BASE_NAME}_chunked - PROPERTIES PASS_REGULAR_EXPRESSION "(beg.end: 0, ${COL_CHUNK_SIZE})" - ) - - CompareNCFiles( - TEST_NAME ${TEST_BASE_NAME}_chunked_vs_not_chunked - SRC_FILE ${TEST_BASE_NAME}_output_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc - TGT_FILE ${TEST_BASE_NAME}_output_not_chunked.INSTANT.nsteps_x${NUM_STEPS}.np${TEST_RANK_END}.${RUN_T0}.nc - LABELS rrtmgp physics - FIXTURES_REQUIRED ${FIXTURES_BASE_NAME}_chunked_np${TEST_RANK_END}_omp1 - ${FIXTURES_BASE_NAME}_not_chunked_np${TEST_RANK_END}_omp1) -endif() diff --git a/components/eamxx/tpls/CMakeLists.txt b/components/eamxx/tpls/CMakeLists.txt index d89f48d4cacd..ee5a6644b135 100644 --- a/components/eamxx/tpls/CMakeLists.txt +++ b/components/eamxx/tpls/CMakeLists.txt @@ -16,90 +16,3 @@ if (NOT SCREAM_LIB_ONLY) include(BuildCprnc) BuildCprnc() endif() - -# MAM aerosol support -if (SCREAM_ENABLE_MAM) - # We use CMake's ExternalProject capability to build and install Haero. - include(ExternalProject) - - if (SCREAM_CIME_BUILD) - # PROJECT_SOURCE_DIR is SCREAM_ROOT/components - set(EXTERNALS_DIR "${PROJECT_SOURCE_DIR}/../externals") - else() - # PROJECT_SOURCE_DIR is SCREAM_ROOT/components/eamxx - set(EXTERNALS_DIR "${PROJECT_SOURCE_DIR}/../../externals") - endif() - - # Normalize CMAKE_BUILD_TYPE. - string(TOLOWER "${CMAKE_BUILD_TYPE}" lc_build_type) - if(${lc_build_type} STREQUAL "release") - set(mam_build_type "Release") - else() - set(mam_build_type "Debug") # when in doubt... - endif() - - # Build and install the Haero aerosol package interface. - if (SCREAM_DOUBLE_PRECISION) - set(HAERO_PRECISION "double") - else() - set(HAERO_PRECISION "single") - endif() - set(HAERO_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/externals/haero") - set(HAERO_CMAKE_OPTS - -DCMAKE_INSTALL_PREFIX=${HAERO_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${mam_build_type} - -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE} - -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DHAERO_ENABLE_GPU=${Kokkos_ENABLE_CUDA} - -DHAERO_ENABLE_MPI=ON - -DHAERO_PRECISION=${HAERO_PRECISION} - -DEKAT_SOURCE_DIR=${EXTERNALS_DIR}/ekat - -DEKAT_BINARY_DIR=${PROJECT_BINARY_DIR}/externals/ekat - -DBUILD_SHARED_LIBS=OFF) - ExternalProject_Add(haero_proj - PREFIX ${PROJECT_BINARY_DIR}/externals/haero - SOURCE_DIR ${EXTERNALS_DIR}/haero - BINARY_DIR ${PROJECT_BINARY_DIR}/externals/haero - CMAKE_ARGS ${HAERO_CMAKE_OPTS} - DEPENDS ekat ekat_test_session ekat_test_main - LOG_CONFIGURE TRUE - BUILD_COMMAND make -j - LOG_BUILD TRUE - INSTALL_COMMAND make install - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE TRUE - BUILD_ALWAYS TRUE) - add_library(haero STATIC IMPORTED GLOBAL) - set_target_properties(haero PROPERTIES IMPORTED_LOCATION ${HAERO_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libhaero.a) - - # Build and install MAM4xx. - set(MAM4XX_INSTALL_PREFIX "${PROJECT_BINARY_DIR}/externals/mam4xx") - set(MAM4XX_CMAKE_OPTS - -DCMAKE_INSTALL_PREFIX=${MAM4XX_INSTALL_PREFIX} - -DCMAKE_BUILD_TYPE=${mam_build_type} - -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE} - -DMAM4XX_HAERO_DIR=${HAERO_INSTALL_PREFIX} - -DBUILD_SHARED_LIBS=OFF - ) - ExternalProject_Add(mam4xx_proj - PREFIX ${PROJECT_BINARY_DIR}/externals/mam4xx - SOURCE_DIR ${EXTERNALS_DIR}/mam4xx - BINARY_DIR ${PROJECT_BINARY_DIR}/externals/mam4xx - CMAKE_ARGS ${MAM4XX_CMAKE_OPTS} - DEPENDS haero_proj - LOG_CONFIGURE TRUE - BUILD_COMMAND make -j - LOG_BUILD TRUE - INSTALL_COMMAND make install - LOG_INSTALL TRUE - LOG_OUTPUT_ON_FAILURE TRUE - BUILD_ALWAYS TRUE) - add_library(mam4xx STATIC IMPORTED GLOBAL) - set_target_properties(mam4xx PROPERTIES IMPORTED_LOCATION ${MAM4XX_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libmam4xx.a) - # Bring in MAM4xx-related targets by including the generated mam4xx.cmake - # list(APPEND CMAKE_MODULE_PATH ${MAM4XX_INSTALL_PREFIX}/share) - # include(mam4xx) - -endif() diff --git a/components/elm/bld/namelist_files/namelist_defaults.xml b/components/elm/bld/namelist_files/namelist_defaults.xml index 356e5be3cdc0..4536a6c61aeb 100644 --- a/components/elm/bld/namelist_files/namelist_defaults.xml +++ b/components/elm/bld/namelist_files/namelist_defaults.xml @@ -274,10 +274,13 @@ ic_tod="0" sim_year="2000" glc_nec="0" use_crop=".true." irrigate=".true." nu_co lnd/clm2/initdata/20220928.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2016-08-01-00000.nc lnd/clm2/initdata/20221218.F2010-CICE.ne30pg2_ne1024pg2.elm.r.2020-01-20-00000.nc +lnd/clm2/initdata/20231226.I2010CRUELM.ne1024pg2_ICOS10.elm.r.1994-10-01-00000.nc +lnd/clm2/initdata/20240104.I2010CRUELM.ne256pg2.elm.r.1994-10-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-01-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-04-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-07-01-00000.nc lnd/clm2/initdata/20220717.I2010CRUELM.ne1024pg2_ICOS10.elm.r.2013-10-01-00000.nc + lnd/clm2/initdata/ICRUCLM45-360x720cru.clm2.r.2016-08-01-00000.nc lnd/clm2/initdata/ICRUCLM45.r0125_oRRS15to5.clm2.r.2016-08-01-00000.nc lnd/clm2/initdata/ICRUCLM45.r0125_oRRS18to6v3.clm2.r.2016-08-01-00000.nc @@ -299,6 +302,8 @@ lnd/clm2/surfdata_map/surfdata_ne30pg2_simyr2010_c210402_with_TOP.nc lnd/clm2/surfdata_map/surfdata_ne120np4_simyr2010_c20181023.nc lnd/clm2/surfdata_map/surfdata_ne120pg2_simyr2010_c230119.nc + +lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr1850_c240131.nc lnd/clm2/surfdata_map/surfdata_ne256pg2_simyr2010_c230207.nc @@ -508,6 +513,10 @@ lnd/clm2/surfdata_map/surfdata_conusx4v1_simyr2000_c160503.nc use_crop=".false." >lnd/clm2/surfdata_map/surfdata.pftdyn_ne16np4_hist_simyr1850-2005_c160803.nc lnd/clm2/surfdata_map/surfdata.pftdyn_ne11np4_hist_simyr1850-2005_c160803.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne1024pg2_historical_simyr1990-2014_c240109.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne256pg2_hist_simyr1850-2015_c240131.nc lnd/clm2/surfdata_map/landuse.timeseries_0.125x0.125_hist_simyr1850-2015_c191004.nc diff --git a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml index 651a7d523835..c635abebc43f 100644 --- a/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/20thC_CMIP6_transient.xml @@ -45,6 +45,10 @@ lnd/clm2/surfdata_map/surfdata_0.5x0.5_simyr1850_c200609_with_TOP.nc + +lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne1024pg2_historical_simyr1990-2014_c240109.nc + .false. .false. diff --git a/components/elm/bld/namelist_files/use_cases/20thC_transient.xml b/components/elm/bld/namelist_files/use_cases/20thC_transient.xml index 283fb35605d1..86035ec6e081 100644 --- a/components/elm/bld/namelist_files/use_cases/20thC_transient.xml +++ b/components/elm/bld/namelist_files/use_cases/20thC_transient.xml @@ -30,5 +30,10 @@ 2010 1850 + +lnd/clm2/surfdata_map/surfdata_ne1024pg2_simyr2010_c211021.nc +lnd/clm2/surfdata_map/landuse.timeseries_ne1024pg2_historical_simyr1990-2014_c240109.nc +.false. +.false. diff --git a/externals/ekat b/externals/ekat index 2ff5853316e1..69a7b84e3963 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 2ff5853316e15d4e8004c21890329fd257fa7459 +Subproject commit 69a7b84e39639aacb63f45861033d698b1cf085a diff --git a/externals/haero b/externals/haero index 574787f9b31c..77c4a45584f2 160000 --- a/externals/haero +++ b/externals/haero @@ -1 +1 @@ -Subproject commit 574787f9b31cc77b7902f07e73e10dcd18cdf29d +Subproject commit 77c4a45584f2bd3bf38a18ea5bc24ef9ce3776a2 diff --git a/externals/mam4xx b/externals/mam4xx index 8f6af492c862..a19bc75891e2 160000 --- a/externals/mam4xx +++ b/externals/mam4xx @@ -1 +1 @@ -Subproject commit 8f6af492c8627ebe5ec7904c3185ab8294e49a12 +Subproject commit a19bc75891e235071426840742a2c530bbea9e52 diff --git a/share/build/buildlib.kokkos b/share/build/buildlib.kokkos index 1b08afa3dd63..035e86b0df2a 100755 --- a/share/build/buildlib.kokkos +++ b/share/build/buildlib.kokkos @@ -99,7 +99,7 @@ def buildlib(bldroot, installpath, case): gmake_cmd = case.get_value("GMAKE") gmake_j = case.get_value("GMAKE_J") - gen_makefile_cmd = f"cmake {kokkos_options} {cxx} -DCMAKE_INSTALL_PREFIX={installpath} {kokkos_dir}" + gen_makefile_cmd = f"cmake {kokkos_options} {cxx} -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX={installpath} {kokkos_dir}" # When later we use find_package to get kokkos in CMake, the folder # install_sharedpath/kokkos (which is bldroot here) gets picked over