From f581e1e81b6e68a06fc5f43272fbe27a0eea5e1b Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 9 Sep 2018 17:10:01 -0500 Subject: [PATCH 01/41] dcmip2016 test2 TC updates --- .../preqx/jobscript-snl.sh | 65 +++++++++++++++---- .../preqx/namelist-r100.nl | 20 +++--- .../preqx/namelist-r400.nl | 18 ++--- .../preqx/plot-horiz-crossx.ncl | 4 +- 4 files changed, 74 insertions(+), 33 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh index 34d3072c53a4..c5be8aa1b8af 100755 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh @@ -1,24 +1,65 @@ -#!/bin/tcsh +#!/bin/bash # #SBATCH --job-name d16-2-preqx #SBATCH --account=FY150001 #SBATCH -N 12 #SBATCH --time=0:20:00 #SBATCH -p ec +#PBS -q acme +#PBS -l walltime=20:00 +#PBS -l nodes=12 +# +# r100 (ne30) 12 nodes, needs 5 min? +# -set OMP_NUM_THREADS = 1 -set NCPU = 40 -if ( ${?SLURM_NNODES} ) then # redsky - set NCPU = $SLURM_NNODES - @ NCPU *= 16 - @ NCPU /= $OMP_NUM_THREADS -endif -date + +OMP_NUM_THREADS=1 +NCPU=40 +if [ -n "$PBS_ENVIRONMENT" ]; then +# NCPU=$PBS_NNODES + [ "$PBS_ENVIRONMENT" = "PBS_BATCH" ] && cd $PBS_O_WORKDIR + NCPU=$PBS_NNODES +fi +if [ -n "$SLURM_NNODES" ]; then + NCPU=$SLURM_NNODES + let NCPU*=16 + let NCPU/=$OMP_NUM_THREADS +fi # hydrostatic preqx -set EXEC = ../../../test_execs/preqx-nlev30-interp/preqx-nlev30-interp # set name of executable -set namelist = namelist-r400.nl +EXEC=../../../test_execs/preqx-nlev30-interp/preqx-nlev30-interp + + +function run { +local NCPU=$1 +echo "NCPU = $NCPU" +namelist=namelist-$prefix.nl \cp -f $namelist input.nl +date mpirun -np $NCPU $EXEC < input.nl - date + +ncl plot-tropical-cyclone-init.ncl # u,t,th,q,pnh,geo,ps, time=0 +ncl plot-horiz-crossx.ncl # contour plot, time=10d, U,V,T,ps,precl,Q,geo +# time series plots from .txt files: +ncl plot-intensity-trace.ncl +ncl plot-precip.ncl +ncl plot-ps.ncl +ncl plot-wvel.ncl + +# save output +\mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc + +\mv -f init.pdf ${prefix}_init.pdf +#wind.pdf +#ps.pdf +#x-sections.pdf +\mv -f min_ps.pdf ${prefix}_min_ps.pdf +\mv -f max_precip.pdf ${prefix}_max_precip.pdf +\mv -f max_w.pdf ${prefix}_max_w.pdf +} + + +prefix=r400 ; run $(($NCPU>384?384:NCPU)) +prefix=r100 ; run $NCPU +#prefix=r50 ; run $NCPU diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl index 98f35f86d5e9..85d3d0708104 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl @@ -7,21 +7,21 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test2" ! test identifier ne = 30 ! number of elements per cube face - qsize = 4 ! num tracer fields + qsize = 3 ! num tracer fields ndays = 10 - statefreq = 10 ! number of steps between screen dumps + statefreq = 720 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 100 ! largest timestep in seconds + tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration - tstep_type = 7 - rsplit = 0 + tstep_type = 5 + rsplit = 3 qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 nu_p = 1e15 nu_top = 0 ! 2.5e5/(120)^(1) - limiter_option = 8 + limiter_option = 9 hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' @@ -36,13 +36,13 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 3 ! every 3 hours + output_frequency = 24 ! every N hours output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file - interp_type = 0 ! 0=native grid, 1=bilinear + interp_type = 1 ! 0=native grid, 1=bilinear output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 - interp_nlon = 360 - interp_nlat = 181 +! interp_nlon = 360 +! interp_nlat = 181 interp_gridtype = 1 / &prof_inparm diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl index ab0443d6e837..d5d18b653897 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl @@ -7,12 +7,12 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test2" ! test identifier ne = 8 ! number of elements per cube face - qsize = 4 ! num tracer fields + qsize = 3 ! num tracer fields ndays = 10 - statefreq = 10 ! number of steps between screen dumps + statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 400.0 ! largest timestep in seconds + tstep = 900.0 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 rsplit = 0 @@ -20,8 +20,8 @@ nu = 4e16 ! default= 1e15*(ne30/ne8)**3.2 = 6.9e16 nu_s = 4e16 nu_p = 4e16 - nu_top = 2.5e5 - limiter_option = 8 + nu_top = 0 ! 2.5e5 + limiter_option = 9 hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' @@ -36,13 +36,13 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 3 + output_frequency = 24 output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file - interp_type = 0 ! 0=native grid, 1=bilinear + interp_type = 1 ! 0=native grid, 1=bilinear output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 - interp_nlon = 360 - interp_nlat = 181 +! interp_nlon = 360 +! interp_nlat = 181 interp_gridtype = 1 / &prof_inparm diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-crossx.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-crossx.ncl index ec8179efe702..583609759d70 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-crossx.ncl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-crossx.ncl @@ -37,8 +37,8 @@ hasTimeIx=True ; does file have time index? timeStep=nt ; If yes, what time index do you want to plot? model="cam-se" ; used for mainStr, but also for model specific if statements mainStr=model+" Day 8" ; main string for plot titles -;out_type="pdf" ; output format, popular options are x11, png, pdf, eps -out_type="x11" ; output format, popular options are x11, png, pdf, eps +out_type="pdf" ; output format, popular options are x11, png, pdf, eps +;out_type="x11" ; output format, popular options are x11, png, pdf, eps Uname="u" ; Variable name of zonal wind (typically "U" or "ua") Vname="v" ; Variable name of meridional wind (typically "V" or "ua") From 5820da3bf818fefb9ea6d29e5a2260e39e9f5bef Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Thu, 13 Sep 2018 20:15:11 -0500 Subject: [PATCH 02/41] dcmip2016 test2 tweaks --- .../preqx/jobscript-snl.sh | 19 ++++++------------- .../preqx/namelist-r100.nl | 1 + .../preqx/plot-intensity-trace.ncl | 2 +- .../preqx/plot-radial-avgs.ncl | 2 +- .../preqx/plot-tropical-cyclone-init.ncl | 18 +++++++++++------- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh index c5be8aa1b8af..97fb40e47c29 100755 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh @@ -41,25 +41,18 @@ date ncl plot-tropical-cyclone-init.ncl # u,t,th,q,pnh,geo,ps, time=0 ncl plot-horiz-crossx.ncl # contour plot, time=10d, U,V,T,ps,precl,Q,geo -# time series plots from .txt files: -ncl plot-intensity-trace.ncl -ncl plot-precip.ncl -ncl plot-ps.ncl -ncl plot-wvel.ncl +ncl plot-intensity-trace.ncl # save output -\mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc +#\mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc \mv -f init.pdf ${prefix}_init.pdf -#wind.pdf -#ps.pdf -#x-sections.pdf -\mv -f min_ps.pdf ${prefix}_min_ps.pdf -\mv -f max_precip.pdf ${prefix}_max_precip.pdf -\mv -f max_w.pdf ${prefix}_max_w.pdf +\mv -f x-sections.pdf ${prefix}_x-sections.pdf +\mv -f wind.pdf ${prefix}_wind.pdf +\mv -f ps.pdf ${prefix}_ps.pdf } -prefix=r400 ; run $(($NCPU>384?384:NCPU)) +#prefix=r400 ; run $(($NCPU>384?384:NCPU)) prefix=r100 ; run $NCPU #prefix=r50 ; run $NCPU diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl index 85d3d0708104..6776139843ce 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl @@ -41,6 +41,7 @@ interp_type = 1 ! 0=native grid, 1=bilinear output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 + interp_lon0 = -180.0 ! shift lon range to [-180,+180) ! interp_nlon = 360 ! interp_nlat = 181 interp_gridtype = 1 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-intensity-trace.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-intensity-trace.ncl index e16bbc15ddfd..90ef01b8b495 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-intensity-trace.ncl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-intensity-trace.ncl @@ -14,7 +14,7 @@ begin ;======================================================================================= top_to_bottom = True ; is data top to bottom? (false for bottom to top) -model = "ACME-A" +model = "PREQX" ; needs to be a catted file ordered time, lev, lat, lon (or time, lat, lon for PS) filename="./movies/dcmip2016_test21.nc" diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-radial-avgs.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-radial-avgs.ncl index ff9e41c66c14..583c5f4681de 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-radial-avgs.ncl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-radial-avgs.ncl @@ -35,7 +35,7 @@ data_on_constant_Z=False ; is data already on CONSTANT Z surfaces? convert_hybridP_to_Z=False ; is data on hybrid pressure levels? hasTimeIx=True ; does file have time index? timeStep=nt ; If yes, what time index do you want to plot? -model="ACME-A" ; used for mainStr, but also for model specific if statements +model="PREQX" ; used for mainStr, but also for model specific if statements mainStr=model+" Day 8" ; main string for plot titles ;outType="x11" ; output format, popular options are x11, png, pdf, eps outType="pdf" ; output format, popular options are x11, png, pdf, eps diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl index 5d4a51a9a2cf..e68f15c847fc 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl @@ -72,8 +72,12 @@ res@gsnYAxisIrregular2Linear = True geo = datafile->geo(0,::-1,:,:) ps = datafile->ps (0,::-1,: ) + nlon = dimsizes(datafile->lon) + ilon0 = 0 ; location of perturbation + ilon1 = nlon/2 ; +180 degrees + print("max u="+max(u)) - print("max u xsection="+max(u(:,:,0))) + print("max u xsection="+max(u(:,:,ilon0))+" lon="+lon(ilon0)) ;_________________________________________________ ; Interpolate fields from eta coord to z coord @@ -99,7 +103,7 @@ res@gsnYAxisIrregular2Linear = True p = p/100 qv = qv*1000 - pp = p(:,:,180)-p(:,:,0) + pp = p(:,:,ilon0)-p(:,:,ilon1) copy_VarCoords(p(:,:,0), pp) ;print("pp="+pp) @@ -120,7 +124,7 @@ res@gsnYAxisIrregular2Linear = True ; p = rho * 287.0 * t / 100.0 ; copy_VarCoords(rho, p) -; pp = p(0,:,:,0) - p(0,:,:,180) +; pp = p(0,:,:,0) - p(0,:,:,ilon0) ; copy_VarCoords(p(0,:,:,0), pp) ; ps = (p(0,0,:,:) * lev(1) - p(0,1,:,:) * lev(0)) / (lev(1) - lev(0)) ; copy_VarCoords(rho(0,0,:,:), ps) @@ -152,7 +156,7 @@ res@gsnYAxisIrregular2Linear = True res_t@tiYAxisString = "Altitude (km)" res_t@gsnCenterString = "Temperature" res_t@gsnRightString = "(K)" - plot(0) = gsn_csm_contour(wks,t(:,:,180),res_t) + plot(0) = gsn_csm_contour(wks,t(:,:,ilon0),res_t) res_theta@cnLevelSelectionMode= "ExplicitLevels" res_theta@cnLevels = (/250., 260., 270., 280., 290., 300., 310., 320., 330., 340., 350., 360., 380., 400., 450., 500., 550., 600., 650., 700./) @@ -160,7 +164,7 @@ res@gsnYAxisIrregular2Linear = True res_theta@tiYAxisString = "Altitude (km)" res_theta@gsnCenterString = "Potential Temperature" res_theta@gsnRightString = "(K)" - plot(1) = gsn_csm_contour(wks,theta(:,:,180),res_theta) + plot(1) = gsn_csm_contour(wks,theta(:,:,ilon0),res_theta) ; --- @@ -174,7 +178,7 @@ res@gsnYAxisIrregular2Linear = True res_u@tiYAxisString = "Altitude (km)" res_u@gsnCenterString = "Zonal velocity" res_u@gsnRightString = "(m/s)" - plot(2) = gsn_csm_contour(wks,u(:,:,180),res_u) + plot(2) = gsn_csm_contour(wks,u(:,:,ilon0),res_u) res_qv@cnLevelSelectionMode= "ManualLevels" res_qv@cnLevelSpacingF = 2.0 @@ -184,7 +188,7 @@ res@gsnYAxisIrregular2Linear = True res_qv@tiYAxisString = "Altitude (km)" res_qv@gsnCenterString = "Specific Humidity" res_qv@gsnRightString = "(g/kg)" - plot(3) = gsn_csm_contour(wks,qv(:,:,180),res_qv) + plot(3) = gsn_csm_contour(wks,qv(:,:,ilon0),res_qv) ; --- From 12114af736357f9e6dd96d54b395cd9fc9fc9aad Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sat, 15 Sep 2018 18:35:10 -0400 Subject: [PATCH 03/41] dcmip2016 test2 configuration with theta-l --- .../preqx/jobscript-snl.sh | 8 +- .../preqx/namelist-r100.nl | 2 +- .../preqx/namelist-r400.nl | 3 +- .../preqx/namelist-r50.nl | 21 +- .../theta-l/CMakeLists.txt | 10 + .../theta-l/build.sh | 6 + .../theta-l/jobscript-snl.sh | 59 +++ .../theta-l/movies/CMakeLists.txt | 0 .../theta-l/namelist-r100.nl | 52 ++ .../theta-l/namelist-r400.nl | 52 ++ .../theta-l/namelist-r50-h.nl | 53 ++ .../theta-l/namelist-r50.nl | 52 ++ .../theta-l/plot-horiz-crossx.ncl | 276 +++++++++++ .../theta-l/plot-intensity-trace-comp.ncl | 74 +++ .../theta-l/plot-intensity-trace.ncl | 72 +++ .../theta-l/plot-precip.ncl | 113 +++++ .../theta-l/plot-ps.ncl | 113 +++++ .../theta-l/plot-radial-avgs.ncl | 282 +++++++++++ .../theta-l/plot-tropical-cyclone-init.ncl | 237 +++++++++ .../theta-l/plot-wvel.ncl | 118 +++++ .../theta-l/radialAvg.ncl | 466 ++++++++++++++++++ 21 files changed, 2053 insertions(+), 16 deletions(-) create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/CMakeLists.txt create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/build.sh create mode 100755 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/movies/CMakeLists.txt create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-crossx.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace-comp.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-precip.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-ps.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-radial-avgs.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-tropical-cyclone-init.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-wvel.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/radialAvg.ncl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh index 97fb40e47c29..d7f92deb279c 100755 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -i # #SBATCH --job-name d16-2-preqx #SBATCH --account=FY150001 @@ -44,7 +44,7 @@ ncl plot-horiz-crossx.ncl # contour plot, time=10d, U,V,T,ps,precl,Q,geo ncl plot-intensity-trace.ncl # save output -#\mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc +\mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc \mv -f init.pdf ${prefix}_init.pdf \mv -f x-sections.pdf ${prefix}_x-sections.pdf @@ -53,6 +53,6 @@ ncl plot-intensity-trace.ncl } -#prefix=r400 ; run $(($NCPU>384?384:NCPU)) -prefix=r100 ; run $NCPU +prefix=r400 ; run $(($NCPU>384?384:NCPU)) +#prefix=r100 ; run $NCPU #prefix=r50 ; run $NCPU diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl index 6776139843ce..77652d2ba998 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl @@ -9,7 +9,7 @@ ne = 30 ! number of elements per cube face qsize = 3 ! num tracer fields ndays = 10 - statefreq = 720 ! number of steps between screen dumps + statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run tstep = 300 ! largest timestep in seconds diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl index d5d18b653897..1d3fed1e75c9 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl @@ -19,7 +19,7 @@ qsplit = 1 nu = 4e16 ! default= 1e15*(ne30/ne8)**3.2 = 6.9e16 nu_s = 4e16 - nu_p = 4e16 + nu_p = 0 ! 4e16 nu_top = 0 ! 2.5e5 limiter_option = 9 hypervis_order = 2 ! 2 = hyperviscosity @@ -39,6 +39,7 @@ output_frequency = 24 output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file interp_type = 1 ! 0=native grid, 1=bilinear + interp_lon0 = -180.0 ! shift lon range to [-180,+180) output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 ! interp_nlon = 360 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl index 88f6b0cf1df8..c317b499ffbf 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl @@ -7,21 +7,21 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test2" ! test identifier ne = 60 ! number of elements per cube face - qsize = 4 ! num tracer fields + qsize = 3 ! num tracer fields ndays = 10 - statefreq = 10 ! number of steps between screen dumps + statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 50 ! largest timestep in seconds + tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 0 + rsplit = 3 qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 - nu_p = 0 !1.1e14 + nu_p = 1.1e14 nu_top = 0 ! 2.5e5/(120)^(1) - limiter_option = 8 + limiter_option = 9 hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' @@ -36,13 +36,14 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 3 ! every 3 hours + output_frequency = 24 ! every N hours output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file - interp_type = 0 ! 0=native grid, 1=bilinear + interp_type = 1 ! 0=native grid, 1=bilinear + interp_lon0 = -180.0 ! shift lon range to [-180,+180) output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 - interp_nlon = 360 - interp_nlat = 181 +! interp_nlon = 360 +! interp_nlat = 181 interp_gridtype = 1 / &prof_inparm diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/build.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/build.sh new file mode 100644 index 000000000000..057e31dbe225 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. + echo "make theta-l-nlev30" + make -j theta-l-nlev30 +cd $cwd diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh new file mode 100755 index 000000000000..677844cb80e2 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh @@ -0,0 +1,59 @@ +#!/bin/bash -i +# +#SBATCH --job-name d16-2-theta-l +#SBATCH --account=FY150001 +#SBATCH -N 12 +#SBATCH --time=0:20:00 +#SBATCH -p ec +#PBS -q acme +#PBS -l walltime=20:00 +#PBS -l nodes=12 +# +# r100 (ne30) 12 nodes, needs 5 min? +# + + +OMP_NUM_THREADS=1 +NCPU=40 +if [ -n "$PBS_ENVIRONMENT" ]; then +# NCPU=$PBS_NNODES + [ "$PBS_ENVIRONMENT" = "PBS_BATCH" ] && cd $PBS_O_WORKDIR + NCPU=$PBS_NNODES +fi +if [ -n "$SLURM_NNODES" ]; then + NCPU=$SLURM_NNODES + let NCPU*=16 + let NCPU/=$OMP_NUM_THREADS +fi + +# theta model +EXEC=../../../test_execs/theta-l-nlev30/theta-l-nlev30 + + +function run { +local NCPU=$1 +echo "NCPU = $NCPU" +namelist=namelist-$prefix.nl +\cp -f $namelist input.nl +date +mpirun -np $NCPU $EXEC < input.nl +date + +ncl plot-tropical-cyclone-init.ncl # u,t,th,q,pnh,geo,ps, time=0 +ncl plot-horiz-crossx.ncl # contour plot, time=10d, U,V,T,ps,precl,Q,geo +ncl plot-intensity-trace.ncl + +# save output +\mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc + +\mv -f init.pdf ${prefix}_init.pdf +\mv -f x-sections.pdf ${prefix}_x-sections.pdf +\mv -f wind.pdf ${prefix}_wind.pdf +\mv -f ps.pdf ${prefix}_ps.pdf +} + + +prefix=r400 ; run $(($NCPU>384?384:NCPU)) +#prefix=r100 ; run $NCPU +#prefix=r50 ; run $NCPU +#prefix=r50-h ; run $NCPU diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl new file mode 100644 index 000000000000..e732631ca979 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl @@ -0,0 +1,52 @@ +! +! theta: namelist for dcmip2016 test2: tropical cyclone +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test2" ! test identifier + ne = 30 ! number of elements per cube face + qsize = 3 ! num tracer fields + ndays = 10 + statefreq = 72 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 300 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 7 + rsplit = 3 + qsplit = 1 + nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 + nu_s = 1e15 + nu_p = 1e15 + nu_top = 0 ! 2.5e5/(120)^(1) + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + moisture = 'wet' + dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 ! every N hours + output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_lon0 = -180.0 ! shift lon range to [-180,+180) +! interp_nlon = 360 +! interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl new file mode 100644 index 000000000000..73d1b121152d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl @@ -0,0 +1,52 @@ +! +! theta: namelist for dcmip2016 test2: tropical cyclone +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test2" ! test identifier + ne = 8 ! number of elements per cube face + qsize = 3 ! num tracer fields + ndays = 10 + statefreq = 24 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 900.0 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 7 + rsplit = 0 + qsplit = 1 + nu = 4e16 ! default= 1e15*(ne30/ne8)**3.2 = 6.9e16 + nu_s = 4e16 + nu_p = 0 ! 4e16 + nu_top = 0 ! 2.5e5 + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + moisture = 'wet' + dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 + output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + interp_lon0 = -180.0 ! shift lon range to [-180,+180) + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +! interp_nlon = 360 +! interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl new file mode 100644 index 000000000000..46055c501374 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl @@ -0,0 +1,53 @@ +! +! theta: namelist for dcmip2016 test2: tropical cyclone +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test2" ! test identifier + ne = 60 ! number of elements per cube face + qsize = 3 ! num tracer fields + ndays = 10 + statefreq = 144 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 150 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 5 + rsplit = 3 + qsplit = 1 + nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 + nu_s = 1.1e14 + nu_p = 1.1e14 + nu_top = 0 ! 2.5e5/(120)^(1) + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + theta_hydrostatic_mode = .true. + moisture = 'wet' + dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 ! every N hours + output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + interp_lon0 = -180.0 ! shift lon range to [-180,+180) + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +! interp_nlon = 360 +! interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl new file mode 100644 index 000000000000..216657f8f832 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl @@ -0,0 +1,52 @@ +! +! theta: namelist for dcmip2016 test2: tropical cyclone +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test2" ! test identifier + ne = 60 ! number of elements per cube face + qsize = 3 ! num tracer fields + ndays = 10 + statefreq = 144 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 150 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 7 + rsplit = 3 + qsplit = 1 + nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 + nu_s = 1.1e14 + nu_p = 1.1e14 + nu_top = 0 ! 2.5e5/(120)^(1) + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + moisture = 'wet' + dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 ! every N hours + output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + interp_lon0 = -180.0 ! shift lon range to [-180,+180) + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +! interp_nlon = 360 +! interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-crossx.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-crossx.ncl new file mode 100644 index 000000000000..583609759d70 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-crossx.ncl @@ -0,0 +1,276 @@ +;======================================================================================= +; This NCL code calculates radially-averaged tangential and radial wind components +; as well as T anomaly for DCMIP test case #2 (cyclone) +; this code requires the accompanying function set "radialAvg.ncl" +; +; Usage: User should modify "user options" for their particular data set. Currently, +; U, V, T, PS are required as variables. +; If variables are on constant Z surfaces, life is easy. +; +; Grepping for "MODELSPEC" will point to possible areas of the code in need of modification +; for model specific output +; +; Written by Colin Zarzycki (zarzycki@ucar.edu) +; Version 0.1 (6/5/2016) - DCMIP-2016 release +;======================================================================================= + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;======================================================================================= +; User options +;======================================================================================= + +filename="./movies/dcmip2016_test21.nc" +f = addfile(filename,"r") + +time = f->time +nt = dimsizes(time)-1 + +data_on_constant_Z=False ; is data already on CONSTANT Z surfaces? +convert_hybridP_to_Z=False ; is data on hybrid pressure levels? +hasTimeIx=True ; does file have time index? +timeStep=nt ; If yes, what time index do you want to plot? +model="cam-se" ; used for mainStr, but also for model specific if statements +mainStr=model+" Day 8" ; main string for plot titles +out_type="pdf" ; output format, popular options are x11, png, pdf, eps +;out_type="x11" ; output format, popular options are x11, png, pdf, eps + +Uname="u" ; Variable name of zonal wind (typically "U" or "ua") +Vname="v" ; Variable name of meridional wind (typically "V" or "ua") +PSname="ps" ; Variable name of surface pressure (typically "PS","PSL","ps",or "slp") +Tname="T" ; Variable name of air temperature (typically "T" or "ta") +Zname="z" ; Variable name of height (typically "Z" or "za") +PRECTname="precl" ; Variable name of height (typically "Z" or "za") + +;======================================================================================= + +;======================================================================================= +; Other settings, required to remain constant for intercomparison +; Generally, don't touch unless experimenting/debugging +;======================================================================================= +stride=1 ; subset in horizontal, leave @ 1 unless debugging +minLat=0. ; max lat to read in (deg) +maxLat=80. ; min lat to read in (deg) +thetaWindMax=60.0 ; max theta wind to plot (m/s) +radMinMax=18.0 ; min/max for radial plot (m/s) +tAnomMax=15.0 ; max for tAnom plots (K) +offAnomDist=5.0 ; offset distance for anom calcs (deg) +zintmin=20.0 ; min height for z interpolation (m) +zintmax=18000.0 ; max height for z interpolation (m) +nzint=100 ; num points for z interpolation +Zmidpt=2500.0 ; half of Z points BELOW this height, half ABOVE +;======================================================================================= + +;======================================================================================= +; Input data checks +;======================================================================================= +if (data_on_constant_Z .and. convert_hybridP_to_Z) then + print("Both data_on_constant_Z and convert_hybridP_to_Z cannot be true, exiting...") + exit +end if + +;======================================================================================= +; Get file, coordinate variables, and U, V, PSL +;======================================================================================= +print("Loading data from file...") + +lat = f->lat({minLat:maxLat:stride}) +lon = f->lon(::stride) +lev = f->lev(:) +nlat = dimsizes(lat) +nlon = dimsizes(lon) +nlev = dimsizes(lev) + +if (hasTimeIx) then + U = f->$Uname$(timeStep,:,{minLat:maxLat:stride},::stride) + V = f->$Vname$(timeStep,:,{minLat:maxLat:stride},::stride) + T = f->$Tname$(timeStep,:,{minLat:maxLat:stride},::stride) + PS = f->$PSname$(timeStep,{minLat:maxLat:stride},::stride) + PRECT = f->$PRECTname$(timeStep,{minLat:maxLat:stride},::stride) +else + U = f->$Uname$(:,{minLat:maxLat:stride},::stride) + V = f->$Vname$(:,{minLat:maxLat:stride},::stride) + T = f->$Tname$(:,{minLat:maxLat:stride},::stride) + PS = f->$PSname$({minLat:maxLat:stride},::stride) + PRECT = f->$PRECTname$({minLat:maxLat:stride},::stride) +end if +; If U and V are not m/s, please convert here +U@units="m/s" +V@units="m/s" +T@units="K" +T@long_name="Temperature" +;======================================================================================= + +;======================================================================================= +; Find center of storm by minimizing PS +;======================================================================================= +print("Finding minimum PS location...") +a = new((/nlat,nlon/),typeof(PS)) +a(:,:) = PS(:,:) +a1D = ndtooned(a) +dsizes_a = dimsizes(a) +a_indices = ind_resolve(minind(a1D),dsizes_a) ; Resolve 1D indices to original array +psminlat = lat(a_indices(0,0)) +psminlon = lon(a_indices(0,1)) +delete([/a,a1D,dsizes_a,a_indices/]) +print("... PS minimum found at lat: "+psminlat+" lon: "+psminlon) +;======================================================================================= + + +;======================================================================================= +; Calculate temperature anomaly +;======================================================================================= +Tanom = T +Tref = T(:,{psminlat+offAnomDist},{psminlon+offAnomDist}) +Tanom = T - conform(T,Tref,0) +Tanom@long_name="Temperature anomaly" +;======================================================================================= + + +;======================================================================================= +; Do Z calculations/interpolations if necessary +;======================================================================================= +if (.not. data_on_constant_Z) then + ;======================================================================================= + ; Convert from hybrid levels to Z (CAM) + ;======================================================================================= + if (convert_hybridP_to_Z) + print("Converting from hybrid P to Z...") + ; If need_to_convert_P_to_Z is true, variables T,Q,hyam,hybm,hyai,hybm,P0 needed + ; all hybrid coefs and TV need to be TOP TO BOTTOM! + + hyam=f->hyam + hybm=f->hybm + hyai=f->hyai + hybi=f->hybi +; P0=f->P0 + P0=100000 + Tconv = f->T(timeStep,:,{minLat:maxLat:stride},::stride) + Qconv = f->Q(timeStep,:,{minLat:maxLat:stride},::stride) + + ; Calculate TV from T (K) and Q (kg/kg) + TV=Tconv*(1.+0.61*Qconv) + + ; PHIS is nlatxnlon array = 0.0 for DCMIP Test 2 + PHIS=PS + PHIS=0.0 + + Z = cz2ccm(PS,PHIS,TV,P0,hyam(::-1),hybm(::-1),hyai(::-1),hybi(::-1)) + Z@units="m" + + delete([/Tconv,Qconv,TV,PHIS/]) + + else + Z = f->geo(timeStep,:,{minLat:maxLat:stride},::stride)/9.81806 + end if + + + ;======================================================================================= + ; If all else fails, try to load Z directly from file + ;======================================================================================= + if(.not. isdefined("Z")) then + if (isfilevar(f, Zname)) then + print("Found Z on file...") + Z = f->Z + else + print("FATAL: Z needs to be either loaded from a file or otherwise defined before continuing...") + exit + end if + end if + ;======================================================================================= + +end if + +;======================================================================================= +; Generate Z levels of interest +;======================================================================================= +print("Generating constant Z levels to interpolate to") +ZlevPBL = fspan(zintmin,Zmidpt,nzint/2) +ZlevABL = fspan(Zmidpt,zintmax,nzint/2) + +Zlev=(/100.,1000.,1500.,2500.,5000.,10000.,15000./) +Zlev@units = "m" +Zlev!0 = "lev" +Zlev&lev = Zlev + +;======================================================================================= +; Interpolate lat/lon variables to constant Z levels +;======================================================================================= +print("Interpolating to Z surfaces") +U_Z = int2p_n_Wrap(Z,U,Zlev,2,0) +V_Z = int2p_n_Wrap(Z,V,Zlev,2,0) +Tanom_Z = int2p_n_Wrap(Z,Tanom,Zlev,2,0) +;======================================================================================= + +WIND_Z = U_Z +WIND_Z = sqrt(U_Z^2+V_Z^2) +WIND_Z@long_name="Horizontal wind" + +PRECT = PRECT*8.64e7 ; convert m/s to mm/day +PRECT@long_name="Precipitation rate" +PRECT@units="mm/d" + +print("Plotting...") + +wks = gsn_open_wks (out_type,"x-sections") ; send graphics to PNG file +contour = new(4,"graphic") + +gsn_define_colormap(wks,"BlAqGrYeOrReVi200") + +res = True +res@gsnDraw = False +res@gsnFrame = False +res@gsnSpreadColors = True ; Span full color map +res@cnFillOn = True ; Turn on contour fill +res@cnLinesOn = False +res@cnLineLabelsOn = False +res@cnInfoLabelOn = False +res@gsnAddCyclic = True + +latWidth = 10.0 +lonWidth = 10.0 +res@mpOutlineOn = False +res@mpMaxLatF = psminlat + latWidth +res@mpMinLatF = psminlat - latWidth +res@mpMinLonF = psminlon - lonWidth +res@mpMaxLonF = psminlon + lonWidth + +res@cnLevelSelectionMode = "ManualLevels" +res@cnLevelSpacingF = 5.0 +res@cnMinLevelValF = 10.0 +res@cnMaxLevelValF = 80.0 +res@tiMainString="1500m Horiz. Wind" +contour(0) = gsn_csm_contour_map(wks,WIND_Z({1500.0},:,:),res) ; create the plot + +res@cnLevelSelectionMode = "ManualLevels" +res@cnLevelSpacingF = 10.0 +res@cnMinLevelValF = -60.0 +res@cnMaxLevelValF = 60.0 +res@tiMainString="100m U-Wind" +contour(1) = gsn_csm_contour_map(wks,U_Z({100.0},:,:),res) ; create the plot + +res@cnLevelSelectionMode = "ManualLevels" +res@cnLevelSpacingF = 1.0 +res@cnMinLevelValF = 0.0 +res@cnMaxLevelValF = 10.0 +res@tiMainString="5000m T anom." +contour(2) = gsn_csm_contour_map(wks,Tanom_Z({5000.0},:,:),res) ; create the plot + +res@cnLevelSelectionMode = "ManualLevels" +res@cnLevelSpacingF = 200.0 +res@cnMinLevelValF = 200.0 +res@cnMaxLevelValF = 4000.0 +res@tiMainString="Precip. rate" +contour(3) = gsn_csm_contour_map(wks,PRECT(:,:),res) ; create the plot + +resP = True ; modify the panel plot +resP@gsnMaximize = True + +gsn_panel(wks,contour,(/2,2/),resP) ; now draw as one plot + +end diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace-comp.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace-comp.ncl new file mode 100644 index 000000000000..5427fc5b1d58 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace-comp.ncl @@ -0,0 +1,74 @@ +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +begin + +whichVar="PS" ; PS or WIND + +;configs=(/"reed_bryan","reed_reed","kessler_bryan","kessler_reed"/) +configs=(/"kessler_bryan","kessler_bryan2"/) +do ii = 0,dimsizes(configs)-1 + +all_files = systemfunc ("ls /glade/u/home/zarzycki/scratch/cam5_work/dcmip2/"+configs(ii)+"/dcmip2*.nc") +a = addfiles (all_files, "r") ; note the "s" of addfile + +;======================== +; choose how files are combined and read in variable across files +;======================== +ListSetType (a, "cat") ; concatenate or "merge" (default) + +if (whichVar .eq. "PS") then + PS=a[:]->PS(:,{0.0:90.0},:) ; just take out NH since we know storm is there + PS=PS/100. ; convert from Pa to hPa/mb + var_vtime = dim_min_n_Wrap(PS,(/1,2/)) ; for now, just find min over lat/lon domain at each time + +else if (whichVar .eq. "WIND") then + UBOT=a[:]->U(:,dimsizes(a->lev)-1,{0.0:90.0},:) ; dimsizes(a->lev)-1 subsets bottom level (CAM ordered top to bottom) + VBOT=a[:]->V(:,dimsizes(a->lev)-1,{0.0:90.0},:) + WIND=UBOT + WIND=sqrt(UBOT^2+VBOT^2) + WIND@long_name="Horizontal wind magnitude" + var_vtime = dim_max_n_Wrap(WIND,(/1,2/)) + +else + print("unsupported var choice... exiting...") + exit +end if +end if + +if (ii .eq. 0) + var_vtime_all=new((/dimsizes(configs),dimsizes(var_vtime)/),typeof(var_vtime)) + time = var_vtime&time +end if + +var_vtime_all(ii,:)=var_vtime + +delete(var_vtime) +delete(a) +delete(all_files) + +end do + +wks = gsn_open_wks ("x11","xy") ; send graphics to PNG file + +res = True ; plot mods desired +res@gsnDraw = False +res@gsnFrame = False +res@tiMainString = "DCMIP2 CAM-SE "+whichVar+" v. time" ; add title + +; add a legend + res@pmLegendDisplayMode = "Always" ; turn on legend + res@pmLegendSide = "Top" ; Change location of + res@pmLegendParallelPosF = .90 ; move units right + res@pmLegendOrthogonalPosF = -0.5 ; more neg = down + res@pmLegendWidthF = 0.12 ; Change width and + res@pmLegendHeightF = 0.25 ; height of legend. + res@lgLabelFontHeightF = .02 ; change font height + res@lgPerimOn = False ; no box around +; labels for the legend + res@xyExplicitLegendLabels = configs + +plot = gsn_csm_xy (wks,time,var_vtime_all,res) ; create plot +draw(plot) +frame(wks) + +end diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace.ncl new file mode 100644 index 000000000000..90ef01b8b495 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-intensity-trace.ncl @@ -0,0 +1,72 @@ +;======================================================================================= +; This NCL +; +; Written by Colin Zarzycki (zarzycki@ucar.edu) +; Version 0.1 (6/5/2016) - DCMIP-2016 release +;======================================================================================= + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" + +begin + +;======================================================================================= + +top_to_bottom = True ; is data top to bottom? (false for bottom to top) +model = "PREQX" + +; needs to be a catted file ordered time, lev, lat, lon (or time, lat, lon for PS) +filename="./movies/dcmip2016_test21.nc" + +;======================================================================================= + +maxLat=90.0 +minLat=0.0 + +;======================================================================================= + +a = addfile(filename,"r") +sfcLev=dimsizes(a->lev)-1 ; dimsizes(a->lev)-1 subsets bottom level +PS = a->ps(:,{minLat:maxLat},:) ; just take out NH since we know storm is there +PS = PS/100. ; convert from Pa to hPa/mb +PS@long_name="Surface pressure" +PS@units="hPa" +var_vtime = dim_min_n_Wrap(PS,(/1,2/)) ; for now, just find min over lat/lon domain at each time + +wks = gsn_open_wks ("pdf","ps") ; send graphics to PNG file + +res = True ; plot mods desired +res@gsnDraw = False +res@gsnFrame = False +res@tiMainString = "DCMIP2 "+model+" PS v. time" ; add title +res@tiXAxisString = "Time (days)" +res@tiYAxisString = var_vtime@long_name+" ("+var_vtime@units+")" +res@xyLineThicknessF = 5.0 +res@xyLineColor ="Red" + +plot = gsn_csm_xy (wks,var_vtime&time,var_vtime,res) ; create plot +draw(plot) +frame(wks) + +UBOT=a->u(:,sfcLev,{minLat:maxLat},:) +VBOT=a->v(:,sfcLev,{minLat:maxLat},:) +WIND=UBOT +WIND=sqrt(UBOT^2+VBOT^2) +WIND@long_name="Lowest level wind" +WIND@units="m/s" +var_vtime = dim_max_n_Wrap(WIND,(/1,2/)) + +wks = gsn_open_wks ("pdf","wind") ; send graphics to PNG file + +res@tiMainString = "DCMIP2 "+model+" WIND vs time" ; add title +res@tiXAxisString = "Time (days)" +res@tiYAxisString = var_vtime@long_name+" ("+var_vtime@units+")" +res@xyLineThicknessF = 5.0 +res@xyLineColor ="Blue" + +plot = gsn_csm_xy (wks,var_vtime&time,var_vtime,res) ; create plot +draw(plot) +frame(wks) + +end diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-precip.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-precip.ncl new file mode 100644 index 000000000000..062340555b62 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-precip.ncl @@ -0,0 +1,113 @@ +;*********************************************** +; plot_supercell_prect.ncl +; +; Version 1.0 +;*********************************************** + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;**************************************************************************** +; DO NOT EDIT ABOVE THIS LINE +;**************************************************************************** +; Read in data and store as follows: +; +; Output: +; prect_rate(time) +; time series precipitation rate (units m/s) +; +; time_s(time) +; time of each sample (units s) +; +; Dimensions: +; time: arbitrary +;---------------------------------------------------------------------------- + +precip = asciiread("./measurement_prect_rate.txt", -1, "double") +time_s = asciiread("./measurement_time.txt", -1, "double") + +; convert from m/s to mm/day +precip = precip*(1000.0)*(24.0*3600) + +;**************************************************************************** +; DO NOT EDIT BELOW THIS LINE +;**************************************************************************** + +;------------------------------------------------ +; Workspace +;------------------------------------------------ + wks = gsn_open_wks("pdf","max_precip") + +;------------------------------------------------ +; Plot resources [options] +;------------------------------------------------ +; Resources: + res = True + + res@gsnDraw = False + res@gsnFrame = False + + res@vpKeepAspect = True + res@vpWidthF = 0.6 + res@vpHeightF = 0.4 + res@gsnMaximize = True + res@gsnPaperOrientation = "portrait" + + res@tmXBLabelsOn = True + res@tmXBOn = True + + res@tmXTLabelsOn = False + res@tmXTOn = False + + res@tmXBMinorOn = True + res@tmYLMinorOn = True + + res@xyLineThicknessF = 3.0 +;res@xyMarkLineMode = "MarkLines" + + resw = res + +;------------------------------------------------ +; Vertical velocity plot +;------------------------------------------------ + + resw@tiXAxisOn = True + resw@tiXAxisString = "time (days)" + + resw@tiYAxisOn = True + resw@tiYAxisString = "max precipitation rate (mm/day)" + +; resw@trXMinF = 0.0 +; resw@trXMaxF = 7200.0 + +; resw@trYMinF = 0.0 +; resw@trYMaxF = 7000.0 + +; resw@tmXBMode = "Manual" +; resw@tmXBTickStartF = 0.0 +; resw@tmXBTickEndF = 7200.0 +; resw@tmXBTickSpacingF = 1800.0 + +; resw@tmYLMode = "Explicit" +; resw@tmYLTickStartF = 0.0 +; resw@tmYLTickEndF = 50.0 + + plot = gsn_csm_xy(wks, time_s/3600.0/24, precip, resw) + +;------------------------------------------------ +; Panel plot +;------------------------------------------------ + + pres = True + pres@gsnPanelBottom = 0.1 + pres@gsnPanelTop = 0.9 + pres@gsnPanelYWhiteSpacePercent = 5 + pres@gsnPanelXWhiteSpacePercent = 5 + gsn_panel(wks, plot, (/1,1/), pres) + +end + diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-ps.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-ps.ncl new file mode 100644 index 000000000000..47846021465d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-ps.ncl @@ -0,0 +1,113 @@ +;*********************************************** +; plot_supercell_wvel.ncl +; +; Version 1.0 +;*********************************************** + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;**************************************************************************** +; DO NOT EDIT ABOVE THIS LINE +;**************************************************************************** +; Read in data and store as follows: +; +; Output: +; maxw(time) +; time series maximum vertical velocity (units m/s) +; +; time_s(time) +; time of each sample (units s) +; +; Dimensions: +; time: arbitrary +;---------------------------------------------------------------------------- + + min_ps = asciiread("./measurement_psmin.txt", -1, "double") + time_s = asciiread("./measurement_time.txt", -1, "double") + +print("min_ps ="+min_ps) +print("time_s="+time_s) + +;**************************************************************************** +; DO NOT EDIT BELOW THIS LINE +;**************************************************************************** + +;------------------------------------------------ +; Workspace +;------------------------------------------------ + wks = gsn_open_wks("pdf","min_ps") + +;------------------------------------------------ +; Plot resources [options] +;------------------------------------------------ +; Resources: + res = True + + res@gsnDraw = False + res@gsnFrame = False + + res@vpKeepAspect = True + res@vpWidthF = 0.8 + res@vpHeightF = 0.6 + res@gsnMaximize = True + res@gsnPaperOrientation = "portrait" + + res@tmXBLabelsOn = True + res@tmXBOn = True + + res@tmXTLabelsOn = False + res@tmXTOn = False + + res@tmXBMinorOn = True + res@tmYLMinorOn = True + + res@xyLineThicknessF = 2.0 +; res@xyMarkLineMode = "MarkLines" + + resw = res + +;------------------------------------------------ +; Vertical velocity plot +;------------------------------------------------ + + resw@tiXAxisOn = True + resw@tiXAxisString = "time (days)" + + resw@tiYAxisOn = True + resw@tiYAxisString = "minimum surface pressure (hPa)" + +; resw@trXMinF = 0.0 +; resw@trXMaxF = 120.0 ;7200.0 ; max(time_s) + +; resw@trYMinF = 0.0 +; resw@trYMaxF = ymax + +; resw@tmXBMode = "Manual" +; resw@tmXBTickStartF = 0.0 +; resw@tmXBTickEndF = 120 ;7200.0 +; resw@tmXBTickSpacingF = 20 ;1800.0 + +; resw@tmYLMode = "Explicit" +; resw@tmYLTickStartF = 0.0 +; resw@tmYLTickEndF = 50.0 + + plot = gsn_csm_xy(wks, time_s/3660.0/24, min_ps/100.0, resw) + +;------------------------------------------------ +; Panel plot +;------------------------------------------------ + + pres = True + pres@gsnPanelBottom = 0.1 + pres@gsnPanelTop = 0.9 + pres@gsnPanelYWhiteSpacePercent = 5 + pres@gsnPanelXWhiteSpacePercent = 5 + gsn_panel(wks, plot, (/1,1/), pres) + +end + diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-radial-avgs.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-radial-avgs.ncl new file mode 100644 index 000000000000..583c5f4681de --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-radial-avgs.ncl @@ -0,0 +1,282 @@ +;======================================================================================= +; This NCL code calculates radially-averaged tangential and radial wind components +; as well as T anomaly for DCMIP test case #2 (cyclone) +; this code requires the accompanying function set "radialAvg.ncl" +; +; Usage: User should modify "user options" for their particular data set. Currently, +; U, V, T, PS are required as variables. +; If variables are on constant Z surfaces, life is easy. +; +; Grepping for "MODELSPEC" will point to possible areas of the code in need of modification +; for model specific output +; +; Written by Colin Zarzycki (zarzycki@ucar.edu) +; Version 0.1 (6/5/2016) - DCMIP-2016 release +;======================================================================================= + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" +load "radialAvg.ncl" + +begin + +;======================================================================================= +; User options +;======================================================================================= +filename="./movies/dcmip2016_test21.nc" +f = addfile(filename,"r") + +time = f->time +nt = dimsizes(time)-1 + +data_on_constant_Z=False ; is data already on CONSTANT Z surfaces? +convert_hybridP_to_Z=False ; is data on hybrid pressure levels? +hasTimeIx=True ; does file have time index? +timeStep=nt ; If yes, what time index do you want to plot? +model="PREQX" ; used for mainStr, but also for model specific if statements +mainStr=model+" Day 8" ; main string for plot titles +;outType="x11" ; output format, popular options are x11, png, pdf, eps +outType="pdf" ; output format, popular options are x11, png, pdf, eps + +Uname="u" ; Variable name of zonal wind (typically "U" or "ua") +Vname="v" ; Variable name of meridional wind (typically "V" or "ua") +PSname="ps" ; Variable name of surface pressure (typically "PS","PSL","ps",or "slp") +Tname="T" ; Variable name of air temperature (typically "T" or "ta") +Zname="z" ; Variable name of height (typically "Z" or "za") +;======================================================================================= + +;======================================================================================= +; Other settings, required to remain constant for intercomparison +; Generally, don't touch unless experimenting/debugging +;======================================================================================= +stride=1 ; subset in horizontal, leave @ 1 unless debugging +minLat=0. ; max lat to read in (deg) +maxLat=80. ; min lat to read in (deg) +thetaWindMax=60.0 ; max theta wind to plot (m/s) +radMinMax=18.0 ; min/max for radial plot (m/s) +tAnomMax=15.0 ; max for tAnom plots (K) +offAnomDist=5.0 ; offset distance for anom calcs (deg) +zintmin=20.0 ; min height for z interpolation (m) +zintmax=18000.0 ; max height for z interpolation (m) +nzint=100 ; num points for z interpolation +Zmidpt=2500.0 ; half of Z points BELOW this height, half ABOVE +;======================================================================================= + +;======================================================================================= +; Input data checks +;======================================================================================= +if (data_on_constant_Z .and. convert_hybridP_to_Z) then + print("Both data_on_constant_Z and convert_hybridP_to_Z cannot be true, exiting...") + exit +end if + +;======================================================================================= +; Get file, coordinate variables, and U, V, PSL +;======================================================================================= +print("Loading data from file...") + + + +lat = f->lat({minLat:maxLat:stride}) +lon = f->lon(::stride) +lev = f->lev(:) +nlat = dimsizes(lat) +nlon = dimsizes(lon) +nlev = dimsizes(lev) + +if (hasTimeIx) then + U = f->$Uname$(timeStep,:,{minLat:maxLat:stride},::stride) + V = f->$Vname$(timeStep,:,{minLat:maxLat:stride},::stride) + T = f->$Tname$(timeStep,:,{minLat:maxLat:stride},::stride) + PS = f->$PSname$(timeStep,{minLat:maxLat:stride},::stride) +else + U = f->$Uname$(:,{minLat:maxLat:stride},::stride) + V = f->$Vname$(:,{minLat:maxLat:stride},::stride) + T = f->$Tname$(:,{minLat:maxLat:stride},::stride) + PS = f->$PSname$({minLat:maxLat:stride},::stride) +end if +; If U and V are not m/s, please convert here +U@units="m/s" +V@units="m/s" +T@units="K" +T@long_name="Temperature" +;======================================================================================= + +;======================================================================================= +; Find center of storm by minimizing PS +;======================================================================================= +print("Finding minimum PS location...") +a = new((/nlat,nlon/),typeof(PS)) +a(:,:) = PS(:,:) +a1D = ndtooned(a) +dsizes_a = dimsizes(a) +a_indices = ind_resolve(minind(a1D),dsizes_a) ; Resolve 1D indices to original array +psminlat = lat(a_indices(0,0)) +psminlon = lon(a_indices(0,1)) +delete([/a,a1D,dsizes_a,a_indices/]) +print("... PS minimum found at lat: "+psminlat+" lon: "+psminlon) +;======================================================================================= + + +;======================================================================================= +; Calculate temperature anomaly +;======================================================================================= +Tanom = T +Tref = T(:,{psminlat+offAnomDist},{psminlon+offAnomDist}) +Tanom = T - conform(T,Tref,0) +Tanom@long_name="Temperature anomaly" +;======================================================================================= + + +;======================================================================================= +; Do Z calculations/interpolations if necessary +;======================================================================================= +if (.not. data_on_constant_Z) then + ;======================================================================================= + ; Convert from hybrid levels to Z (CAM) + ;======================================================================================= + if (convert_hybridP_to_Z) + print("Converting from hybrid P to Z...") + ; If need_to_convert_P_to_Z is true, variables T,Q,hyam,hybm,hyai,hybm,P0 needed + ; all hybrid coefs and TV need to be TOP TO BOTTOM! + + hyam=f->hyam + hybm=f->hybm + hyai=f->hyai + hybi=f->hybi + ; P0=f->P0 + P0=100000 + Tconv = f->T(timeStep,:,{minLat:maxLat:stride},::stride) + Qconv = f->Q(timeStep,:,{minLat:maxLat:stride},::stride) + + ; Calculate TV from T (K) and Q (kg/kg) + TV=Tconv*(1.+0.61*Qconv) + + ; PHIS is nlatxnlon array = 0.0 for DCMIP Test 2 + PHIS=PS + PHIS=0.0 + + Z = cz2ccm(PS,PHIS,TV,P0,hyam(::-1),hybm(::-1),hyai(::-1),hybi(::-1)) + Z@units="m" + + delete([/Tconv,Qconv,TV,PHIS/]) + end if + + Z = f->geo(timeStep,:,{minLat:maxLat:stride},::stride)/9.81806 + + ;======================================================================================= + ; Generate uniform Z levs to interpolate to + ; Extra density (half of pts) is specified in PBL, controlled by Zmidpt + ;======================================================================================= + print("Generating constant Z levels to interpolate to") + ZlevPBL = fspan(zintmin,Zmidpt,nzint/2) + ZlevABL = fspan(Zmidpt,zintmax,nzint/2) + + Zlev=new(nzint-1,"float") + Zlev(0:(nzint/2)-1)=ZlevPBL + Zlev((nzint/2)-1:nzint-2)=ZlevABL + Zlev@units = "m" + Zlev!0 = "lev" + Zlev&lev = Zlev + delete([/ZlevPBL,ZlevABL/]) + print(Zlev) + + ;======================================================================================= + + ;======================================================================================= + ; Interpolate lat/lon variables to constant Z levels + ;======================================================================================= + print("Interpolating from Z model levels to constant Z surfaces") + U_Z = int2p_n_Wrap(Z,U,Zlev,2,0) + V_Z = int2p_n_Wrap(Z,V,Zlev,2,0) + Tanom_Z = int2p_n_Wrap(Z,Tanom,Zlev,2,0) + ;======================================================================================= +else + print("Data already on Z surfaces, copying to new vars") + U_Z = U + V_Z = V + Tanom_Z = Tanom +end if + +;======================================================================================= +; Calculate radial and tangential wind components on native grid +;======================================================================================= +print("Separating winds into radial and tangential components given PS min loc...") +vComps = calcWindComponents(U_Z,V_Z,lat,lon,psminlat,psminlon) +v_rad = vComps[0] +v_theta = vComps[1] +delete(vComps) +;======================================================================================= + + +;======================================================================================= +; Perform radial integrations +;======================================================================================= +print("Do radial integrations...") +rad_v_theta = radialAvg3D(v_theta,lat,lon,v_theta&lev,psminlat,psminlon,600.,True) +rad_v_rad = radialAvg3D(v_rad ,lat,lon,v_rad&lev ,psminlat,psminlon,600.,True) +rad_t_anom = radialAvg3D(Tanom_Z,lat,lon,Tanom_Z&lev ,psminlat,psminlon,600.,True) +;======================================================================================= + + + +print("Plotting...") + +res = True +res@gsnDraw = False +res@gsnFrame = False +res@gsnSpreadColors = True ; Span full color map +res@cnFillOn = True ; Turn on contour fill +res@cnLinesOn = False +res@cnLineLabelsOn = False +res@cnInfoLabelOn = False +res@tiYAxisString = "Height (m)" +res@gsnYAxisIrregular2Linear = True +res@gsnXAxisIrregular2Linear = True + +wks = gsn_open_wks (outType,"t_anom_"+model) +gsn_define_colormap(wks,"matlab_jet") +res_rad = res +res_rad@cnLevelSelectionMode = "ExplicitLevels" +res_rad@cnLevels = ispan(0,toint(tAnomMax),1) +plot = gsn_csm_contour(wks,rad_t_anom,res_rad) +draw(plot) +frame(wks) + +delete(plot) +delete(wks) +delete(res_rad) + +wks = gsn_open_wks (outType,"v_theta_"+model) +gsn_define_colormap(wks,"BlAqGrYeOrReVi200") +res_rad = res +res_rad@tiMainString = mainStr+" tangential wind" +res_rad@cnLevelSelectionMode = "ExplicitLevels" +res_rad@cnLevels = fspan(0.0,thetaWindMax,21) +plot = gsn_csm_contour(wks,rad_v_theta,res_rad) +draw(plot) +frame(wks) + +delete(plot) +delete(wks) +delete(res_rad) + +wks = gsn_open_wks (outType,"v_rad_"+model) +gsn_define_colormap(wks,"hotcolr_19lev") +res_rad = res +res_rad@tiMainString = mainStr+" radial wind" +res_rad@cnLevelSelectionMode = "ExplicitLevels" +res_rad@cnLevels = ispan(toint(-radMinMax),toint(radMinMax),1) +plot = gsn_csm_contour(wks,rad_v_rad,res_rad) +draw(plot) +frame(wks) + +delete(plot) +delete(wks) +delete(res_rad) + + + +end diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-tropical-cyclone-init.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-tropical-cyclone-init.ncl new file mode 100644 index 000000000000..e68f15c847fc --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-tropical-cyclone-init.ncl @@ -0,0 +1,237 @@ +;*********************************************** +; plot_tropicalcyclone_init.ncl +;*********************************************** +; https://github.com/ClimateGlobalChange/DCMIP2016/ + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +;************************************************ +begin + +;************************************************ +; Input parameters +;************************************************ + +datafilename = "./movies/dcmip2016_test21.nc" + +;************************************************ +; Initialize workspace +;************************************************ + + wks = gsn_open_wks("pdf","init.pdf") + +;************************************************ +; Plot resources [options] +;************************************************ + + res = True + ;res@mpCenterLonF = 180 + res@cnFillOn = True ; color [default=False] + res@cnLinesOn = True ; draw contour lines [default=True] + res@lbLabelAutoStride = True + res@vpKeepAspect = True + res@vpWidthF = 0.38 ; user specified shape + res@vpHeightF = 0.19 + res@gsnSpreadColors = True ; use all colors + res@gsnSpreadColorStart = 2 ; default=2 + res@gsnSpreadColorEnd = 23 ; final color of "gui-default" +; res@gsnSpreadColorStart = 2 ; default=2 +; res@gsnSpreadColorEnd = 11 ; final color of "cosam" + + res@gsnDraw = False ; default=True + res@gsnFrame = False ; default=True +res@gsnYAxisIrregular2Linear = True + + res_p = res ; pressure plot resources + res_ps = res ; surface pressure plot resources + res_u = res ; zonal velocity plot resources + res_pp = res ; zonal velocity pert plot resources + res_t = res ; temperature plot resources + res_qv = res ; water vapor plot resources + res_theta = res ; potential temperature plot resources + +;************************************************ +; Panel plot +;************************************************ + plot = new(6,graphic) + gsn_merge_colormaps(wks,"gui_default", "BlRe") +; gsn_merge_colormaps(wks,"cosam", "BlRe") + + datafile = addfile(datafilename, "r") + + lev = datafile->lev + lon = datafile->lon + u = datafile->u (0,::-1,:,:) + t = datafile->T (0,::-1,:,:) + theta = datafile->Th (0,::-1,:,:) + qv = datafile->Q (0,::-1,:,:) + p = datafile->pnh(0,::-1,:,:) + geo = datafile->geo(0,::-1,:,:) + ps = datafile->ps (0,::-1,: ) + + nlon = dimsizes(datafile->lon) + ilon0 = 0 ; location of perturbation + ilon1 = nlon/2 ; +180 degrees + + print("max u="+max(u)) + print("max u xsection="+max(u(:,:,ilon0))+" lon="+lon(ilon0)) + + ;_________________________________________________ + ; Interpolate fields from eta coord to z coord + + z_eta = geo/9.81/1000 + z = lev + z = 1.0d-3*ispan(500,29550,1000) + + ;print("z="+z) + print("z_eta="+z_eta(:,0,0)) + print("interpolating from geopotential levels to evenly spaced z") + theta = linint1_n_Wrap(z_eta,theta,False,z,0,0) + t = linint1_n_Wrap(z_eta,t ,False,z,0,0) + u = linint1_n_Wrap(z_eta,u ,False,z,0,0) + p = linint1_n_Wrap(z_eta,p ,False,z,0,0) + qv = linint1_n_Wrap(z_eta,qv,False,z,0,0) + + delete(u@long_name) + delete(p@long_name) + delete(qv@long_name) + delete(theta@long_name) + delete(t@long_name) + + p = p/100 + qv = qv*1000 + pp = p(:,:,ilon0)-p(:,:,ilon1) + + copy_VarCoords(p(:,:,0), pp) +;print("pp="+pp) + + nlon = dimsizes(u(0,0,:)) + up = u + do i=0,nlon-1 + up(0,:,i) = up(0,:,i) - u(0,:,0) + end do + +; lev = datafile->lev +; rho = datafile->Rho +; t = datafile->T +; theta = datafile->Theta +; u = datafile->U +; qv = datafile->RhoQv / rho * 1000.0 +; copy_VarCoords(rho, qv) +; p = rho * 287.0 * t / 100.0 +; copy_VarCoords(rho, p) + +; pp = p(0,:,:,0) - p(0,:,:,ilon0) +; copy_VarCoords(p(0,:,:,0), pp) +; ps = (p(0,0,:,:) * lev(1) - p(0,1,:,:) * lev(0)) / (lev(1) - lev(0)) +; copy_VarCoords(rho(0,0,:,:), ps) +; nlon = dimsizes(u(0,0,0,:)) +; up = u +; do i=0,nlon-1 +; up(0,0,:,i) = up(0,0,:,i) - u(0,0,:,0) +; end do + +; --- +; z = lev; * 30.0 +;z = geo(:,0,0) / 9.81 / 1000.0 +;print("z="+z) +; p&lev = z +; t&lev = z +; theta&lev = z +; qv&lev = z +; u&lev = z +; pp&lev = z + + + +; --- + res_t@cnLevelSelectionMode= "ManualLevels" + res_t@cnLevelSpacingF = 10. + res_t@cnMinLevelValF = 200. + res_t@cnMaxLevelValF = 310. + + res_t@tiYAxisString = "Altitude (km)" + res_t@gsnCenterString = "Temperature" + res_t@gsnRightString = "(K)" + plot(0) = gsn_csm_contour(wks,t(:,:,ilon0),res_t) + + res_theta@cnLevelSelectionMode= "ExplicitLevels" + res_theta@cnLevels = (/250., 260., 270., 280., 290., 300., 310., 320., 330., 340., 350., 360., 380., 400., 450., 500., 550., 600., 650., 700./) + + res_theta@tiYAxisString = "Altitude (km)" + res_theta@gsnCenterString = "Potential Temperature" + res_theta@gsnRightString = "(K)" + plot(1) = gsn_csm_contour(wks,theta(:,:,ilon0),res_theta) + +; --- + + res_u@cnLevelSelectionMode= "ManualLevels" + res_u@cnLevelSpacingF = 2.0 + res_u@cnMinLevelValF = -26.0 + res_u@cnMaxLevelValF = 26.0 + + res_u@trXMinF = 0. + res_u@trXMaxF = 20. + res_u@tiYAxisString = "Altitude (km)" + res_u@gsnCenterString = "Zonal velocity" + res_u@gsnRightString = "(m/s)" + plot(2) = gsn_csm_contour(wks,u(:,:,ilon0),res_u) + + res_qv@cnLevelSelectionMode= "ManualLevels" + res_qv@cnLevelSpacingF = 2.0 + res_qv@cnMinLevelValF = 2.0 + res_qv@cnMaxLevelValF = 18.0 + + res_qv@tiYAxisString = "Altitude (km)" + res_qv@gsnCenterString = "Specific Humidity" + res_qv@gsnRightString = "(g/kg)" + plot(3) = gsn_csm_contour(wks,qv(:,:,ilon0),res_qv) + +; --- + + res_pp@cnLevelSelectionMode= "ManualLevels" + res_pp@cnLevelSpacingF = 1.0 + res_pp@cnMinLevelValF = -10.0 + res_pp@cnMaxLevelValF = -1.0 + + res_pp@trXMinF = 0. + res_pp@trXMaxF = 20. + + res_pp@tiYAxisString = "Altitude (km)" + res_pp@gsnCenterString = "Pressure perturbation" + res_pp@gsnRightString = "(hPa)" + plot(4) = gsn_csm_contour(wks,pp(:,:),res_pp) + + res_p@cnLevelSelectionMode= "ExplicitLevels" + res_p@cnLevels = (/1.e1, 2.e1, 5.e1, 1.e2, 2.e2, 3.e2, 4.e2, 5.e2, 6.e2, 7.e2, 8.e2, 9.e2, 1.e3/) + + res_p@tiYAxisString = "Altitude (km)" + res_p@gsnCenterString = "Background pressure" + res_p@gsnRightString = "(hPa)" + plot(5) = gsn_csm_contour(wks,p(:,:,0),res_p) + + ; res_ps@cnLevelSelectionMode= "ManualLevels" + ; res_ps@cnLevelSpacingF = 10. + ; res_ps@cnMinLevelValF = 950. + ; res_ps@cnMaxLevelValF = 1050. + + ; res_ps@gsnCenterString = "Surfae Pressure" + ; res_ps@gsnRightString = "(hPa)" + ; plot(5) = gsn_csm_contour(wks,ps(:,:),res_ps) + +; --- + + pres = True + pres@gsnFrame = False + pres@gsnPanelYWhiteSpacePercent = 5 + pres@gsnPanelBottom = 0.1 + + gsn_panel(wks,plot,(/3,2/),pres) + + frame(wks) + + +end diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-wvel.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-wvel.ncl new file mode 100644 index 000000000000..2a84178d95ef --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-wvel.ncl @@ -0,0 +1,118 @@ +;*********************************************** +; plot_supercell_wvel.ncl +; +; Version 1.0 +;*********************************************** + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;**************************************************************************** +; DO NOT EDIT ABOVE THIS LINE +;**************************************************************************** +; Read in data and store as follows: +; +; Output: +; maxw(time) +; time series maximum vertical velocity (units m/s) +; +; time_s(time) +; time of each sample (units s) +; +; Dimensions: +; time: arbitrary +;---------------------------------------------------------------------------- + + maxw = asciiread("./measurement_wmax.txt", -1, "double") + time_s = asciiread("./measurement_time.txt", -1, "double") + +ymax = max( (/3.0, max(maxw) /) ) +print("ymax="+ymax) + +print("maxw="+maxw) +print("time_s="+time_s) + +;time_s = fspan(0.0, 7200.0, 25) + +;**************************************************************************** +; DO NOT EDIT BELOW THIS LINE +;**************************************************************************** + +;------------------------------------------------ +; Workspace +;------------------------------------------------ + wks = gsn_open_wks("pdf","max_w") + +;------------------------------------------------ +; Plot resources [options] +;------------------------------------------------ +; Resources: + res = True + + res@gsnDraw = False + res@gsnFrame = False + + res@vpKeepAspect = True + res@vpWidthF = 0.6 + res@vpHeightF = 0.6 + res@gsnMaximize = True + res@gsnPaperOrientation = "portrait" + + res@tmXBLabelsOn = True + res@tmXBOn = True + + res@tmXTLabelsOn = False + res@tmXTOn = False + + res@tmXBMinorOn = True + res@tmYLMinorOn = True + + res@xyLineThicknessF = 2.0 + res@xyMarkLineMode = "MarkLines" + + resw = res + +;------------------------------------------------ +; Vertical velocity plot +;------------------------------------------------ + + resw@tiXAxisOn = True + resw@tiXAxisString = "time (days)" + + resw@tiYAxisOn = True + resw@tiYAxisString = "maximum vertical velocity (m/s)" + +;resw@trXMinF = 0.0 +; resw@trXMaxF = 7200.0 ; max(time_s) + + resw@trYMinF = 0.0 + resw@trYMaxF = ymax + +; resw@tmXBMode = "Manual" +; resw@tmXBTickStartF = 0.0 +; resw@tmXBTickEndF = 7200.0 +; resw@tmXBTickSpacingF = 1800.0 + +; resw@tmYLMode = "Explicit" +; resw@tmYLTickStartF = 0.0 +; resw@tmYLTickEndF = 50.0 + + plot = gsn_csm_xy(wks, time_s/3600.0/24.0, maxw, resw) + +;------------------------------------------------ +; Panel plot +;------------------------------------------------ + + pres = True + pres@gsnPanelBottom = 0.1 + pres@gsnPanelTop = 0.9 + pres@gsnPanelYWhiteSpacePercent = 5 + pres@gsnPanelXWhiteSpacePercent = 5 + gsn_panel(wks, plot, (/1,1/), pres) + +end + diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/radialAvg.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/radialAvg.ncl new file mode 100644 index 000000000000..38a434b2a8be --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/radialAvg.ncl @@ -0,0 +1,466 @@ +; This file contains various functions relating to TC plotting in radial space +; Colin Zarzycki (zarzycki@ucar.edu) +; History: +; 6/5/2016 - Released for DCMIP-2016 +; +; contains: radialAvg3D, radialAvg2D, calcWindComponents, radialInt3D, calc_vertmax_from_rad +; +; +; radialAvg3D +; Usage: radialAvg3D(data,lat,lon,lev,psminlat,psminlon,outerRad,mergeInnerBins) +; data (numeric): 3-D (lev,lat,lon) of variable to transform to radial space +; lat (numeric): 1-D array of lats (deg) +; lon (numeric): 1-D array of lons (deg) +; lev (numeric): 1-D array of levs (Pa or m) +; psminlat (numeric): lat of TC center (deg) +; psminlon (numeric): lon of TC center (deg) +; outerRad (numeric) : outermost radius to calculate (km) (typically 500-800km) +; mergeInnerBins (logical) :merge bins at storm center for sampling purpose? (generally True) +; +; ex: radialAvg3D(v_theta,lat,lon,v_theta&lev,psminlat,psminlon,500.,True) +; +; +; radialAvg2D +; Usage: radialAvg2D(data,lat,lon,psminlat,psminlon,outerRad,mergeInnerBins) +; data (numeric): 2-D (lat,lon) of variable to transform to radial space +; lat (numeric): 1-D array of lats (deg) +; lon (numeric): 1-D array of lons (deg) +; psminlat (numeric): lat of TC center (deg) +; psminlon (numeric): lon of TC center (deg) +; outerRad (numeric) : outermost radius to calculate (km) (typically 500-800km) +; mergeInnerBins (logical) :merge bins at storm center for sampling purpose? (generally True) +; +; ex: +; +; +; calcWindComponents +; Usage: radialAvg3D(U,V,lat,lon,psminlat,psminlon) +; U (numeric): 2-D (lat,lon) or 3-D (lev, lat, lon) or 4-D (time, lev, lat, lon) zonal wind +; V (numeric): 2-D (lat,lon) or 3-D (lev, lat, lon) or 4-D (time, lev, lat, lon) zonal wind +; lat (numeric): 1-D array of lats (deg) +; lon (numeric): 1-D array of lons (deg) +; psminlat (numeric): lat of TC center (deg) +; psminlon (numeric): lon of TC center (deg) +; +; ex: calcWindComponents(U_Z,V_Z,lat,lon,psminlat,psminlon) +; +; +; radialInt3D +; Usage: radialInt3D(data,lat,lon,psminlat,psminlon,outerRad) +; data (numeric): 3-D (lev,lat,lon) +; lat (numeric): 1-D array of lats (deg) +; lon (numeric): 1-D array of lons (deg) +; lev (numeric): 1-D array of levs (Pa or m) +; psminlat (numeric): lat of TC center (deg) +; psminlon (numeric): lon of TC center (deg) +; outerRad (numeric) : outermost radius to integrate (km) (typically 500-800km) +; +; ex: radialInt3D(tofloat(DTCOND),tofloat(lat),tofloat(lon),tofloat(lev),psminlat,psminlon,radius_3D) + +undef ( "radialAvg3D" ) +function radialAvg3D ( \ + data[*][*][*] : numeric, \ + lat[*] : numeric, \ + lon[*] : numeric, \ + lev[*] : numeric, \ + psminlat : numeric, \ + psminlon : numeric, \ + outerRad : numeric, \ + mergeInnerBins : logical \ +) +local None + +; lat (deg) +; lon (deg) +; center_lat(deg) +; center_lon(deg) +; outerRad (km) + +begin + +deltaLat = lat(1)-lat(0) +deltaLon = lon(1)-lon(0) +deltaMax = max((/deltaLon, deltaLat/)) +deltaMin = min((/deltaLon, deltaLat/)) +kmInDeg = 111.32 +kmGrid = kmInDeg * deltaMax +print("The max lat/lon km grid spacing at equator is "+kmGrid+" km") + +nlat = dimsizes(lat) +nlon = dimsizes(lon) +nlev = dimsizes(lev) + +pi = 3.14159 +d2r = pi/180. +r2d = 180./pi +lonr = lon*d2r +latr = lat*d2r +copy_VarCoords(lon,lonr) +copy_VarCoords(lat,latr) + +psminLocY = ind_nearest_coord(psminlat,lat,0) +psminLocX = ind_nearest_coord(psminlon,lon,0) + +;;;;; DO RADIAL AVERAGES + +loopWidth = toint(ceil((outerRad*1.10) / (kmInDeg * deltaMin))) +; for standard lat/lon grid +;1.25 gives n ~= 5 in inner grid if numMerge = 2 +;1.4 gives n ~= 9 in inner grid if numMerge = 2 +timesGrid = 1.1 ; we want each radius bin to be timeGrid times kmGrid + +nx = toint(outerRad/(timesGrid*kmGrid)) +print("Number of bins is equal to "+nx) + +if (mergeInnerBins) then +; origRadiusArr = fspan(0,outerRad,nx) +; radiusArr = new(dimsizes(origRadiusArr)-1,double) +; radiusArr(0) = origRadiusArr(0) +; radiusArr(1:dimsizes(radiusArr)-1) = origRadiusArr(2:dimsizes(origRadiusArr)-1) + numMerge = 2 ; numMerge is the number of innermost radial bins we want to merge + print("Merging innermost "+numMerge+" bins because radial average is so small.") + numMergeMinusOne = numMerge-1 + origRadiusArr = fspan(0,outerRad,nx+numMergeMinusOne) + radiusArr = new((dimsizes(origRadiusArr)-numMergeMinusOne),double) + radiusArr(0) = origRadiusArr(0) + print("Going from "+dimsizes(origRadiusArr)+" to "+dimsizes(radiusArr)+" bins...") + radiusArr(1:dimsizes(radiusArr)-1) = origRadiusArr(numMerge:dimsizes(origRadiusArr)-1) +else + print("Not merging any innermost bins -- be careful that your inner bins have > 1 pt.") + radiusArr = fspan(0,outerRad,nx) +end if + +numRadBins = dimsizes(radiusArr) + +rad_thevar_hit = new((/nlev,numRadBins/),integer) +rad_thevar_cum = new((/nlev,numRadBins/),double) + +; NEEDS to be 1 or 0 +rad_thevar_hit!1 = "radius" +rad_thevar_cum!1 = "radius" +rad_thevar_hit&radius = radiusArr +rad_thevar_cum&radius = radiusArr + +rad_thevar_hit!0 = "lev" +rad_thevar_cum!0 = "lev" +rad_thevar_hit&lev = lev +rad_thevar_cum&lev = lev + +rad_thevar_hit = 0 +rad_thevar_cum = 0 + +; NEED TO CORRECT NLEV +print("Starting loop") +do i = psminLocY-loopWidth,psminLocY+loopWidth + print("Doing outer loop #: "+(i-psminLocY+loopWidth)+" of "+(loopWidth*2)+" ("+((tofloat(i-psminLocY+loopWidth)/tofloat(loopWidth*2))*100.)+"%)") + do j = psminLocX-loopWidth,psminLocX+loopWidth + do k = 0,dimsizes(lev)-1 + gcdist = tofloat(gc_latlon(psminlat,psminlon,lat(i),lon(j),2,4)) + if (gcdist .le. outerRad) + rad_thevar_hit(k,{gcdist}) = rad_thevar_hit(k,{gcdist}) + 1 + rad_thevar_cum(k,{gcdist}) = rad_thevar_cum(k,{gcdist}) + data(k,i,j) + end if + end do + end do +end do + +print("Minimum number of hits per gridbox: "+min(rad_thevar_hit)) +print("Maximum number of hits per gridbox: "+max(rad_thevar_hit)) +rad_thevar = rad_thevar_cum/rad_thevar_hit + +copy_VarCoords(rad_thevar_hit,rad_thevar) + +rad_thevar@units = data@units +rad_thevar@long_name=data@long_name + +return(rad_thevar) + +end + +; ==================================================================================================== + +undef ( "radialAvg2D" ) +function radialAvg2D ( \ + data[*][*] : numeric, \ + lat[*] : numeric, \ + lon[*] : numeric, \ + psminlat : numeric, \ + psminlon : numeric, \ + outerRad : numeric, \ + mergeInnerBins : logical \ +) +local None + +; lat (deg) +; lon (deg) +; center_lat(deg) +; center_lon(deg) +; outerRad (km) + +begin + +deltaLat = lat(1)-lat(0) +deltaLon = lon(1)-lon(0) +deltaMax = max((/deltaLon, deltaLat/)) +deltaMin = min((/deltaLon, deltaLat/)) +kmInDeg = 111.32 +kmGrid = kmInDeg * deltaMax +print("The max lat/lon km grid spacing at equator is "+kmGrid+" km") + +nlat = dimsizes(lat) +nlon = dimsizes(lon) + +pi = 3.14159 +d2r = pi/180. +r2d = 180./pi +lonr = lon*d2r +latr = lat*d2r +copy_VarCoords(lon,lonr) +copy_VarCoords(lat,latr) + +psminLocY = ind_nearest_coord(psminlat,lat,0) +psminLocX = ind_nearest_coord(psminlon,lon,0) + +;;;;; DO RADIAL AVERAGES + +loopWidth = toint(ceil((outerRad*1.10) / (kmInDeg * deltaMin))) +; for standard lat/lon grid +;1.25 gives n ~= 5 in inner grid if numMerge = 2 +;1.4 gives n ~= 9 in inner grid if numMerge = 2 +timesGrid = 1.1 ; we want each radius bin to be timeGrid times kmGrid + +nx = toint(outerRad/(timesGrid*kmGrid)) +print("Number of bins is equal to "+nx) + +if (mergeInnerBins) then +; origRadiusArr = fspan(0,outerRad,nx) +; radiusArr = new(dimsizes(origRadiusArr)-1,double) +; radiusArr(0) = origRadiusArr(0) +; radiusArr(1:dimsizes(radiusArr)-1) = origRadiusArr(2:dimsizes(origRadiusArr)-1) + numMerge = 2 ; numMerge is the number of innermost radial bins we want to merge + print("Merging innermost "+numMerge+" bins because radial average is so small.") + numMergeMinusOne = numMerge-1 + origRadiusArr = fspan(0,outerRad,nx+numMergeMinusOne) + radiusArr = new((dimsizes(origRadiusArr)-numMergeMinusOne),double) + radiusArr(0) = origRadiusArr(0) + print("Going from "+dimsizes(origRadiusArr)+" to "+dimsizes(radiusArr)+" bins...") + radiusArr(1:dimsizes(radiusArr)-1) = origRadiusArr(numMerge:dimsizes(origRadiusArr)-1) +else + print("Not merging any innermost bins -- be careful that your inner bins have > 1 pt.") + radiusArr = fspan(0,outerRad,nx) +end if + +numRadBins = dimsizes(radiusArr) + +rad_thevar_hit = new((/numRadBins/),integer) +rad_thevar_cum = new((/numRadBins/),double) + +; NEEDS to be 1 or 0 +rad_thevar_hit!0 = "radius" +rad_thevar_cum!0 = "radius" +rad_thevar_hit&radius = radiusArr +rad_thevar_cum&radius = radiusArr + +rad_thevar_hit = 0 +rad_thevar_cum = 0 + +; NEED TO CORRECT NLEV +print("Starting loop") +do i = psminLocY-loopWidth,psminLocY+loopWidth + print("Doing outer loop #: "+(i-psminLocY+loopWidth)+" of "+(loopWidth*2)+" ("+((tofloat(i-psminLocY+loopWidth)/tofloat(loopWidth*2))*100.)+"%)") + do j = psminLocX-loopWidth,psminLocX+loopWidth + gcdist = tofloat(gc_latlon(psminlat,psminlon,lat(i),lon(j),2,4)) + if (gcdist .le. outerRad) + rad_thevar_hit({gcdist}) = rad_thevar_hit({gcdist}) + 1 + rad_thevar_cum({gcdist}) = rad_thevar_cum({gcdist}) + data(i,j) + end if + end do +end do + +print("Minimum number of hits per gridbox: "+min(rad_thevar_hit)) +print("Maximum number of hits per gridbox: "+max(rad_thevar_hit)) +rad_thevar = rad_thevar_cum/rad_thevar_hit + +copy_VarCoords(rad_thevar_hit,rad_thevar) + +rad_thevar@units = data@units +rad_thevar@long_name=data@long_name + +return(rad_thevar) + +end + +; ==================================================================================================== + +undef ( "calcWindComponents" ) +function calcWindComponents ( \ + U[*][*][*] : numeric, \ + V[*][*][*] : numeric, \ + lat[*] : numeric, \ + lon[*] : numeric, \ + psminlat : numeric, \ + psminlon : numeric \ +) +local None +begin + +pi = 3.14159 +d2r = pi/180. +r2d = 180./pi +lonr = lon*d2r +latr = lat*d2r +copy_VarCoords(lon,lonr) +copy_VarCoords(lat,latr) + +deltalong = lonr(:)-(psminlon*d2r) + +arr1 = conform(U,sin(deltalong),2) * conform(U,cos(latr),1) +arr2 = cos(psminlat*d2r)*conform(U,sin(latr),1) - sin(psminlat*d2r)*conform(U,cos(latr),1)*conform(U,cos(deltalong),2) +dir_angle_r = atan2(arr1,arr2) +; need to conform here +dir_angle_d = dir_angle_r*r2d + +copy_VarCoords(U,dir_angle_r) +copy_VarCoords(U,dir_angle_d) + +eps = 10.e-6 +phi_r = atan2(U,V) +phi_d = phi_r*r2d + +copy_VarCoords(U,phi_r) +copy_VarCoords(U,phi_d) + +WIND = U +WIND = sqrt(U^2+V^2) + +v_theta = WIND*sin(dir_angle_r - phi_r) +v_rad = -WIND*cos(dir_angle_r - phi_r) + +copy_VarCoords(U,v_theta) +copy_VarCoords(U,v_rad) + +v_theta@units = U@units +v_theta@long_name="Tangential wind" +v_rad@units = U@units +v_rad@long_name="Radial wind" + +return( [/ v_rad, v_theta /] ) + +end + + +undef ( "radialInt3D" ) +function radialInt3D ( \ + data[*][*][*] : numeric, \ + lat[*] : numeric, \ + lon[*] : numeric, \ + lev[*] : numeric, \ + psminlat : numeric, \ + psminlon : numeric, \ + outerRad : numeric \ +) +local None + +; lat (deg) +; lon (deg) +; center_lat(deg) +; center_lon(deg) +; outerRad (km) + +begin + +deltaLat = lat(1)-lat(0) +deltaLon = lon(1)-lon(0) +deltaMax = max((/deltaLon, deltaLat/)) +deltaMin = min((/deltaLon, deltaLat/)) +kmInDeg = 111.32 +kmGrid = kmInDeg * deltaMax +print("The max lat/lon km grid spacing at equator is "+kmGrid+" km") + +nlat = dimsizes(lat) +nlon = dimsizes(lon) +nlev = dimsizes(lev) + +pi = 3.14159 +d2r = pi/180. +r2d = 180./pi +lonr = lon*d2r +latr = lat*d2r +copy_VarCoords(lon,lonr) +copy_VarCoords(lat,latr) + +psminLocY = ind_nearest_coord(psminlat,lat,0) +psminLocX = ind_nearest_coord(psminlon,lon,0) + +; Get cosine weights +rad = 4.0*atan(1.0)/180.0 +clat = cos(lat*rad) + +loopWidth = toint(ceil((outerRad*1.25) / (kmInDeg * deltaMin))) +; make sure we are more than 1 to do a few rows of buffer cells + +rad_thevar_hit = new((/nlev/),float) +rad_thevar_cum = new((/nlev/),float) + +rad_thevar_hit!0 = "lev" +rad_thevar_cum!0 = "lev" +rad_thevar_hit&lev = lev +rad_thevar_cum&lev = lev + +rad_thevar_hit = 0 +rad_thevar_cum = 0 + +; NEED TO CORRECT NLEV +print("Starting loop") +do i = psminLocY-loopWidth,psminLocY+loopWidth + print("Doing outer loop #: "+(i-psminLocY+loopWidth)+" of "+(loopWidth*2)+" ("+((tofloat(i-psminLocY+loopWidth)/tofloat(loopWidth*2))*100.)+"%)") + do j = psminLocX-loopWidth,psminLocX+loopWidth + do k = 0,dimsizes(lev)-1 + gcdist = tofloat(gc_latlon(psminlat,psminlon,lat(i),lon(j),2,4)) + if (gcdist .le. outerRad) + rad_thevar_hit(k) = rad_thevar_hit(k) + data(k,i,j)*clat(i) + rad_thevar_cum(k) = rad_thevar_cum(k) + clat(i) + end if + end do + end do +end do + +rad_thevar = rad_thevar_hit/rad_thevar_cum + +copy_VarCoords(rad_thevar_hit,rad_thevar) + +rad_thevar@units = data@units +rad_thevar@long_name=data@long_name + +return(rad_thevar) + +end + + + +undef ( "calc_vertmax_from_rad" ) +function calc_vertmax_from_rad ( \ + vrad[*][*] : numeric \ +) +local None + +begin + +lev = vrad&lev +nlev = dimsizes(lev) +rad = vrad&radius +nrad = dimsizes(rad) + +rmw = new((/nlev/),"float") + +do k = 0,dimsizes(lev)-1 + maxix = maxind(vrad(k,:)) + rmw(k) = tofloat(rad(maxix)) +end do + +rmw!0 = "lev" +rmw&lev = lev + +return(rmw) + +end From d778956b5551deaea796fefa9fb62d088e0a2fc6 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sat, 15 Sep 2018 18:48:13 -0400 Subject: [PATCH 04/41] remove (not working) preqx supercell configuration --- .../preqx/CMakeLists.txt | 10 --- .../dcmip2016_test3_supercell/preqx/build.sh | 6 -- .../preqx/convert_eta_to_z.ncl | 78 ------------------- .../preqx/jobscript-cori.sh | 21 ----- .../preqx/jobscript-darwin.sh | 9 --- .../preqx/jobscript-edison.sh | 19 ----- .../preqx/jobscript-snl.sh | 24 ------ .../preqx/movies/CMakeLists.txt | 0 .../preqx/namelist-default.nl | 50 ------------ .../preqx/namelist-lowres.nl | 50 ------------ 10 files changed, 267 deletions(-) delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/CMakeLists.txt delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/build.sh delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/convert_eta_to_z.ncl delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-cori.sh delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-darwin.sh delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-edison.sh delete mode 100755 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-snl.sh delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/movies/CMakeLists.txt delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-default.nl delete mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-lowres.nl diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/CMakeLists.txt deleted file mode 100644 index ecfb00937223..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts -file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts -file(GLOB NAMELISTS "*.nl") # get namelists -file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts -file(GLOB M_FILES "*.m") # get matlab scripts - -# Copy test files to build -install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - -add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/build.sh b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/build.sh deleted file mode 100644 index 4872939cf746..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -cwd=`pwd` -cd ../../.. - echo "make preqx-nlev40-interp" - make -j preqx-nlev40-interp -cd $cwd diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/convert_eta_to_z.ncl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/convert_eta_to_z.ncl deleted file mode 100644 index 032562536e4f..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/convert_eta_to_z.ncl +++ /dev/null @@ -1,78 +0,0 @@ -load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" -load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" -load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" -load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" - -begin - - ; copy data file and append "_z" - print("cp ./movies/dcmip2016_test31.nc ./movies/dcmip2016_test31_z.nc") - system("cp ./movies/dcmip2016_test31.nc ./movies/dcmip2016_test31_z.nc") - - ; open in read/write mode - f = addfile("./movies/dcmip2016_test31_z.nc", "w") - - ;_________________________________________________ - ; read in fields and invert vertical order (so k=1 is bottom level) - ; output_varnames1 ='T','p','ps','pnh','geo','u','v','omega','Th','Q','Q2','Q3','Q4' ! variables to write to file - - - time = f->time - nt = dimsizes(time) - print("nt="+nt) - - geo = f->geo(:,::-1,:,:) - z_eta = geo/9.81 - z = f->lev - z = ispan(250,19750,500) - - print("z="+z) - print("z_eta="+z_eta(0,:,0,0)) - print("interpolating from geopotential levels to evenly spaced z") - - ; interpolate to z and write to file - print("converting T") - T = f->T (:,::-1,:,:) - f->T = linint1_n_Wrap(z_eta,T ,False,z,0,1) - - print("converting p") - p = f->pnh(:,::-1,:,:) - f->p = linint1_n_Wrap(z_eta,p ,False,z,0,1) - - print("converting pnh") - pnh = f->pnh(:,::-1,:,:) - f->pnh = linint1_n_Wrap(z_eta,pnh ,False,z,0,1) - - print("converting geo") - geo = f->geo(:,::-1,:,:) - f->geo = linint1_n_Wrap(z_eta,geo ,False,z,0,1) - - print("converting u") - u = f->u (:,::-1,:,:) - f->u = linint1_n_Wrap(z_eta,u ,False,z,0,1) - - print("converting v") - v = f->v (:,::-1,:,:) - f->v = linint1_n_Wrap(z_eta,v ,False,z,0,1) - - print("converting omega") - omega = f->w (:,::-1,:,:) - f->w = linint1_n_Wrap(z_eta,w ,False,z,0,1) - - print("converting Th") - Th = f->Th (:,::-1,:,:) - f->Th = linint1_n_Wrap(z_eta,Th ,False,z,0,1) - - print("converting qv") - qv = f->Q (:,::-1,:,:) - f->Q = linint1_n_Wrap(z_eta,qv ,False,z,0,1) - - print("converting qc") - qc = f->Q2 (:,::-1,:,:) - f->Q2 = linint1_n_Wrap(z_eta,qc ,False,z,0,1) - - print("converting qr") - qr = f->Q3 (:,::-1,:,:) - f->Q3 = linint1_n_Wrap(z_eta,qr ,False,z,0,1) - -end diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-cori.sh b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-cori.sh deleted file mode 100644 index fc41f83d652c..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-cori.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2016 test3 on the NERSC Cori machine -# -# usage: sbatch jobscript-... - -#SBATCH -J d16-3-preqx # job name -#SBATCH -o out_dcmip16-3.o%j # output and error file name (%j expands to jobID) -#SBATCH -n 192 # total number of mpi tasks requested -#SBATCH -p debug # queue (partition) -- normal, development, etc. -#SBATCH -t 00:10:00 # run time (hh:mm:ss) -#SBATCH -A acme # charge hours to account 1 -#SBATCH -C haswell # use Haswell nodes -date - -EXEC=../../../test_execs/preqx-nlev40-interp/preqx-nlev40-interp # set name of executable -cp ./namelist-default.nl input.nl -srun -n 192 $EXEC < input.nl # launch simulation -#ncl plot_lon_vs_z.ncl - -date diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-darwin.sh b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-darwin.sh deleted file mode 100644 index 8e5d98ac01b3..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-darwin.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2016 test 3 on a mac running Darwin -# -# usage: ./jobscript-... - -# launch the simulation -EXEC=../../../test_execs/preqx-nlev40-interp/preqx-nlev40-interp # set name of executable -openmpiexec -n 6 $EXEC < ./namelist-lowres.nl # launch simulation diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-edison.sh b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-edison.sh deleted file mode 100644 index b8e18748865e..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-edison.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# -# Jobscript for launching dcmip2016 test3 on the NERSC Edison machine -# -# usage: sbatch jobscript-... - -#SBATCH -J d16-3-preqx # job name -#SBATCH -o out_dcmip16-3.o%j # output and error file name (%j expands to jobID) -#SBATCH -n 192 # total number of mpi tasks requested -#SBATCH -p debug # queue (partition) -- normal, development, etc. -#SBATCH -t 00:10:00 # run time (hh:mm:ss) -#SBATCH -A acme # charge hours to account 1 -date - -EXEC=../../../test_execs/preqx-nlev40-interp/preqx-nlev40-interp # set name of executable -cp ./namelist-default.nl input.nl -srun -n 192 $EXEC < input.nl # launch simulation - -date diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-snl.sh deleted file mode 100755 index 4246b65c1338..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/jobscript-snl.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/tcsh -# -#SBATCH --job-name d16-3-preqx -#SBATCH --account=FY150001 -#SBATCH -N 12 -#SBATCH --time=0:20:00 -#SBATCH -p ec - -set OMP_NUM_THREADS = 1 -set NCPU = 40 -if ( ${?SLURM_NNODES} ) then # redsky - set NCPU = $SLURM_NNODES - @ NCPU *= 16 - @ NCPU /= $OMP_NUM_THREADS -endif -date - -# hydrostatic preqx -set EXEC = ../../../test_execs/preqx-nlev40-interp/preqx-nlev40-interp # set name of executable -set namelist = namelist-default.nl -\cp -f $namelist input.nl -mpirun -np $NCPU $EXEC < input.nl - -date diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/movies/CMakeLists.txt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-default.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-default.nl deleted file mode 100644 index dd3bc99e0231..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-default.nl +++ /dev/null @@ -1,50 +0,0 @@ -! -! preqx: namelist for dcmip2016 test 3: supercell storm (small planet X=120) -!_______________________________________________________________________ -&ctl_nl - nthreads = 1 - partmethod = 4 ! mesh parition method: 4 = space filling curve - topology = "cube" ! mesh type: cubed sphere - test_case = "dcmip2016_test3" ! test identifier - ne = 30 ! number of elements per cube face - qsize = 4 ! num tracer fields - nmax = 36000 ! 7200s(120min)/tstep - statefreq = 100 ! number of steps between screen dumps - restartfreq = -1 ! don't write restart files if < 0 - runtype = 0 ! 0 => new run - tstep = 0.2 ! largest timestep in seconds - integration = 'explicit' ! explicit time integration - tstep_type = 5 - rsplit = 1 - qsplit = 1 - nu = 2e9 !5.8e8 ! 1e15/(120)^3 - nu_s = 2e9 !5.8e8 - nu_p = 2e9 !5.8e8 - hypervis_order = 2 ! 2 = hyperviscosity - hypervis_subcycle = 1 - rearth = 53133 ! 6.376E6 / 120 - omega = 0 - se_ftype = 0 - moisture = 'dry' -/ -&vert_nl - vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain - vanalytic = 1 ! set vcoords in initialization routine - vtop = 5e-2 ! vertical coordinate at top of atm (z=20km) -/ -&analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file - output_timeunits = 3 ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 300 !900, ! 300 (5min) !900 seconds (15 minutes) - output_varnames1 ='T','p','ps','pnh','geo','rho','u','v','omega','Th','Q','Q2','Q3','Q4' ! variables to write to file - interp_nlon = 360 - interp_nlat = 181 - interp_gridtype = 1 - interp_type = 0 ! 0=native grid, 1=bilinear - output_type ='netcdf' ! netcdf or pnetcdf - num_io_procs = 16 -/ -&prof_inparm - profile_outpe_num = 100 - profile_single_file = .true. -/ diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-lowres.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-lowres.nl deleted file mode 100644 index 931fd8d58e6e..000000000000 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/preqx/namelist-lowres.nl +++ /dev/null @@ -1,50 +0,0 @@ -! -! preqx: namelist for dcmip2016 test 3: supercell storm (small planet X=120) -!_______________________________________________________________________ -&ctl_nl - nthreads = 1 - partmethod = 4 ! mesh parition method: 4 = space filling curve - topology = "cube" ! mesh type: cubed sphere - test_case = "dcmip2016_test3" ! test identifier - ne = 7 ! number of elements per cube face - qsize = 4 ! num tracer fields: qv,qc,qr - nmax = 36000 ! 7200s(120min)/tstep - statefreq = 10 ! number of steps between screen dumps - restartfreq = -1 ! don't write restart files if < 0 - runtype = 0 ! 0 => new run - tstep = 0.2 ! largest timestep in seconds - integration = 'explicit' ! explicit time integration - tstep_type = 5 - rsplit = 1 - qsplit = 1 - nu = 4.57e10 ! 1e15/(120)^3 *(ne30/ne7) - nu_s = 4.57e10 - nu_p = 4.57e10 - hypervis_order = 2 ! 2 = hyperviscosity - hypervis_subcycle = 1 ! 1 = no hyperviz subcycling - rearth = 53133 ! 6.376E6 / 120 - omega = 0 - se_ftype = 0 - moisture = 'dry' -/ -&vert_nl - vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain - vanalytic = 1 ! set vcoords in initialization routine - vtop = 5e-2 ! vertical coordinate at top of atm (z=20km) -/ -&analysis_nl - output_dir = "./movies/" ! destination dir for netcdf file - output_timeunits = 0 !3, ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 100 !900, ! 900 seconds (15 minutes) - output_varnames1 ='T','p','ps','pnh','geo','u','v','omega','Th','Q','Q2','Q3','Q4' ! variables to write to file - interp_nlon = 360 - interp_nlat = 181 - interp_gridtype = 1 - interp_type = 0 ! 0=native grid, 1=bilinear - output_type ='netcdf' ! netcdf or pnetcdf - num_io_procs = 16 -/ -&prof_inparm - profile_outpe_num = 100 - profile_single_file = .true. -/ From 5d261ce73da9b3394e43bcbebfe88543e4271f58 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Thu, 20 Sep 2018 12:11:58 -0600 Subject: [PATCH 05/41] theta-l model will use correct nu_div scaling and not duplicate the buggy nu_div scaling from PREQX --- components/homme/src/theta-l/viscosity_theta.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/homme/src/theta-l/viscosity_theta.F90 b/components/homme/src/theta-l/viscosity_theta.F90 index 54e1fc82d80b..833c1d643bde 100644 --- a/components/homme/src/theta-l/viscosity_theta.F90 +++ b/components/homme/src/theta-l/viscosity_theta.F90 @@ -85,7 +85,7 @@ subroutine biharmonic_wk_theta(elem,stens,vtens,deriv,edgebuf,hybrid,nt,nets,net ! we have a problem with the tensor in that we cant seperate ! div and curl components. So we do, with tensor V: ! nu * (del V del ) * ( nu_ratio * grad(div) - curl(curl)) - nu_ratio1=(nu_div/nu)**2 ! preserve buggy scaling + nu_ratio1=(nu_div/nu) ! do not match buggy scaling used by PREQX model nu_ratio2=1 else nu_ratio1=nu_div/nu From d45eae2e199f9c1637625417da802da1cb1ae016 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Thu, 20 Sep 2018 12:22:07 -0600 Subject: [PATCH 06/41] DCMIP 2012 test2.0 theta-l configuration --- .../theta-l/CMakeLists.txt | 10 + .../theta-l/build.sh | 6 + .../theta-l/jobscript-snl.sh | 51 +++++ .../theta-l/movies/CMakeLists.txt | 0 .../theta-l/namelist-h-lowres.nl | 48 +++++ .../theta-l/namelist-h.nl | 47 +++++ .../theta-l/namelist-nh.nl | 47 +++++ .../theta-l/plot_z_lon.ncl | 198 ++++++++++++++++++ .../theta-l/test200-range.ncl | 191 +++++++++++++++++ 9 files changed, 598 insertions(+) create mode 100644 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/CMakeLists.txt create mode 100644 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/build.sh create mode 100755 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh create mode 100644 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/movies/CMakeLists.txt create mode 100644 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl create mode 100644 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl create mode 100644 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl create mode 100755 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/plot_z_lon.ncl create mode 100644 components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/test200-range.ncl diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/CMakeLists.txt new file mode 100644 index 000000000000..ecfb00937223 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/CMakeLists.txt @@ -0,0 +1,10 @@ +file(GLOB NCL_SCRIPTS "*.ncl") # get ncl-scripts +file(GLOB SHELL_SCRIPTS "*.sh") # get shell-scripts +file(GLOB NAMELISTS "*.nl") # get namelists +file(GLOB PYTHON_SCRIPTS "*.py") # get python scripts +file(GLOB M_FILES "*.m") # get matlab scripts + +# Copy test files to build +install(PROGRAMS ${NCL_SCRIPTS} ${SHELL_SCRIPTS} ${NAMELISTS} ${PYTHON_SCRIPTS} ${M_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + +add_subdirectory(movies) diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/build.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/build.sh new file mode 100644 index 000000000000..fa5fb18a0df8 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cwd=`pwd` +cd ../../.. + echo "make -j4 theta-l-nlev30" + make -j4 theta-l-nlev30 +cd $cwd diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh new file mode 100755 index 000000000000..3e0119463a92 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh @@ -0,0 +1,51 @@ +#!/bin/tcsh +# +# hydrostatic: 4 nodes: 3min +# NH: 10 nodes ? +# +#SBATCH -p ec +#SBATCH --job-name d20-theta +#SBATCH --account=FY150001 +#SBATCH -N 25 +#SBATCH --time=3:00:00 +#XXSBATCH -N 4 +#XXSBATCH --time=0:10:00 +#PBS -q acme +#PBS -l walltime=0:30:00 +#PBS -l nodes=10 + + +set OMP_NUM_THREADS = 1 +set NCPU = 40 +if ( ${?PBS_ENVIRONMENT} ) then # anvil + set NCPU = $PBS_NNODES + if ( $PBS_ENVIRONMENT == PBS_BATCH ) cd $PBS_O_WORKDIR +endif +if ( ${?SLURM_NNODES} ) then + set NCPU = $SLURM_NNODES + @ NCPU *= 16 + @ NCPU /= $OMP_NUM_THREADS +endif + +# hydrostatic theta +set EXEC = ../../../test_execs/theta-l-nlev30/theta-l-nlev30 +set namelist = namelist-h.nl +\cp -f $namelist input.nl +mpirun -np $NCPU $EXEC < input.nl +ncl plot_z_lon.ncl +ncl test200-range.ncl +\mv -f dcmip2012_test2_0_u_t6.00.pdf hydro_test2_0_u_z.pdf +\mv -f movies/dcmip2012_test2_01.nc.pdf hydro_test2_0_u.pdf +\mv -f movies/dcmip2012_test2_01.nc movies/hydro_dcmip2012_test2_01.nc + +# nonhydrostatic theta +set EXEC = ../../../test_execs/theta-nlev30/theta-nlev30 +set namelist = namelist-nh.nl +\cp -f $namelist input.nl +mpirun -np $NCPU $EXEC < input.nl +ncl plot_z_lon.ncl +ncl test200-range.ncl +\mv -f dcmip2012_test2_0_u_t6.00.pdf nonhydro_test2_0_u_t6.00.pdf +\mv -f movies/dcmip2012_test2_01.nc.pdf nonhydro_test2_0_u.pdf +\mv -f movies/dcmip2012_test2_01.nc movies/nonhydro_dcmip2012_test2_01.nc + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/movies/CMakeLists.txt b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/movies/CMakeLists.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl new file mode 100644 index 000000000000..9832d7ba910b --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl @@ -0,0 +1,48 @@ +! +! namelist for dcmip2012 test2-0: steady-state atmosphere with orography +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_0" ! test identifier + theta_hydrostatic_mode = .true. + ne = 7 ! number of elements per cube face + qsize = 0 ! num tracer fields + ndays = 6 ! num simulation days: 0 = use nmax steps + statefreq = 25 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 300 ! largest timestep in seconds + rsplit = 3 + integration = 'explicit' ! explicit time integration + tstep_type = 5 ! 1 => default method + nu = 3.4e17 ! hyperviscosity + nu_s = 3.4e17 + nu_p = 3.4e17 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + omega = 0.0 ! earth angular speed = 0.0 + dcmip2_0_zetam = 0.785 ! mountain half-width = pi/4 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 2.05e-1 ! vertical coordinate at top of atm (z=12000m) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 12, ! output every 12 hours + output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 91 + interp_nlon = 360 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl new file mode 100644 index 000000000000..26586507ce4d --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl @@ -0,0 +1,47 @@ +! +! namelist for dcmip2012 test2-0: steady-state atmosphere with orography +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_0" ! test identifier + theta_hydrostatic_mode = .true. + ne = 30 ! number of elements per cube face + qsize = 0 ! num tracer fields + ndays = 6 ! num simulation days: 0 = use nmax steps + statefreq = 288 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 300 ! largest timestep in seconds + rsplit = 3 + integration = 'explicit' ! explicit time integration + tstep_type = 5 ! 1 => default method + nu = 1e15 ! hyperviscosity + nu_s = 1e15 + nu_p = 1e15 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + omega = 0.0 ! earth angular speed = 0.0 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 2.05e-1 ! vertical coordinate at top of atm (z=12000m) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 12, ! output every 12 hours + output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 91 + interp_nlon = 360 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl new file mode 100644 index 000000000000..50e8e6d6bfea --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl @@ -0,0 +1,47 @@ +! +! namelist for dcmip2012 test2-0: steady-state atmosphere with orography +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2012_test2_0" ! test identifier + theta_hydrostatic_mode = .false. + ne = 30 ! number of elements per cube face + qsize = 0 ! num tracer fields + ndays = 6 ! num simulation days: 0 = use nmax steps + statefreq = 288 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 = new run + tstep = 180.0 ! largest timestep in seconds + rsplit = 5 + integration = 'explicit' ! explicit time integration + tstep_type = 6 ! + nu = 1e15 ! hyperviscosity + nu_s = 1e15 + nu_p = 1e15 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + omega = 0.0 ! earth angular speed = 0.0 +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine + vtop = 2.05e-1 ! vertical coordinate at top of atm (z=12000m) +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 1=days, 2=hours, 0=timesteps + output_frequency = 12, ! output every 12 hours + output_varnames1 ='T','ps','u','v','omega','geo' ! variables to write to file + interp_type = 0 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlat = 91 + interp_nlon = 360 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/plot_z_lon.ncl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/plot_z_lon.ncl new file mode 100755 index 000000000000..7ac8e1981445 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/plot_z_lon.ncl @@ -0,0 +1,198 @@ +; plot_z_lon +; +; Plots vertical cross section from interpolated data +; +; usage plot_z_lon t=(time_index) n=(variable_number) +; n = (0,1,2) => (omega,u,v) + + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + + T0 = 300.d0 ; temperature (K) + lapse = 0.0065d0 ; temperature lapse rate (K/m) + ztop = 12000.d0 ; model top (m) + Rd = 287.0d0 ; Ideal gas const dry air (J kg^-1 K^1) + g = 9.80616d0 ; Gravity (m s^2) + Cp = 1004.5d0 ; Specific heat capacity (J kg^-1 K^1) + pi = 4.d0*atan(1.d0) ; pi + p0 = 100000.0 ; reference pressure + + exponent = g/(Rd*lapse) + exponent_r= 1.0/exponent + + ;_________________________________________________ + ; open file and read in data(time,lev,lat,long) + + f = addfile("movies/dcmip2012_test2_01.nc","r") + lat = f->lat + lon = f->lon + lev = f->lev + time = f->time + hyam = f->hyam + hybm = f->hybm + + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + nt = dimsizes(time) + + ;_________________________________________________ + ; get time index from command line + t_index = nt-1 + if(isvar("t")) then + t_index = t + end if + print ("t_index: time = " + f->time(t_index)) + + equator = nlat/2 + + ;_________________________________________________ + ; load data + + var_num = 1 + if(isvar("n")) then + var_num = n + end if + + omega = f->omega(t_index,:,equator,:) + u = f->u(t_index,:,equator,:) + v = f->v(t_index,:,equator,:) + + print("max u ="+max(u)+" min u="+min(u)) + print("max v ="+max(v)+" min v="+min(v)) + print("max omega="+max(omega)+" min omega="+min(omega)) + + if(var_num .eq. 0) then + var = f->omega (t_index,:,equator,:) + var_name = "omega" + end if + + if(var_num .eq. 1) then + var = f->u(t_index,:,equator,:) + var_name = "u" + end if + + if(var_num .eq. 2) then + var = f->v(t_index,:,equator,:) + var_name = "v" + end if + + ps = f->ps(t_index,equator,:) + + norm = max(abs(var)) + + amax = 0.25 + step = amax/10 + +; var = var/norm + ;var@long_name = " " + + ;_________________________________________________ + ; Set up height coordinate with constant z spacing + + z = new((/nlev/),double) + dz = 12000.0/nlev + + do kk=0,nlev-1 + z(kk) = (12000.0-0.5*dz)-(kk)*dz + end do + + ; Get surface elevation curve + + zs = (T0/lapse)*(1.0-(ps/p0)^exponent_r) + + ;_________________________________________________ + ; Get pressure and z position of eta-level midpoints + + z_m = new((/nlev,nlon/),double) + p_m = new((/nlev,nlon/),double) + + do k =0, nlev-1 + p_m(k,:) = hyam(k)*p0 + hybm(k)*ps + z_m(k,:) = (T0/lapse)*(1.0-(p_m(k,:)/p0)^exponent_r) + end do + + ;_________________________________________________ + ; Interpolate var field from eta coord to z coord + + var_z = new( (/nlev,nlon/), double) + + do i = 0, nlon-1 + z_col = z_m(::-1,i) ; single column of z field, inverted + var_col = var(::-1,i) ; single column of var field, inverted + var_z(:,i) = ftcurv(z_col, var_col, z) ; interpolate to evenly spaced z using splines + end do + + ;_________________________________________________ + ; Setup the plot + + res1 = True + res1@gsnDraw = False + res1@gsnFrame = False ; don't draw yet + res1@gsnMaximize = True + res1@cnFillOn = True + res1@cnLinesOn = False ; turn off contour lines + res1@vpWidthF = 1 + res1@vpHeightF = 0.5 + res1@cnInfoLabelOn = False + res1@cnLineLabelsOn = False + res1@lbLabelBarOn = True + res1@sfXArray = lon ; use lon as plot x-axis + res1@sfYArray = z ; use z for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiXAxisString = "longitude" ; x-axis title + res1@tiYAxisString = "height" ; y-axis title + res1@gsnPaperOrientation = "portrait" + res1@lbBoxLinesOn = False ; Turn off labelbar box lines + res1@lbLabelStride = 1 + res1@lbAutoManage = False + res1@lbTitlePosition = "Bottom" + res1@lbOrientation = "Vertical" + res1@lbTitleString = sprintf("max=%7.3f", norm) + res1@cnFillPalette = "temp_diff_18lev"; "WhBlGrYeRe"; "BlWhRe"; "BlueWhiteOrangeRed"; "BlueRed"; "cmp_b2r"; + res1@cnSpanFillPalette = True + res1@tiXAxisFontHeightF = 0.020 + res1@tmXBLabelFontHeightF = 0.015 + res1@lbTitleFontHeightF = 0.020 + res1@lbLabelFontHeightF = 0.020 + res1@tiMainFont = 22 + res1@tiMainFontHeightF = 0.020 + res1@gsnCenterStringFontHeightF = 0.025 ; to set the gsnLeft/Center/RightString font heights, + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnMaxLevelValF = amax; -0.5*step; 0.9; ; max contour color label + res1@cnMinLevelValF =-amax; +0.5*step; -0.9; ; min contour color label + res1@cnLevelSpacingF = step; 0.1; ; contour color spacing + res1@gsnCenterString ="" + res1@tiMainString = "DCMIP 2012 Test 2-0 "+var_name+" t = "+sprintf("%5.2f", time(t_index)) + " days" + + wks_type="pdf" + wks_type@wkPaperWidthF = 11 + wks_type@wkPaperHeightF = 7.5 + filename ="dcmip2012_test2_0_"+var_name+"_t"+sprintf("%.2f", time(t_index)) + wks = gsn_open_wks(wks_type,filename) + print("outputfilename = "+filename+".pdf") + + gsres = True + gsres@gsLineColor = "black" + gsres@gsFillColor = "black" + gsres@tfPolyDrawOrder = "PostDraw" + gsres@gsLineThicknessF = 5.0 + + zs(0)=0 + zs(nlon-1)=0 + + plot1 = gsn_csm_contour (wks,var(:,:),res1) + mountains = gsn_add_polygon (wks, plot1,lon, zs, gsres) + mt = gsn_add_polyline(wks, plot1, lon, zs, gsres) + + draw(plot1) + + frame(wks) + +end + diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/test200-range.ncl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/test200-range.ncl new file mode 100644 index 000000000000..48b982e4e9f0 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/test200-range.ncl @@ -0,0 +1,191 @@ +;================================================; +; test200-height-lon.ncl +; +; Example ncl script to produce the height-lon +; plots for test 200. +; Created by James Kent, Christiane Jablonowski +; and Paul Ullrich (University of Michigan). +; +;================================================; +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" +; ================================================; + + ; This produces the height-lon plots for test 200 + +begin + +;=================================================; +; open file and read in data +; GIVEN AS (time,lev,lat,long) from 0 to n-1 +;=================================================; + + ; replace your models file name below + +; f = addfile("../cam-fv.200.medium.L30.latlon.hydro.4th_order_div_damping.nc","r") + +; f = addfile("d2-hydro.nc", "r") +; f= addfile("d2preqx-ne30.nc","r") + + +; name = "d2preqx-ne30" +; name = "d2thetah-ne30" +; name = "d2thetah-ne30-upwind" +; title = "Hydrostatic eqns, original formulation" +; title = "Hydrostatic eqns, Exner p., upwinding" +; name = "nonh-ne30" +; title = "Nonhydrostatic eqns" +; name = "d2non-ne30-discrete-phi" +; title = "Nonhydro, discrete phi" + name = "movies/dcmip2012_test2_01.nc" + title = "" + + + f = addfile(name+".nc","r") + minuplot = -.25 + maxuplot = .25 + ; Input useful parameters + + lat = f->lat + lon = f->lon + lev = f->lev + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + + ; We want to plot zonal and vertical velocity at the equator + ; Note that CAM-FV uses vertical pressure velocity + + varload = f->u(:,:,:,:) + maxu = max(varload) + minu = min(varload) + print("max u ="+maxu+" min u="+minu) + varload2 = f->omega(:,:,:,:) + + ; We want the plots at the equator at time 6 days + ; For 6hr intervals this corresponds to index 24 + ; The equator is at nlat/2 + + equator = nlat/2 + + ; Input the variables at this time + + day6 = closest_val( 6.0, f->time) + if (day6 .gt. dimsizes(f->time)-1 ) then + day6 = dimsizes(f->time) -1 + end if + ;print("times = "+f->time) + print("plotting n="+day6+" time(n) = "+f->time(day6)+"d") + + + var1 = varload(day6,:,equator,:) + var2 = varload2(day6,:,equator,:) + + ; Delete loaded data + + delete(varload) + delete(varload2) + + ; Set up height coordinate (assuming constant spacing) + ; Note that for CAM-FV pressure coordinates are used, + ; therefore the height levels are upside-down + ; i.e. for 30 vertical levels index 0 is at 11800 m + ; and index 29 is at 200 m. + + maxu = max(var1) + minu = min(var1) + print("crosssection: max u ="+maxu+" min u="+minu) + + + z = new((/nlev/),double) + dz = 12000.0/nlev + + do kk=0,nlev-1 + + z(kk) = (12000.0-0.5*dz)-(kk)*dz + + end do + + ; We don't want long-name in the plots + + var1@long_name = " " + var2@long_name = " " + + ; We now produce the plot + + plot = new (1, graphic) ; define plot - need 2 panels + + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@cnFillOn = True + res1@cnLinesOn = False ; turn off contour lines + res1@gsnSpreadColors = True + res1@lbLabelAutoStride = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 0.38 + res1@vpHeightF = 0.19 + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnInfoLabelOn = False ; give each + res1@cnLineLabelsOn = False ; panel its own + res1@lbLabelBarOn = True ; label bar + + pres = True + pres@gsnMaximize = True + + res1@sfXArray = lon ; uses lon as plot x-axis + res1@sfYArray = z ; uses z for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiYAxisString = " " ; y-axis title + res1@tiXAxisString = " " ; x-axis title + + pltTitle ="test 200, t=6 days, min/max="+sprintf("%5.2f", minu)+"/"+sprintf("%5.2f", maxu) ; Plot title if required + pres@txString = pltTitle + + ; Change the output type and name + + ;wks = gsn_open_wks("X11","cam-fv-test200heightlon") ; output using X11 + wks = gsn_open_wks("pdf",name) ; output using eps + ;wks = gsn_open_wks("png","cam-fv-test200heightlon") ; output using png + gsn_define_colormap(wks,"gui_default") ; Colormap + + ; Add height label + + txres3 = True + txres3@txAngleF = 90. + txres3@txFontHeightF = 0.02 + gsn_text_ndc(wks,"Height (m)",0.03,0.48,txres3) + + txres3@txAngleF = 0. + gsn_text_ndc(wks,"Longitude",0.5,0.02,txres3) + + ; Contour spacing for u + + res1@cnMaxLevelValF = maxuplot;0.1 ; max contour color label + res1@cnMinLevelValF = minuplot; -0.1 ; min contour color label + res1@cnLevelSpacingF = (maxuplot-minuplot)/20. ; contour color spacing + + res1@gsnCenterString = "a) U" + plot(0) = gsn_csm_contour(wks,var1(:,:),res1) ; plot var1 + + ; Contour spacing for w + +; res1@cnMaxLevelValF = 0.0003 ; max contour color label +; res1@cnMinLevelValF = -0.0003 ; min contour color label +; res1@cnLevelSpacingF = 0.0001 ; contour color spacing + + ;res1@gsnCenterString = "b) OMEGA" + ;plot(1) = gsn_csm_contour(wks,var2(:,:),res1) ; plot var2 + ; this is with omega + ;gsn_panel(wks,plot,(/2,1/),pres) ; 2x1 plot + + gsn_panel(wks,plot,(/1,1/),pres) ; 2x1 plot + frame(wks) + + + + +end From e77b36fa876b79caafd4dd7f77b8638912ff8c1d Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Wed, 3 Oct 2018 23:54:15 -0500 Subject: [PATCH 07/41] new KGS542 IMEX scheme and reference PHI bugfix new KGS542 scheme from A. Steyer with better stability properties PHI_ref used in hyperviscosity computed from dp_ref, theta_ref instead of from prognostic variables created configurations and/or updated namelists for theta-l: dcmip2012 test2.0 dcmip2012 test4.1 dcmip2016 test1, test2, test3 non-BFB for theta model --- .../theta-l/jobscript-snl.sh | 6 +- .../theta-l/namelist-h-lowres.nl | 4 +- .../theta-l/namelist-h.nl | 4 +- .../theta-l/namelist-nh.nl | 8 +- .../theta-l/h-x1.nl | 4 +- .../theta-l/jobscript-snl.sh | 2 +- .../theta-l/nh-x1.nl | 6 +- .../theta-l/nh-x10.nl | 12 +-- .../theta-l/nh-x100.nl | 12 +-- .../theta-l/nh-x1000.nl | 16 ++-- .../theta-l/preqx-x1.nl | 66 ------------- .../theta-l/namelist-r100-dry.nl | 2 +- .../theta-l/namelist-r100-h.nl | 2 +- .../theta-l/namelist-r100.nl | 2 +- .../preqx/plot-tropical-cyclone-init.ncl | 8 ++ .../theta-l/namelist-r100.nl | 2 +- .../theta-l/namelist-r50-h.nl | 2 +- .../theta-l/namelist-r50.nl | 2 +- .../theta-l/jobscript-snl.sh | 1 + .../theta-l/namelist-r100.nl | 18 ++-- .../theta-l/namelist-r200.nl | 16 ++-- .../theta-l/namelist-r400.nl | 18 ++-- .../theta-l/namelist-r50.nl | 18 ++-- components/homme/src/theta-l/eos.F90 | 4 +- .../homme/src/theta-l/prim_advance_mod.F90 | 96 +++++++++++-------- 25 files changed, 144 insertions(+), 187 deletions(-) delete mode 100644 components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/preqx-x1.nl diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh index 3e0119463a92..9a17859fb50b 100755 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/jobscript-snl.sh @@ -12,7 +12,7 @@ #XXSBATCH --time=0:10:00 #PBS -q acme #PBS -l walltime=0:30:00 -#PBS -l nodes=10 +#PBS -l nodes=20 set OMP_NUM_THREADS = 1 @@ -27,8 +27,9 @@ if ( ${?SLURM_NNODES} ) then @ NCPU /= $OMP_NUM_THREADS endif -# hydrostatic theta set EXEC = ../../../test_execs/theta-l-nlev30/theta-l-nlev30 + +# hydrostatic theta set namelist = namelist-h.nl \cp -f $namelist input.nl mpirun -np $NCPU $EXEC < input.nl @@ -39,7 +40,6 @@ ncl test200-range.ncl \mv -f movies/dcmip2012_test2_01.nc movies/hydro_dcmip2012_test2_01.nc # nonhydrostatic theta -set EXEC = ../../../test_execs/theta-nlev30/theta-nlev30 set namelist = namelist-nh.nl \cp -f $namelist input.nl mpirun -np $NCPU $EXEC < input.nl diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl index 9832d7ba910b..c767eb324c57 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h-lowres.nl @@ -10,10 +10,10 @@ ne = 7 ! number of elements per cube face qsize = 0 ! num tracer fields ndays = 6 ! num simulation days: 0 = use nmax steps - statefreq = 25 ! number of steps between screen dumps + statefreq = 36 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 = new run - tstep = 300 ! largest timestep in seconds + tstep = 1200 ! largest timestep in seconds rsplit = 3 integration = 'explicit' ! explicit time integration tstep_type = 5 ! 1 => default method diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl index 26586507ce4d..25b8b159c9ef 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-h.nl @@ -10,11 +10,11 @@ ne = 30 ! number of elements per cube face qsize = 0 ! num tracer fields ndays = 6 ! num simulation days: 0 = use nmax steps - statefreq = 288 ! number of steps between screen dumps + statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 = new run tstep = 300 ! largest timestep in seconds - rsplit = 3 + rsplit = 6 integration = 'explicit' ! explicit time integration tstep_type = 5 ! 1 => default method nu = 1e15 ! hyperviscosity diff --git a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl index 50e8e6d6bfea..591308962f51 100644 --- a/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl +++ b/components/homme/dcmip_tests/dcmip2012_test2.0_steady_state_with_orography/theta-l/namelist-nh.nl @@ -10,13 +10,13 @@ ne = 30 ! number of elements per cube face qsize = 0 ! num tracer fields ndays = 6 ! num simulation days: 0 = use nmax steps - statefreq = 288 ! number of steps between screen dumps + statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 = new run - tstep = 180.0 ! largest timestep in seconds - rsplit = 5 + tstep = 300.0 ! largest timestep in seconds + rsplit = 6 ! unstable with desired rsplit=6 - why? integration = 'explicit' ! explicit time integration - tstep_type = 6 ! + tstep_type = 7 ! nu = 1e15 ! hyperviscosity nu_s = 1e15 nu_p = 1e15 diff --git a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/h-x1.nl b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/h-x1.nl index 714fe59dc00c..d2310d86f2f8 100644 --- a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/h-x1.nl +++ b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/h-x1.nl @@ -11,11 +11,11 @@ rotate_grid = 0 ne=30 qsize = 0 ndays = 15 -statefreq=288 +statefreq=144 runtype = 0 mesh_file='/dev/null' tstep=300 -rsplit=3 +rsplit=6 qsplit = 1 tstep_type = 5 integration = "explicit" diff --git a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/jobscript-snl.sh index 8dbe18cc0df3..463f0d911576 100755 --- a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/jobscript-snl.sh @@ -7,7 +7,7 @@ #SBATCH --time=1:30:00 #XXSBATCH -N 20 #XXSBATCH --time=5:00:00 -#PBS -l walltime=60:00 +#PBS -l walltime=20:00 #PBS -l nodes=20 #PBS -q acme # diff --git a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1.nl b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1.nl index 18078f8e3e74..99367f12eee3 100644 --- a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1.nl +++ b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1.nl @@ -11,11 +11,11 @@ rotate_grid = 0 ne=30 qsize = 0 ndays = 15 -statefreq=60 +statefreq=144 runtype = 0 mesh_file='/dev/null' -tstep=120 -rsplit=10 +tstep=300 +rsplit=6 qsplit = 1 tstep_type = 7 integration = "explicit" diff --git a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x10.nl b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x10.nl index 1276d1aa1299..5e347249ef64 100644 --- a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x10.nl +++ b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x10.nl @@ -10,12 +10,12 @@ u_perturb = 1 rotate_grid = 0 ne=30 qsize = 0 -nmax = 10800 ! need to run 129600s -statefreq=60 +nmax = 4320 ! need to run 129600s +statefreq=72 runtype = 0 mesh_file='/dev/null' -tstep=12.0 -rsplit=10 +tstep=30.0 +rsplit=6 qsplit = 1 tstep_type = 7 integration = "explicit" @@ -51,8 +51,8 @@ profile_single_file = .true. interp_gridtype=2 output_timeunits=0 ! 1 is days, 2- hours, 0 - tsteps - output_frequency=720 - output_start_time=5040 + output_frequency=288 + output_start_time=2016 output_end_time=300000000 output_varnames1='ps','zeta','u','v','T' num_io_procs = 16 diff --git a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x100.nl b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x100.nl index c98646e5685b..49974110c82b 100644 --- a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x100.nl +++ b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x100.nl @@ -10,12 +10,12 @@ u_perturb = 1 rotate_grid = 0 ne=30 qsize = 0 -nmax = 10800 ! need to run 12960s -statefreq=600 +nmax = 4320 ! need to run 12960s +statefreq=72 runtype = 0 mesh_file='/dev/null' -tstep=1.2 -rsplit=10 +tstep=3.0 +rsplit=6 qsplit = 1 tstep_type = 7 integration = "explicit" @@ -51,8 +51,8 @@ profile_single_file = .true. interp_gridtype=2 output_timeunits=0 ! 1 is days, 2- hours, 0 - tsteps - output_frequency=720 - output_start_time=5040 + output_frequency=288 + output_start_time=2016 output_end_time=300000000 output_varnames1='ps','zeta','u','v','T' num_io_procs = 16 diff --git a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1000.nl b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1000.nl index 2e3fa2c4329d..f4eb3b2e0e76 100644 --- a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1000.nl +++ b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/nh-x1000.nl @@ -10,14 +10,14 @@ u_perturb = 1 rotate_grid = 0 ne=30 qsize = 0 -nmax = 10800 ! need to run 1296s. -statefreq=60 +nmax = 4320 ! need to run 1296s. +statefreq=72 runtype = 0 mesh_file='/dev/null' -tstep=0.12 ! this case can run with tstep=0.2 if tstep_type=5 -rsplit=6 ! aborts with dt_remap=1.2 (rsplit=10) +tstep=0.3 +rsplit=6 qsplit = 1 -tstep_type = 7 +tstep_type = 5 integration = "explicit" nu=1e6 nu_div=1e6 @@ -49,9 +49,11 @@ profile_single_file = .true. ! interp_nlon = 1024 interp_gridtype=2 +! output daily, from day 7 to 30 +! 1 day = 86.4/dt = 288 steps output_timeunits=0 ! 1 is days, 2- hours, 0 - tsteps - output_frequency=720 - output_start_time=5040 + output_frequency=288 + output_start_time=2016 output_end_time=300000000 output_varnames1='ps','zeta','u','v','T' num_io_procs = 16 diff --git a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/preqx-x1.nl b/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/preqx-x1.nl deleted file mode 100644 index 98e36a545596..000000000000 --- a/components/homme/dcmip_tests/dcmip2012_test4.1_baroclinic_instability/theta-l/preqx-x1.nl +++ /dev/null @@ -1,66 +0,0 @@ -&ctl_nl -theta_hydrostatic_mode = .true. -dcmip4_moist = 0 -dcmip4_X = 1.0 -NThreads=1 -partmethod = 4 -topology = "cube" -test_case = "dcmip2012_test4" -u_perturb = 1 -rotate_grid = 0 -ne=30 -qsize = 0 -ndays = 15 -statefreq=288 -runtype = 0 -mesh_file='/dev/null' -tstep=300 -rsplit=3 -qsplit = 1 -tstep_type = 5 -integration = "explicit" -nu=1e15 -nu_div=1e15 -nu_p=1e15 -nu_q=1e15 -nu_s=1e15 -nu_top = 0 -se_ftype = 0 -limiter_option = 9 -vert_remap_q_alg = 0 -hypervis_scaling=0 -hypervis_order = 2 -hypervis_subcycle = 1 -hypervis_subcycle = 1 -/ -&vert_nl -vform = "ccm" -vfile_mid = '../vcoord/camm-30.ascii' -vfile_int = '../vcoord/cami-30.ascii' -/ - -&prof_inparm -profile_outpe_num = 100 -profile_single_file = .true. -/ - -&analysis_nl -! to compare with EUL ref solution: -! interp_nlat = 512 -! interp_nlon = 1024 - interp_gridtype=2 - - output_timeunits=1 ! 1- days, 2 hours, 0 - tsteps - output_frequency=1 - output_start_time=7 - output_end_time=3000 - output_varnames1='ps','zeta','u','v','T' - num_io_procs = 16 - output_type = 'netcdf' - output_prefix = 'preqx-X1-' -/ - - - - - diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl index f7d435ffcea7..fe10d2833a12 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl index 18bf533e4b51..927ea6e4f680 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl index e7e426c688ea..64beee587d08 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 3 ! may require ftype=4 to get to rsplit=6? qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl index e68f15c847fc..d3c09747a1a0 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-tropical-cyclone-init.ncl @@ -76,6 +76,14 @@ res@gsnYAxisIrregular2Linear = True ilon0 = 0 ; location of perturbation ilon1 = nlon/2 ; +180 degrees +; original dcmip 2016 NCL script plots: +; T, theta,qv at ilon1 +; u,p at ilon0 +; pp perturbation from ilon0 and ilon1 +; but then we dont see the purturbation in T and theta, so +; we plot at ilon0 +; + print("max u="+max(u)) print("max u xsection="+max(u(:,:,ilon0))+" lon="+lon(ilon0)) diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl index e732631ca979..465441f64eda 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl index 46055c501374..f4e7ba22028a 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl index 216657f8f832..7f0808172837 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 3 ! aborts with rsplit=6. needs ftype=4? qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh index ad60110324e7..f61a2e94684e 100755 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh @@ -1,6 +1,7 @@ #!/bin/tcsh # # Anvil, 25 nodes, all 4 resolutions run in 25min +# need at least 11 nodes, since r400 is hardcoded as mpirun -np 384 # #SBATCH --job-name d16-3-theta #SBATCH --account=FY150001 diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl index 0090ff5e4c88..58d79de52232 100644 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl @@ -7,16 +7,16 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test3" ! test identifier ne = 30 ! number of elements per cube face - qsize = 4 ! num tracer fields - nmax = 28800 ! 7200s(120min)/tstep - statefreq = 100 ! number of steps between screen dumps + qsize = 3 ! num tracer fields + nmax = 3200 ! 7200s(120min)/tstep + statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 1.0 ! largest timestep in seconds + tstep = 2.00 ! 2.25 stable. 2.5 unstable integration = 'explicit' ! explicit time integration tstep_type = 7 rsplit = 3 - qsplit = 3 ! dt_tracer <= 3s + qsplit = 2 ! dt_tracer <= 3s nu = 5.8e8 ! default= 1e15/(120)^3 *(ne30/ne30)**3.2 nu_s = 0 nu_p = 0 @@ -41,12 +41,12 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 3 ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 300 ! 300 seconds + output_frequency = 600 ! 900 seconds output_varnames1 ='T','p','ps','pnh','geo','u','v','w','Th','Q','Q2','Q3','precl' ! variables to write to file - interp_nlon = 360 - interp_nlat = 181 +! interp_nlon = 360 +! interp_nlat = 181 interp_gridtype = 1 - interp_type = 0 ! 0=native grid, 1=bilinear + interp_type = 1 ! 0=native grid, 1=bilinear interp_lon0 = -180.0 ! shift lon range to [-180,+180) output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl index a99d3bd309d7..2defe82045ca 100644 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl @@ -7,16 +7,16 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test3" ! test identifier ne = 15 ! number of elements per cube face - qsize = 4 ! num tracer fields - nmax = 14400 ! 7200s(120min)/tstep + qsize = 3 ! num tracer fields + nmax = 1800 ! 7200s(120min)/tstep statefreq = 100 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 2. ! largest timestep in seconds + tstep = 4. ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 rsplit = 3 - qsplit = 3 + qsplit = 2 nu = 5.3e9 ! default= 1e15/(120)^3 *(ne30/ne15)**3.2 nu_s = 0 nu_p = 0 @@ -41,12 +41,12 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 3 ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 300 ! 300 seconds + output_frequency = 600 ! 300 seconds output_varnames1 ='T','p','ps','pnh','geo','u','v','w','Th','Q','Q2','Q3','precl' ! variables to write to file - interp_nlon = 360 - interp_nlat = 181 +! interp_nlon = 360 +! interp_nlat = 181 interp_gridtype = 1 - interp_type = 0 ! 0=native grid, 1=bilinear + interp_type = 1 ! 0=native grid, 1=bilinear interp_lon0 = -180.0 ! shift lon range to [-180,+180) output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r400.nl index c4284131464a..ef9b3caa7391 100644 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r400.nl @@ -7,16 +7,16 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test3" ! test identifier ne = 8 ! number of elements per cube face - qsize = 4 ! num tracer fields: qv,qc,qr - nmax = 7200 ! 7200s/tstep + qsize = 3 ! num tracer fields: qv,qc,qr + nmax = 1200 ! 7200s/tstep statefreq = 100 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 4.0 ! largest timestep in seconds + tstep = 6.0 integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 - qsplit = 3 + rsplit = 2 ! dt_remap=48 unstable, 32 stable. + qsplit = 2 nu = 1.5e10 ! default= 1e15/(120)^3 *(ne30/ne8)**3.2 = 4e10 nu_s = 0 nu_p = 0 @@ -41,12 +41,12 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 3 ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 300 ! 300 seconds + output_frequency = 600 ! 300 seconds output_varnames1 ='T','p','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','precl' ! variables to write to file - interp_nlon = 360 - interp_nlat = 181 +! interp_nlon = 360 +! interp_nlat = 181 interp_gridtype = 1 - interp_type = 0 ! 0=native grid, 1=bilinear + interp_type = 1 ! 0=native grid, 1=bilinear interp_lon0 = -180.0 ! shift lon range to [-180,+180) output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r50.nl index 70caaa57d873..2bb5daffbc44 100644 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r50.nl @@ -7,16 +7,16 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test3" ! test identifier ne = 60 ! number of elements per cube face - qsize = 4 ! num tracer fields - nmax = 28800 ! 7200s(120min)/tstep - statefreq = 100 ! number of steps between screen dumps + qsize = 3 ! num tracer fields + nmax = 7200 ! 7200s(120min)/tstep + statefreq = 360 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 0.5 ! largest timestep in seconds + tstep = 1.0 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 rsplit = 3 - qsplit = 3 ! dt_tracer <= 1.5s + qsplit = 2 ! dt_tracer <= 1.5s nu = 6.3e7 ! default= 1e15/(120)^3 *(ne30/ne60)**3.2 nu_s = 0 nu_p = 0 @@ -41,12 +41,12 @@ &analysis_nl output_dir = "./movies/" ! destination dir for netcdf file output_timeunits = 3 ! 0=timesteps, 1=days, 2=hours, 3=seconds - output_frequency = 300 ! 300 seconds + output_frequency = 600 ! 300 seconds output_varnames1 ='T','p','ps','pnh','geo','u','v','w','Th','Q','Q2','Q3','precl' ! variables to write to file - interp_nlon = 360 - interp_nlat = 181 +! interp_nlon = 360 +! interp_nlat = 181 interp_gridtype = 1 - interp_type = 0 ! 0=native grid, 1=bilinear + interp_type = 1 ! 0=native grid, 1=bilinear interp_lon0 = -180.0 ! shift lon range to [-180,+180) output_type ='netcdf' ! netcdf or pnetcdf num_io_procs = 16 diff --git a/components/homme/src/theta-l/eos.F90 b/components/homme/src/theta-l/eos.F90 index e5f85b68ed82..fc33177786f9 100644 --- a/components/homme/src/theta-l/eos.F90 +++ b/components/homme/src/theta-l/eos.F90 @@ -224,8 +224,8 @@ subroutine get_dirk_jacobian(JacL,JacD,JacU,dt2,dp3d,phi_i,pnh,exact,& ! The rule-of-thumb optimal epsie is epsie = norm(elem)*sqrt(macheps) !=================================================================================== real (kind=real_kind), intent(out) :: JacD(nlev,np,np) - real (kind=real_kind), intent(out) :: JacL(nlev-1,np,np),JacU(nlev-1,np,np), phi_i(np,np,nlevp) - real (kind=real_kind), intent(in) :: dp3d(np,np,nlev) + real (kind=real_kind), intent(out) :: JacL(nlev-1,np,np),JacU(nlev-1,np,np) + real (kind=real_kind), intent(in) :: dp3d(np,np,nlev), phi_i(np,np,nlevp) real (kind=real_kind), intent(inout) :: pnh(np,np,nlev) real (kind=real_kind), intent(in) :: dt2 diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index ccc06ad2ca68..dda3f4beed6a 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -47,8 +47,6 @@ module prim_advance_mod applycamforcing_ps, applycamforcing_dp3d, & applyCAMforcing_dynamics, vertical_mesh_init2 - real (kind=real_kind), allocatable :: ur_weights(:) - contains @@ -63,20 +61,6 @@ subroutine prim_advance_init1(par, elem,integration) integer :: i integer :: ie - ! compute averaging weights for RK+LF (tstep_type=1) timestepping: - allocate(ur_weights(qsplit)) - ur_weights(:)=0.0d0 - - if(mod(qsplit,2).NE.0)then - ur_weights(1)=1.0d0/qsplit - do i=3,qsplit,2 - ur_weights(i)=2.0d0/qsplit - enddo - else - do i=2,qsplit,2 - ur_weights(i)=2.0d0/qsplit - enddo - endif end subroutine prim_advance_init1 @@ -175,6 +159,22 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co ! leapfrog: u(dt) = u(0) + dt RHS(dt/2) (store in u(np1)) call compute_andor_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,& deriv,nets,nete,.false.,eta_ave_w,1.d0,1.d0,1.d0) + + + else if (tstep_type==4) then ! explicit table from IMEX-KG254 method + call compute_andor_apply_rhs(np1,n0,n0,qn0,dt/4,elem,hvcoord,hybrid,& + deriv,nets,nete,compute_diagnostics,0d0,1.d0,1.d0,1.d0) + call compute_andor_apply_rhs(np1,n0,np1,qn0,dt/6,elem,hvcoord,hybrid,& + deriv,nets,nete,compute_diagnostics,0d0,1.d0,1.d0,1.d0) + call compute_andor_apply_rhs(np1,n0,np1,qn0,3*dt/8,elem,hvcoord,hybrid,& + deriv,nets,nete,compute_diagnostics,0d0,1.d0,1.d0,1.d0) + call compute_andor_apply_rhs(np1,n0,np1,qn0,dt/2,elem,hvcoord,hybrid,& + deriv,nets,nete,compute_diagnostics,0d0,1.d0,1.d0,1.d0) + call compute_andor_apply_rhs(np1,n0,np1,qn0,dt,elem,hvcoord,hybrid,& + deriv,nets,nete,compute_diagnostics,eta_ave_w*1d0,1.d0,1.d0,1.d0) + + + else if (tstep_type==5) then ! Ullrich 3nd order 5 stage: CFL=sqrt( 4^2 -1) = 3.87 ! u1 = u0 + dt/5 RHS(u0) (save u1 in timelevel nm1) @@ -269,22 +269,32 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co max_itercnt_perstep = 0 max_itererr_perstep = 0.0 - a1 = 1./4. - a2 = 1./6. - a3 = 3./8. - a4 = 1./2. + a1 = 1/4d0 + a2 = 1/6d0 + a3 = 3/8d0 + a4 = 1/2d0 a5 = 1 - ahat1 = 0. - ahat5 = 1. + ahat1 = 0 + ahat5 = 1 - ! IMEX-KGO254c coefficients - dhat2 = 5./6. - dhat3 = 5./6. - dhat4 = 2./3. - ahat4 = 1./2.-dhat4 - dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& + + ! IMEX-KGO254 most stable coefficients + dhat2 = 1d0 + dhat3 = 1d0 + dhat4 = 1d0 + ahat4 = 1d0/2d0-dhat4 + dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& (ahat5-dhat3-dhat2-dhat4) + + ! IMEX-KGO254c coefficients + !dhat2 = 5/6d0 + !dhat3 = 5/6d0 + !dhat4 = 2/3d0 + !ahat4 = 1/2d0-dhat4 + ! dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& + ! (ahat5-dhat3-dhat2-dhat4) + ! IMEX-KGO254b coefficients NOT GOOD ! dhat2 = 1./6. ! dhat3 = 1./6. @@ -586,18 +596,22 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w (hvcoord%hybi(k+1)-hvcoord%hybi(k))*ps_ref(:,:) enddo - call get_phinh(hvcoord,elem(ie)%state%phis,& - elem(ie)%state%vtheta_dp(:,:,:,nt),elem(ie)%state%dp3d(:,:,:,nt),& - phi_ref(:,:,:,ie)) - do k=1,nlev ! convert vtheta_dp -> theta elem(ie)%state%vtheta_dp(:,:,k,nt)=& elem(ie)%state%vtheta_dp(:,:,k,nt)/elem(ie)%state%dp3d(:,:,k,nt) enddo - call set_theta_ref(hvcoord,elem(ie)%state%dp3d(:,:,:,nt),theta_ref(:,:,:,ie)) +! call set_theta_ref(hvcoord,elem(ie)%state%dp3d(:,:,:,nt),theta_ref(:,:,:,ie)) + call set_theta_ref(hvcoord,dp_ref(:,:,:,ie),theta_ref(:,:,:,ie)) + + ! compute vtheta_dp_ref, store in 'heating' as temp array: + heating(:,:,:)=theta_ref(:,:,:,ie)*dp_ref(:,:,:,ie) + call get_phinh(hvcoord,elem(ie)%state%phis,& + heating(:,:,:),dp_ref(:,:,:,ie),phi_ref(:,:,:,ie)) + #if 0 + ! disable reference background states theta_ref(:,:,:,ie)=0 phi_ref(:,:,:,ie)=0 dp_ref(:,:,:,ie)=0 @@ -776,8 +790,9 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w endif elem(ie)%state%vtheta_dp(:,:,k,nt)=elem(ie)%state%vtheta_dp(:,:,k,nt) & - +stens(:,:,k,2,ie)*hvcoord%dp0(k)*exner0(k)/(exner(:,:,k)*elem(ie)%state%dp3d(:,:,k,nt))& - -heating(:,:,k) + +stens(:,:,k,2,ie)*hvcoord%dp0(k)*exner0(k)/(exner(:,:,k)*elem(ie)%state%dp3d(:,:,k,nt)) +!& +! -heating(:,:,k) enddo enddo @@ -1070,8 +1085,7 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& vtheta(:,:,:) = vtheta_dp(:,:,:)/dp3d(:,:,:) phi_i => elem(ie)%state%phinh_i(:,:,:,n0) - call get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_i,& - pnh,exner,dpnh_dp_i) + call get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_i,pnh,exner,dpnh_dp_i) dp3d_i(:,:,1) = dp3d(:,:,1) dp3d_i(:,:,nlevp) = dp3d(:,:,nlev) @@ -1631,7 +1645,6 @@ subroutine compute_stage_value_dirk(np1,qn0,dt2,elem,hvcoord,hybrid,& real (kind=real_kind) :: phi_n0(np,np,nlevp) real (kind=real_kind) :: Ipiv(nlev,np,np) real (kind=real_kind) :: Fn(np,np,nlev),x(nlev,np,np) - real (kind=real_kind) :: pnh_i(np,np,nlevp) real (kind=real_kind) :: itererr,itererrtemp(np,np) real (kind=real_kind) :: itercountmax,itererrmax real (kind=real_kind) :: norminfr0(np,np),norminfJ0(np,np) @@ -1659,8 +1672,7 @@ subroutine compute_stage_value_dirk(np1,qn0,dt2,elem,hvcoord,hybrid,& phi_np1 => elem(ie)%state%phinh_i(:,:,:,np1) phis => elem(ie)%state%phis(:,:) - call get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_np1,& - pnh,exner,dpnh_dp_i,pnh_i_out=pnh_i) + call get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_np1,pnh,exner,dpnh_dp_i) dp3d_i(:,:,1) = dp3d(:,:,1) dp3d_i(:,:,nlevp) = dp3d(:,:,nlev) @@ -1706,6 +1718,7 @@ subroutine compute_stage_value_dirk(np1,qn0,dt2,elem,hvcoord,hybrid,& maxnorminfJ0r0=max(maxval(norminfJ0(:,:)),maxval(norminfr0(:,:))) itererr=maxval(itererrtemp(:,:))/maxnorminfJ0r0 + do while ((itercount < maxiter).and.(itererr > itertol)) info(:,:) = 0 @@ -1730,8 +1743,7 @@ subroutine compute_stage_value_dirk(np1,qn0,dt2,elem,hvcoord,hybrid,& end do end do - call get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_np1,& - pnh,exner,dpnh_dp_i,pnh_i_out=pnh_i) + call get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_np1,pnh,exner,dpnh_dp_i) ! update approximate solution of w elem(ie)%state%w_i(:,:,1:nlev,np1) = w_n0(:,:,1:nlev) - g*dt2 * & From 044fec772416ce5723e61036889bfcad9e97f738 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Thu, 4 Oct 2018 11:22:51 -0500 Subject: [PATCH 08/41] add q=1 tracer to DCMIP2016 test1 to check consistency --- .../preqx/namelist-r100-dry.nl | 2 +- .../preqx/namelist-r100.nl | 2 +- .../preqx/namelist-r400-dry.nl | 2 +- .../preqx/namelist-r400.nl | 2 +- .../preqx/namelist-r50-dry.nl | 2 +- .../preqx/namelist-r50.nl | 2 +- .../theta-l/namelist-r100-dry.nl | 2 +- .../theta-l/namelist-r100-h.nl | 2 +- .../theta-l/namelist-r100.nl | 2 +- .../theta-l/namelist-r400.nl | 2 +- .../theta-l/namelist-r50.nl | 2 +- components/homme/src/test_src/dcmip16_wrapper.F90 | 7 ++++--- components/homme/src/theta-l/prim_advance_mod.F90 | 10 +++++----- .../test_execs/preqx-nlev30-interp/CMakeLists.txt | 2 +- .../test_execs/preqx-nlev30-native/CMakeLists.txt | 2 +- .../homme/test_execs/theta-l-nlev30/CMakeLists.txt | 2 +- .../homme/test_execs/theta-nlev30/CMakeLists.txt | 2 +- 17 files changed, 24 insertions(+), 23 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100-dry.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100-dry.nl index 41c7249e7643..9aca3c4a7382 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100-dry.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100-dry.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl index 18bf533e4b51..56871271186c 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400-dry.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400-dry.nl index b60e3fc19a50..040fd0fb80fa 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400-dry.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400-dry.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 8 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl index e02226ddc348..36e23e480e50 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 8 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50-dry.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50-dry.nl index c5fc86be1420..99504702a6e2 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50-dry.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50-dry.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 60 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl index 425085cdd5a4..625c53671a93 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 60 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl index fe10d2833a12..94320a1ce506 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl index 927ea6e4f680..b8046b7ed350 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl index 64beee587d08..977e0b71f295 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl index e0a52a12a4bf..a8447fab6234 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 8 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl index 2db8686d0449..d5e9fac383be 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 60 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index c6e28a7d46ef..55f9d7f10916 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -81,7 +81,7 @@ subroutine dcmip2016_test1(elem,hybrid,hvcoord,nets,nete) real(rl), dimension(np,np,nlev):: p,z,u,v,w,T,thetav,rho,dp ! field values real(rl), dimension(np,np,nlevp):: p_i,w_i,z_i real(rl), dimension(np,np):: ps, phis - real(rl), dimension(np,np,nlev,5):: q + real(rl), dimension(np,np,nlev,6):: q real(rl) :: min_thetav, max_thetav min_thetav = +huge(rl) @@ -118,7 +118,8 @@ subroutine dcmip2016_test1(elem,hybrid,hvcoord,nets,nete) lon = elem(ie)%spherep(i,j)%lon lat = elem(ie)%spherep(i,j)%lat - q(i,j,k,:) = 0.0d0 + q(i,j,k,1:5) = 0.0d0 + q(i,j,k,6) = 1 w(i,j,k) = 0.0d0 call baroclinic_wave_test(is_deep,moist,pertt,dcmip_X,lon,lat,p(i,j,k),& @@ -126,7 +127,7 @@ subroutine dcmip2016_test1(elem,hybrid,hvcoord,nets,nete) ! initialize tracer chemistry call initial_value_terminator( lat*rad2dg, lon*rad2dg, q(i,j,k,4), q(i,j,k,5) ) - call set_tracers(q(i,j,k,1:5),5,dp(i,j,k),i,j,k,lat,lon,elem(ie)) + call set_tracers(q(i,j,k,1:6),6,dp(i,j,k),i,j,k,lat,lon,elem(ie)) min_thetav = min( min_thetav, thetav(i,j,k) ) max_thetav = max( max_thetav, thetav(i,j,k) ) diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index dda3f4beed6a..c3a13aba4674 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -785,14 +785,13 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w elem(ie)%state%v(:,:,2,k,nt)*vtens(:,:,2,k,ie) +& (elem(ie)%state%w_i(:,:,k,nt)*stens(:,:,k,3,ie) +& elem(ie)%state%w_i(:,:,k2,nt)*stens(:,:,k2,3,ie))/2 +& - stens(:,:,k,4,ie) ) / & + ( stens(:,:,k,4,ie)+stens(:,:,k2,4,ie))/2 ) / & (exner(:,:,k)*Cp) endif elem(ie)%state%vtheta_dp(:,:,k,nt)=elem(ie)%state%vtheta_dp(:,:,k,nt) & - +stens(:,:,k,2,ie)*hvcoord%dp0(k)*exner0(k)/(exner(:,:,k)*elem(ie)%state%dp3d(:,:,k,nt)) -!& -! -heating(:,:,k) + +stens(:,:,k,2,ie)*hvcoord%dp0(k)*exner0(k)/(exner(:,:,k)*elem(ie)%state%dp3d(:,:,k,nt))& + -heating(:,:,k) enddo enddo @@ -1066,6 +1065,7 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& do ie=nets,nete +#if 0 if (.not. theta_hydrostatic_mode) then temp(:,:,1) = (elem(ie)%state%v(:,:,1,nlev,n0)*elem(ie)%derived%gradphis(:,:,1) + & elem(ie)%state%v(:,:,2,nlev,n0)*elem(ie)%derived%gradphis(:,:,2))/g @@ -1079,7 +1079,7 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& elem(ie)%state%w_i(:,:,nlevp,n0) = (elem(ie)%state%v(:,:,1,nlev,n0)*elem(ie)%derived%gradphis(:,:,1) + & elem(ie)%state%v(:,:,2,nlev,n0)*elem(ie)%derived%gradphis(:,:,2))/g endif - +#endif dp3d => elem(ie)%state%dp3d(:,:,:,n0) vtheta_dp => elem(ie)%state%vtheta_dp(:,:,:,n0) vtheta(:,:,:) = vtheta_dp(:,:,:)/dp3d(:,:,:) diff --git a/components/homme/test_execs/preqx-nlev30-interp/CMakeLists.txt b/components/homme/test_execs/preqx-nlev30-interp/CMakeLists.txt index 151484295a8d..3b611e78f18e 100644 --- a/components/homme/test_execs/preqx-nlev30-interp/CMakeLists.txt +++ b/components/homme/test_execs/preqx-nlev30-interp/CMakeLists.txt @@ -3,5 +3,5 @@ preqx_setup() # name target NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D -createTestExec(preqx-nlev30-interp preqx 4 4 30 FALSE TRUE 5) +createTestExec(preqx-nlev30-interp preqx 4 4 30 FALSE TRUE 6) diff --git a/components/homme/test_execs/preqx-nlev30-native/CMakeLists.txt b/components/homme/test_execs/preqx-nlev30-native/CMakeLists.txt index 3017f27d9fd5..284fabcc9d63 100644 --- a/components/homme/test_execs/preqx-nlev30-native/CMakeLists.txt +++ b/components/homme/test_execs/preqx-nlev30-native/CMakeLists.txt @@ -3,4 +3,4 @@ preqx_setup() # name target NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D -createTestExec(preqx-nlev30-native preqx 4 4 30 TRUE FALSE 4) +createTestExec(preqx-nlev30-native preqx 4 4 30 TRUE FALSE 6) diff --git a/components/homme/test_execs/theta-l-nlev30/CMakeLists.txt b/components/homme/test_execs/theta-l-nlev30/CMakeLists.txt index fc428da98298..586f67bf3d4e 100644 --- a/components/homme/test_execs/theta-l-nlev30/CMakeLists.txt +++ b/components/homme/test_execs/theta-l-nlev30/CMakeLists.txt @@ -3,5 +3,5 @@ thetal_setup() # name target NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D -createTestExec(theta-l-nlev30 theta-l 4 4 30 FALSE TRUE 5) +createTestExec(theta-l-nlev30 theta-l 4 4 30 FALSE TRUE 6) diff --git a/components/homme/test_execs/theta-nlev30/CMakeLists.txt b/components/homme/test_execs/theta-nlev30/CMakeLists.txt index 4756a0f61605..97e6eced78f8 100644 --- a/components/homme/test_execs/theta-nlev30/CMakeLists.txt +++ b/components/homme/test_execs/theta-nlev30/CMakeLists.txt @@ -3,5 +3,5 @@ theta_setup() # name target NP NC PLEV USE_PIO WITH_ENERGY QSIZE_D -createTestExec(theta-nlev30 theta 4 4 30 FALSE TRUE 5) +createTestExec(theta-nlev30 theta 4 4 30 FALSE TRUE 6) From ddf0fa6d2918d58d862844bf7c6a82e371260aa1 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Thu, 4 Oct 2018 12:55:09 -0500 Subject: [PATCH 09/41] clean up omega diagnostic calcluation --- components/homme/src/theta-l/element_ops.F90 | 5 ++--- components/homme/src/theta-l/eos.F90 | 1 - .../homme/src/theta-l/prim_advance_mod.F90 | 19 +++++++++---------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/components/homme/src/theta-l/element_ops.F90 b/components/homme/src/theta-l/element_ops.F90 index 5bd1389078e3..b1a3f205aa07 100644 --- a/components/homme/src/theta-l/element_ops.F90 +++ b/components/homme/src/theta-l/element_ops.F90 @@ -103,8 +103,7 @@ recursive subroutine get_field(elem,name,field,hvcoord,nt,ntQ) enddo case ('omega'); - call get_field(elem,'p',p,hvcoord,nt,ntQ) - field = elem%derived%omega_p*p + field = elem%derived%omega_p case('rho') @@ -471,7 +470,7 @@ subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) if(theta_hydrostatic_mode) then ! overwrite w and phi_i computed above - w = -(elem%derived%omega_p*pnh)/(rho*g) + w = -(elem%derived%omega_p)/(rho*g) do k=nlev,1,-1 temp(:,:,k) = Rgas*elem%state%vtheta_dp(:,:,k,nt)*exner(:,:,k)/pnh(:,:,k) diff --git a/components/homme/src/theta-l/eos.F90 b/components/homme/src/theta-l/eos.F90 index fc33177786f9..d623088ec347 100644 --- a/components/homme/src/theta-l/eos.F90 +++ b/components/homme/src/theta-l/eos.F90 @@ -26,7 +26,6 @@ module eos use parallel_mod, only: abortmp use physical_constants, only : p0, kappa, g, Rgas use control_mod, only: theta_hydrostatic_mode - use prim_si_mod, only: preq_hydrostatic_v2, preq_omega_ps implicit none diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index c3a13aba4674..aa3ca5f78aa9 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -29,7 +29,7 @@ module prim_advance_mod use parallel_mod, only: abortmp, global_shared_buf, global_shared_sum, iam, parallel_t use physical_constants, only: Cp, cp, cpwater_vapor, g, kappa, Rgas, Rwater_vapor, p0 use physics_mod, only: virtual_specific_heat, virtual_temperature - use prim_si_mod, only: preq_vertadv_v1, preq_omega_ps + use prim_si_mod, only: preq_vertadv_v1 use reduction_mod, only: parallelmax, reductionbuffer_ordered_1d_t use time_mod, only: timelevel_qdp, timelevel_t use test_mod, only: set_prescribed_wind @@ -1004,7 +1004,8 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& real (kind=real_kind) :: vtheta(np,np,nlev) real (kind=real_kind) :: vtheta_i(np,np,nlevp) - real (kind=real_kind) :: omega_p(np,np,nlev) + real (kind=real_kind) :: omega_i(np,np,nlevp) + real (kind=real_kind) :: omega(np,np,nlev) real (kind=real_kind) :: vort(np,np,nlev) ! vorticity real (kind=real_kind) :: divdp(np,np,nlev) real (kind=real_kind) :: phi(np,np,nlev) @@ -1127,23 +1128,21 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& vort(:,:,k)=vorticity_sphere(elem(ie)%state%v(:,:,:,k,n0),deriv,elem(ie)) enddo - ! Compute omega_p = 1/pi Dpi/Dt - ! first compute hydrostatic pressure + ! Compute omega = Dpi/Dt Used only as a DIAGNOSTIC + ! for historical reasons, we actually compute w/pi pi_i(:,:,1)=hvcoord%hyai(1)*hvcoord%ps0 + omega_i(:,:,1)=0 do k=1,nlev pi_i(:,:,k+1)=pi_i(:,:,k) + dp3d(:,:,k) + omega_i(:,:,k+1)=omega_i(:,:,k)+divdp(:,:,k) enddo do k=1,nlev pi(:,:,k)=pi_i(:,:,k) + dp3d(:,:,k)/2 - enddo - do k=1,nlev - ! get hydrostatic pressure flux to compute omega=dp/dt vtemp(:,:,:,k) = gradient_sphere( pi(:,:,k), deriv, elem(ie)%Dinv); vgrad_p(:,:,k) = elem(ie)%state%v(:,:,1,k,n0)*vtemp(:,:,1,k)+& elem(ie)%state%v(:,:,2,k,n0)*vtemp(:,:,2,k) + omega(:,:,k) = (vgrad_p(:,:,k) - ( omega_i(:,:,k)+omega_i(:,:,k+1))/2) enddo - call preq_omega_ps(omega_p,hvcoord,pi,vgrad_p,divdp) - ! ================================================== ! Compute eta_dot_dpdn @@ -1241,7 +1240,7 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& elem(ie)%derived%eta_dot_dpdn(:,:,k) = & elem(ie)%derived%eta_dot_dpdn(:,:,k) + eta_ave_w*eta_dot_dpdn(:,:,k) elem(ie)%derived%omega_p(:,:,k) = & - elem(ie)%derived%omega_p(:,:,k) + eta_ave_w*omega_p(:,:,k) + elem(ie)%derived%omega_p(:,:,k) + eta_ave_w*omega(:,:,k) enddo elem(ie)%derived%eta_dot_dpdn(:,:,nlev+1) = & elem(ie)%derived%eta_dot_dpdn(:,:,nlev+1) + eta_ave_w*eta_dot_dpdn(:,:,nlev+1) From 56f7b7cb11595da5508f9453bf9977c94c4523e5 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 5 Oct 2018 08:47:11 -0500 Subject: [PATCH 10/41] remove heating term heating term appears to be buggy. it destablizes dcmip2016 test1 --- components/homme/src/theta-l/element_ops.F90 | 4 ++-- components/homme/src/theta-l/prim_advance_mod.F90 | 4 ++-- components/homme/src/theta-l/prim_state_mod.F90 | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/homme/src/theta-l/element_ops.F90 b/components/homme/src/theta-l/element_ops.F90 index b1a3f205aa07..427f7d2eae4e 100644 --- a/components/homme/src/theta-l/element_ops.F90 +++ b/components/homme/src/theta-l/element_ops.F90 @@ -53,7 +53,7 @@ module element_ops use perf_mod, only: t_startf, t_stopf, t_barrierf, t_adj_detailf ! _EXTERNAL use parallel_mod, only: abortmp use physical_constants, only : p0, Cp, Rgas, Rwater_vapor, Cpwater_vapor, kappa, g, dd_pi - use control_mod, only: use_moisture, use_cpstar, theta_hydrostatic_mode + use control_mod, only: use_moisture, theta_hydrostatic_mode use eos, only: get_pnh_and_exner, get_phinh use prim_si_mod, only: preq_hydrostatic_v2 implicit none @@ -596,7 +596,7 @@ subroutine get_cp_star(cp_star,Q) real (kind=real_kind), intent(in) :: Q(np,np,nlev) integer :: k - if (use_moisture .and. use_cpstar==1) then + if (use_moisture) then #if (defined COLUMN_OPENMP) !$omp parallel do default(shared), private(k) #endif diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index aa3ca5f78aa9..c15379afec62 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -790,8 +790,8 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w endif elem(ie)%state%vtheta_dp(:,:,k,nt)=elem(ie)%state%vtheta_dp(:,:,k,nt) & - +stens(:,:,k,2,ie)*hvcoord%dp0(k)*exner0(k)/(exner(:,:,k)*elem(ie)%state%dp3d(:,:,k,nt))& - -heating(:,:,k) + +stens(:,:,k,2,ie)*hvcoord%dp0(k)*exner0(k)/(exner(:,:,k)*elem(ie)%state%dp3d(:,:,k,nt)& + ) ! -heating(:,:,k) enddo enddo diff --git a/components/homme/src/theta-l/prim_state_mod.F90 b/components/homme/src/theta-l/prim_state_mod.F90 index 7eff30f53e11..128781ea3e95 100644 --- a/components/homme/src/theta-l/prim_state_mod.F90 +++ b/components/homme/src/theta-l/prim_state_mod.F90 @@ -11,7 +11,7 @@ module prim_state_mod use global_norms_mod, only: wrap_repro_sum use hybrid_mod, only: hybrid_t use time_mod, only: tstep, secpday, timelevel_t, TimeLevel_Qdp, time_at - use control_mod, only: integration, test_case, moisture, & + use control_mod, only: integration, test_case, use_moisture, & qsplit, ftype, rsplit,& theta_hydrostatic_mode use hybvcoord_mod, only: hvcoord_t @@ -642,7 +642,7 @@ subroutine prim_printstate(elem, tl,hybrid,hvcoord,nets,nete) if(hybrid%masterthread) then - if(moisture /= "dry")then + if(use_moisture)then if (qsize>=1) then write(iulog,'(a,E23.15,a,E23.15,a)') " dry M = ",Mass-Q1mass(1),' kg/m^2' endif From 30c53fddc06e58ad204aec9251f111b138df3ab5 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 5 Oct 2018 08:50:28 -0500 Subject: [PATCH 11/41] remove tracer 6 (q=1, for consistency check) from non-dry test cases consistency check only makes sense in dry test cases. in moist test cases, q1 forcing is also applied to dp3d and so it is not possible to check consistency with a q=1 tracer. --- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl index 56871271186c..18bf533e4b51 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 6 ! num tracer fields + qsize = 5 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl index 36e23e480e50..e02226ddc348 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 8 ! number of elements per cube face - qsize = 6 ! num tracer fields + qsize = 5 ! num tracer fields ndays = 30 statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl index 625c53671a93..425085cdd5a4 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 60 ! number of elements per cube face - qsize = 6 ! num tracer fields + qsize = 5 ! num tracer fields ndays = 30 statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl index b8046b7ed350..927ea6e4f680 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 6 ! num tracer fields + qsize = 5 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl index 977e0b71f295..64beee587d08 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 6 ! num tracer fields + qsize = 5 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl index a8447fab6234..e0a52a12a4bf 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 8 ! number of elements per cube face - qsize = 6 ! num tracer fields + qsize = 5 ! num tracer fields ndays = 30 statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl index d5e9fac383be..2db8686d0449 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 60 ! number of elements per cube face - qsize = 6 ! num tracer fields + qsize = 5 ! num tracer fields ndays = 30 statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 From 9692b3248b3de905f97cc64a50cf7fe316da12b9 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 5 Oct 2018 09:09:07 -0500 Subject: [PATCH 12/41] Revert "remove tracer 6 (q=1, for consistency check) from non-dry test cases" This reverts commit 30c53fddc06e58ad204aec9251f111b138df3ab5. test case IC code sets 6 tracers, so we need qsize>=6 for all namelists --- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl index 18bf533e4b51..56871271186c 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl index e02226ddc348..36e23e480e50 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 8 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl index 425085cdd5a4..625c53671a93 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 60 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl index 927ea6e4f680..b8046b7ed350 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl index 64beee587d08..977e0b71f295 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 30 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 72 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl index e0a52a12a4bf..a8447fab6234 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 8 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 24 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl index 2db8686d0449..d5e9fac383be 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl @@ -7,7 +7,7 @@ topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier ne = 60 ! number of elements per cube face - qsize = 5 ! num tracer fields + qsize = 6 ! num tracer fields ndays = 30 statefreq = 144 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 From e96e1f9ebfae62d5f30fdc0c4109f3bdc15b6aa9 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 5 Oct 2018 15:48:53 -0500 Subject: [PATCH 13/41] disable heating term, revert to original reference profile Instabilities were being caused by a buggy heating term. Disabling for now. The heating term appears to also have been responsible for instabilities previously blamed on the reference profiles used in the hyperviscosity operator. With heating term disabled, reverting back to the original more accurate reference profiles. --- .../homme/src/theta-l/prim_advance_mod.F90 | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index c15379afec62..7945dc7d9e23 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -277,7 +277,7 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co ahat1 = 0 ahat5 = 1 - +#if 1 ! IMEX-KGO254 most stable coefficients dhat2 = 1d0 dhat3 = 1d0 @@ -285,24 +285,27 @@ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, co ahat4 = 1d0/2d0-dhat4 dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& (ahat5-dhat3-dhat2-dhat4) +#endif - +#if 0 ! IMEX-KGO254c coefficients - !dhat2 = 5/6d0 - !dhat3 = 5/6d0 - !dhat4 = 2/3d0 - !ahat4 = 1/2d0-dhat4 - ! dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& - ! (ahat5-dhat3-dhat2-dhat4) + dhat2 = 5/6d0 + dhat3 = 5/6d0 + dhat4 = 2/3d0 + ahat4 = 1/2d0-dhat4 + dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& + (ahat5-dhat3-dhat2-dhat4) +#endif +#if 0 ! IMEX-KGO254b coefficients NOT GOOD -! dhat2 = 1./6. -! dhat3 = 1./6. -! dhat4 = 1./6. -! ahat4 = 1./2.-dhat4 -!! dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& -! (ahat5-dhat3-dhat2-dhat4) - + dhat2 = 1./6. + dhat3 = 1./6. + dhat4 = 1./6. + ahat4 = 1./2.-dhat4 + dhat1= (ahat4*ahat5 - ahat5*dhat3 - ahat5*dhat2 + dhat3*dhat2+ dhat3*dhat4 + dhat2*dhat4)/& + (ahat5-dhat3-dhat2-dhat4) +#endif ! IMEX-KG254 ahat3 = (- ahat4*ahat5*dhat1 - ahat4*ahat5*dhat2+ ahat5*dhat1*dhat2 + ahat5*dhat1*dhat3 +& @@ -602,13 +605,21 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w elem(ie)%state%vtheta_dp(:,:,k,nt)/elem(ie)%state%dp3d(:,:,k,nt) enddo -! call set_theta_ref(hvcoord,elem(ie)%state%dp3d(:,:,:,nt),theta_ref(:,:,:,ie)) +#if 0 + ! reference state based only on ps: call set_theta_ref(hvcoord,dp_ref(:,:,:,ie),theta_ref(:,:,:,ie)) ! compute vtheta_dp_ref, store in 'heating' as temp array: heating(:,:,:)=theta_ref(:,:,:,ie)*dp_ref(:,:,:,ie) call get_phinh(hvcoord,elem(ie)%state%phis,& heating(:,:,:),dp_ref(:,:,:,ie),phi_ref(:,:,:,ie)) +#endif +#if 1 + ! most realistic reference state based on hydrostatic pressure + call set_theta_ref(hvcoord,elem(ie)%state%dp3d(:,:,:,nt),theta_ref(:,:,:,ie)) + call get_phinh(hvcoord,elem(ie)%state%phis,& + elem(ie)%state%vtheta_dp(:,:,:,nt),elem(ie)%state%dp3d(:,:,:,nt),phi_ref(:,:,:,ie)) +#endif #if 0 ! disable reference background states @@ -757,11 +768,6 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w ! we want exner*cp*dp3d*heating = dp3d*U dot diss(U) ! and thus heating = U dot diss(U) / exner*cp ! - ! PE dissipation - ! d(PE)/dt = dp3d diss(phi) - ! we want dp3d diss(phi) = exner*cp*dp3d*heating - ! heating = diss(phi) / exner*cp - ! ! use hydrostatic pressure for simplicity p_i(:,:,1)=hvcoord%hyai(1)*hvcoord%ps0 do k=1,nlev @@ -784,14 +790,13 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w heating(:,:,k)= (elem(ie)%state%v(:,:,1,k,nt)*vtens(:,:,1,k,ie) + & elem(ie)%state%v(:,:,2,k,nt)*vtens(:,:,2,k,ie) +& (elem(ie)%state%w_i(:,:,k,nt)*stens(:,:,k,3,ie) +& - elem(ie)%state%w_i(:,:,k2,nt)*stens(:,:,k2,3,ie))/2 +& - ( stens(:,:,k,4,ie)+stens(:,:,k2,4,ie))/2 ) / & + elem(ie)%state%w_i(:,:,k2,nt)*stens(:,:,k2,3,ie))/2 ) / +& (exner(:,:,k)*Cp) endif elem(ie)%state%vtheta_dp(:,:,k,nt)=elem(ie)%state%vtheta_dp(:,:,k,nt) & +stens(:,:,k,2,ie)*hvcoord%dp0(k)*exner0(k)/(exner(:,:,k)*elem(ie)%state%dp3d(:,:,k,nt)& - ) ! -heating(:,:,k) + ) ! -heating(:,:,k) enddo enddo From 3eab88d9e96e55cc67968cc1fcf24450989a489d Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 5 Oct 2018 19:01:09 -0500 Subject: [PATCH 14/41] added pressure based TOM sponge layer option from Lauritzen also moved vertical_levels_init2 out of prim_advance_mod.F90 and into model_init_mod.F90 --- components/homme/src/pese/model_init_mod.F90 | 17 ++++++ .../homme/src/pese/prim_advance_mod.F90 | 15 ------ components/homme/src/preqx/model_init_mod.F90 | 13 +++++ .../src/preqx/share/prim_advance_mod.F90 | 16 +----- .../homme/src/preqx_acc/model_init_mod.F90 | 13 +++++ .../homme/src/share/prim_driver_base.F90 | 8 +-- .../homme/src/theta-l/element_state.F90 | 3 ++ .../homme/src/theta-l/model_init_mod.F90 | 53 +++++++++++++++++-- .../homme/src/theta-l/prim_advance_mod.F90 | 48 +++++------------ components/homme/src/theta/model_init_mod.F90 | 13 +++++ .../homme/src/theta/prim_advance_mod.F90 | 16 +----- 11 files changed, 129 insertions(+), 86 deletions(-) diff --git a/components/homme/src/pese/model_init_mod.F90 b/components/homme/src/pese/model_init_mod.F90 index decc4643dbbf..a4b63a4472b5 100644 --- a/components/homme/src/pese/model_init_mod.F90 +++ b/components/homme/src/pese/model_init_mod.F90 @@ -19,6 +19,23 @@ module model_init_mod contains + !_____________________________________________________________________ + subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) + + type (element_t), intent(inout), target :: elem(:) ! array of element_t structures + integer, intent(in) :: nets,nete ! start and end element indices + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + + if (hybrid%masterthread) print *,"initializing PESE dynamics solver" + + ! initialize vertical operators and coordinates + call make_vertical_mesh(hybrid, hvcoord) + + end subroutine vertical_mesh_init2 + + + subroutine model_init2( elem , hybrid, deriv,hvcoord,tl,nets,nete ) type(element_t) , intent(in) :: elem(:) diff --git a/components/homme/src/pese/prim_advance_mod.F90 b/components/homme/src/pese/prim_advance_mod.F90 index 7c97ee8bfcee..6aba36c26be7 100644 --- a/components/homme/src/pese/prim_advance_mod.F90 +++ b/components/homme/src/pese/prim_advance_mod.F90 @@ -53,21 +53,6 @@ subroutine prim_advance_init1(par, elem, integration) end subroutine prim_advance_init1 - !_____________________________________________________________________ - subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) - - type (element_t), intent(inout), target :: elem(:) ! array of element_t structures - integer, intent(in) :: nets,nete ! start and end element indices - type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct - type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct - - if (hybrid%masterthread) print *,"initializing PESE dynamics solver" - - ! initialize vertical operators and coordinates - call make_vertical_mesh(hybrid, hvcoord) - - end subroutine vertical_mesh_init2 - !_____________________________________________________________________ subroutine prim_advance_exp(elem,deriv,hvcoord,hybrid,dt,tl,nets,nete,compute_diagnostics,single_column) diff --git a/components/homme/src/preqx/model_init_mod.F90 b/components/homme/src/preqx/model_init_mod.F90 index bb3b13f4494b..07a8caf3f631 100644 --- a/components/homme/src/preqx/model_init_mod.F90 +++ b/components/homme/src/preqx/model_init_mod.F90 @@ -20,6 +20,19 @@ module model_init_mod contains + !_____________________________________________________________________ + subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) + + ! additional solver specific initializations (called from prim_init2) + + type (element_t), intent(inout), target :: elem(:)! array of element_t structures + integer, intent(in) :: nets,nete ! start and end element indices + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + + end subroutine vertical_mesh_init2 + + subroutine model_init2( elem , hybrid, deriv,hvcoord,tl,nets,nete ) type(element_t) , intent(in) :: elem(:) diff --git a/components/homme/src/preqx/share/prim_advance_mod.F90 b/components/homme/src/preqx/share/prim_advance_mod.F90 index d9aff8827704..9a101f356e8c 100644 --- a/components/homme/src/preqx/share/prim_advance_mod.F90 +++ b/components/homme/src/preqx/share/prim_advance_mod.F90 @@ -26,8 +26,7 @@ module prim_advance_mod private save public :: prim_advance_exp, prim_advance_init1, & - applyCAMforcing_ps, applyCAMforcing_dp3d, & - vertical_mesh_init2 + applyCAMforcing_ps, applyCAMforcing_dp3d real (kind=real_kind), allocatable :: ur_weights(:) @@ -60,19 +59,6 @@ subroutine prim_advance_init1(par, elem,integration) end subroutine prim_advance_init1 - !_____________________________________________________________________ - subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) - - ! additional solver specific initializations (called from prim_init2) - - type (element_t), intent(inout), target :: elem(:)! array of element_t structures - integer, intent(in) :: nets,nete ! start and end element indices - type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct - type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct - - end subroutine vertical_mesh_init2 - - #ifndef CAM !_____________________________________________________________________ subroutine set_prescribed_wind(elem,deriv,hybrid,hv,dt,tl,nets,nete,eta_ave_w) diff --git a/components/homme/src/preqx_acc/model_init_mod.F90 b/components/homme/src/preqx_acc/model_init_mod.F90 index 0fcd9b31b9b9..3a68ef0bd0b8 100644 --- a/components/homme/src/preqx_acc/model_init_mod.F90 +++ b/components/homme/src/preqx_acc/model_init_mod.F90 @@ -19,6 +19,19 @@ module model_init_mod contains + !_____________________________________________________________________ + subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) + + ! additional solver specific initializations (called from prim_init2) + + type (element_t), intent(inout), target :: elem(:)! array of element_t structures + integer, intent(in) :: nets,nete ! start and end element indices + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + + end subroutine vertical_mesh_init2 + + subroutine model_init2( elem , hybrid, deriv ,hvcoord,tl,nets,nete) use element_state, only: state_qdp, derived_vn0, derived_divdp, derived_divdp_proj use dimensions_mod, only: nelemd diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 71e5794d948d..22c52849958a 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -537,8 +537,7 @@ end subroutine prim_init1 !_____________________________________________________________________ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) - use control_mod, only: runtype, integration, test_case, & - debug_level, vfile_int, vform, vfile_mid, & + use control_mod, only: runtype, integration, & topology,rsplit, qsplit, rk_stage_user,& sub_case, limiter_option, nu, nu_q, nu_div, tstep_type, hypervis_subcycle, & hypervis_subcycle_q, moisture, use_moisture @@ -547,9 +546,8 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) use parallel_mod, only: parallel_t, haltmp, syncmp, abortmp use prim_state_mod, only: prim_printstate, prim_diag_scalars use prim_si_mod, only: prim_set_mass - use prim_advance_mod, only: vertical_mesh_init2 use prim_advection_mod, only: prim_advec_init2 - use model_init_mod, only: model_init2 + use model_init_mod, only: model_init2, vertical_mesh_init2 use time_mod, only: timelevel_t, tstep, phys_tscale, timelevel_init, nendstep, smooth, nsplit, TimeLevel_Qdp #ifndef CAM @@ -617,6 +615,7 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) integer :: qn0 real (kind=real_kind) :: eta_ave_w + interface subroutine noxinit(vectorSize,vector,comm,v_container,p_container,j_container) & bind(C,name='noxinit') @@ -822,6 +821,7 @@ end subroutine noxinit ! timesteps to use for advective stability: tstep*qsplit and tstep call print_cfl(elem,hybrid,nets,nete,dtnu) + if (hybrid%masterthread) then ! CAM has set tstep based on dtime before calling prim_init2(), ! so only now does HOMME learn the timstep. print them out: diff --git a/components/homme/src/theta-l/element_state.F90 b/components/homme/src/theta-l/element_state.F90 index 246476656372..d16482ce944b 100644 --- a/components/homme/src/theta-l/element_state.F90 +++ b/components/homme/src/theta-l/element_state.F90 @@ -18,6 +18,9 @@ module element_state ! maximum error of Newton iteration for an IMEX-RK stage per time-step real (kind=real_kind), public :: max_itererr_perstep + ! pressure based TOM sponge layer + real (kind=real_kind),public :: nu_scale_top(nlev) + ! =========== PRIMITIVE-EQUATION DATA-STRUCTURES ===================== diff --git a/components/homme/src/theta-l/model_init_mod.F90 b/components/homme/src/theta-l/model_init_mod.F90 index 59d7e0d27221..d273c5ba5054 100644 --- a/components/homme/src/theta-l/model_init_mod.F90 +++ b/components/homme/src/theta-l/model_init_mod.F90 @@ -9,6 +9,8 @@ ! most models do nothing. introduced for preqx_acc to initialize ! GPU related data ! +! 2018/10 MT: adding TOM pressure-based sponge layer dissipation from P. Lauritzen +! module model_init_mod use element_mod, only: element_t @@ -17,7 +19,7 @@ module model_init_mod use hybrid_mod, only: hybrid_t use dimensions_mod, only: np,nlev,nlevp,nelemd use eos , only: get_pnh_and_exner,get_dirk_jacobian - use element_state, only: timelevels + use element_state, only: timelevels, nu_scale_top use viscosity_mod, only: make_c0_vector use kinds, only: real_kind,iulog use control_mod, only: qsplit,theta_hydrostatic_mode @@ -38,8 +40,9 @@ subroutine model_init2(elem,hybrid,deriv,hvcoord,tl,nets,nete ) integer :: nets,nete ! local variables - integer :: ie,t + integer :: ie,t,k real (kind=real_kind) :: gradtemp(np,np,2,nelemd) + real (kind=real_kind) :: ptop_over_press ! other theta specific model initialization should go here @@ -65,10 +68,54 @@ subroutine model_init2(elem,hybrid,deriv,hvcoord,tl,nets,nete ) ! unit test for analytic jacobian used by IMEX methods if (.not. theta_hydrostatic_mode) & call test_imex_jacobian(elem,hybrid,hvcoord,tl,nets,nete) - + + + + ! + ! compute scaling of sponge layer damping + ! + if (hybrid%masterthread) write(iulog,*) "sponge layer nu_top viscosity scaling factor" + do k=1,nlev + !press = (hvcoord%hyam(k)+hvcoord%hybm(k))*hvcoord%ps0 + !ptop = hvcoord%hyai(1)*hvcoord%ps0 + ! sponge layer starts at p=4*ptop + ! + ! some test cases have ptop=200mb + ptop_over_press = hvcoord%etai(1) / hvcoord%etam(k) ! pure sigma coordinates has etai(1)=0 + + ! active for p<4*ptop (following cd_core.F90 in CAM-FV) + ! CAM 26L and 30L: top 2 levels + ! E3SM 72L: top 4 levels + nu_scale_top(k) = 8*(1+tanh(log(ptop_over_press))) ! active for p<4*ptop + + ! active for p<7*ptop + ! CAM 26L and 30L: top 3 levels + ! E3SM 72L: top 5 levels + !nu_scale_top(k) = 8*(1+.911*tanh(log(ptop_over_press))) ! active for p<6.5*ptop + + if (hybrid%masterthread) then + if (nu_scale_top(k)>1) write(iulog,*) " nu_scale_top ",k,nu_scale_top(k) + end if + end do + end subroutine + + subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) + + ! additional solver specific initializations (called from prim_init2) + + type (element_t), intent(inout), target :: elem(:)! array of element_t structures + integer, intent(in) :: nets,nete ! start and end element indices + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + + + end subroutine vertical_mesh_init2 + + + subroutine test_imex_jacobian(elem,hybrid,hvcoord,tl,nets,nete) ! the following code compares the analytic vs exact imex Jacobian ! can test over more elements if desired diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index 7945dc7d9e23..5f8365a6b72e 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -5,6 +5,8 @@ ! ! Man dynamics routines for "theta" nonhydrostatic model ! Original version: Mark Taylor 2017/1 +! +! 2018/8 TOM sponger layer scaling from P. Lauritzen ! module prim_advance_mod @@ -19,7 +21,7 @@ module prim_advance_mod use edge_mod, only: edge_g, edgevpack_nlyr, edgevunpack_nlyr use edgetype_mod, only: EdgeBuffer_t, EdgeDescriptor_t, edgedescriptor_t use element_mod, only: element_t - use element_state, only: max_itercnt_perstep,avg_itercnt,max_itererr_perstep + use element_state, only: max_itercnt_perstep,avg_itercnt,max_itererr_perstep, nu_scale_top use element_ops, only: get_temperature, set_theta_ref, state0, get_R_star use eos, only: get_pnh_and_exner,get_phinh,get_dirk_jacobian use hybrid_mod, only: hybrid_t @@ -45,7 +47,9 @@ module prim_advance_mod save public :: prim_advance_exp, prim_advance_init1, & applycamforcing_ps, applycamforcing_dp3d, & - applyCAMforcing_dynamics, vertical_mesh_init2 + applyCAMforcing_dynamics + + contains @@ -68,20 +72,6 @@ end subroutine prim_advance_init1 - subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) - - ! additional solver specific initializations (called from prim_init2) - - type (element_t), intent(inout), target :: elem(:)! array of element_t structures - integer, intent(in) :: nets,nete ! start and end element indices - type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct - type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct - - end subroutine vertical_mesh_init2 - - - - !_____________________________________________________________________ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, compute_diagnostics, & single_column) @@ -540,7 +530,6 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w ! local real (kind=real_kind) :: eta_ave_w ! weighting for mean flux terms - real (kind=real_kind) :: nu_scale_top integer :: k2,k,kptr,i,j,ie,ic,nt,nlyr_tot,ssize real (kind=real_kind), dimension(np,np,2,nlev,nets:nete) :: vtens real (kind=real_kind), dimension(np,np,nlev,4,nets:nete) :: stens ! dp3d,theta,w,phi @@ -660,34 +649,25 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w elem(ie)%derived%dpdiss_biharmonic(:,:,:)=elem(ie)%derived%dpdiss_biharmonic(:,:,:)+& eta_ave_w*stens(:,:,:,1,ie)/hypervis_subcycle endif -#if (defined COLUMN_OPENMP) -!$omp parallel do private(k,lap_s,lap_v,nu_scale_top) -#endif do k=1,nlev ! advace in time. ! note: DSS commutes with time stepping, so we can time advance and then DSS. ! note: weak operators alreayd have mass matrix "included" - ! add regular diffusion in top 3 layers: - if (nu_top>0 .and. k<=3) then + ! biharmonic terms need a negative sign: + if (nu_top>0 .and. nu_scale_top(k)>1) then + ! add regular diffusion near top lap_s(:,:,1)=laplace_sphere_wk(elem(ie)%state%dp3d (:,:,k,nt),deriv,elem(ie),var_coef=.false.) lap_s(:,:,2)=laplace_sphere_wk(elem(ie)%state%vtheta_dp (:,:,k,nt),deriv,elem(ie),var_coef=.false.) lap_s(:,:,3)=laplace_sphere_wk(elem(ie)%state%w_i (:,:,k,nt),deriv,elem(ie),var_coef=.false.) lap_s(:,:,4)=laplace_sphere_wk(elem(ie)%state%phinh_i (:,:,k,nt),deriv,elem(ie),var_coef=.false.) lap_v=vlaplace_sphere_wk(elem(ie)%state%v(:,:,:,k,nt),deriv,elem(ie),var_coef=.false.) - endif - - nu_scale_top = 1 - if (k==1) nu_scale_top=4 - if (k==2) nu_scale_top=2 - ! biharmonic terms need a negative sign: - if (nu_top>0 .and. k<=3) then - vtens(:,:,:,k,ie)=( -nu*vtens(:,:,:,k,ie) + nu_scale_top*nu_top*lap_v(:,:,:)) ! u and v - stens(:,:,k,1,ie)=(-nu_p*stens(:,:,k,1,ie) + nu_scale_top*nu_top*lap_s(:,:,1)) ! dp3d - stens(:,:,k,2,ie)=( -nu*stens(:,:,k,2,ie) + nu_scale_top*nu_top*lap_s(:,:,2)) ! theta - stens(:,:,k,3,ie)=( -nu*stens(:,:,k,3,ie) + nu_scale_top*nu_top*lap_s(:,:,3)) ! w - stens(:,:,k,4,ie)=(-nu_s*stens(:,:,k,4,ie) + nu_scale_top*nu_top*lap_s(:,:,4)) ! phi + vtens(:,:,:,k,ie)=( -nu*vtens(:,:,:,k,ie) + nu_scale_top(k)*nu_top*lap_v(:,:,:)) ! u and v + stens(:,:,k,1,ie)=(-nu_p*stens(:,:,k,1,ie) + nu_scale_top(k)*nu_top*lap_s(:,:,1)) ! dp3d + stens(:,:,k,2,ie)=( -nu*stens(:,:,k,2,ie) + nu_scale_top(k)*nu_top*lap_s(:,:,2)) ! theta + stens(:,:,k,3,ie)=( -nu*stens(:,:,k,3,ie) + nu_scale_top(k)*nu_top*lap_s(:,:,3)) ! w + stens(:,:,k,4,ie)=(-nu_s*stens(:,:,k,4,ie) + nu_scale_top(k)*nu_top*lap_s(:,:,4)) ! phi else vtens(:,:,:,k,ie)=-nu *vtens(:,:,:,k,ie) ! u,v stens(:,:,k,1,ie)=-nu_p*stens(:,:,k,1,ie) ! dp3d diff --git a/components/homme/src/theta/model_init_mod.F90 b/components/homme/src/theta/model_init_mod.F90 index ed7f0691719c..da92d8a82a1b 100644 --- a/components/homme/src/theta/model_init_mod.F90 +++ b/components/homme/src/theta/model_init_mod.F90 @@ -26,6 +26,19 @@ module model_init_mod contains + subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) + + ! additional solver specific initializations (called from prim_init2) + + type (element_t), intent(inout), target :: elem(:)! array of element_t structures + integer, intent(in) :: nets,nete ! start and end element indices + type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct + type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct + + end subroutine vertical_mesh_init2 + + + subroutine model_init2(elem,hybrid,deriv,hvcoord,tl,nets,nete ) type(element_t) , intent(in) :: elem(:) diff --git a/components/homme/src/theta/prim_advance_mod.F90 b/components/homme/src/theta/prim_advance_mod.F90 index ec163b31b736..1bd28de8ec06 100644 --- a/components/homme/src/theta/prim_advance_mod.F90 +++ b/components/homme/src/theta/prim_advance_mod.F90 @@ -45,7 +45,7 @@ module prim_advance_mod private save public :: prim_advance_exp, prim_advance_init1, & - applyCAMforcing_dp3d, applyCAMforcing_ps, vertical_mesh_init2 + applyCAMforcing_dp3d, applyCAMforcing_ps ! type (EdgeBuffer_t) :: edge5 type (EdgeBuffer_t) :: edge6 @@ -89,20 +89,6 @@ end subroutine prim_advance_init1 - subroutine vertical_mesh_init2(elem, nets, nete, hybrid, hvcoord) - - ! additional solver specific initializations (called from prim_init2) - - type (element_t), intent(inout), target :: elem(:)! array of element_t structures - integer, intent(in) :: nets,nete ! start and end element indices - type (hybrid_t), intent(in) :: hybrid ! mpi/omp data struct - type (hvcoord_t), intent(inout) :: hvcoord ! hybrid vertical coord data struct - - end subroutine vertical_mesh_init2 - - - - !_____________________________________________________________________ subroutine prim_advance_exp(elem, deriv, hvcoord, hybrid,dt, tl, nets, nete, compute_diagnostics, & single_column) From be4038a4cbefa76dafa7d4d146ab4709cd8284e8 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sat, 6 Oct 2018 09:32:06 -0500 Subject: [PATCH 15/41] adding high-res DCMIP2016 test1 namelists --- .../theta-l/jobscript-snl.sh | 9 ++ .../theta-l/namelist-ne120.nl | 56 +++++++++ .../theta-l/namelist-ne256.nl | 56 +++++++++ .../theta-l/namelist-ne512.nl | 56 +++++++++ .../theta-l/jobscript-snl.sh | 115 ++++++------------ .../theta-l/namelist-explicit-r100.nl | 57 +++++++++ 6 files changed, 268 insertions(+), 81 deletions(-) create mode 100755 components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl create mode 100755 components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl create mode 100644 components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl create mode 100644 components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh index 2e68cb3d8975..a29b815ac665 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh @@ -60,6 +60,15 @@ prefix=r100 ; run $NCPU prefix=r50 ; run $NCPU +# high res cases +#prefix=ne120 ; run $NCPU # 72 nodes, 2h Anvil: 6s +#prefix=ne256 ; run $NCPU # 72 nodes, 2h Anvil: 55s + # 50 nodes ? + # 25 nodes ? +#prefix=ne512 ; run $NCPU # 72 nodes killed. should work with +#prefix=ne1024 ; run $NCPU # + + diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl new file mode 100755 index 000000000000..4b30b8dfd4bd --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl @@ -0,0 +1,56 @@ +! +! theta: namelist for dcmip2016 test1: moist baroclinic wave +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test1" ! test identifier + ne = 120 ! number of elements per cube face + qsize = 6 ! num tracer fields +! ndays = 1 + nmax = 96 ! run for 7200s (2h) + statefreq = 24 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 75 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 7 + rsplit = 3 + qsplit = 1 + nu = 1e13 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 + nu_s = 1e13 + nu_p = 1e13 + nu_top = 0 ! default = 2.5e5 + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + moisture = 'wet' + theta_hydrostatic_mode = .false. + dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl +! output_prefix = "r50-" + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 ! every N hours + output_start_time = 48 ! start after N hours +! output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','Q4','Q5','rho','precl','zeta' ! variables to write to file + output_varnames1 ='T','ps','pnh','geo','u','Th','Q','Q2','Q3','Q4','Q5','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlon = 360 ! low-res output grid to save init time + interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl new file mode 100755 index 000000000000..51e9b0cd3500 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl @@ -0,0 +1,56 @@ +! +! theta: namelist for dcmip2016 test1: moist baroclinic wave +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test1" ! test identifier + ne = 256 ! number of elements per cube face + qsize = 6 ! num tracer fields +! ndays = 1 + nmax = 180 ! run for 7200s (2h) + statefreq = 45 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 40 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 7 + rsplit = 3 + qsplit = 1 + nu = 1e12 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 + nu_s = 1e12 + nu_p = 1e12 + nu_top = 0 ! default = 2.5e5 + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + moisture = 'wet' + theta_hydrostatic_mode = .false. + dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl +! output_prefix = "r50-" + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 ! every N hours + output_start_time = 48 ! start after N hours +! output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','Q4','Q5','rho','precl','zeta' ! variables to write to file + output_varnames1 ='T','ps','pnh','geo','u','Th','Q','Q2','Q3','Q4','Q5','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlon = 360 ! low-res output grid to save init time + interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl new file mode 100644 index 000000000000..86b056d2b208 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl @@ -0,0 +1,56 @@ +! +! theta: namelist for dcmip2016 test1: moist baroclinic wave +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test1" ! test identifier + ne = 512 ! number of elements per cube face + qsize = 6 ! num tracer fields +! ndays = 1 + nmax = 360 ! run for 7200s (2h) + statefreq = 20 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 40 ! largest timestep in seconds + integration = 'explicit' ! explicit time integration + tstep_type = 7 + rsplit = 3 + qsplit = 1 + nu = 1e11 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 + nu_s = 1e11 + nu_p = 1e11 + nu_top = 0 ! default = 2.5e5 + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + moisture = 'wet' + theta_hydrostatic_mode = .false. + dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl +! output_prefix = "r50-" + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 ! every N hours + output_start_time = 48 ! start after N hours +! output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','Q4','Q5','rho','precl','zeta' ! variables to write to file + output_varnames1 ='T','ps','pnh','geo','u','Th','Q','Q2','Q3','Q4','Q5','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlon = 360 ! low-res output grid to save init time + interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh index f61a2e94684e..d5ac49482c12 100755 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/jobscript-snl.sh @@ -1,4 +1,4 @@ -#!/bin/tcsh +#!/bin/bash # # Anvil, 25 nodes, all 4 resolutions run in 25min # need at least 11 nodes, since r400 is hardcoded as mpirun -np 384 @@ -12,100 +12,53 @@ #PBS -l walltime=30:00 #PBS -l nodes=25 -set OMP_NUM_THREADS = 1 -set NCPU = 640 -if ( ${?PBS_ENVIRONMENT} ) then # anvil - set NCPU = $PBS_NNODES - if ( $PBS_ENVIRONMENT == PBS_BATCH ) cd $PBS_O_WORKDIR -endif -if ( ${?SLURM_NNODES} ) then # redsky - set NCPU = $SLURM_NNODES - @ NCPU *= 16 - @ NCPU /= $OMP_NUM_THREADS -endif +OMP_NUM_THREADS=1 +NCPU=640 +if [ -n "$PBS_ENVIRONMENT" ]; then +# NCPU=$PBS_NNODES + [ "$PBS_ENVIRONMENT" = "PBS_BATCH" ] && cd $PBS_O_WORKDIR + NCPU=$PBS_NNODES +fi +if [ -n "$SLURM_NNODES" ]; then + NCPU=$SLURM_NNODES + let NCPU*=16 + let NCPU/=$OMP_NUM_THREADS +fi -set EXEC = ../../../test_execs/theta-l-nlev40/theta-l-nlev40 +EXEC=../../../test_execs/theta-l-nlev40/theta-l-nlev40 -date - -# 4dg resolution -\cp -f namelist-r400.nl input.nl -mpirun -np 384 $EXEC < input.nl -ncl plot_supercell_wvel.ncl -ncl plot_supercell_2.5km_wvel_xsec.ncl -ncl plot_supercell_5km_xsec.ncl -ncl plot_supercell_prect.ncl -\mv movies/dcmip2016_test31.nc movies/dcmip2016_test3_r400.nc -\mv HommeTime HommeTime_r400 -\mv max_w.pdf max_w_r400.pdf -\mv max_precip.pdf max_precip_r400.pdf -\mv 2.5km_wvel_xsec.pdf 2.5km_wvel_xsec_r400.pdf -\mv 5km_xsec.pdf 5km_xsec_r400.pdf -\mv measurement_wmax.txt measurement_wmax_r400.txt -\mv measurement_time.txt measurement_time_r400.txt -\mv measurement_prect_rate.txt measurement_prect_rate_r400.txt +function run { +local NCPU=$1 +echo "NCPU = $NCPU" +namelist=namelist-$prefix.nl +\cp -f $namelist input.nl date - -# 2dg resolution -\cp -f namelist-r200.nl input.nl mpirun -np $NCPU $EXEC < input.nl -ncl plot_supercell_wvel.ncl -ncl plot_supercell_2.5km_wvel_xsec.ncl -ncl plot_supercell_5km_xsec.ncl -ncl plot_supercell_prect.ncl - -\mv movies/dcmip2016_test31.nc movies/dcmip2016_test3_r200.nc -\mv HommeTime HommeTime_r200 -\mv max_w.pdf max_w_r200.pdf -\mv max_precip.pdf max_precip_r200.pdf -\mv 2.5km_wvel_xsec.pdf 2.5km_wvel_xsec_r200.pdf -\mv 5km_xsec.pdf 5km_xsec_r200.pdf -\mv measurement_wmax.txt measurement_wmax_r200.txt -\mv measurement_time.txt measurement_time_r200.txt -\mv measurement_prect_rate.txt measurement_prect_rate_r200.txt - date -# 1dg resolution -\cp -f namelist-r100.nl input.nl -mpirun -np $NCPU $EXEC < input.nl ncl plot_supercell_wvel.ncl ncl plot_supercell_2.5km_wvel_xsec.ncl ncl plot_supercell_5km_xsec.ncl ncl plot_supercell_prect.ncl -\mv movies/dcmip2016_test31.nc movies/dcmip2016_test3_r100.nc -\mv HommeTime HommeTime_r100 -\mv max_w.pdf max_w_r100.pdf -\mv max_precip.pdf max_precip_r100.pdf -\mv 2.5km_wvel_xsec.pdf 2.5km_wvel_xsec_r100.pdf -\mv 5km_xsec.pdf 5km_xsec_r100.pdf -\mv measurement_wmax.txt measurement_wmax_r100.txt -\mv measurement_time.txt measurement_time_r100.txt -\mv measurement_prect_rate.txt measurement_prect_rate_r100.txt +\mv movies/dcmip2016_test31.nc movies/dcmip2016_test3_${prefix}.nc +\mv HommeTime HommeTime_${prefix} +\mv max_w.pdf max_w_${prefix}.pdf +\mv max_precip.pdf max_precip_${prefix}.pdf +\mv 2.5km_wvel_xsec.pdf 2.5km_wvel_xsec_${prefix}.pdf +\mv 5km_xsec.pdf 5km_xsec_${prefix}.pdf +\mv measurement_wmax.txt measurement_wmax_${prefix}.txt +\mv measurement_time.txt measurement_time_${prefix}.txt +\mv measurement_prect_rate.txt measurement_prect_rate_${prefix}.txt -date +} +prefix=r400 ; run $(($NCPU>384?384:NCPU)) +prefix=r200 ; run $NCPU +prefix=r100 ; run $NCPU +prefix=r50 ; run $NCPU -# 0.5dg resolution -\cp -f namelist-r50.nl input.nl -mpirun -np $NCPU $EXEC < input.nl -ncl plot_supercell_wvel.ncl -ncl plot_supercell_2.5km_wvel_xsec.ncl -ncl plot_supercell_5km_xsec.ncl -ncl plot_supercell_prect.ncl - -\mv movies/dcmip2016_test31.nc movies/dcmip2016_test3_r50.nc -\mv HommeTime HommeTime_r50 -\mv max_w.pdf max_w_r50.pdf -\mv max_precip.pdf max_precip_r50.pdf -\mv 2.5km_wvel_xsec.pdf 2.5km_wvel_xsec_r50.pdf -\mv 5km_xsec.pdf 5km_xsec_r50.pdf -\mv measurement_wmax.txt measurement_wmax_r50.txt -\mv measurement_time.txt measurement_time_r50.txt -\mv measurement_prect_rate.txt measurement_prect_rate_r50.txt - -date +prefix=explicit-r100 ; run $NCPU diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl new file mode 100644 index 000000000000..7d9bb9e8b6c0 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl @@ -0,0 +1,57 @@ +! +! theta: namelist for dcmip2016 test 3: supercell storm (small planet X=120) +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test3" ! test identifier + ne = 30 ! number of elements per cube face + qsize = 3 ! num tracer fields + nmax = 3200 ! 7200s(120min)/tstep + statefreq = 180 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 2.00 ! 2.25 stable. 2.5 unstable + integration = 'explicit' ! explicit time integration + tstep_type = 5 + rsplit = 3 + qsplit = 2 ! dt_tracer <= 3s + nu = 5.8e8 ! default= 1e15/(120)^3 *(ne30/ne30)**3.2 + nu_s = 0 + nu_p = 0 + nu_q = 0 + nu_top = 0 ! 2.5e5/(120)^(1) + vert_remap_q_alg = -1 + limiter_option = 4 + dcmip16_mu = 500.0d0 ! additional uniform viscosity + dcmip16_mu_s = 1500.0d0 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + rearth = 53133 ! 6.376E6 / 120 + omega = 0 + se_ftype = 0 + moisture = 'wet' + theta_hydrostatic_mode = .false. +/ +&vert_nl + vform = "ccm" ! vertical coordinate type "ccm"=hybrid pressure/terrain + vanalytic = 1 ! set vcoords in initialization routine +/ +&analysis_nl + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 3 ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 600 ! 900 seconds + output_varnames1 ='T','p','ps','pnh','geo','u','v','w','Th','Q','Q2','Q3','precl' ! variables to write to file +! interp_nlon = 360 +! interp_nlat = 181 + interp_gridtype = 1 + interp_type = 1 ! 0=native grid, 1=bilinear + interp_lon0 = -180.0 ! shift lon range to [-180,+180) + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ From b43cd230d75c3fa48d5c4d6a133a635970155ecc Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 7 Oct 2018 15:27:30 -0500 Subject: [PATCH 16/41] sequentialize MPI rank execution per node to reduce memory usage create subcommunicator for each node workaround for non-scalable memory used by GridVertex and GridEdge arrays This sequentilizes the domain decomposition and scheduling operations. Timing is given in new timer, ScheduleTime Need to perform ne512 and ne1024 benchmarks and see how much this slows down the initialization --- components/homme/src/share/cube_mod.F90 | 5 +- components/homme/src/share/metagraph_mod.F90 | 28 ++- components/homme/src/share/parallel_mod.F90 | 39 +++- .../homme/src/share/prim_driver_base.F90 | 169 ++++++++---------- components/homme/src/share/schedtype_mod.F90 | 2 - components/homme/src/share/spacecurve_mod.F90 | 74 +------- components/homme/src/sweqx/init_mod.F90 | 4 +- 7 files changed, 151 insertions(+), 170 deletions(-) diff --git a/components/homme/src/share/cube_mod.F90 b/components/homme/src/share/cube_mod.F90 index 52bfaa165a69..466afbdf6e9f 100644 --- a/components/homme/src/share/cube_mod.F90 +++ b/components/homme/src/share/cube_mod.F90 @@ -76,7 +76,7 @@ module cube_mod public :: cube_assemble public :: vmap,dmap public :: set_corner_coordinates - public :: assign_node_numbers_to_elem +! public :: assign_node_numbers_to_elem public :: CubeEdgeCount @@ -850,7 +850,7 @@ subroutine set_corner_coordinates(elem) #endif end subroutine set_corner_coordinates - +#if 0 subroutine assign_node_numbers_to_elem(elements, GridVertex) use element_mod, only : element_t use control_mod, only : north, south, east, west, neast, seast, swest, nwest @@ -927,6 +927,7 @@ subroutine assign_node_numbers_to_elem(elements, GridVertex) ! elements(el)%node_numbers = connectivity(elements(el)%vertex%number, :) ! end do end subroutine assign_node_numbers_to_elem +#endif ! ================================================ diff --git a/components/homme/src/share/metagraph_mod.F90 b/components/homme/src/share/metagraph_mod.F90 index b2f5b59f1896..adc3f57f41c5 100644 --- a/components/homme/src/share/metagraph_mod.F90 +++ b/components/homme/src/share/metagraph_mod.F90 @@ -11,7 +11,8 @@ module metagraph_mod ! use kinds, only : int_kind, iulog use gridgraph_mod, only : gridvertex_t, gridedge_t, & - allocate_gridvertex_nbrs, assignment ( = ) + deallocate_gridvertex_nbrs, allocate_gridvertex_nbrs, assignment ( = ) + use pio_types ! _EXTERNAL implicit none @@ -47,6 +48,7 @@ module metagraph_mod public :: edge_uses_vertex public :: PrintMetaEdge, PrintMetaVertex public :: LocalElemCount + public :: deallocate_metavertex_data !public :: MetaEdgeCount public :: initMetaGraph @@ -176,6 +178,30 @@ function LocalElemCount(Vertex) result(nelemd) end function LocalElemCount + + subroutine deallocate_metavertex_data(Vertex) + implicit none + + type (MetaVertex_t),intent(in) :: Vertex + integer :: j + + do j = 1, Vertex%nmembers + call deallocate_gridvertex_nbrs(Vertex%members(j)) + end do + do j = 1, Vertex%nedges + deallocate(Vertex%edges(j)%members) + deallocate(Vertex%edges(j)%edgeptrP) + deallocate(Vertex%edges(j)%edgeptrS) + deallocate(Vertex%edges(j)%edgeptrP_ghost) + end do + deallocate(Vertex%edges) + deallocate(Vertex%members) + + end subroutine + + + + function edge_uses_vertex(Vertex,Edge) result(log) implicit none diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index f62c9cba4b06..7fc411f236f2 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -45,6 +45,9 @@ module parallel_mod integer :: root ! local root integer :: nprocs ! number of processes in group integer :: comm ! local communicator + integer :: node_comm ! local communicator of all procs per node + integer :: node_rank ! local rank in node_comm + integer :: node_nprocs ! local rank in node_comm logical :: masterproc logical :: dynproc ! Designation of a dynamics processor - AaronDonahue end type @@ -120,6 +123,7 @@ function initmp(npes_in,npes_stride) result(par) integer(kind=int_kind),allocatable :: tarray(:) integer(kind=int_kind) :: namelen,i + integer :: node_color #ifdef CAM integer :: color = 1 integer :: iam_cam, npes_cam @@ -199,12 +203,20 @@ function initmp(npes_in,npes_stride) result(par) ! ====================================================================== ! Calculate how many other MPI processes are on my node ! ====================================================================== + node_color=0 nmpi_per_node = 0 do i=1,par%nprocs if( TRIM(ADJUSTL(my_name)) .eq. TRIM(ADJUSTL(the_names(i))) ) then nmpi_per_node = nmpi_per_node + 1 + if (node_color==0) node_color=i endif enddo + if (node_color==0) call abortmp("initmp: Errror computing procs per node") + + ! create a communicator of all procs per node + call mpi_comm_split(par%comm, node_color, par%rank, par%node_comm, ierr) + call MPI_comm_rank(par%node_comm,par%node_rank,ierr) + call MPI_comm_size(par%node_comm,par%node_nprocs,ierr) ! ======================================================================= ! Verify that everybody agrees on this number otherwise do not do @@ -217,10 +229,10 @@ function initmp(npes_in,npes_stride) result(par) PartitionForNodes=.FALSE. else PartitionForNodes=.TRUE. + if (par%masterproc) write(iulog,*)'number of MPI processes per node: ',nmpi_per_node endif - deallocate(the_names) #else @@ -320,6 +332,31 @@ subroutine syncmp(par) #endif end subroutine syncmp +! ===================================== +! syncmp_comm: +! +! same as above, but allow user to specify communicator +! +! ===================================== + subroutine syncmp_comm(comm) + + integer :: comm + +#ifdef _MPI +#include + integer :: errorcode,errorlen,ierr + character(len=MPI_MAX_ERROR_STRING) :: errorstring + + call MPI_barrier(comm,ierr) + + if(ierr.eq.MPI_ERROR) then + errorcode=ierr + call MPI_Error_String(errorcode,errorstring,errorlen,ierr) + call abortmp(errorstring) + endif +#endif + end subroutine syncmp_comm + ! ============================================= ! pmin_1d: ! 1D version of the parallel MIN diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 22c52849958a..0ce462d1ab91 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -22,6 +22,11 @@ module prim_driver_base use quadrature_mod, only: quadrature_t, test_gauss, test_gausslobatto, gausslobatto use reduction_mod, only: reductionbuffer_ordered_1d_t, red_min, red_max, red_max_int, & red_sum, red_sum_int, red_flops, initreductionbuffer + use parallel_mod, only : iam, parallel_t, syncmp, syncmp_comm, abortmp, global_shared_buf, nrepro_vars +#ifdef _MPI + use parallel_mod, only : mpiinteger_t, mpi_max +#endif + #ifndef CAM use prim_restart_mod, only : initrestartfile use restart_io_mod , only : RestFile,readrestart @@ -59,17 +64,18 @@ subroutine prim_init1(elem, par, dom_mt, Tl) use mass_matrix_mod, only : mass_matrix ! -------------------------------- use cube_mod, only : cubeedgecount , cubeelemcount, cubetopology, cube_init_atomic, & - set_corner_coordinates, assign_node_numbers_to_elem, & + set_corner_coordinates, & set_area_correction_map0, set_area_correction_map2 ! -------------------------------- use mesh_mod, only : MeshSetCoordinates, MeshUseMeshFile, MeshCubeTopology, & MeshCubeElemCount, MeshCubeEdgeCount, MeshCubeTopologyCoords ! -------------------------------- - use metagraph_mod, only : metavertex_t, metaedge_t, localelemcount, initmetagraph, printmetavertex + use metagraph_mod, only : metavertex_t, metaedge_t, localelemcount, initmetagraph, & + deallocate_metavertex_data, printmetavertex ! -------------------------------- use gridgraph_mod, only : gridvertex_t, gridedge_t, allocate_gridvertex_nbrs, deallocate_gridvertex_nbrs ! -------------------------------- - use schedtype_mod, only : schedule + use schedtype_mod, only : schedule, schedule_t ! -------------------------------- use schedule_mod, only : genEdgeSched, PrintSchedule ! -------------------------------- @@ -79,11 +85,6 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! -------------------------------- #ifdef TRILINOS use prim_implicit_mod, only : prim_implicit_init -#endif - ! -------------------------------- - use parallel_mod, only : iam, parallel_t, syncmp, abortmp, global_shared_buf, nrepro_vars -#ifdef _MPI - use parallel_mod, only : mpiinteger_t, mpireal_t, mpi_max, mpi_sum, haltmp #endif ! -------------------------------- use metis_mod, only : genmetispart @@ -124,21 +125,17 @@ subroutine prim_init1(elem, par, dom_mt, Tl) type (GridVertex_t), target,allocatable :: GridVertex(:) type (GridEdge_t), target,allocatable :: Gridedge(:) - type (MetaVertex_t), target,allocatable :: MetaVertex(:) + type (MetaVertex_t) :: MetaVertex - integer :: ii,ie, ith + integer :: i,ii,ie, ith integer :: nets, nete integer :: nelem_edge,nedge integer :: nstep integer :: nlyr integer :: iMv - integer :: err, ierr, l, j + integer :: err, ierr, l, j, task_id logical, parameter :: Debug = .FALSE. - integer :: i - integer,allocatable :: TailPartition(:) - integer,allocatable :: HeadPartition(:) - integer total_nelem real(kind=real_kind) :: approx_elements_per_task type (quadrature_t) :: gp ! element GLL points @@ -207,28 +204,41 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! ================================== call derivinit(deriv1) + if (MeshUseMeshFile) then + nelem = MeshCubeElemCount() + nelem_edge = MeshCubeEdgeCount() + else + nelem = CubeElemCount() + nelem_edge = CubeEdgeCount() + end if + + ! we want to exit elegantly when we are using too many processors. + if (nelem < par%nprocs) then + call abortmp('Error: too many MPI tasks. set dyn_npes <= nelem') + end if + + + ! ==================================================== + ! start domain decomposition + ! non-scalable global arrays: GridVertex, GridEdge + ! OOM errors with 64 MPI tasks per node at ne512 on 96GB/node system + ! possible solutions: + ! 1. have only one MPI task per node compute MetaVertex for all other tasks + ! and then MPI_send data. This is quite difficult because MetaVertex + ! contains nested tree of structs, including pointers to other structs that may + ! not exist on the receiving MPI task + ! 2. Have only one MPI task per node compute Schedule for all other tasks + ! Schedule is relatively easy to flatten and send via MPI + ! but genEdgeSched() would need to be split into two, a routine to compute Schedule + ! and a routine to initialize elem() data from Schedule + ! 3. hack for now: loop over MPI tasks per node and only allow one to run at a time + ! deallocating global errays at the end of each loop ! =============================================================== - ! Allocate and initialize the graph (array of GridVertex_t types) - ! =============================================================== + call t_startf('ScheduleTime') + do task_id=1,par%node_nprocs + if ( task_id == par%node_rank + 1 ) then if (topology=="cube") then - - if (par%masterproc) then - write(iulog,*)"creating cube topology..." - end if - - if (MeshUseMeshFile) then - nelem = MeshCubeElemCount() - nelem_edge = MeshCubeEdgeCount() - else - nelem = CubeElemCount() - nelem_edge = CubeEdgeCount() - end if - - ! we want to exit elegantly when we are using too many processors. - if (nelem < par%nprocs) then - call abortmp('Error: too many MPI tasks. set dyn_npes <= nelem') - end if - + if (par%masterproc) write(iulog,*)"creating cube topology..." allocate(GridVertex(nelem)) allocate(GridEdge(nelem_edge)) @@ -286,66 +296,60 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! =========================================================== ! given partition, count number of local element descriptors ! =========================================================== - allocate(MetaVertex(1)) allocate(Schedule(1)) - nelem_edge=SIZE(GridEdge) - - allocate(TailPartition(nelem_edge)) - allocate(HeadPartition(nelem_edge)) - do i=1,nelem_edge - TailPartition(i)=GridEdge(i)%tail%processor_number - HeadPartition(i)=GridEdge(i)%head%processor_number - enddo ! ==================================================== ! Generate the communication graph ! ==================================================== - call initMetaGraph(iam,MetaVertex(1),GridVertex,GridEdge) + call initMetaGraph(iam,MetaVertex,GridVertex,GridEdge) - nelemd = LocalElemCount(MetaVertex(1)) + nelemd = LocalElemCount(MetaVertex) if(par%masterproc .and. Debug) then - call PrintMetaVertex(MetaVertex(1)) + call PrintMetaVertex(MetaVertex) endif if(nelemd .le. 0) then call abortmp('Not yet ready to handle nelemd = 0 yet' ) stop endif -#ifdef _MPI - call mpi_allreduce(nelemd,nelemdmax,1,MPIinteger_t,MPI_MAX,par%comm,ierr) -#else - nelemdmax=nelemd -#endif - if (nelemd>0) then - allocate(elem(nelemd)) - call setup_element_pointers(elem) - call allocate_element_desc(elem) - endif + allocate(elem(nelemd)) + call allocate_element_desc(elem) ! ==================================================== ! Generate the communication schedule ! ==================================================== + call genEdgeSched(elem,iam,Schedule(1),MetaVertex) + deallocate(GridEdge) + do j =1,nelem + call deallocate_gridvertex_nbrs(GridVertex(j)) + end do + deallocate(GridVertex) + call deallocate_metavertex_data(MetaVertex) - call genEdgeSched(elem,iam,Schedule(1),MetaVertex(1)) + endif ! active MPI task + if (is_zoltan_partition(partmethod) .or. is_zoltan_task_mapping(z2_map_method)) then + ! no barrier since zoltan algorithms are distributed + else + ! barrier forces sequency execution per MPI task on this node. + call syncmp_comm(par%node_comm) + endif + enddo ! loop over all MPI tasks on this node + call t_stopf('ScheduleTime') - allocate(global_shared_buf(nelemd,nrepro_vars)) - global_shared_buf=0.0_real_kind - ! nlyr=edge3p1%nlyr - ! call MessageStats(nlyr) - ! call testchecksum(par,GridEdge) +#ifdef _MPI + call mpi_allreduce(nelemd,nelemdmax,1,MPIinteger_t,MPI_MAX,par%comm,ierr) +#else + nelemdmax=nelemd +#endif - ! ======================================================== - ! load graph information into local element descriptors - ! ======================================================== + call setup_element_pointers(elem) - ! do ii=1,nelemd - ! elem(ii)%vertex = MetaVertex(iam)%members(ii) - ! enddo + allocate(global_shared_buf(nelemd,nrepro_vars)) + global_shared_buf=0.0_real_kind - call syncmp(par) ! ================================================================= ! Set number of domains (for 'decompose') equal to number of threads @@ -402,7 +406,6 @@ subroutine prim_init1(elem, par, dom_mt, Tl) do ie=1,nelemd call set_corner_coordinates(elem(ie)) end do - call assign_node_numbers_to_elem(elem, GridVertex) end if do ie=1,nelemd call cube_init_atomic(elem(ie),gp%points) @@ -482,27 +485,6 @@ subroutine prim_init1(elem, par, dom_mt, Tl) #endif !DBG write(iulog,*) 'prim_init: after call to initRestartFile' - deallocate(GridEdge) - do j =1,nelem - call deallocate_gridvertex_nbrs(GridVertex(j)) - end do - deallocate(GridVertex) - - do j = 1, MetaVertex(1)%nmembers - call deallocate_gridvertex_nbrs(MetaVertex(1)%members(j)) - end do - do j = 1, MetaVertex(1)%nedges - deallocate(MetaVertex(1)%edges(j)%members) - deallocate(MetaVertex(1)%edges(j)%edgeptrP) - deallocate(MetaVertex(1)%edges(j)%edgeptrS) - deallocate(MetaVertex(1)%edges(j)%edgeptrP_ghost) - end do - deallocate(MetaVertex(1)%edges) - deallocate(MetaVertex(1)%members) - deallocate(MetaVertex) - deallocate(TailPartition) - deallocate(HeadPartition) - ! single global edge buffer for all models: ! hydrostatic 4*nlev NH: 6*nlev+1 ! SL tracers: (qsize+1)*nlev @@ -543,7 +525,6 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) hypervis_subcycle_q, moisture, use_moisture use global_norms_mod, only: test_global_integral, print_cfl use hybvcoord_mod, only: hvcoord_t - use parallel_mod, only: parallel_t, haltmp, syncmp, abortmp use prim_state_mod, only: prim_printstate, prim_diag_scalars use prim_si_mod, only: prim_set_mass use prim_advection_mod, only: prim_advec_init2 @@ -873,7 +854,6 @@ subroutine prim_run_subcycle(elem, hybrid,nets,nete, dt, single_column, tl, hvco use control_mod, only: statefreq, ftype, qsplit, rsplit, disable_diagnostics use hybvcoord_mod, only: hvcoord_t - use parallel_mod, only: abortmp use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes use vertremap_mod, only: vertical_remap use reduction_mod, only: parallelmax @@ -1087,7 +1067,6 @@ subroutine prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord, compute_diagnostic use control_mod, only: statefreq, integration, ftype, qsplit, nu_p, rsplit use control_mod, only: use_semi_lagrange_transport use hybvcoord_mod, only : hvcoord_t - use parallel_mod, only: abortmp use prim_advance_mod, only: prim_advance_exp use prim_advection_mod, only: prim_advec_tracers_remap use reduction_mod, only: parallelmax diff --git a/components/homme/src/share/schedtype_mod.F90 b/components/homme/src/share/schedtype_mod.F90 index 13e46de24073..4878b5552845 100644 --- a/components/homme/src/share/schedtype_mod.F90 +++ b/components/homme/src/share/schedtype_mod.F90 @@ -41,8 +41,6 @@ module schedtype_mod end type Schedule_t type (Schedule_t), public, allocatable, target :: Schedule(:) - type (Schedule_t), public, allocatable, target :: gSchedule(:) - type (Schedule_t), public, allocatable, target :: sSchedule(:) end module schedtype_mod diff --git a/components/homme/src/share/spacecurve_mod.F90 b/components/homme/src/share/spacecurve_mod.F90 index c538fa60b3b3..4162f414f131 100644 --- a/components/homme/src/share/spacecurve_mod.F90 +++ b/components/homme/src/share/spacecurve_mod.F90 @@ -7,6 +7,7 @@ module spacecurve_mod ! ! Revisions: ! Mark Taylor: 2018/3 Remove memory leaks, make most routines private +! Mark Taylor: 2018/10 add more deallocates ! use kinds, only : iulog implicit none @@ -18,9 +19,8 @@ module spacecurve_mod end type factor_t - integer,public, dimension(:,:), allocatable :: ordered - integer,public, dimension(:,:), allocatable :: dir ! direction to move along each level - integer,public, dimension(:) , allocatable :: pos ! position along each of the axes + integer, dimension(:,:), allocatable :: ordered + integer, dimension(:) , allocatable :: pos ! position along each of the axes integer,public :: maxdim ! dimensionality of entire space integer,public :: vcnt ! visitation count @@ -781,67 +781,6 @@ function IncrementCurve(ja,jd) result(ierr) ierr = 0 end function IncrementCurve !--------------------------------------------------------- - recursive function hilbert_old(l,d,ma,md,ja,jd) result(ierr) - - integer :: l,d ! log base 2 of levels and dimensions left - integer :: ma,md ! main axis and direction - integer :: ja,jd ! joiner axis and direction - - integer :: ierr - integer :: axis - integer :: ll - - if(verbose) write(iulog,10) l,d,ma,md,ja,jd,pos(0),pos(1) - ll = l ! Copy this to a temporary variable - if(d == 0) then - ll=ll-1 - if(ll == 0) then - return - endif - axis = ja - if(dir(ll,axis) /= jd) then ! do not move away from joiner plane - axis = MOD(axis+1,maxdim) ! next axis - endif - if(verbose) write(iulog,*)'hilbert_old: call hilbert_old(l,d) #1:' - ierr = hilbert_old(ll,maxdim,axis,dir(ll,axis),ja,jd) - dir(ll,ja) = -dir(ll,ja) - return - endif - axis = MOD(ma+1,maxdim) - if(verbose) write(iulog,*)'hilbert_old: before call hilbert_old(l,d) #2:' - ierr = hilbert_old(ll,d-1,axis,dir(ll,axis),ma,md) - if(verbose) write(iulog,*)'hilbert_old: after call hilbert_old(l,d) #2:' - if(verbose) write(iulog,30) l,d,ma,md,ja,jd,pos(0),pos(1) - - - pos(ma) = pos(ma) + md - dir(ll,ma) = - dir(ll,ma) - - !---------------------------------- - ! Mark this node as visited - !---------------------------------- - if(verbose) write(iulog,20) l,d,ma,md,ja,jd,pos(0),pos(1) - vcnt=vcnt+1 - if(verbose) write(iulog,15) pos(0)+1,pos(1)+1,vcnt - if(verbose) write(iulog,*)' ' - if(verbose) write(iulog,*)' ' - ordered(pos(0)+1,pos(1)+1)=vcnt - - if(verbose) write(iulog,*)'hilbert_old: before call hilbert_old(l,d) #3:' - ierr = hilbert_old(ll,d-1,axis,dir(ll,axis),ja,jd) - if(verbose) write(iulog,*)'hilbert_old: after call hilbert_old(l,d) #3:' - -10 format('hilbert_old: Entering hilbert_old (l,d,ma,md,ja,jd) are: ', & - 2(i4),' [',2(i3),'][',2(i3),']',2(i3)) -15 format('hilbert_old: mark element {x,y,ordered}:',3(i4)) -20 format('hilbert_old: Before visit code (l,d,ma,md,ja,jd) are:', & - 2(i4),' [',2(i3),'][',2(i3),']',2(i3)) - -30 format('hilbert_old: after call hilbert_old(l,d) #2: (l,d,ma,md,ja,jd are:', & - 2(i4),' [',2(i3),'][',2(i3),']',2(i3)) - - end function hilbert_old - !--------------------------------------------------------- function log2( n) implicit none @@ -1037,11 +976,12 @@ subroutine GenSpaceCurve(Mesh) ordered(:,:) = 0 call map(level) - deallocate(fact%factors) - - Mesh(:,:) = ordered(:,:) + deallocate(fact%factors) + deallocate(ordered) + deallocate(pos) + end subroutine GenSpaceCurve !------------------------------------------------------------------------------------------------------- subroutine PrintCurve(Mesh) diff --git a/components/homme/src/sweqx/init_mod.F90 b/components/homme/src/sweqx/init_mod.F90 index 47f93e563bc2..be6e89f8dd3c 100644 --- a/components/homme/src/sweqx/init_mod.F90 +++ b/components/homme/src/sweqx/init_mod.F90 @@ -34,7 +34,7 @@ subroutine init(elem, edge1,edge2,edge3,red,par, dom_mt) MeshCubeEdgeCount ! -------------------------------- use cube_mod, only : cube_init_atomic, set_corner_coordinates, & - assign_node_numbers_to_elem, set_area_correction_map2 + set_area_correction_map2 ! -------------------------------- use edge_mod, only : initedgebuffer @@ -282,7 +282,7 @@ subroutine init(elem, edge1,edge2,edge3,red,par, dom_mt) do ie=1,nelemd call set_corner_coordinates(elem(ie)) enddo - call assign_node_numbers_to_elem(elem, GridVertex) + !call assign_node_numbers_to_elem(elem, GridVertex) endif do ie=1,nelemd From 19ecd5fc086fd4f60a690476c150eaa7837b230d Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 7 Oct 2018 17:18:52 -0500 Subject: [PATCH 17/41] tweak init sequentialization option only force sequential initialization if we will be processing more than 40M elements per node. --- .../theta-l/jobscript-snl.sh | 12 +- .../theta-l/namelist-ne512.nl | 6 +- components/homme/src/share/parallel_mod.F90 | 5 +- .../homme/src/share/prim_driver_base.F90 | 200 +++++++++--------- 4 files changed, 111 insertions(+), 112 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh index a29b815ac665..39af5d1d03cc 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh @@ -63,15 +63,9 @@ prefix=r50 ; run $NCPU # high res cases #prefix=ne120 ; run $NCPU # 72 nodes, 2h Anvil: 6s #prefix=ne256 ; run $NCPU # 72 nodes, 2h Anvil: 55s - # 50 nodes ? - # 25 nodes ? -#prefix=ne512 ; run $NCPU # 72 nodes killed. should work with -#prefix=ne1024 ; run $NCPU # - - - - - +#prefix=ne512 ; run $NCPU # 100 nodes 2h Anvil: 611s + # 25 nodes: (runs) +#prefix=ne1024 ; run $NCPU # diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl index 86b056d2b208..8a35bdb23c5e 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl @@ -9,11 +9,11 @@ ne = 512 ! number of elements per cube face qsize = 6 ! num tracer fields ! ndays = 1 - nmax = 360 ! run for 7200s (2h) - statefreq = 20 ! number of steps between screen dumps + nmax = 402 ! run for 7200s (2h) + statefreq = 18 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 40 ! largest timestep in seconds + tstep = 18 ! 20 was unstable integration = 'explicit' ! explicit time integration tstep_type = 7 rsplit = 3 diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index 7fc411f236f2..6d52272b0c3f 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -6,8 +6,11 @@ module parallel_mod use kinds, only : real_kind, int_kind, iulog ! --------------------------- use dimensions_mod, only : nmpi_per_node, nlev, qsize_d +! +! Revisions: +! 2018/10: M. Taylor adding MPI tasks per node subcommunicator - +! implicit none public diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 0ce462d1ab91..30a7a1bbdd96 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -5,6 +5,7 @@ ! 08/2016: O. Guba Inserting code for "espilon bubble" reference element map ! 03/2018: M. Taylor fix memory leak ! 06/2018: O. Guba code for new ftypes +! 10/2018: M. Taylor sequentialize domain decmposition and schedule setup ! #ifdef HAVE_CONFIG_H #include "config.h" @@ -133,7 +134,7 @@ subroutine prim_init1(elem, par, dom_mt, Tl) integer :: nstep integer :: nlyr integer :: iMv - integer :: err, ierr, l, j, task_id + integer :: err, ierr, l, j, task_id, task_id_stride logical, parameter :: Debug = .FALSE. integer total_nelem @@ -235,109 +236,110 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! deallocating global errays at the end of each loop ! =============================================================== call t_startf('ScheduleTime') - do task_id=1,par%node_nprocs - if ( task_id == par%node_rank + 1 ) then - if (topology=="cube") then - if (par%masterproc) write(iulog,*)"creating cube topology..." - allocate(GridVertex(nelem)) - allocate(GridEdge(nelem_edge)) - - do j =1,nelem - call allocate_gridvertex_nbrs(GridVertex(j)) - end do - - if (MeshUseMeshFile) then - if (par%masterproc) then - write(iulog,*) "Set up grid vertex from mesh..." - end if - call MeshCubeTopologyCoords(GridEdge, GridVertex, coord_dim1, coord_dim2, coord_dim3, coord_dimension) - !MD:TODO: still need to do the coordinate transformation for this case. - + ! On Anvil (64GB/node), we can handle 27 copies of GridVertex/GridEdge at ne=512 + ! So we need to keep nelem*task_id_stride < 42M + ! if initialization time is too slow, and memory per node > 64GB, this can be increased + task_id_stride=max(40000000/nelem,1) + if (task_id_stride Date: Sun, 7 Oct 2018 17:38:34 -0500 Subject: [PATCH 18/41] tweaked initialization settings so Anvil can run ne=1024 --- .../theta-l/jobscript-snl.sh | 2 +- .../theta-l/namelist-ne1024.nl | 56 +++++++++++++++++++ .../homme/src/share/prim_driver_base.F90 | 13 +++-- 3 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh index 39af5d1d03cc..9e87c5cd0836 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh @@ -65,7 +65,7 @@ prefix=r50 ; run $NCPU #prefix=ne256 ; run $NCPU # 72 nodes, 2h Anvil: 55s #prefix=ne512 ; run $NCPU # 100 nodes 2h Anvil: 611s # 25 nodes: (runs) -#prefix=ne1024 ; run $NCPU # +#prefix=ne1024 ; run $NCPU # 100 nodes 2h Anvi: RUNNING. diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl new file mode 100644 index 000000000000..5748d7887e96 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl @@ -0,0 +1,56 @@ +! +! theta: namelist for dcmip2016 test1: moist baroclinic wave +!_______________________________________________________________________ +&ctl_nl + nthreads = 1 + partmethod = 4 ! mesh parition method: 4 = space filling curve + topology = "cube" ! mesh type: cubed sphere + test_case = "dcmip2016_test1" ! test identifier + ne = 1024 ! number of elements per cube face + qsize = 6 ! num tracer fields +! ndays = 1 + nmax = 801 ! run for 7200s (2h) + statefreq = 18 ! number of steps between screen dumps + restartfreq = -1 ! don't write restart files if < 0 + runtype = 0 ! 0 => new run + tstep = 9 ! 20 was unstable + integration = 'explicit' ! explicit time integration + tstep_type = 7 + rsplit = 3 + qsplit = 1 + nu = 1e10 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 + nu_s = 1e10 + nu_p = 1e10 + nu_top = 0 ! default = 2.5e5 + limiter_option = 9 + hypervis_order = 2 ! 2 = hyperviscosity + hypervis_subcycle = 1 ! 1 = no hyperviz subcycling + moisture = 'wet' + theta_hydrostatic_mode = .false. + dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none +/ +&vert_nl + vform = "ccm" + vfile_mid = "../vcoord/camm-30.ascii" + vfile_int = "../vcoord/cami-30.ascii" +/ +&analysis_nl +! output_prefix = "r50-" + output_dir = "./movies/" ! destination dir for netcdf file + output_timeunits = 2, ! 0=timesteps, 1=days, 2=hours, 3=seconds + output_frequency = 24 ! every N hours + output_start_time = 48 ! start after N hours +! output_varnames1 ='T','ps','pnh','geo','u','v','w','omega','Th','Q','Q2','Q3','Q4','Q5','rho','precl','zeta' ! variables to write to file + output_varnames1 ='T','ps','pnh','geo','u','Th','Q','Q2','Q3','Q4','Q5','precl','zeta' ! variables to write to file + interp_type = 1 ! 0=native grid, 1=bilinear + output_type ='netcdf' ! netcdf or pnetcdf + num_io_procs = 16 + interp_nlon = 360 ! low-res output grid to save init time + interp_nlat = 181 + interp_gridtype = 1 +/ +&prof_inparm + profile_outpe_num = 100 + profile_single_file = .true. +/ diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 30a7a1bbdd96..c42d67275f96 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -232,18 +232,19 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! Schedule is relatively easy to flatten and send via MPI ! but genEdgeSched() would need to be split into two, a routine to compute Schedule ! and a routine to initialize elem() data from Schedule - ! 3. hack for now: loop over MPI tasks per node and only allow one to run at a time - ! deallocating global errays at the end of each loop + ! 3. hack for now: loop over MPI tasks per node and only allow "task_id_stride" + ! to run at a time, deallocating global errays at the end of each loop ! =============================================================== call t_startf('ScheduleTime') - ! On Anvil (64GB/node), we can handle 27 copies of GridVertex/GridEdge at ne=512 - ! So we need to keep nelem*task_id_stride < 42M + ! Tuned on Anvil (64GB/node). for ne=1024, keep nelem*task_id_stide < 20M ! if initialization time is too slow, and memory per node > 64GB, this can be increased - task_id_stride=max(40000000/nelem,1) + task_id_stride=max(20000000/nelem,1) if (task_id_stride Date: Sun, 7 Oct 2018 22:42:00 -0500 Subject: [PATCH 19/41] add namelists for DCMIP2016 test1 ne1024 resolution --- .../theta-l/jobscript-snl.sh | 16 +++++++++++----- .../theta-l/namelist-ne1024.nl | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh index 9e87c5cd0836..1facaa01d83f 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh @@ -61,12 +61,18 @@ prefix=r100 ; run $NCPU prefix=r50 ; run $NCPU # high res cases -#prefix=ne120 ; run $NCPU # 72 nodes, 2h Anvil: 6s -#prefix=ne256 ; run $NCPU # 72 nodes, 2h Anvil: 55s -#prefix=ne512 ; run $NCPU # 100 nodes 2h Anvil: 611s - # 25 nodes: (runs) -#prefix=ne1024 ; run $NCPU # 100 nodes 2h Anvi: RUNNING. +#prefix=ne120 ; run $NCPU +#prefix=ne256 ; run $NCPU +#prefix=ne512 ; run $NCPU +#prefix=ne1024 ; run $NCPU +# timings on ANVIL +#ne120 72 nodes, 2h Anvil: 6s +#ne256 72 nodes, 2h Anvil: 55s +#ne512 100 nodes 2h Anvil: 611s (can run on 25 nodes) +#ne1024 100 nodes 2h Anvi: 4642s (6min init) +# 100 nodes 270 timesteps: 1836s (6min init) +# diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl index 5748d7887e96..e679926ba0f9 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl @@ -9,7 +9,7 @@ ne = 1024 ! number of elements per cube face qsize = 6 ! num tracer fields ! ndays = 1 - nmax = 801 ! run for 7200s (2h) + nmax = 270 ! statefreq = 18 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run From f4f59ff942a42c6ad704686f168f886d709e5a25 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 7 Oct 2018 20:43:22 -0700 Subject: [PATCH 20/41] add KNL jobscript for dcmip2016 test1 --- .../theta-l/jobscript-knl.sh | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh new file mode 100644 index 000000000000..dcbffb2e0035 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# +#SBATCH --job-name dcmip2016-1 +#SBATCH -N 128 +#SBATCH -C knl +#SBATCH --time=0:30:00 +#SBATCH -q debug +# +# + + +# +# mpi run command +# +export OMP_STACKSIZE=16M # Cori has 96GB per node. had to lower to 8M on 3K nodes +export OMP_NUM_THREADS=2 +PER_NODE=64 # MPI per node + +# number of virtual cores per MPI task +VC_PER_MPI=256 # Set this to 272 if using PER_NODE divides 272 instead of 256 +let VC_PER_MPI/=$PER_NODE + +export KMP_AFFINITY="granularity=core,scatter" +bind=--cpu_bind=core + +# compute number of MPI tasks +if [ -n "$SLURM_NNODES" ]; then + NNODES=$SLURM_NNODES +else + NNODES=1 +fi + + +NMPI=$NNODES +let NMPI*=$PER_NODE + +echo NODES = $NNODES +echo NMPI_PER_NODE = $PER_NODE +echo NTHREADS_PER_MPI = $OMP_NUM_THREADS +mpirun="srun -n $NMPI -N $NNODES -c $VC_PER_MPI $bind" +echo mpi commnand: +echo $mpirun + + +# hydrostatic preqx +EXEC=../../../test_execs/theta-l-nlev30/theta-l-nlev30 + +function run { +local NCPU=$1 +echo "NCPU = $NCPU" +namelist=namelist-$prefix.nl +\cp -f $namelist input.nl +date +$mpirun $EXEC < input.nl +date + +ncl plot-baroclinicwave-init.ncl +ncl plot-lat-lon-TPLSPS.ncl 'var_choice=1' +ncl plot-lat-lon-TPLSPS.ncl 'var_choice=2' +ncl plot-lat-lon-TPLSPS.ncl 'var_choice=3' +\mv -f plot_baroclinicwave_init.pdf ${prefix}_init.pdf +\mv -f preqx-test16-1latlonT850.pdf ${prefix}_T850.pdf +\mv -f preqx-test16-1latlonPS.pdf ${prefix}_PS.pdf +\mv -f preqx-test16-1latlonPRECL.pdf ${prefix}_PRECL.pdf + +\mv -f movies/dcmip2016_test11.nc movies/${prefix}_dcmip2016_test11.nc +} + +prefix=r400 ; run $(($NCPU>384?384:NCPU)) + +prefix=r100-dry; run $NCPU +prefix=r100-h ; run $NCPU +prefix=r100 ; run $NCPU + +prefix=r50 ; run $NCPU + +# high res cases +#prefix=ne120 ; run $NCPU +#prefix=ne256 ; run $NCPU +#prefix=ne512 ; run $NCPU +#prefix=ne1024 ; run $NCPU + +# cori-KNL timings +# ne=120 25 nodes, 2h: 23s +# ne=256 25 nodes, 2h: 142s +# ne=512 64 nodes, 2h: 533s +# ne=1024 512 nodes, 270 timsteps: 596s + init +# init time if run 16x8: 400s +# init time if run 64x2: 1400s +# +# +# + + + + + + From 991044d990e2041609297025a4b3998c4ea7be0d Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 8 Oct 2018 11:18:28 -0500 Subject: [PATCH 21/41] cleaned up initialization timers --- .../homme/src/share/prim_driver_base.F90 | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index c42d67275f96..c57927407a2f 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -5,7 +5,7 @@ ! 08/2016: O. Guba Inserting code for "espilon bubble" reference element map ! 03/2018: M. Taylor fix memory leak ! 06/2018: O. Guba code for new ftypes -! 10/2018: M. Taylor sequentialize domain decmposition and schedule setup +! 10/2018: M. Taylor serialize domain decmposition and schedule setup ! #ifdef HAVE_CONFIG_H #include "config.h" @@ -223,24 +223,25 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! start domain decomposition ! non-scalable global arrays: GridVertex, GridEdge ! OOM errors with 64 MPI tasks per node at ne512 on 96GB/node system - ! possible solutions: - ! 1. have only one MPI task per node compute MetaVertex for all other tasks - ! and then MPI_send data. This is quite difficult because MetaVertex - ! contains nested tree of structs, including pointers to other structs that may - ! not exist on the receiving MPI task - ! 2. Have only one MPI task per node compute Schedule for all other tasks - ! Schedule is relatively easy to flatten and send via MPI - ! but genEdgeSched() would need to be split into two, a routine to compute Schedule - ! and a routine to initialize elem() data from Schedule - ! 3. hack for now: loop over MPI tasks per node and only allow "task_id_stride" - ! to run at a time, deallocating global errays at the end of each loop + ! cost breakdown (Anvil, ne1024): 2x higher on KNL + ! compute GridVertex, GridEDge: 22s + ! compute MetaVertex for 1 task: 8s + ! compute Schedule for 1 task: .1s + ! consider 36 MPI tasks per node. some possible solutions: + ! 1. serialize entire calcluation: 36x(22+8) = 1080s + ! 2. root computes & bcasts MetaVertex: 22 + 36x8 = 310s + ! 3. serialize, but allow 3 tasks to be active per node: (36/3)*(22+8) = 360s + ! 4. Compute MetaVertex offline + ! + ! we are using #3 for now. ! =============================================================== - call t_startf('ScheduleTime') + call t_startf('MeshSetup') + allocate(Schedule(1)) ! Tuned on Anvil (64GB/node). for ne=1024, keep nelem*task_id_stide < 20M ! if initialization time is too slow, and memory per node > 64GB, this can be increased task_id_stride=max(20000000/nelem,1) if (task_id_stride Date: Mon, 8 Oct 2018 11:46:30 -0700 Subject: [PATCH 22/41] a few more tweaks for KNL jobscripts tuned number of active MPI tasks per node during init serialization to minimize init time on KNL --- .../theta-l/jobscript-knl.sh | 40 ++++++++++--------- .../homme/src/share/prim_driver_base.F90 | 30 +++++++------- .../homme/src/theta-l/prim_advance_mod.F90 | 7 ++-- 3 files changed, 40 insertions(+), 37 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh index dcbffb2e0035..5e275acd7ae0 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-knl.sh @@ -1,7 +1,7 @@ #!/bin/bash # #SBATCH --job-name dcmip2016-1 -#SBATCH -N 128 +#SBATCH -N 43 #SBATCH -C knl #SBATCH --time=0:30:00 #SBATCH -q debug @@ -34,6 +34,15 @@ fi NMPI=$NNODES let NMPI*=$PER_NODE + + +# hydrostatic preqx +EXEC=../../../test_execs/theta-l-nlev30/theta-l-nlev30 + + +function run { +local NMPI=$1 + echo NODES = $NNODES echo NMPI_PER_NODE = $PER_NODE echo NTHREADS_PER_MPI = $OMP_NUM_THREADS @@ -41,13 +50,6 @@ mpirun="srun -n $NMPI -N $NNODES -c $VC_PER_MPI $bind" echo mpi commnand: echo $mpirun - -# hydrostatic preqx -EXEC=../../../test_execs/theta-l-nlev30/theta-l-nlev30 - -function run { -local NCPU=$1 -echo "NCPU = $NCPU" namelist=namelist-$prefix.nl \cp -f $namelist input.nl date @@ -66,27 +68,27 @@ ncl plot-lat-lon-TPLSPS.ncl 'var_choice=3' \mv -f movies/dcmip2016_test11.nc movies/${prefix}_dcmip2016_test11.nc } -prefix=r400 ; run $(($NCPU>384?384:NCPU)) +prefix=r400 ; run $(($NMPI>384?384:NMPI)) -prefix=r100-dry; run $NCPU -prefix=r100-h ; run $NCPU -prefix=r100 ; run $NCPU +prefix=r100-dry; run $(($NMPI>5400?5400:NMPI)) +prefix=r100-h ; run $(($NMPI>5400?5400:NMPI)) +prefix=r100 ; run $(($NMPI>5400?5400:NMPI)) -prefix=r50 ; run $NCPU +prefix=r50 ; run $NMPI # high res cases -#prefix=ne120 ; run $NCPU -#prefix=ne256 ; run $NCPU -#prefix=ne512 ; run $NCPU -#prefix=ne1024 ; run $NCPU +#prefix=ne120 ; run $NMPI +#prefix=ne256 ; run $NMPI +#prefix=ne512 ; run $NMPI +#prefix=ne1024 ; run $NMPI # cori-KNL timings # ne=120 25 nodes, 2h: 23s # ne=256 25 nodes, 2h: 142s # ne=512 64 nodes, 2h: 533s # ne=1024 512 nodes, 270 timsteps: 596s + init -# init time if run 16x8: 400s -# init time if run 64x2: 1400s +# init time if run 16x8: 133s +# init time if run 64x2: 600s # # # diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index c57927407a2f..78b92b082d74 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -227,30 +227,30 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! compute GridVertex, GridEDge: 22s ! compute MetaVertex for 1 task: 8s ! compute Schedule for 1 task: .1s - ! consider 36 MPI tasks per node. some possible solutions: - ! 1. serialize entire calcluation: 36x(22+8) = 1080s - ! 2. root computes & bcasts MetaVertex: 22 + 36x8 = 310s - ! 3. serialize, but allow 3 tasks to be active per node: (36/3)*(22+8) = 360s - ! 4. Compute MetaVertex offline ! - ! we are using #3 for now. + ! in order to not run out of memory, only allow task_id_stride MPI tasks to be + ! active per node, and active tasks must deallocate GridVertex and GridEdge data + ! before the next tasks start ! =============================================================== call t_startf('MeshSetup') allocate(Schedule(1)) - ! Tuned on Anvil (64GB/node). for ne=1024, keep nelem*task_id_stide < 20M - ! if initialization time is too slow, and memory per node > 64GB, this can be increased - task_id_stride=max(20000000/nelem,1) + ! Anvil (64GB/node). for ne=1024, keep nelem*task_id_stide < 20M + ! cori-KNL: 60M works. 80M OOM. set default to 50M + ! if OOM errors during initialization, decrease this number + ! if initialization time is too slow, and memory per node > 64GB, increase this number + task_id_stride=max(51000000/nelem,1) if (task_id_stride Date: Mon, 8 Oct 2018 18:50:10 -0500 Subject: [PATCH 23/41] fix thread race condition in dcmip2016 scalar I/O --- .../theta-l/jobscript-snl.sh | 7 +++++-- components/homme/src/test_src/dcmip16_wrapper.F90 | 11 ++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh index 1facaa01d83f..dda6d0a352b1 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh @@ -13,18 +13,21 @@ # 12 nodes, 10min for r400 an r100 # -OMP_NUM_THREADS=1 +export OMP_NUM_THREADS=1 +export OMP_STACKSIZE=16M # Cori has 96GB per node. had to lower to 8M on 3K nodes NCPU=40 if [ -n "$PBS_ENVIRONMENT" ]; then # NCPU=$PBS_NNODES [ "$PBS_ENVIRONMENT" = "PBS_BATCH" ] && cd $PBS_O_WORKDIR NCPU=$PBS_NNODES + let NCPU/=$OMP_NUM_THREADS fi if [ -n "$SLURM_NNODES" ]; then NCPU=$SLURM_NNODES let NCPU*=16 let NCPU/=$OMP_NUM_THREADS fi +let PPN=36/$OMP_NUM_THREADS # hydrostatic preqx EXEC=../../../test_execs/theta-l-nlev30/theta-l-nlev30 @@ -37,7 +40,7 @@ echo "NCPU = $NCPU" namelist=namelist-$prefix.nl \cp -f $namelist input.nl date -mpirun -np $NCPU $EXEC < input.nl +mpirun -bind-to=core -ppn $PPN -np $NCPU $EXEC < input.nl date ncl plot-baroclinicwave-init.ncl diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index 55f9d7f10916..d3a54f0852f2 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -21,7 +21,7 @@ module dcmip16_wrapper use hybrid_mod, only: hybrid_t use hybvcoord_mod, only: hvcoord_t, set_layer_locations use kinds, only: rl=>real_kind, iulog -use parallel_mod, only: abortmp +use parallel_mod, only: abortmp,iam use element_ops, only: set_state, set_state_i, set_elem_state, get_state, tests_finalize,& set_forcing_rayleigh_friction, set_thermostate use physical_constants, only: p0, g, Rgas, kappa, Cp, Rwater_vapor, pi=>dd_pi @@ -333,9 +333,8 @@ subroutine dcmip2016_append_measurements(max_w,max_precl,min_ps,tl,hybrid) real(rl) :: next_sample_time = 0.0 time = time_at(tl%nstep) - ! initialize output file - if(next_sample_time == 0.0) then + if(next_sample_time == 0.0 .and. hybrid%masterthread) then open(unit=10,file=w_filename, form="formatted",status="UNKNOWN") close(10) @@ -348,11 +347,13 @@ subroutine dcmip2016_append_measurements(max_w,max_precl,min_ps,tl,hybrid) open(unit=12,file=ps_filename, form="formatted",status="UNKNOWN") close(13) endif - ! append measurements at regular intervals if(time .ge. next_sample_time) then - +!$OMP BARRIER +!$OMP MASTER next_sample_time = next_sample_time + sample_period +!$OMP END MASTER +!$OMP BARRIER pmax_w = parallelMax(max_w, hybrid) pmax_precl = parallelMax(max_precl,hybrid) pmin_ps = parallelMin(min_ps, hybrid) From d369a507985e8cbafe37924afef7ed967fd33494 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 8 Oct 2018 16:51:15 -0700 Subject: [PATCH 24/41] update namelists so number of threads comes from OMP_NUM_THREADS --- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl | 2 +- .../theta-l/namelist-r100-dry.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl index e679926ba0f9..808dee0b2e9e 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl @@ -2,7 +2,7 @@ ! theta: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl index 4b30b8dfd4bd..f397bd3f3b1f 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl @@ -2,7 +2,7 @@ ! theta: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl index 51e9b0cd3500..77a431689039 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl @@ -2,7 +2,7 @@ ! theta: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl index 8a35bdb23c5e..422b23c1da45 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl @@ -2,7 +2,7 @@ ! theta: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl index 94320a1ce506..c664352eed1e 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-dry.nl @@ -2,7 +2,7 @@ ! preqx: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl index b8046b7ed350..579ff194ebbd 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl @@ -2,7 +2,7 @@ ! preqx: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl index 977e0b71f295..779572a8f032 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl @@ -2,7 +2,7 @@ ! preqx: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl index a8447fab6234..b782a9aa18a0 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl @@ -2,7 +2,7 @@ ! preqx: namelist for dcmip2016 test1: moist baroclininc wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl index d5e9fac383be..e786dc5b7210 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl @@ -2,7 +2,7 @@ ! theta: namelist for dcmip2016 test1: moist baroclinic wave !_______________________________________________________________________ &ctl_nl - nthreads = 1 + nthreads = -1 ! use OMP_NUM_THREADS partmethod = 4 ! mesh parition method: 4 = space filling curve topology = "cube" ! mesh type: cubed sphere test_case = "dcmip2016_test1" ! test identifier From 7615d560094565b7010dff57116dc38c92e18615 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 8 Oct 2018 21:10:27 -0500 Subject: [PATCH 25/41] fix threading bug in theta-l model incorrect array dimension when calling make_c0_vector() --- components/homme/src/theta-l/eos.F90 | 2 +- components/homme/src/theta-l/model_init_mod.F90 | 4 ++-- components/homme/src/theta-l/prim_advance_mod.F90 | 4 +--- components/homme/src/theta-l/prim_state_mod.F90 | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/components/homme/src/theta-l/eos.F90 b/components/homme/src/theta-l/eos.F90 index d623088ec347..6a6df5ff82dc 100644 --- a/components/homme/src/theta-l/eos.F90 +++ b/components/homme/src/theta-l/eos.F90 @@ -18,7 +18,7 @@ ! module eos - use dimensions_mod, only: np, nlev, nlevp, nelemd + use dimensions_mod, only: np, nlev, nlevp use element_mod, only: element_t use element_state, only: timelevels, elem_state_t use hybvcoord_mod, only: hvcoord_t diff --git a/components/homme/src/theta-l/model_init_mod.F90 b/components/homme/src/theta-l/model_init_mod.F90 index d273c5ba5054..91570ffc0831 100644 --- a/components/homme/src/theta-l/model_init_mod.F90 +++ b/components/homme/src/theta-l/model_init_mod.F90 @@ -17,7 +17,7 @@ module model_init_mod use derivative_mod, only: derivative_t,gradient_sphere use hybvcoord_mod, only: hvcoord_t use hybrid_mod, only: hybrid_t - use dimensions_mod, only: np,nlev,nlevp,nelemd + use dimensions_mod, only: np,nlev,nlevp use eos , only: get_pnh_and_exner,get_dirk_jacobian use element_state, only: timelevels, nu_scale_top use viscosity_mod, only: make_c0_vector @@ -41,7 +41,7 @@ subroutine model_init2(elem,hybrid,deriv,hvcoord,tl,nets,nete ) ! local variables integer :: ie,t,k - real (kind=real_kind) :: gradtemp(np,np,2,nelemd) + real (kind=real_kind) :: gradtemp(np,np,2,nets:nete) real (kind=real_kind) :: ptop_over_press diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index 98eee835c435..3e1b71aba2a8 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -17,7 +17,7 @@ module prim_advance_mod use derivative_mod, only: derivative_t, divergence_sphere, gradient_sphere, laplace_sphere_wk,& laplace_z, vorticity_sphere, vlaplace_sphere_wk use derivative_mod, only: subcell_div_fluxes, subcell_dss_fluxes - use dimensions_mod, only: max_corner_elem, nelemd, nlev, nlevp, np, qsize + use dimensions_mod, only: max_corner_elem, nlev, nlevp, np, qsize use edge_mod, only: edge_g, edgevpack_nlyr, edgevunpack_nlyr use edgetype_mod, only: EdgeBuffer_t, EdgeDescriptor_t, edgedescriptor_t use element_mod, only: element_t @@ -1040,8 +1040,6 @@ subroutine compute_andor_apply_rhs(np1,nm1,n0,qn0,dt2,elem,hvcoord,hybrid,& integer :: i,j,k,kptr,ie, nlyr_tot - real (kind=real_kind) :: wtemp(np,np,nelemd) - call t_startf('compute_andor_apply_rhs') if (theta_hydrostatic_mode) then diff --git a/components/homme/src/theta-l/prim_state_mod.F90 b/components/homme/src/theta-l/prim_state_mod.F90 index 128781ea3e95..8994b2bcf04c 100644 --- a/components/homme/src/theta-l/prim_state_mod.F90 +++ b/components/homme/src/theta-l/prim_state_mod.F90 @@ -5,7 +5,7 @@ module prim_state_mod use kinds, only: real_kind, iulog - use dimensions_mod, only: nlev, np, qsize_d, qsize, nelemd, nlevp + use dimensions_mod, only: nlev, np, qsize_d, qsize, nlevp use parallel_mod, only: iam, ordered, parallel_t, syncmp use parallel_mod, only: global_shared_buf, global_shared_sum use global_norms_mod, only: wrap_repro_sum From 2beeeb025104f1d21f50e96fddde4170a75ede1e Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 8 Oct 2018 21:48:41 -0500 Subject: [PATCH 26/41] adding missing NCL files for dcmip2016 test1 --- .../theta-l/plot-baroclinicwave-init.ncl | 211 ++++++++++++++ .../theta-l/plot-lat-lon-TPLSPS.ncl | 269 ++++++++++++++++++ components/homme/src/share/parallel_mod.F90 | 10 +- 3 files changed, 486 insertions(+), 4 deletions(-) create mode 100755 components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-baroclinicwave-init.ncl create mode 100644 components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-lat-lon-TPLSPS.ncl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-baroclinicwave-init.ncl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-baroclinicwave-init.ncl new file mode 100755 index 000000000000..da63299d3fe3 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-baroclinicwave-init.ncl @@ -0,0 +1,211 @@ +;*********************************************** +; plot_baroclinicwave_init.ncl +;*********************************************** + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +;************************************************ +begin + +;************************************************ +; Input parameters +;************************************************ + + datafilename = "./movies/dcmip2016_test11.nc" + +;************************************************ +; Initialize workspace +;************************************************ + + wks = gsn_open_wks("pdf","plot_baroclinicwave_init") + +;************************************************ +; Plot resources [options] +;************************************************ + + res = True + ;res@mpCenterLonF = 180 + res@cnFillOn = True ; color [default=False] + res@cnLinesOn = True ; draw contour lines [default=True] + res@lbLabelAutoStride = True + res@vpKeepAspect = True + res@vpWidthF = 0.38 ; user specified shape + res@vpHeightF = 0.19 + res@gsnSpreadColors = True ; use all colors + res@gsnSpreadColorStart = 2 ; default=2 + res@gsnSpreadColorEnd = 23 ; final color of "gui-default" +; res@gsnSpreadColorStart = 2 ; default=2 +; res@gsnSpreadColorEnd = 11 ; final color of "cosam" + + res@gsnDraw = False ; default=True + res@gsnFrame = False ; default=True + + res_p = res ; pressure plot resources + res_ps = res ; surface pressure plot resources + res_u = res ; zonal velocity plot resources + res_up = res ; zonal velocity pert plot resources + res_t = res ; temperature plot resources + res_qv = res ; water vapor plot resources + res_theta = res ; potential temperature plot resources + +;************************************************ +; Panel plot +;************************************************ + plot = new(6,graphic) + gsn_merge_colormaps(wks,"gui_default", "BlRe") +; gsn_merge_colormaps(wks,"cosam", "BlRe") + + datafile = addfile(datafilename, "r") + + lev = datafile->lev + lon = datafile->lon + + nlon = dimsizes(lon) + + t = datafile->T(0,::-1,:,:) + theta = datafile->Th(0,::-1,:,:) + u = datafile->u(0,::-1,:,:) + qv = datafile->Q(0,::-1,:,:) + p = datafile->pnh(0,::-1,:,:) + geo = datafile->geo(0,::-1,:,:) + qv = qv*1000.0 + p = p/100.0 + ps = datafile->ps(0,:,:) + +delete(u@long_name) +delete(p@long_name) +delete(qv@long_name) +delete(theta@long_name) +delete(t@long_name) + + print("lon(0)="+lon(0)) + if(lon(0) .eq. 0) then + print("lonFlip") + t = lonFlip(t) + theta = lonFlip(theta) + u = lonFlip(u) + qv = lonFlip(qv) + p = lonFlip(p) + ps = lonFlip(ps) + geo = lonFlip(geo) + end if + + + up = u + do i=0,nlon-1 + up(0,:,i) = up(0,:,i) - u(0,:,0) + end do + + ;_________________________________________________ + ; Interpolate fields from eta coord to z coord + + z_eta = geo/9.81 + z = lev + z = ispan(500,29500,1000) + + print("interpolating from geopotential levels to evenly spaced z") + theta = linint1_n_Wrap(z_eta,theta,False,z,0,0) + t = linint1_n_Wrap(z_eta,t ,False,z,0,0) + u = linint1_n_Wrap(z_eta,u ,False,z,0,0) + qv = linint1_n_Wrap(z_eta,qv,False,z,0,0) + p = linint1_n_Wrap(z_eta,p ,False,z,0,0) + +; --- +; z = lev * 30.0 + p&lev = z/1000 + t&lev = z/1000 + theta&lev = z/1000 + qv&lev = z/1000 + u&lev = z/1000 + +; --- + res_t@cnLevelSelectionMode= "ManualLevels" + res_t@cnLevelSpacingF = 10. + res_t@cnMinLevelValF = 180. + res_t@cnMaxLevelValF = 310. + + res_t@tiYAxisString = "Altitude (km)" + res_t@gsnCenterString = "Temperature" + res_t@gsnRightString = "(K)" + plot(0) = gsn_csm_contour(wks,t(:,:,0),res_t) + + res_theta@cnLevelSelectionMode= "ExplicitLevels" + res_theta@cnLevels = (/250., 260., 270., 280., 290., 300., 310., 320., 330., 340., 350., 360., 380., 400., 450., 500., 550., 600., 650., 700./) + + res_theta@tiYAxisString = "Altitude (km)" + res_theta@gsnCenterString = "Potential Temperature" + res_theta@gsnRightString = "(K)" + plot(1) = gsn_csm_contour(wks,theta(:,:,0),res_theta) + +; --- + + res_u@cnLevelSelectionMode= "ManualLevels" + res_u@cnLevelSpacingF = 2.0 + res_u@cnMinLevelValF = 2.0 + res_u@cnMaxLevelValF = 26.0 + + res_u@tiYAxisString = "Altitude (km)" + res_u@gsnCenterString = "Zonal velocity" + res_u@gsnRightString = "(m/s)" + plot(2) = gsn_csm_contour(wks,u(:,:,0),res_u) + + res_qv@cnLevelSelectionMode= "ManualLevels" + res_qv@cnLevelSpacingF = 2.0 + res_qv@cnMinLevelValF = 2.0 + res_qv@cnMaxLevelValF = 18.0 + + res_qv@tiYAxisString = "Altitude (km)" + res_qv@gsnCenterString = "Specific Humidity" + res_qv@gsnRightString = "(g/kg)" + plot(3) = gsn_csm_contour(wks,qv(:,:,0),res_qv) + +; --- + + res_up@cnLevelSelectionMode= "ManualLevels" + res_up@cnLevelSpacingF = 0.1 + res_up@cnMinLevelValF = 0.1 + res_up@cnMaxLevelValF = 1.0 + + res_up@trXMinF = 0. + res_up@trXMaxF = 40. + res_up@trYMinF = 30. + res_up@trYMaxF = 50. + + res_up@tiYAxisString = "Latitude" + res_up@gsnCenterString = "Zonal wind perturbation" + res_up@gsnRightString = "(m/s)" + plot(4) = gsn_csm_contour(wks,up(0,:,:),res_up) + + res_p@cnLevelSelectionMode= "ExplicitLevels" + res_p@cnLevels = (/1.e1, 2.e1, 5.e1, 1.e2, 2.e2, 3.e2, 4.e2, 5.e2, 6.e2, 7.e2, 8.e2, 9.e2, 1.e3/) + + res_p@tiYAxisString = "Altitude (km)" + res_p@gsnCenterString = "Pressure" + res_p@gsnRightString = "(hPa)" + plot(5) = gsn_csm_contour(wks,p(:,:,0),res_p) + + ; res_ps@cnLevelSelectionMode= "ManualLevels" + ; res_ps@cnLevelSpacingF = 10. + ; res_ps@cnMinLevelValF = 950. + ; res_ps@cnMaxLevelValF = 1050. + + ; res_ps@gsnCenterString = "Surfae Pressure" + ; res_ps@gsnRightString = "(hPa)" + ; plot(5) = gsn_csm_contour(wks,ps(:,:),res_ps) + +; --- + + pres = True + pres@gsnFrame = False + pres@gsnPanelYWhiteSpacePercent = 5 + pres@gsnPanelBottom = 0.1 + + gsn_panel(wks,plot,(/3,2/),pres) + + frame(wks) + + +end diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-lat-lon-TPLSPS.ncl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-lat-lon-TPLSPS.ncl new file mode 100644 index 000000000000..ce69bf08eda8 --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/plot-lat-lon-TPLSPS.ncl @@ -0,0 +1,269 @@ +;================================================; +; Example ncl script to produce the set of +; lat-lon plots for DCMIP-2016 test case 1 +; Created by James Kent, Christiane Jablonowski +; and Paul Ullrich (University of Michigan) for DCMIP-2012 +; +; Modified by Colin Zarzycki for DCMIP-2016 +; +; v1.01 - 6/7/16 +; v1.02 - 6/8/16 - Typo in var_choice options +;================================================; +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" +; ================================================; + +; NOTE, if your model does not output T850 OR P at level +; midpoints OR you cannot calculate P from hybrid levels, you +; will need to make model specific additions denoted by +; CALCPHERE below + +; PRECL needs to be in m/s for contours to plot correctly + +begin + +;=================================================; +; open file and read in data +; GIVEN AS (time,lev,lat,lon) from 0 to n-1 +;=================================================; + + ; NOTE, that this file needs to be a catted file including all times at daily interval + ; if you are outputting 1 file per time, you can run "ncrcat dcmip1.files.*.nc cat.nc" + ; for one file + + f = addfile("./movies/dcmip2016_test11.nc","r") + + ; Input useful parameters + + lat = f->lat + lon = f->lon + lev = f->lev + nlat = getfilevardimsizes(f, "lat" ) + nlon = getfilevardimsizes(f, "lon" ) + nlev = getfilevardimsizes(f, "lev" ) + + ; We want plots of temperature, vertical velocity and relative vorticity + ; at the 850 hPa level. We also want surface pressure. + + ; Select var_choice + ; 1 - Temperature at 850 hPa + ; 2 - Large-scale precipitation + ; 3 - Surface Pressure + + if(isvar("var_choice")) then + else + var_choice = 1 + end if + + if (var_choice .eq. 1) then ; Select T + if (isfilevar(f,"T850")) then + varload = f->T850 + + else + + ; we interp T to the 850 hPa level + varload0 = f->T ; T is a 4D var + varload = f->T(:,0,:,:) + ; Calculate P using hybrid coefficients + ; If you have pressure as a variable just do + ; P = f->P + ; CALCPHERE +; hyam = f->hyam +; hybm = f->hybm +; P0 = f->P0 +; ps = f->ps +; P = 0.0*varload0 +; P = pres_hybrid_ccm(PS,P0,hyam,hybm) +; P!0 = "time" +; P!1 = "lev" +; P!2 = "lat" +; P!3 = "lon" + + P = f->pnh + ; interp to 850 hPa level + plevel = 85000.0 + varload = int2p_n(P,varload0,plevel,2,1) + + end if + + else if (var_choice .eq. 2) then ; Select Large-scale precipitation + + varload = f->precl + varload=varload*8.64e7 + + else if (var_choice .eq. 3) then ; Select PS + + varload = f->ps + + end if + end if + end if + + ; We want the output at days 6, 9, 12 and 15. If your model + ; output is in 6 hourly intervals this becomes indices 24, 36, 48 and 60 + +time = f->time +nt = dimsizes(time) +print("nt="+nt) + +day6 = closest_val( 6, time) +day9 = closest_val( 9, time) +day12 = closest_val( 12, time) +day15 = closest_val( 15, time) + + var1 = varload(day6,:,:) + var2 = varload(day9,:,:) + var3 = varload(day12,:,:) + var4 = varload(day15,:,:) + + ; We don't want long-name in the plots + + var1@long_name = " " + var2@long_name = " " + var3@long_name = " " + var4@long_name = " " + + ; Delete loaded data + + delete(varload) + + ; We now produce the plot + + plot = new (4, graphic) ; define plot - need 4 panels + + res1 = True + res1@gsnDraw = False ; panel plot + res1@gsnFrame = False ; don't draw yet + res1@cnFillOn = True + res1@cnLinesOn = True + res1@gsnSpreadColors = True + res1@lbLabelAutoStride = True + res1@gsnCenterString = "" + res1@tiMainString = "" + res1@vpWidthF = 0.38 + res1@vpHeightF = 0.19 + res1@cnLevelSelectionMode = "ManualLevels" + res1@cnInfoLabelOn = False ; don't give each + res1@cnLineLabelsOn = False ; panel its own + res1@lbLabelBarOn = False ; label bar + + pres = True + pres@gsnMaximize = True + pres@gsnPanelLabelBar = True ; Communal label bar + pres@gsnPanelLeft = 0.1 + pres@gsnPanelRight = 0.9 + pres@pmLabelBarOrthogonalPosF = -0.03 + pres@gsnFrame = False + pres@lbLabelStride = 1 + + res1@sfXArray = lon ; uses lon as plot x-axis + res1@sfYArray = lat ; uses lat for y axis + res1@trYReverse = False ; reverses y-axis, false + res1@tiYAxisString = "" ; y-axis title + res1@tiXAxisString = "" ; x-axis title + + if (var_choice .eq. 1) then ;================ Temperature plot ==================== + + pltTitle="PREQX Test 16-1, T850 hPa" ; Plot title if required + pres@txString = pltTitle + + ; Change the output type and name + + ;wks = gsn_open_wks("X11","acme-test16-1-latlonT850") ; output using X11 + wks = gsn_open_wks("pdf","preqx-test16-1latlonT850") ; output using eps + gsn_define_colormap(wks,"gui_default") + + res1@cnMaxLevelValF = 300.0 ; max contour color label + res1@cnMinLevelValF = 230.0 ; min contour color label + res1@cnLevelSpacingF = 10.0 ; contour color spacing ; choose a colormap + + else if (var_choice .eq. 2) then ;================ Large-scale precipitation plot ==================== + + pltTitle="PREQX Test 16-1, Large-scale precipitation" ; Plot title if required + pres@txString = pltTitle + + var1@units = " " + var2@units = " " + var3@units = " " + var4@units = " " + + res1@cnLinesOn = False ; Turn lines off for clearer plot + + ; Change the output type and name + + ;wks = gsn_open_wks("X11","acme-test16-1latlonPRECL") ; output using X11 + wks = gsn_open_wks("pdf","preqx-test16-1latlonPRECL") ; output using eps + gsn_define_colormap(wks,"gui_default") + + ; Note that the maximum might be larger than these contour spacing values + + res1@cnMaxLevelValF = 50.0 ; max contour color label + res1@cnMinLevelValF = 0.0 ; min contour color label + res1@cnLevelSpacingF = 4.0 ; contour color spacing ; choose a colormap + + else if (var_choice .eq. 3) then ;================ Surface Pressure plot ==================== + + pltTitle="PREQX Test 16-1, PS" ; Plot title if required + pres@txString = pltTitle + + ; Convert to hPa + + var1=var1/100.0 + var2=var2/100.0 + var3=var3/100.0 + var4=var4/100.0 + + var1@units = "hPa " + var2@units = "hPa " + var3@units = "hPa " + var4@units = "hPa " + + ; Change the output type and name + + ;wks = gsn_open_wks("X11","acme-test16-1latlonPS") ; output using X11 + wks = gsn_open_wks("pdf","preqx-test16-1latlonPS") ; output using eps + gsn_define_colormap(wks,"gui_default") + + res1@cnMaxLevelValF = 1020.0 ; max contour color label + res1@cnMinLevelValF = 920.0 ; min contour color label + res1@cnLevelSpacingF = 10.0 ; contour color spacing ; choose a colormap + + end if ;============== end plot choice =============== + end if ;============== end plot choice =============== + end if ;============== end plot choice =============== + + res1@gsnCenterString = "t = 6" + plot(0) = gsn_csm_contour(wks,var1(:,:),res1) ; plot var1 + + res1@gsnCenterString = "t = 9" + plot(1) = gsn_csm_contour(wks,var2(:,:),res1) ; plot var2 + + res1@gsnCenterString = "t = 12" + plot(2) = gsn_csm_contour(wks,var3(:,:),res1) ; plot var3 + + res1@gsnCenterString = "t = 15" + plot(3) = gsn_csm_contour(wks,var4(:,:),res1) ; plot var4 + + gsn_panel(wks,plot,(/2,2/),pres) ; 2x2 plot + + ; Add latitude and longitude labels + + txres3 = True + txres3@txAngleF = 90. + txres3@txFontHeightF = 0.02 + gsn_text_ndc(wks,"Latitude",0.08,0.49,txres3) + + txres3@txAngleF = 0. + gsn_text_ndc(wks,"Longitude",0.5,0.22,txres3) + + frame(wks) + +end + + + + + + diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index 6d52272b0c3f..ad6e2a6ae1fb 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -118,7 +118,7 @@ function initmp(npes_in,npes_stride) result(par) #include - integer(kind=int_kind) :: ierr,tmp + integer(kind=int_kind) :: ierr,tmp_min,tmp_max integer(kind=int_kind) :: FrameNumber logical :: running ! state of MPI at beginning of initmp call character(len=MPI_MAX_PROCESSOR_NAME) :: my_name @@ -225,14 +225,16 @@ function initmp(npes_in,npes_stride) result(par) ! Verify that everybody agrees on this number otherwise do not do ! the multi-level partitioning ! ======================================================================= - call MPI_Allreduce(nmpi_per_node,tmp,1,MPIinteger_t,MPI_BAND,par%comm,ierr) - if(tmp .ne. nmpi_per_node) then + call MPI_Allreduce(nmpi_per_node,tmp_min,1,MPIinteger_t,MPI_MIN,par%comm,ierr) + call MPI_Allreduce(nmpi_per_node,tmp_max,1,MPIinteger_t,MPI_MAX,par%comm,ierr) + if (par%masterproc) write(iulog,*)'number of MPI processes per node: min,max=',& + tmp_min,tmp_max + if(tmp_min .ne. tmp_max) then if (par%masterproc) write(iulog,*)'initmp: disagrement accross nodes for nmpi_per_node' nmpi_per_node = 1 PartitionForNodes=.FALSE. else PartitionForNodes=.TRUE. - if (par%masterproc) write(iulog,*)'number of MPI processes per node: ',nmpi_per_node endif From 6be01c990a78a22c6baee50bbd173b2d758b0040 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 8 Oct 2018 23:00:24 -0500 Subject: [PATCH 27/41] tweak to Anvil jobscript --- .../dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh index dda6d0a352b1..894a9a135f22 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/jobscript-snl.sh @@ -15,6 +15,7 @@ export OMP_NUM_THREADS=1 export OMP_STACKSIZE=16M # Cori has 96GB per node. had to lower to 8M on 3K nodes +export MV2_ENABLE_AFFINITY=0 NCPU=40 if [ -n "$PBS_ENVIRONMENT" ]; then # NCPU=$PBS_NNODES From 178446318ca04c73625457685a331bbcc1818f44 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Wed, 10 Oct 2018 11:47:50 -0500 Subject: [PATCH 28/41] dcmip2016 initial condition bugfixes two bug fixes: TC (test2) was ignoring mosture when computing initial vtheta_dp dcmip2016 wrapper code was computing z_i (interface position) from z_m. this was changed to use the model's native versoin of z_i --- .../homme/src/preqx/share/element_ops.F90 | 53 +++++++++++- .../homme/src/test_src/dcmip16_wrapper.F90 | 82 +++++++++++-------- components/homme/src/theta-l/element_ops.F90 | 11 ++- components/homme/src/theta/element_ops.F90 | 67 +++++++++++++-- 4 files changed, 167 insertions(+), 46 deletions(-) diff --git a/components/homme/src/preqx/share/element_ops.F90 b/components/homme/src/preqx/share/element_ops.F90 index c07bb2eb367c..663a63329afa 100644 --- a/components/homme/src/preqx/share/element_ops.F90 +++ b/components/homme/src/preqx/share/element_ops.F90 @@ -127,6 +127,54 @@ subroutine get_phi(elem,phi,hvcoord,nt,ntQ) end subroutine get_phi + !_____________________________________________________________________ + subroutine get_phi_i(elem,phi_i,hvcoord,nt,ntQ) + implicit none + + type (element_t), intent(in) :: elem + real (kind=real_kind), intent(out) :: phi_i(np,np,nlevp) + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + ! local + real (kind=real_kind) :: pfull(np,np,nlev) + real (kind=real_kind) :: dp(np,np,nlev) + real (kind=real_kind) :: T_v(np,np,nlev) + real (kind=real_kind) :: Qt + integer :: k,i,j + + + do k=1,nlev + pfull(:,:,k) = hvcoord%hyam(k)*hvcoord%ps0 & + + hvcoord%hybm(k)*elem%state%ps_v(:,:,nt) + dp(:,:,k) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & + ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem%state%ps_v(:,:,nt) + + + if (.not. use_moisture ) then + T_v(:,:,k) = elem%state%T(:,:,k,nt) + else + do j=1,np + do i=1,np + Qt = elem%state%Qdp(i,j,k,1,ntQ)/dp(i,j,k) + T_v(i,j,k) = Virtual_Temperature(elem%state%T(i,j,k,nt),Qt) + end do + end do + endif + + enddo + + phi_i(:,:,nlevp) = elem%state%phis(:,:) + ! traditional Hydrostatic integral + do k=nlev,1,-1 + phi_i(:,:,k)=phi_i(:,:,k+1)+Rgas*T_v(:,:,k)*dp(:,:,k)/pfull(:,:,k) + enddo + + + end subroutine get_phi_i + + !_____________________________________________________________________ subroutine get_pottemp(elem,pottemp,hvcoord,nt,ntQ) implicit none @@ -252,12 +300,13 @@ subroutine set_elem_state(u,v,w,w_i,T,ps,phis,p,dp,zm,zi,g,elem,n0,n1,ntQ) end subroutine set_elem_state !_____________________________________________________________________ - subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) + subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,zi,g,elem,hvcoord,nt,ntQ) ! get state variables at layer midpoints ! used by tests to compute idealized physics forcing terms real(real_kind), dimension(np,np,nlev), intent(inout) :: u,v,w,T,pnh,dp,zm,rho + real(real_kind), dimension(np,np,nlevp), intent(inout) :: zi real(real_kind), dimension(np,np), intent(inout) :: ps real(real_kind), intent(in) :: g integer, intent(in) :: nt,ntQ @@ -268,6 +317,8 @@ subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) integer :: k call get_phi(elem,phi,hvcoord,nt,ntQ) + call get_phi_i(elem,zi,hvcoord,nt,ntQ) + zi=zi/g ! set prognostic state variables at level midpoints u = elem%state%v (:,:,1,:,nt) diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index d3a54f0852f2..bb7f8db472d8 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -156,57 +156,59 @@ subroutine dcmip2016_test2(elem,hybrid,hvcoord,nets,nete) integer :: i,j,k,ie , ierr ! loop indices real(rl):: lon,lat,ntop ! pointwise coordiantes - real(rl):: p,z,u,v,w,T,thetav,phis,ps,rho,q(3),dp ! pointwise field values + real(rl), dimension(np,np,nlev):: p,z,u,v,w,T,rho,dp ! pointwise field values + real(rl), dimension(np,np):: ps,phis + real(rl), dimension(np,np,nlevp):: w_i,z_i,p_i + real(rl) :: thetav,q(3) if (hybrid%masterthread) write(iulog,*) 'initializing dcmip2016 test 2: tropical cyclone' !use vertical levels specificed in cam30 file ! set initial conditions do ie = nets,nete - do k=1,nlev; do j=1,np; do i=1,np + do k=1,nlevp; do j=1,np; do i=1,np - ! get surface pressure + ! get surface pressure ps(i,j) at lat, lon, z=0, ignore all other output lon = elem(ie)%spherep(i,j)%lon lat = elem(ie)%spherep(i,j)%lat - z=0; call tropical_cyclone_test(lon,lat,p,z,1,u,v,T,thetav,phis,ps,rho,q(1)) + z_i(i,j,k)=0; call tropical_cyclone_test(lon,lat,p(i,j,1),z_i(i,j,k),1,u(i,j,1),& + v(i,j,1),T(i,j,1),thetav,phis(i,j),ps(i,j),rho(i,j,1),q(1)) - ! get pressure at level midpoints - p = p0*hvcoord%hyam(k) + ps*hvcoord%hybm(k) + ! get hydrostatic pressure at level k + p_i(i,j,k) = p0*hvcoord%hyai(k) + ps(i,j)*hvcoord%hybi(k) - ! get initial conditions at pressure level p - call tropical_cyclone_test(lon,lat,p,z,0,u,v,T,thetav,phis,ps,rho,q(1)) + ! call this only to compute z_i, will ignore all other output + call tropical_cyclone_test(lon,lat,p_i(i,j,k),z_i(i,j,k),0,u(i,j,1),v(i,j,1),& + T(i,j,1),thetav,phis(i,j),ps(i,j),rho(i,j,1),q(1)) - dp = pressure_thickness(ps,k,hvcoord) - w = 0 - q(2)=0 - q(3)=0 + w_i(i,j,k) = 0 - call set_state(u,v,w,T,ps,phis,p,dp,z,g,i,j,k,elem(ie),1,nt) - call set_tracers(q,3,dp,i,j,k,lat,lon,elem(ie)) enddo; enddo; enddo; - - - do k=1,nlevp; do j=1,np; do i=1,np + do k=1,nlev; do j=1,np; do i=1,np ! get surface pressure lon = elem(ie)%spherep(i,j)%lon lat = elem(ie)%spherep(i,j)%lat - z=0; call tropical_cyclone_test(lon,lat,p,z,1,u,v,T,thetav,phis,ps,rho,q(1)) + z=0; call tropical_cyclone_test(lon,lat,p(i,j,k),z(i,j,k),1,u(i,j,k),v(i,j,k),& + T(i,j,k),thetav,phis(i,j),ps(i,j),rho(i,j,k),q(1)) ! get pressure at level midpoints - p = p0*hvcoord%hyai(k) + ps*hvcoord%hybi(k) + p(i,j,k) = p0*hvcoord%hyam(k) + ps(i,j)*hvcoord%hybm(k) ! get initial conditions at pressure level p - call tropical_cyclone_test(lon,lat,p,z,0,u,v,T,thetav,phis,ps,rho,q(1)) + call tropical_cyclone_test(lon,lat,p(i,j,k),z(i,j,k),0,u(i,j,k),v(i,j,k),& + T(i,j,k),thetav,phis(i,j),ps(i,j),rho(i,j,k),q(1)) - dp = pressure_thickness(ps,k,hvcoord) - w = 0 + dp(i,j,k) = pressure_thickness(ps(i,j),k,hvcoord) + w(i,j,k) = 0 + q(2)=0 + q(3)=0 - call set_state_i(u,v,w,T,ps,phis,p,dp,z,g,i,j,k,elem(ie),1,nt) + call set_tracers(q(:),3,dp(i,j,k),i,j,k,lat,lon,elem(ie)) enddo; enddo; enddo; - + call set_elem_state(u,v,w,w_i,T,ps,phis,p,dp,z,z_i,g,elem(ie),1,nt,ntQ=1) call tests_finalize(elem(ie),hvcoord,1,nt) enddo @@ -415,7 +417,7 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) precl(:,:,ie) = -1.0d0 ! get current element state - call get_state(u,v,w,T,p,dp,ps,rho,z,g,elem(ie),hvcoord,nt,ntQ) + call get_state(u,v,w,T,p,dp,ps,rho,z,zi,g,elem(ie),hvcoord,nt,ntQ) ! compute form of exner pressure expected by Kessler physics exner_kess = (p/p0)**(Rgas/Cp) @@ -438,10 +440,10 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) u0=u; v0=v; T0=T; qv0=qv; qc0=qc; qr0=qr ! get zi from z - zi(:,:,nlevp) = 0 ! phis=0 - zi(:,:,2:nlev)= (z(:,:,2:nlev)+z(:,:,1:nlev-1))/2.0d0 - dz_top = dp(:,:,1)/(rho(:,:,1)*g) - zi(:,:,1) = zi(:,:,2)+dz_top +! zi(:,:,nlevp) = 0 ! phis=0 +! zi(:,:,2:nlev)= (z(:,:,2:nlev)+z(:,:,1:nlev-1))/2.0d0 +! dz_top = dp(:,:,1)/(rho(:,:,1)*g) +! zi(:,:,1) = zi(:,:,2)+dz_top ! apply forcing to columns do j=1,np; do i=1,np @@ -542,7 +544,7 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t precl(:,:,ie) = -1.0d0 ! get current element state - call get_state(u,v,w,T,p,dp,ps,rho,z,g,elem(ie),hvcoord,nt,ntQ) + call get_state(u,v,w,T,p,dp,ps,rho,z,zi,g,elem(ie),hvcoord,nt,ntQ) ! compute form of exner pressure expected by Kessler physics exner_kess = (p/p0)**(Rgas/Cp) @@ -553,6 +555,8 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t qc = elem(ie)%state%Qdp(:,:,:,2,ntQ)/dp qr = elem(ie)%state%Qdp(:,:,:,3,ntQ)/dp + !rho = (1-qv)*rho ! convert to dry density + ! ensure positivity where(qv<0); qv=0; endwhere where(qc<0); qc=0; endwhere @@ -562,10 +566,10 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t u0=u; v0=v; T0=T; qv0=qv; qc0=qc; qr0=qr ! get zi from z - zi(:,:,nlevp) = 0 ! phis=0 - zi(:,:,2:nlev)= (z(:,:,2:nlev)+z(:,:,1:nlev-1))/2.0d0 - dz_top = dp(:,:,1)/(rho(:,:,1)*g) - zi(:,:,1) = zi(:,:,2)+dz_top + !zi(:,:,nlevp) = 0 ! phis=0 + !zi(:,:,2:nlev)= (z(:,:,2:nlev)+z(:,:,1:nlev-1))/2.0d0 + !dz_top = dp(:,:,1)/(rho(:,:,1)*g) + !zi(:,:,1) = zi(:,:,2)+dz_top ! apply forcing to columns do j=1,np; do i=1,np @@ -598,6 +602,13 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t enddo; enddo; ! get temperature from new pressure + ! rho(dry mass) didn't change + ! rho R* Theta = p/exner + !rho = rho/(1-qv) ! convert to full desnsity + !p_over_exner = (rho*Rgas*theta_kess) + !p = (p_over_exner * p0**(-kappa))**(1/1-kappa) + !exner = p/p_over_exner + T = theta_kess*exner_kess ! set dynamics forcing @@ -637,6 +648,7 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) real(rl), dimension(np,np,nlev) :: u,v,w,T,theta_kess,rho_kess,exner_kess,p,dp,rho,z,qv,qc,qr real(rl), dimension(np,np,nlev) :: T0,qv0,qc0,qr0 real(rl), dimension(np,np,nlev) :: theta_inv,qv_inv,qc_inv,qr_inv,rho_inv,exner_inv,z_inv ! inverted columns + real(rl), dimension(np,np,nlevp):: zi real(rl), dimension(np,np) :: ps real(rl) :: max_w, max_precl, min_ps @@ -649,7 +661,7 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) precl(:,:,ie) = 0.0 ! get current element state - call get_state(u,v,w,T,p,dp,ps,rho,z,g,elem(ie),hvcoord,nt,ntQ) + call get_state(u,v,w,T,p,dp,ps,rho,z,zi,g,elem(ie),hvcoord,nt,ntQ) ! get mixing ratios qv = elem(ie)%state%Qdp(:,:,:,1,ntQ)/dp diff --git a/components/homme/src/theta-l/element_ops.F90 b/components/homme/src/theta-l/element_ops.F90 index 427f7d2eae4e..a10130293041 100644 --- a/components/homme/src/theta-l/element_ops.F90 +++ b/components/homme/src/theta-l/element_ops.F90 @@ -15,7 +15,7 @@ ! copy state variables from one timelevel to another timelevel ! set_thermostate() ! initial condition interface used by DCMIP 2008 tests, old HOMME tests -! set_state() +! set_state(), set_state_i() ! initial condition interface used by DCMIP 2012 tests ! set_elem_state() ! initial condition interface used by DCMIP 2016 tests @@ -28,6 +28,8 @@ ! tests_finalize() ! initialize geopotential to be in hydrostatic balance ! used by DCMIP2012, 2016 tests +! set_forcing_rayleigh_friction() +! apply rayleigh friction to prognostic variables, used by some DCMIP2016 tests ! ! Accessory routines used by the above: ! get_pottemp() @@ -55,7 +57,6 @@ module element_ops use physical_constants, only : p0, Cp, Rgas, Rwater_vapor, Cpwater_vapor, kappa, g, dd_pi use control_mod, only: use_moisture, theta_hydrostatic_mode use eos, only: get_pnh_and_exner, get_phinh - use prim_si_mod, only: preq_hydrostatic_v2 implicit none private @@ -427,7 +428,7 @@ subroutine set_elem_state(u,v,w,w_i,T,ps,phis,p,dp,zm,zi,g,elem,n0,n1,ntQ) end subroutine set_elem_state !_____________________________________________________________________ - subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) + subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,zi,g,elem,hvcoord,nt,ntQ) ! get state variables at layer midpoints ! used by idealized tests to compute idealized physics forcing terms ! currently all forcing is done on u,v and T/theta - no forcing @@ -435,6 +436,7 @@ subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) ! if we add a test case that computes forcing for interface variables real(real_kind), dimension(np,np,nlev), intent(inout) :: u,v,w,T,pnh,dp,zm,rho + real(real_kind), dimension(np,np,nlevp), intent(inout) :: zi real(real_kind), dimension(np,np), intent(inout) :: ps real(real_kind), intent(in) :: g integer, intent(in) :: nt,ntQ @@ -481,6 +483,9 @@ subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) do k=1,nlev zm(:,:,k) = (phi_i(:,:,k)+phi_i(:,:,k+1))/(2*g) end do + do k=1,nlevp + zi(:,:,k) = phi_i(:,:,k)/g + end do end subroutine get_state diff --git a/components/homme/src/theta/element_ops.F90 b/components/homme/src/theta/element_ops.F90 index f4683e8e10a3..c3daf4d9b0f4 100644 --- a/components/homme/src/theta/element_ops.F90 +++ b/components/homme/src/theta/element_ops.F90 @@ -305,6 +305,60 @@ subroutine get_phi(elem,phi,hvcoord,nt,ntQ) + subroutine get_phi_i(elem,phi_i,hvcoord,nt,ntQ) + implicit none + + type (element_t), intent(in) :: elem + type (hvcoord_t), intent(in) :: hvcoord + real (kind=real_kind), intent(out) :: phi_i(np,np,nlevp) + integer, intent(in) :: nt + integer, intent(in) :: ntQ + + real (kind=real_kind), dimension(np,np,nlev) :: dp,dpnh,kappa_star + real (kind=real_kind) :: phi(np,np,nlev) + real (kind=real_kind) :: pnh(np,np,nlev) + real (kind=real_kind) :: exner(np,np,nlev) + real (kind=real_kind) :: temp(np,np,nlev) + integer :: k + + ! compute hydrostatic version first: + + do k=1,nlev + dp(:,:,k) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & + (hvcoord%hybi(k+1)-hvcoord%hybi(k))*elem%state%ps_v(:,:,nt) + enddo + + call get_kappa_star(kappa_star,elem%state%Q(:,:,:,1)) + + call get_pnh_and_exner(hvcoord,elem%state%theta_dp_cp(:,:,:,nt),& + dp,elem%state%phinh(:,:,:,nt),elem%state%phis(:,:),kappa_star,& + pnh,dpnh,exner) + + do k=1,nlev + !temp(:,:,k) = theta_dp_cp(:,:,k)*(exner_i(:,:,k+1)-exner_i(:,:,k))/dp3d(:,:,k) + temp(:,:,k) = kappa_star(:,:,k)*elem%state%theta_dp_cp(:,:,k,nt)*exner(:,:,k)/pnh(:,:,k) + enddo + + phi_i(:,:,nlevp) = elem%state%phis(:,:) + ! traditional Hydrostatic integral + do k=nlev,1,-1 + phi_i(:,:,k)=phi_i(:,:,k+1)+temp(:,:,k) + enddo + + if (.not. theta_hydrostatic_mode) then + phi = elem%state%phinh(:,:,:,nt) + ! average phinh + phi_i(:,:,2:nlev)= (phi(:,:,2:nlev)+phi(:,:,1:nlev-1))/2 + + ! boundaries: + phi_i(:,:,nlevp) = elem%state%phis(:,:) + phi_i(:,:,1) = phi(:,:,1)+temp(:,:,1)/2 + endif + + end subroutine + + + @@ -454,12 +508,13 @@ subroutine set_elem_state(u,v,w,w_i,T,ps,phis,p,dp,zm,zi,g,elem,n0,n1,ntQ) end subroutine set_elem_state !_____________________________________________________________________ - subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) + subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,zi,g,elem,hvcoord,nt,ntQ) ! get state variables at layer midpoints ! used by idealized tests to compute idealized physics forcing terms real(real_kind), dimension(np,np,nlev), intent(inout) :: u,v,w,T,pnh,dp,zm,rho + real(real_kind), dimension(np,np,nlevp), intent(inout) :: zi real(real_kind), dimension(np,np), intent(inout) :: ps real(real_kind), intent(in) :: g integer, intent(in) :: nt,ntQ @@ -492,13 +547,11 @@ subroutine get_state(u,v,w,T,pnh,dp,ps,rho,zm,g,elem,hvcoord,nt,ntQ) if(theta_hydrostatic_mode) then w = -(elem%derived%omega_p*pnh)/(rho*g) - do k=1,nlev - !temp(:,:,k) = theta_dp_cp(:,:,k)*(exner_i(:,:,k+1)-exner_i(:,:,k))/dp3d(:,:,k) - temp(:,:,k) = kappa_star(:,:,k)*elem%state%theta_dp_cp(:,:,k,nt)*exner(:,:,k)/pnh(:,:,k) - enddo - call preq_hydrostatic_v2(phi,elem%state%phis,temp) endif - zm = phi/g + call get_phi(elem,zm,hvcoord,nt,ntQ); zm=zm/g + call get_phi_i(elem,zi,hvcoord,nt,ntQ); zi=zi/g + + end subroutine get_state From 77e93fda8583abbdb8c5c15941509facd9108fb2 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 12 Oct 2018 10:39:29 -0500 Subject: [PATCH 29/41] add PS contour plots to TC test case --- .../theta-l/jobscript-snl.sh | 4 +- .../theta-l/plot-horiz-ps.ncl | 99 +++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-ps.ncl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh index 677844cb80e2..971fde49dc53 100755 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh @@ -42,14 +42,14 @@ date ncl plot-tropical-cyclone-init.ncl # u,t,th,q,pnh,geo,ps, time=0 ncl plot-horiz-crossx.ncl # contour plot, time=10d, U,V,T,ps,precl,Q,geo ncl plot-intensity-trace.ncl - +ncl plot-horis-ps.ncl # save output \mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc \mv -f init.pdf ${prefix}_init.pdf \mv -f x-sections.pdf ${prefix}_x-sections.pdf \mv -f wind.pdf ${prefix}_wind.pdf -\mv -f ps.pdf ${prefix}_ps.pdf +\mv -f psmap.pdf ${prefix}_psmap.pdf } diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-ps.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-ps.ncl new file mode 100644 index 000000000000..08ee8097a12e --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/plot-horiz-ps.ncl @@ -0,0 +1,99 @@ +;======================================================================================= +; This NCL code calculates radially-averaged tangential and radial wind components +; as well as T anomaly for DCMIP test case #2 (cyclone) +; this code requires the accompanying function set "radialAvg.ncl" +; +; Usage: User should modify "user options" for their particular data set. Currently, +; U, V, T, PS are required as variables. +; If variables are on constant Z surfaces, life is easy. +; +; Grepping for "MODELSPEC" will point to possible areas of the code in need of modification +; for model specific output +; +; Written by Colin Zarzycki (zarzycki@ucar.edu) +; Version 0.1 (6/5/2016) - DCMIP-2016 release +;======================================================================================= + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;======================================================================================= +; User options +;======================================================================================= + +filename="./movies/dcmip2016_test21.nc" +f = addfile(filename,"r") + + + +nlat = dimsizes(f->lat) +nlon = dimsizes(f->lon) +nlev = dimsizes(f->lev) + +t10 = closest_val( 10.0, f->time) +t9 = closest_val( 9.0, f->time) +t8 = closest_val( 8.0, f->time) +t0 = closest_val( 0.0, f->time) + + + + +print("Plotting...") + +wks = gsn_open_wks ("pdf","psmap") ; send graphics to PNG file +contour = new(4,"graphic") + +gsn_define_colormap(wks,"BlAqGrYeOrReVi200") + +res = True +res@gsnDraw = False +res@gsnFrame = False +res@gsnSpreadColors = True ; Span full color map +res@cnFillOn = True ; Turn on contour fill +res@cnLinesOn = False +res@cnLineLabelsOn = False +res@cnInfoLabelOn = False +res@gsnAddCyclic = True + +res@mpOutlineOn = False +res@mpMaxLatF = 90. +res@mpMinLatF = -45. +res@mpCenterLonF = 160. +res@mpMinLonF = 160.-135. +res@mpMaxLonF = 160+135. + + +res@cnLevelSelectionMode = "ManualLevels" +dx=20.0 +clev=20 +res@cnLevelSpacingF = dx +res@cnMinLevelValF = 101500-(clev+.5)*dx +res@cnMaxLevelValF = 101500+(clev+.5)*dx + +res@tiMainString="PS t="+f->time(t0) +ps = f->ps(t0,:,:) +print("ps min/max = "+min(ps)+","+max(ps)) +contour(0) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +res@tiMainString="PS t="+f->time(t8) +ps = f->ps(t8,:,:) +contour(1) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +res@tiMainString="PS t="+f->time(t9) +ps = f->ps(t9,:,:) +contour(2) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +res@tiMainString="PS t="+f->time(t10) +ps = f->ps(t10,:,:) +contour(3) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +resP = True ; modify the panel plot +resP@gsnMaximize = True + +gsn_panel(wks,contour,(/2,2/),resP) ; now draw as one plot + +end From b75f5f7909894245ee61bfa2eca6056b70229ae4 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 12 Oct 2018 10:57:04 -0500 Subject: [PATCH 30/41] fix typo --- .../theta-l/jobscript-snl.sh | 8 ++++---- .../theta-l/namelist-r50-h.nl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh index 971fde49dc53..225845194d27 100755 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/jobscript-snl.sh @@ -42,7 +42,7 @@ date ncl plot-tropical-cyclone-init.ncl # u,t,th,q,pnh,geo,ps, time=0 ncl plot-horiz-crossx.ncl # contour plot, time=10d, U,V,T,ps,precl,Q,geo ncl plot-intensity-trace.ncl -ncl plot-horis-ps.ncl +ncl plot-horiz-ps.ncl # save output \mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc @@ -54,6 +54,6 @@ ncl plot-horis-ps.ncl prefix=r400 ; run $(($NCPU>384?384:NCPU)) -#prefix=r100 ; run $NCPU -#prefix=r50 ; run $NCPU -#prefix=r50-h ; run $NCPU +prefix=r100 ; run $NCPU +prefix=r50 ; run $NCPU +prefix=r50-h ; run $NCPU diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl index f4e7ba22028a..46055c501374 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 6 + rsplit = 3 qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 From 9a356c9664f5c80756dc937ea39cba4aed253f5a Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Fri, 12 Oct 2018 20:30:05 -0500 Subject: [PATCH 31/41] add ps contour plots to theta model, dcmip2016 test2 --- .../preqx/jobscript-snl.sh | 2 + .../preqx/plot-horiz-ps.ncl | 99 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-ps.ncl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh index d7f92deb279c..dd46d76d7b75 100755 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh @@ -42,6 +42,7 @@ date ncl plot-tropical-cyclone-init.ncl # u,t,th,q,pnh,geo,ps, time=0 ncl plot-horiz-crossx.ncl # contour plot, time=10d, U,V,T,ps,precl,Q,geo ncl plot-intensity-trace.ncl +ncl plot-horiz-ps.ncl # save output \mv -f movies/dcmip2016_test21.nc movies/${prefix}_dcmip2016_test21.nc @@ -50,6 +51,7 @@ ncl plot-intensity-trace.ncl \mv -f x-sections.pdf ${prefix}_x-sections.pdf \mv -f wind.pdf ${prefix}_wind.pdf \mv -f ps.pdf ${prefix}_ps.pdf +\mv -f psmap.pdf ${prefix}_psmap.pdf } diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-ps.ncl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-ps.ncl new file mode 100644 index 000000000000..08ee8097a12e --- /dev/null +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/plot-horiz-ps.ncl @@ -0,0 +1,99 @@ +;======================================================================================= +; This NCL code calculates radially-averaged tangential and radial wind components +; as well as T anomaly for DCMIP test case #2 (cyclone) +; this code requires the accompanying function set "radialAvg.ncl" +; +; Usage: User should modify "user options" for their particular data set. Currently, +; U, V, T, PS are required as variables. +; If variables are on constant Z surfaces, life is easy. +; +; Grepping for "MODELSPEC" will point to possible areas of the code in need of modification +; for model specific output +; +; Written by Colin Zarzycki (zarzycki@ucar.edu) +; Version 0.1 (6/5/2016) - DCMIP-2016 release +;======================================================================================= + +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_code.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/gsn_csm.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/shea_util.ncl" + +begin + +;======================================================================================= +; User options +;======================================================================================= + +filename="./movies/dcmip2016_test21.nc" +f = addfile(filename,"r") + + + +nlat = dimsizes(f->lat) +nlon = dimsizes(f->lon) +nlev = dimsizes(f->lev) + +t10 = closest_val( 10.0, f->time) +t9 = closest_val( 9.0, f->time) +t8 = closest_val( 8.0, f->time) +t0 = closest_val( 0.0, f->time) + + + + +print("Plotting...") + +wks = gsn_open_wks ("pdf","psmap") ; send graphics to PNG file +contour = new(4,"graphic") + +gsn_define_colormap(wks,"BlAqGrYeOrReVi200") + +res = True +res@gsnDraw = False +res@gsnFrame = False +res@gsnSpreadColors = True ; Span full color map +res@cnFillOn = True ; Turn on contour fill +res@cnLinesOn = False +res@cnLineLabelsOn = False +res@cnInfoLabelOn = False +res@gsnAddCyclic = True + +res@mpOutlineOn = False +res@mpMaxLatF = 90. +res@mpMinLatF = -45. +res@mpCenterLonF = 160. +res@mpMinLonF = 160.-135. +res@mpMaxLonF = 160+135. + + +res@cnLevelSelectionMode = "ManualLevels" +dx=20.0 +clev=20 +res@cnLevelSpacingF = dx +res@cnMinLevelValF = 101500-(clev+.5)*dx +res@cnMaxLevelValF = 101500+(clev+.5)*dx + +res@tiMainString="PS t="+f->time(t0) +ps = f->ps(t0,:,:) +print("ps min/max = "+min(ps)+","+max(ps)) +contour(0) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +res@tiMainString="PS t="+f->time(t8) +ps = f->ps(t8,:,:) +contour(1) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +res@tiMainString="PS t="+f->time(t9) +ps = f->ps(t9,:,:) +contour(2) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +res@tiMainString="PS t="+f->time(t10) +ps = f->ps(t10,:,:) +contour(3) = gsn_csm_contour_map(wks,ps(:,:),res) ; create the plot + +resP = True ; modify the panel plot +resP@gsnMaximize = True + +gsn_panel(wks,contour,(/2,2/),resP) ; now draw as one plot + +end From 0891021480d2cb44bcdf9b362f09d29f68426519 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 14 Oct 2018 18:35:47 -0500 Subject: [PATCH 32/41] update DCMIP2016 forcing routines to use dry mixing ratios DCMIP2016 forcing functions are expecting dry mixing rations and we were sending wet mixing ratios --- .../homme/src/test_src/dcmip16_wrapper.F90 | 128 +++++++++++------- 1 file changed, 79 insertions(+), 49 deletions(-) diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index bb7f8db472d8..b1a20b02c488 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -10,7 +10,7 @@ module dcmip16_wrapper ! Implementation of the dcmip2012 dycore tests for the preqx dynamics target use dcmip12_wrapper, only: pressure_thickness, set_tracers, get_evenly_spaced_z, set_hybrid_coefficients -use control_mod, only: test_case, dcmip16_pbl_type, dcmip16_prec_type, use_moisture +use control_mod, only: test_case, dcmip16_pbl_type, dcmip16_prec_type, use_moisture, theta_hydrostatic_mode use baroclinic_wave, only: baroclinic_wave_test use supercell, only: supercell_init, supercell_test, supercell_z use tropical_cyclone, only: tropical_cyclone_test @@ -398,6 +398,7 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) integer :: i,j,k,ie ! loop indices real(rl), dimension(np,np,nlev) :: u,v,w,T,exner_kess,theta_kess,p,dp,rho,z,qv,qc,qr real(rl), dimension(np,np,nlev) :: u0,v0,T0,qv0,qc0,qr0,cl,cl2,ddt_cl,ddt_cl2 + real(rl), dimension(np,np,nlev) :: theta0,exner_new,rho_dry,rho_new,Rstar,p_pk real(rl), dimension(nlev) :: u_c,v_c,p_c,qv_c,qc_c,qr_c,rho_c,z_c, th_c real(rl) :: max_w, max_precl, min_ps real(rl) :: lat, lon, dz_top(np,np), zi(np,np,nlevp),zi_c(nlevp), ps(np,np) @@ -436,14 +437,17 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) where(qc<0); qc=0; endwhere where(qr<0); qr=0; endwhere + + rho_dry = (1-qv)*rho ! convert to dry density using wet mixing ratio + + ! convert to dry mixing ratios + qv = qv*rho/rho_dry + qc = qc*rho/rho_dry + qr = qr*rho/rho_dry + ! save un-forced prognostics u0=u; v0=v; T0=T; qv0=qv; qc0=qc; qr0=qr - - ! get zi from z -! zi(:,:,nlevp) = 0 ! phis=0 -! zi(:,:,2:nlev)= (z(:,:,2:nlev)+z(:,:,1:nlev-1))/2.0d0 -! dz_top = dp(:,:,1)/(rho(:,:,1)*g) -! zi(:,:,1) = zi(:,:,2)+dz_top + theta0 = theta_kess ! apply forcing to columns do j=1,np; do i=1,np @@ -455,7 +459,7 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) qc_c = qc (i,j,nlev:1:-1) qr_c = qr (i,j,nlev:1:-1) p_c = p (i,j,nlev:1:-1) - rho_c= rho(i,j,nlev:1:-1) + rho_c= rho_dry(i,j,nlev:1:-1) z_c = z (i,j,nlev:1:-1) zi_c = zi (i,j,nlevp:1:-1) th_c = theta_kess(i,j,nlev:1:-1) @@ -482,18 +486,24 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) enddo; enddo; - ! get temperature from new pressure - T = theta_kess*exner_kess + + rho_new = rho_dry*(1+qv) + Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) + !exner_new = p/(rho_new*Rstar*theta_kess) + p_pk = rho_new*Rstar*theta_kess + exner_new = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) ! set dynamics forcing elem(ie)%derived%FM(:,:,1,:) = (u - u0)/dt elem(ie)%derived%FM(:,:,2,:) = (v - v0)/dt - elem(ie)%derived%FT(:,:,:) = (T - T0)/dt + !elem(ie)%derived%FT(:,:,:) = (T - T0)/dt + elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt ! good! ! set tracer-mass forcing - qi=1; elem(ie)%derived%FQ(:,:,:,qi) = dp*(qv-qv0)/dt - qi=2; elem(ie)%derived%FQ(:,:,:,qi) = dp*(qc-qc0)/dt - qi=3; elem(ie)%derived%FQ(:,:,:,qi) = dp*(qr-qr0)/dt + ! set tracer-mass forcing. conserve tracer mass + elem(ie)%derived%FQ(:,:,:,1) = (rho_dry/rho)*dp*(qv-qv0)/dt + elem(ie)%derived%FQ(:,:,:,2) = (rho_dry/rho)*dp*(qc-qc0)/dt + elem(ie)%derived%FQ(:,:,:,3) = (rho_dry/rho)*dp*(qr-qr0)/dt qi=4; elem(ie)%derived%FQ(:,:,:,qi) = dp*ddt_cl qi=5; elem(ie)%derived%FQ(:,:,:,qi) = dp*ddt_cl2 @@ -524,6 +534,7 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t integer :: i,j,k,ie ! loop indices real(rl), dimension(np,np,nlev) :: u,v,w,T,exner_kess,theta_kess,p,dp,rho,z,qv,qc,qr real(rl), dimension(np,np,nlev) :: u0,v0,T0,qv0,qc0,qr0 + real(rl), dimension(np,np,nlev) :: theta0,exner_new,rho_dry,rho_new,Rstar,p_pk real(rl), dimension(nlev) :: u_c,v_c,p_c,qv_c,qc_c,qr_c,rho_c,z_c, th_c real(rl) :: max_w, max_precl, min_ps real(rl) :: lat, dz_top(np,np), zi(np,np,nlevp),zi_c(nlevp), ps(np,np) @@ -550,26 +561,28 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t exner_kess = (p/p0)**(Rgas/Cp) theta_kess = T/exner_kess - ! get mixing ratios + ! get wet mixing ratios qv = elem(ie)%state%Qdp(:,:,:,1,ntQ)/dp qc = elem(ie)%state%Qdp(:,:,:,2,ntQ)/dp qr = elem(ie)%state%Qdp(:,:,:,3,ntQ)/dp - !rho = (1-qv)*rho ! convert to dry density - ! ensure positivity where(qv<0); qv=0; endwhere where(qc<0); qc=0; endwhere where(qr<0); qr=0; endwhere - ! save un-forced prognostics + rho_dry = (1-qv)*rho ! convert to dry density using wet mixing ratio + + ! convert to dry mixing ratios + qv = qv*rho/rho_dry + qc = qc*rho/rho_dry + qr = qr*rho/rho_dry + + + ! save un-forced prognostics (DRY) u0=u; v0=v; T0=T; qv0=qv; qc0=qc; qr0=qr + theta0=theta_kess - ! get zi from z - !zi(:,:,nlevp) = 0 ! phis=0 - !zi(:,:,2:nlev)= (z(:,:,2:nlev)+z(:,:,1:nlev-1))/2.0d0 - !dz_top = dp(:,:,1)/(rho(:,:,1)*g) - !zi(:,:,1) = zi(:,:,2)+dz_top ! apply forcing to columns do j=1,np; do i=1,np @@ -581,7 +594,7 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t qc_c = qc (i,j,nlev:1:-1) qr_c = qr (i,j,nlev:1:-1) p_c = p (i,j,nlev:1:-1) - rho_c= rho(i,j,nlev:1:-1) + rho_c= rho_dry(i,j,nlev:1:-1) z_c = z (i,j,nlev:1:-1) zi_c = zi (i,j,nlevp:1:-1) th_c = theta_kess(i,j,nlev:1:-1) @@ -600,26 +613,25 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t theta_kess(i,j,:) = th_c(nlev:1:-1) enddo; enddo; + rho_new = rho_dry*(1+qv) + Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) + !exner_new = p/(rho_new*Rstar*theta_kess) + p_pk = rho_new*Rstar*theta_kess + exner_new = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) - ! get temperature from new pressure - ! rho(dry mass) didn't change - ! rho R* Theta = p/exner - !rho = rho/(1-qv) ! convert to full desnsity - !p_over_exner = (rho*Rgas*theta_kess) - !p = (p_over_exner * p0**(-kappa))**(1/1-kappa) - !exner = p/p_over_exner - - T = theta_kess*exner_kess ! set dynamics forcing elem(ie)%derived%FM(:,:,1,:) = (u - u0)/dt elem(ie)%derived%FM(:,:,2,:) = (v - v0)/dt - elem(ie)%derived%FT(:,:,:) = (T - T0)/dt + !elem(ie)%derived%FT(:,:,:) = exner_kess*(theta_kess - theta0)/dt + elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt ! a little better than above + + + ! set tracer-mass forcing. conserve tracer mass + elem(ie)%derived%FQ(:,:,:,1) = (rho_dry/rho)*dp*(qv-qv0)/dt + elem(ie)%derived%FQ(:,:,:,2) = (rho_dry/rho)*dp*(qc-qc0)/dt + elem(ie)%derived%FQ(:,:,:,3) = (rho_dry/rho)*dp*(qr-qr0)/dt - ! set tracer-mass forcing - elem(ie)%derived%FQ(:,:,:,1) = dp*(qv-qv0)/dt - elem(ie)%derived%FQ(:,:,:,2) = dp*(qc-qc0)/dt - elem(ie)%derived%FQ(:,:,:,3) = dp*(qr-qr0)/dt ! perform measurements of max w, and max prect max_w = max( max_w , maxval(w ) ) @@ -645,8 +657,9 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) integer :: i,j,k,ie ! loop indices real(rl):: lat - real(rl), dimension(np,np,nlev) :: u,v,w,T,theta_kess,rho_kess,exner_kess,p,dp,rho,z,qv,qc,qr + real(rl), dimension(np,np,nlev) :: u,v,w,T,theta_kess,exner_kess,p,dp,rho,z,qv,qc,qr real(rl), dimension(np,np,nlev) :: T0,qv0,qc0,qr0 + real(rl), dimension(np,np,nlev) :: theta0,exner_new,rho_dry,rho_new,Rstar,p_pk real(rl), dimension(np,np,nlev) :: theta_inv,qv_inv,qc_inv,qr_inv,rho_inv,exner_inv,z_inv ! inverted columns real(rl), dimension(np,np,nlevp):: zi real(rl), dimension(np,np) :: ps @@ -673,20 +686,29 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) where(qc<0); qc=0; endwhere where(qr<0); qr=0; endwhere - ! save un-forced prognostics - T0=T; qv0=qv; qc0=qc; qr0=qr + rho_dry = (1-qv)*rho ! convert to dry density using wet mixing ratio + + ! convert to dry mixing ratios + qv = qv*rho/rho_dry + qc = qc*rho/rho_dry + qr = qr*rho/rho_dry + ! compute form of exner pressure expected by Kessler physics exner_kess = (p/p0)**(Rgas/Cp) - rho_kess = p/(Rgas*T) theta_kess = T/exner_kess + ! save un-forced prognostics + T0=T; qv0=qv; qc0=qc; qr0=qr + theta0 = theta_kess + + ! invert columns (increasing z) theta_inv= theta_kess(:,:,nlev:1:-1) qv_inv = qv (:,:,nlev:1:-1) qc_inv = qc (:,:,nlev:1:-1) qr_inv = qr (:,:,nlev:1:-1) - rho_inv = rho_kess (:,:,nlev:1:-1) + rho_inv = rho_dry (:,:,nlev:1:-1) exner_inv= exner_kess(:,:,nlev:1:-1) z_inv = z (:,:,nlev:1:-1) @@ -713,17 +735,25 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) qv = qv_inv (:,:,nlev:1:-1) qc = qc_inv (:,:,nlev:1:-1) qr = qr_inv (:,:,nlev:1:-1) - T = theta_kess*exner_kess + + + + rho_new = rho_dry*(1+qv) + Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) + !exner_new = p/(rho_new*Rstar*theta_kess) + p_pk = rho_new*Rstar*theta_kess + exner_new = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + ! set dynamics forcing elem(ie)%derived%FM(:,:,1,:) = 0 elem(ie)%derived%FM(:,:,2,:) = 0 - elem(ie)%derived%FT(:,:,:) = (T - T0)/dt + elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt - ! set tracer-mass forcing - elem(ie)%derived%FQ(:,:,:,1) = dp*(qv-qv0)/dt - elem(ie)%derived%FQ(:,:,:,2) = dp*(qc-qc0)/dt - elem(ie)%derived%FQ(:,:,:,3) = dp*(qr-qr0)/dt + ! set tracer-mass forcing. conserve tracer mass + elem(ie)%derived%FQ(:,:,:,1) = (rho_dry/rho)*dp*(qv-qv0)/dt + elem(ie)%derived%FQ(:,:,:,2) = (rho_dry/rho)*dp*(qc-qc0)/dt + elem(ie)%derived%FQ(:,:,:,3) = (rho_dry/rho)*dp*(qr-qr0)/dt ! perform measurements of max w, and max prect max_w = max( max_w , maxval(w ) ) From 647c422b0f54f643c41a24a23f9f713b78d32df7 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 15 Oct 2018 11:11:46 -0500 Subject: [PATCH 33/41] tweaks to theta-l hypervis reference states --- .../theta-l/namelist-r100-h.nl | 2 +- .../theta-l/namelist-r50.nl | 2 +- .../homme/src/test_src/dcmip16_wrapper.F90 | 4 +-- .../homme/src/theta-l/prim_advance_mod.F90 | 26 +++++++++++-------- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl index 579ff194ebbd..3700d48ab4ee 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 6 + rsplit = 3 qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl index e786dc5b7210..5cb4238ec6a8 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 1 ! r50 crashes with rsplit>1 qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index b1a20b02c488..e72794af8c83 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -497,7 +497,8 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) elem(ie)%derived%FM(:,:,1,:) = (u - u0)/dt elem(ie)%derived%FM(:,:,2,:) = (v - v0)/dt !elem(ie)%derived%FT(:,:,:) = (T - T0)/dt - elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt ! good! + elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt +! elem(ie)%derived%FT(:,:,:) = exner_kess*(theta_kess - theta0)/dt ! set tracer-mass forcing ! set tracer-mass forcing. conserve tracer mass @@ -626,7 +627,6 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t !elem(ie)%derived%FT(:,:,:) = exner_kess*(theta_kess - theta0)/dt elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt ! a little better than above - ! set tracer-mass forcing. conserve tracer mass elem(ie)%derived%FQ(:,:,:,1) = (rho_dry/rho)*dp*(qv-qv0)/dt elem(ie)%derived%FQ(:,:,:,2) = (rho_dry/rho)*dp*(qc-qc0)/dt diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index 3e1b71aba2a8..dd6ffbe38c08 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -588,14 +588,9 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w (hvcoord%hybi(k+1)-hvcoord%hybi(k))*ps_ref(:,:) enddo - do k=1,nlev - ! convert vtheta_dp -> theta - elem(ie)%state%vtheta_dp(:,:,k,nt)=& - elem(ie)%state%vtheta_dp(:,:,k,nt)/elem(ie)%state%dp3d(:,:,k,nt) - enddo #if 0 - ! reference state based only on ps: + ! phi_ref,theta_ref depend only on ps: call set_theta_ref(hvcoord,dp_ref(:,:,:,ie),theta_ref(:,:,:,ie)) ! compute vtheta_dp_ref, store in 'heating' as temp array: @@ -604,18 +599,27 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w heating(:,:,:),dp_ref(:,:,:,ie),phi_ref(:,:,:,ie)) #endif #if 1 - ! most realistic reference state based on hydrostatic pressure - call set_theta_ref(hvcoord,elem(ie)%state%dp3d(:,:,:,nt),theta_ref(:,:,:,ie)) + ! phi_ref depends only on ps, theta_ref depends on dp3d + call set_theta_ref(hvcoord,dp_ref(:,:,:,ie),theta_ref(:,:,:,ie)) + exner(:,:,:)=theta_ref(:,:,:,ie)*dp_ref(:,:,:,ie) ! use as temp array call get_phinh(hvcoord,elem(ie)%state%phis,& - elem(ie)%state%vtheta_dp(:,:,:,nt),elem(ie)%state%dp3d(:,:,:,nt),phi_ref(:,:,:,ie)) -#endif + exner(:,:,:),dp_ref(:,:,:,ie),phi_ref(:,:,:,ie)) + call set_theta_ref(hvcoord,elem(ie)%state%dp3d(:,:,:,nt),theta_ref(:,:,:,ie)) +#endif #if 0 - ! disable reference background states + ! no reference state, for testing theta_ref(:,:,:,ie)=0 phi_ref(:,:,:,ie)=0 dp_ref(:,:,:,ie)=0 #endif + + + ! convert vtheta_dp -> theta + do k=1,nlev + elem(ie)%state%vtheta_dp(:,:,k,nt)=& + elem(ie)%state%vtheta_dp(:,:,k,nt)/elem(ie)%state%dp3d(:,:,k,nt) + enddo enddo From 1caeee68c7104734cf6c64ec75f5d50e78146b92 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 15 Oct 2018 11:28:04 -0500 Subject: [PATCH 34/41] use different temp variable name, for clairity --- components/homme/src/theta-l/prim_advance_mod.F90 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index dd6ffbe38c08..232d702f55e1 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -592,11 +592,9 @@ subroutine advance_hypervis(elem,hvcoord,hybrid,deriv,nt,nets,nete,dt2,eta_ave_w #if 0 ! phi_ref,theta_ref depend only on ps: call set_theta_ref(hvcoord,dp_ref(:,:,:,ie),theta_ref(:,:,:,ie)) - - ! compute vtheta_dp_ref, store in 'heating' as temp array: - heating(:,:,:)=theta_ref(:,:,:,ie)*dp_ref(:,:,:,ie) + exner(:,:,:)=theta_ref(:,:,:,ie)*dp_ref(:,:,:,ie) ! use as temp array call get_phinh(hvcoord,elem(ie)%state%phis,& - heating(:,:,:),dp_ref(:,:,:,ie),phi_ref(:,:,:,ie)) + exner(:,:,:),dp_ref(:,:,:,ie),phi_ref(:,:,:,ie)) #endif #if 1 ! phi_ref depends only on ps, theta_ref depends on dp3d From b9cd7cefd2c15d02807575eac0da57c97c4e98bf Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Thu, 18 Oct 2018 12:52:48 -0500 Subject: [PATCH 35/41] fix run length for dcmip2016 test3 namelists --- .../theta-l/namelist-explicit-r100.nl | 2 +- .../theta-l/namelist-r100.nl | 2 +- components/homme/src/theta-l/eos.F90 | 14 ++++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl index 7d9bb9e8b6c0..b16ae9c08dea 100644 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-explicit-r100.nl @@ -8,7 +8,7 @@ test_case = "dcmip2016_test3" ! test identifier ne = 30 ! number of elements per cube face qsize = 3 ! num tracer fields - nmax = 3200 ! 7200s(120min)/tstep + nmax = 3600 ! 7200s(120min)/tstep statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl index 58d79de52232..0d11fb92dd41 100644 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r100.nl @@ -8,7 +8,7 @@ test_case = "dcmip2016_test3" ! test identifier ne = 30 ! number of elements per cube face qsize = 3 ! num tracer fields - nmax = 3200 ! 7200s(120min)/tstep + nmax = 3600 ! 7200s(120min)/tstep statefreq = 180 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run diff --git a/components/homme/src/theta-l/eos.F90 b/components/homme/src/theta-l/eos.F90 index 6a6df5ff82dc..ea069e0fc68d 100644 --- a/components/homme/src/theta-l/eos.F90 +++ b/components/homme/src/theta-l/eos.F90 @@ -64,7 +64,7 @@ subroutine get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_i,pnh,exner,& real (kind=real_kind) :: pnh_i(np,np,nlevp) real (kind=real_kind) :: dp3d_i(np,np,nlevp) real (kind=real_kind) :: pi_i(np,np,nlevp) - integer :: i,j,k + integer :: i,j,k,k2 ! hydrostatic pressure pi_i(:,:,1)=hvcoord%hyai(1)*hvcoord%ps0 @@ -104,7 +104,17 @@ subroutine get_pnh_and_exner(hvcoord,vtheta_dp,dp3d,phi_i,pnh,exner,& endif enddo enddo - call abortmp('error: rho<0') + do i=1,np + do j=1,np + if ( p_over_exner(i,j,k)<0 ) then + print *,'vertical column:' + do k2=1,nlev + write(*,'(i3,4f14.4)') k2,phi_i(i,j,k2),dp3d(i,j,k2),vtheta_dp(i,j,k2) + enddo + call abortmp('error: rho<0') + endif + enddo + enddo endif pnh(:,:,k) = p0 * (p_over_exner(:,:,k)/p0)**(1/(1-kappa)) From d8bb7bf1a82d734f173182741471cdb8e511768e Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sat, 20 Oct 2018 17:03:26 -0500 Subject: [PATCH 36/41] fixed bug in converting from T to theta with bug fix, update DCMIP forcing functions to return correct Temperature --- .../homme/src/test_src/dcmip16_wrapper.F90 | 33 +- .../dcmip2016-baroclinic.F90_original | 459 ------------------ components/homme/src/theta-l/eos.F90 | 52 ++ .../homme/src/theta-l/prim_advance_mod.F90 | 25 +- 4 files changed, 70 insertions(+), 499 deletions(-) delete mode 100644 components/homme/src/test_src/dcmip2016-baroclinic.F90_original diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index e72794af8c83..df5c8ba94e32 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -398,7 +398,7 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) integer :: i,j,k,ie ! loop indices real(rl), dimension(np,np,nlev) :: u,v,w,T,exner_kess,theta_kess,p,dp,rho,z,qv,qc,qr real(rl), dimension(np,np,nlev) :: u0,v0,T0,qv0,qc0,qr0,cl,cl2,ddt_cl,ddt_cl2 - real(rl), dimension(np,np,nlev) :: theta0,exner_new,rho_dry,rho_new,Rstar,p_pk + real(rl), dimension(np,np,nlev) :: rho_dry,rho_new,Rstar,p_pk real(rl), dimension(nlev) :: u_c,v_c,p_c,qv_c,qc_c,qr_c,rho_c,z_c, th_c real(rl) :: max_w, max_precl, min_ps real(rl) :: lat, lon, dz_top(np,np), zi(np,np,nlevp),zi_c(nlevp), ps(np,np) @@ -447,7 +447,6 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) ! save un-forced prognostics u0=u; v0=v; T0=T; qv0=qv; qc0=qc; qr0=qr - theta0 = theta_kess ! apply forcing to columns do j=1,np; do i=1,np @@ -489,18 +488,15 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) rho_new = rho_dry*(1+qv) Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) - !exner_new = p/(rho_new*Rstar*theta_kess) p_pk = rho_new*Rstar*theta_kess - exner_new = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + T = exner_kess*theta_kess ! set dynamics forcing elem(ie)%derived%FM(:,:,1,:) = (u - u0)/dt elem(ie)%derived%FM(:,:,2,:) = (v - v0)/dt - !elem(ie)%derived%FT(:,:,:) = (T - T0)/dt - elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt -! elem(ie)%derived%FT(:,:,:) = exner_kess*(theta_kess - theta0)/dt + elem(ie)%derived%FT(:,:,:) = (T - T0)/dt - ! set tracer-mass forcing ! set tracer-mass forcing. conserve tracer mass elem(ie)%derived%FQ(:,:,:,1) = (rho_dry/rho)*dp*(qv-qv0)/dt elem(ie)%derived%FQ(:,:,:,2) = (rho_dry/rho)*dp*(qc-qc0)/dt @@ -535,7 +531,7 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t integer :: i,j,k,ie ! loop indices real(rl), dimension(np,np,nlev) :: u,v,w,T,exner_kess,theta_kess,p,dp,rho,z,qv,qc,qr real(rl), dimension(np,np,nlev) :: u0,v0,T0,qv0,qc0,qr0 - real(rl), dimension(np,np,nlev) :: theta0,exner_new,rho_dry,rho_new,Rstar,p_pk + real(rl), dimension(np,np,nlev) :: rho_dry,rho_new,Rstar,p_pk real(rl), dimension(nlev) :: u_c,v_c,p_c,qv_c,qc_c,qr_c,rho_c,z_c, th_c real(rl) :: max_w, max_precl, min_ps real(rl) :: lat, dz_top(np,np), zi(np,np,nlevp),zi_c(nlevp), ps(np,np) @@ -582,7 +578,6 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t ! save un-forced prognostics (DRY) u0=u; v0=v; T0=T; qv0=qv; qc0=qc; qr0=qr - theta0=theta_kess ! apply forcing to columns @@ -616,16 +611,16 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t enddo; enddo; rho_new = rho_dry*(1+qv) Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) - !exner_new = p/(rho_new*Rstar*theta_kess) p_pk = rho_new*Rstar*theta_kess - exner_new = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + T = exner_kess*theta_kess ! set dynamics forcing elem(ie)%derived%FM(:,:,1,:) = (u - u0)/dt elem(ie)%derived%FM(:,:,2,:) = (v - v0)/dt - !elem(ie)%derived%FT(:,:,:) = exner_kess*(theta_kess - theta0)/dt - elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt ! a little better than above + elem(ie)%derived%FT(:,:,:) = (T-T0)/dt + ! set tracer-mass forcing. conserve tracer mass elem(ie)%derived%FQ(:,:,:,1) = (rho_dry/rho)*dp*(qv-qv0)/dt @@ -659,7 +654,7 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) real(rl):: lat real(rl), dimension(np,np,nlev) :: u,v,w,T,theta_kess,exner_kess,p,dp,rho,z,qv,qc,qr real(rl), dimension(np,np,nlev) :: T0,qv0,qc0,qr0 - real(rl), dimension(np,np,nlev) :: theta0,exner_new,rho_dry,rho_new,Rstar,p_pk + real(rl), dimension(np,np,nlev) :: rho_dry,rho_new,Rstar,p_pk real(rl), dimension(np,np,nlev) :: theta_inv,qv_inv,qc_inv,qr_inv,rho_inv,exner_inv,z_inv ! inverted columns real(rl), dimension(np,np,nlevp):: zi real(rl), dimension(np,np) :: ps @@ -700,7 +695,6 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) ! save un-forced prognostics T0=T; qv0=qv; qc0=qc; qr0=qr - theta0 = theta_kess ! invert columns (increasing z) @@ -740,15 +734,14 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) rho_new = rho_dry*(1+qv) Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) - !exner_new = p/(rho_new*Rstar*theta_kess) p_pk = rho_new*Rstar*theta_kess - exner_new = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) - + exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + T = exner_kess*theta_kess ! set dynamics forcing elem(ie)%derived%FM(:,:,1,:) = 0 elem(ie)%derived%FM(:,:,2,:) = 0 - elem(ie)%derived%FT(:,:,:) = exner_new*(theta_kess - theta0)/dt + elem(ie)%derived%FT(:,:,:) = (T-T0)/dt ! set tracer-mass forcing. conserve tracer mass elem(ie)%derived%FQ(:,:,:,1) = (rho_dry/rho)*dp*(qv-qv0)/dt diff --git a/components/homme/src/test_src/dcmip2016-baroclinic.F90_original b/components/homme/src/test_src/dcmip2016-baroclinic.F90_original deleted file mode 100644 index f7131281b02f..000000000000 --- a/components/homme/src/test_src/dcmip2016-baroclinic.F90_original +++ /dev/null @@ -1,459 +0,0 @@ -MODULE baroclinic_wave - -!======================================================================= -! -! Date: July 29, 2015 -! -! Functions for setting up idealized initial conditions for the -! Ullrich, Melvin, Staniforth and Jablonowski baroclinic instability. -! -! SUBROUTINE baroclinic_wave_sample( -! deep,moist,pertt,X,lon,lat,p,z,zcoords,u,v,w,t,phis,ps,rho,q) -! -! Options: -! deep deep atmosphere (1 = yes or 0 = no) -! moist include moisture (1 = yes or 0 = no) -! pertt type of perturbation (0 = exponential, 1 = stream function) -! X Earth scaling factor -! -! Given a point specified by: -! lon longitude (radians) -! lat latitude (radians) -! p/z pressure (Pa) / height (m) -! zcoords 1 if z is specified, 0 if p is specified -! -! the functions will return: -! p pressure if z is specified and zcoords = 1 (Pa) -! u zonal wind (m s^-1) -! v meridional wind (m s^-1) -! t temperature (K) -! thetav virtual potential temperature (K) -! phis surface geopotential (m^2 s^-2) -! ps surface pressure (Pa) -! rho density (kj m^-3) -! q water vapor mixing ratio (kg/kg) -! -! -! Author: Paul Ullrich -! University of California, Davis -! Email: paullrich@ucdavis.edu -! -!======================================================================= - -use physical_constants, only: g0=>g,kappa0=>kappa,Rgas,Cp0=>Cp,Rwater_vapor,rearth0,omega0, dd_pi - - IMPLICIT NONE - - -!======================================================================= -! Physical constants -!======================================================================= - - ! REAL(8), PARAMETER :: & - ! a = 6371220.0d0, & ! Reference Earth's Radius (m) - ! Rd = 287.0d0, & ! Ideal gas const dry air (J kg^-1 K^1) - ! g = 9.80616d0, & ! Gravity (m s^2) - ! cp = 1004.5d0, & ! Specific heat capacity (J kg^-1 K^1) - ! Lvap = 2.5d6, & ! Latent heat of vaporization of water - ! Rvap = 461.5d0, & ! Ideal gas constnat for water vapor - ! Mvap = 0.608d0, & ! Ratio of molar mass of dry air/water - ! pi = 3.14159265358979d0, & ! pi - ! p0 = 100000.0d0, & ! surface pressure (Pa) - ! kappa = 2.d0/7.d0, & ! Ratio of Rd to cp - ! omega = 7.29212d-5, & ! Reference rotation rate of the Earth (s^-1) - ! deg2rad = pi/180.d0 ! Conversion factor of degrees to radians - - REAL(8), PARAMETER :: & - a = rearth0, & ! Reference Earth's Radius (m) - Rd = Rgas, & ! Ideal gas const dry air (J kg^-1 K^1) - g = g0, & ! Gravity (m s^2) - cp = Cp0, & ! Specific heat capacity (J kg^-1 K^1) - Lvap = 2.5d6, & ! Latent heat of vaporization of water - Rvap = Rwater_vapor, & ! Ideal gas constnat for water vapor - Mvap = Rvap/Rd-1.0d0, & ! Ratio of molar mass of dry air/water - pi = dd_pi, & ! pi - p0 = 100000.0d0, & ! surface pressure (Pa) - kappa = kappa0, & ! Ratio of Rd to cp - omega = omega0, & ! Reference rotation rate of the Earth (s^-1) - deg2rad = pi/180.d0 ! Conversion factor of degrees to radians - -!======================================================================= -! Test case parameters -!======================================================================= - REAL(8), PARAMETER :: & - T0E = 310.d0 , & ! temperature at equatorial surface (K) - T0P = 240.d0 , & ! temperature at polar surface (K) - B = 2.d0 , & ! jet half-width parameter - K = 3.d0 , & ! jet width parameter - lapse = 0.005d0 ! lapse rate parameter - - REAL(8), PARAMETER :: & - pertu0 = 0.5d0 , & ! SF Perturbation wind velocity (m/s) - pertr = 1.d0/6.d0 , & ! SF Perturbation radius (Earth radii) - pertup = 1.0d0 , & ! Exp. perturbation wind velocity (m/s) - pertexpr = 0.1d0 , & ! Exp. perturbation radius (Earth radii) - pertlon = pi/9.d0 , & ! Perturbation longitude - pertlat = 2.d0*pi/9.d0, & ! Perturbation latitude - pertz = 15000.d0 , & ! Perturbation height cap - dxepsilon = 1.d-5 ! Small value for numerical derivatives - - REAL(8), PARAMETER :: & - moistqlat = 2.d0*pi/9.d0, & ! Humidity latitudinal width - moistqp = 34000.d0, & ! Humidity vertical pressure width - moisttr = 0.1d0, & ! Vertical cut-off pressure for humidity - moistqs = 1.d-12, & ! Humidity above cut-off - moistq0 = 0.018d0, & ! Maximum specific humidity - moistqr = 0.9d0, & ! Maximum saturation ratio - moisteps = 0.622d0, & ! Ratio of gas constants - moistT0 = 273.16d0, & ! Reference temperature (K) - moistE0Ast = 610.78d0 ! Saturation vapor pressure at T0 (Pa) - -CONTAINS - -!======================================================================= -! Generate the baroclinic instability initial conditions -!======================================================================= - SUBROUTINE baroclinic_wave_test(deep,moist,pertt,X,lon,lat,p,z,zcoords,u,v,t,thetav,phis,ps,rho,q) & - BIND(c, name = "baroclinic_wave_test") - - IMPLICIT NONE - -!----------------------------------------------------------------------- -! input/output params parameters at given location -!----------------------------------------------------------------------- - INTEGER, INTENT(IN) :: & - deep, & ! Deep (1) or Shallow (0) test case - moist, & ! Moist (1) or Dry (0) test case - pertt ! Perturbation type - - REAL(8), INTENT(IN) :: & - lon, & ! Longitude (radians) - lat, & ! Latitude (radians) - X ! Earth scaling parameter - - REAL(8), INTENT(INOUT) :: & - p, & ! Pressure (Pa) - z ! Altitude (m) - - INTEGER, INTENT(IN) :: zcoords ! 1 if z coordinates are specified - ! 0 if p coordinates are specified - - REAL(8), INTENT(OUT) :: & - u, & ! Zonal wind (m s^-1) - v, & ! Meridional wind (m s^-1) - t, & ! Temperature (K) - thetav, & ! Virtual potential temperature (K) - phis, & ! Surface Geopotential (m^2 s^-2) - ps, & ! Surface Pressure (Pa) - rho, & ! density (kg m^-3) - q ! water vapor mixing ratio (kg/kg) - - !------------------------------------------------ - ! Local variables - !------------------------------------------------ - REAL(8) :: aref, omegaref - REAL(8) :: T0, constH, constC, scaledZ, inttau2, rratio - REAL(8) :: inttermU, bigU, rcoslat, omegarcoslat - REAL(8) :: eta, qratio, qnum, qden - - !------------------------------------------------ - ! Pressure and temperature - !------------------------------------------------ - if (zcoords .eq. 1) then - CALL evaluate_pressure_temperature(deep, X, lon, lat, z, p, t) - else - CALL evaluate_z_temperature(deep, X, lon, lat, p, z, t) - end if - - !------------------------------------------------ - ! Compute test case constants - !------------------------------------------------ - aref = a / X - omegaref = omega * X - - T0 = 0.5d0 * (T0E + T0P) - - constH = Rd * T0 / g - - constC = 0.5d0 * (K + 2.d0) * (T0E - T0P) / (T0E * T0P) - - scaledZ = z / (B * constH) - - inttau2 = constC * z * exp(- scaledZ**2) - - ! radius ratio - if (deep .eq. 0) then - rratio = 1.d0 - else - rratio = (z + aref) / aref; - end if - - !----------------------------------------------------- - ! Initialize surface pressure - !----------------------------------------------------- - ps = p0 - - !----------------------------------------------------- - ! Initialize velocity field - !----------------------------------------------------- - inttermU = (rratio * cos(lat))**(K - 1.d0) - (rratio * cos(lat))**(K + 1.d0) - bigU = g / aref * K * inttau2 * inttermU * t - if (deep .eq. 0) then - rcoslat = aref * cos(lat) - else - rcoslat = (z + aref) * cos(lat) - end if - - omegarcoslat = omegaref * rcoslat - - u = - omegarcoslat + sqrt(omegarcoslat**2 + rcoslat * bigU) - v = 0.d0 - - !----------------------------------------------------- - ! Add perturbation to the velocity field - !----------------------------------------------------- - - ! Exponential type - if (pertt .eq. 0) then - u = u + evaluate_exponential(lon, lat, z) - - ! Stream function type - elseif (pertt .eq. 1) then - u = u - 1.d0 / (2.d0 * dxepsilon) * & - ( evaluate_streamfunction(lon, lat + dxepsilon, z) & - - evaluate_streamfunction(lon, lat - dxepsilon, z)) - - v = v + 1.d0 / (2.d0 * dxepsilon * cos(lat)) * & - ( evaluate_streamfunction(lon + dxepsilon, lat, z) & - - evaluate_streamfunction(lon - dxepsilon, lat, z)) - end if - - !----------------------------------------------------- - ! Initialize surface geopotential - !----------------------------------------------------- - phis = 0.d0 - - !----------------------------------------------------- - ! Initialize density - !----------------------------------------------------- - rho = p / (Rd * t) - - !----------------------------------------------------- - ! Initialize specific humidity - !----------------------------------------------------- - if (moist .eq. 1) then - eta = p/p0 - - if (eta .gt. moisttr) then - q = moistq0 * exp(- (lat/moistqlat)**4) & - * exp(- ((eta-1.d0)*p0/moistqp)**2) - else - q = moistqs - end if - - ! Convert virtual temperature to temperature - t = t / (1.d0 + Mvap * q) - - else - q = 0.d0 - end if - - !----------------------------------------------------- - ! Initialize virtual potential temperature - !----------------------------------------------------- - thetav = t * (1.d0 + Mvap * q) * (p0 / p)**(Rd / cp) - - END SUBROUTINE baroclinic_wave_test - -!----------------------------------------------------------------------- -! Calculate pointwise pressure and temperature -!----------------------------------------------------------------------- - SUBROUTINE evaluate_pressure_temperature(deep, X, lon, lat, z, p, t) - - INTEGER, INTENT(IN) :: deep ! Deep (1) or Shallow (0) test case - - REAL(8), INTENT(IN) :: & - X, & ! Earth scaling ratio - lon, & ! Longitude (radians) - lat, & ! Latitude (radians) - z ! Altitude (m) - - REAL(8), INTENT(OUT) :: & - p, & ! Pressure (Pa) - t ! Temperature (K) - - REAL(8) :: aref, omegaref - REAL(8) :: T0, constA, constB, constC, constH, scaledZ - REAL(8) :: tau1, tau2, inttau1, inttau2 - REAL(8) :: rratio, inttermT - - !-------------------------------------------- - ! Constants - !-------------------------------------------- - aref = a / X - omegaref = omega * X - - T0 = 0.5d0 * (T0E + T0P) - constA = 1.d0 / lapse - constB = (T0 - T0P) / (T0 * T0P) - constC = 0.5d0 * (K + 2.d0) * (T0E - T0P) / (T0E * T0P) - constH = Rd * T0 / g - - scaledZ = z / (B * constH) - - !-------------------------------------------- - ! tau values - !-------------------------------------------- - tau1 = constA * lapse / T0 * exp(lapse * z / T0) & - + constB * (1.d0 - 2.d0 * scaledZ**2) * exp(- scaledZ**2) - tau2 = constC * (1.d0 - 2.d0 * scaledZ**2) * exp(- scaledZ**2) - - inttau1 = constA * (exp(lapse * z / T0) - 1.d0) & - + constB * z * exp(- scaledZ**2) - inttau2 = constC * z * exp(- scaledZ**2) - - !-------------------------------------------- - ! radius ratio - !-------------------------------------------- - if (deep .eq. 0) then - rratio = 1.d0 - else - rratio = (z + aref) / aref; - end if - - !-------------------------------------------- - ! interior term on temperature expression - !-------------------------------------------- - inttermT = (rratio * cos(lat))**K & - - K / (K + 2.d0) * (rratio * cos(lat))**(K + 2.d0) - - !-------------------------------------------- - ! temperature - !-------------------------------------------- - t = 1.d0 / (rratio**2 * (tau1 - tau2 * inttermT)) - - !-------------------------------------------- - ! hydrostatic pressure - !-------------------------------------------- - p = p0 * exp(- g / Rd * (inttau1 - inttau2 * inttermT)) - - END SUBROUTINE evaluate_pressure_temperature - -!----------------------------------------------------------------------- -! Calculate pointwise z and temperature given pressure -!----------------------------------------------------------------------- - SUBROUTINE evaluate_z_temperature(deep, X, lon, lat, p, z, t) - - INTEGER, INTENT(IN) :: deep ! Deep (1) or Shallow (0) test case - - REAL(8), INTENT(IN) :: & - X, & ! Earth scaling ratio - lon, & ! Longitude (radians) - lat, & ! Latitude (radians) - p ! Pressure (Pa) - - REAL(8), INTENT(OUT) :: & - z, & ! Altitude (m) - t ! Temperature (K) - - INTEGER :: ix - - REAL(8) :: z0, z1, z2 - REAL(8) :: p0, p1, p2 - - z0 = 0.d0 - z1 = 10000.d0 - - CALL evaluate_pressure_temperature(deep, X, lon, lat, z0, p0, t) - CALL evaluate_pressure_temperature(deep, X, lon, lat, z1, p1, t) - - DO ix = 1, 1000 - z2 = z1 - (p1 - p) * (z1 - z0) / (p1 - p0) - - CALL evaluate_pressure_temperature(deep, X, lon, lat, z2, p2, t) - - IF (ABS((p2 - p)/p) .lt. 1.0d-13) THEN - EXIT - END IF - - z0 = z1 - p0 = p1 - - z1 = z2 - p1 = p2 - END DO - - z = z2 - - CALL evaluate_pressure_temperature(deep, X, lon, lat, z, p0, t) - - END SUBROUTINE evaluate_z_temperature - -!----------------------------------------------------------------------- -! Exponential perturbation function -!----------------------------------------------------------------------- - REAL(8) FUNCTION evaluate_exponential(lon, lat, z) - - REAL(8), INTENT(IN) :: & - lon, & ! Longitude (radians) - lat, & ! Latitude (radians) - z ! Altitude (meters) - - REAL(8) :: greatcircler, perttaper - - ! Great circle distance - greatcircler = 1.d0 / pertexpr & - * acos(sin(pertlat) * sin(lat) + cos(pertlat) * cos(lat) * cos(lon - pertlon)) - - ! Vertical tapering of stream function - if (z < pertz) then - perttaper = 1.d0 - 3.d0 * z**2 / pertz**2 + 2.d0 * z**3 / pertz**3 - else - perttaper = 0.d0 - end if - - ! Zonal velocity perturbation - if (greatcircler < 1.d0) then - evaluate_exponential = pertup * perttaper * exp(- greatcircler**2) - else - evaluate_exponential = 0.d0 - end if - - END FUNCTION evaluate_exponential - -!----------------------------------------------------------------------- -! Stream function perturbation function -!----------------------------------------------------------------------- - REAL(8) FUNCTION evaluate_streamfunction(lon, lat, z) - - REAL(8), INTENT(IN) :: & - lon, & ! Longitude (radians) - lat, & ! Latitude (radians) - z ! Altitude (meters) - - REAL(8) :: greatcircler, perttaper, cospert - - ! Great circle distance - greatcircler = 1.d0 / pertr & - * acos(sin(pertlat) * sin(lat) + cos(pertlat) * cos(lat) * cos(lon - pertlon)) - - ! Vertical tapering of stream function - if (z < pertz) then - perttaper = 1.d0 - 3.d0 * z**2 / pertz**2 + 2.d0 * z**3 / pertz**3 - else - perttaper = 0.d0 - end if - - ! Horizontal tapering of stream function - if (greatcircler .lt. 1.d0) then - cospert = cos(0.5d0 * pi * greatcircler) - else - cospert = 0.d0 - end if - - evaluate_streamfunction = & - (- pertu0 * pertr * perttaper * cospert**4) - - END FUNCTION evaluate_streamfunction - -END MODULE baroclinic_wave diff --git a/components/homme/src/theta-l/eos.F90 b/components/homme/src/theta-l/eos.F90 index ea069e0fc68d..e601d1498dbb 100644 --- a/components/homme/src/theta-l/eos.F90 +++ b/components/homme/src/theta-l/eos.F90 @@ -161,6 +161,58 @@ end subroutine get_pnh_and_exner +subroutine get_theta_from_T(hvcoord,Rstar,temperature,dp3d,phi_i,vtheta_dp) +implicit none +! +! Use Equation of State to compute vtheta_dp from Temperature +! +! p_over_exner = -Rstar T dp / (dphi/ds) +! +! input: dp3d, phi, phis, T, Rstar +! output: vtheta_dp +! + type (hvcoord_t), intent(in) :: hvcoord ! hybrid vertical coordinate struct + real (kind=real_kind), intent(in) :: Rstar(np,np,nlev) + real (kind=real_kind), intent(in) :: temperature(np,np,nlev) + real (kind=real_kind), intent(in) :: dp3d(np,np,nlev) + real (kind=real_kind), intent(in) :: phi_i(np,np,nlevp) + real (kind=real_kind), intent(out) :: vtheta_dp(np,np,nlev) + + ! local + real (kind=real_kind) :: exner(np,np,nlev) + real (kind=real_kind) :: pi(np,np,nlev) + real (kind=real_kind) :: pi_i(np,np,nlevp) + integer :: k + + if (theta_hydrostatic_mode) then + ! hydrostatic pressure + pi_i(:,:,1)=hvcoord%hyai(1)*hvcoord%ps0 + do k=1,nlev + pi_i(:,:,k+1)=pi_i(:,:,k) + dp3d(:,:,k) + enddo + do k=1,nlev + pi(:,:,k)=pi_i(:,:,k) + dp3d(:,:,k)/2 + enddo + + ! hydrostatic pressure + exner = (pi/p0)**kappa + else + + do k=1,nlev + pi(:,:,k) = Rstar(:,:,k)*temperature(:,:,k)*dp3d(:,:,k)/ & + (phi_i(:,:,k)-phi_i(:,:,k+1)) + enddo + exner = (pi/p0)**kappa + + endif ! hydrostatic/nonhydrostatic version + + vtheta_dp = & + (Rstar(:,:,:)/Rgas)*temperature(:,:,:)*dp3d(:,:,:)/exner(:,:,:) + + end subroutine get_theta_from_T + + + !_____________________________________________________________________ subroutine get_phinh(hvcoord,phis,vtheta_dp,dp,phi_i) ! diff --git a/components/homme/src/theta-l/prim_advance_mod.F90 b/components/homme/src/theta-l/prim_advance_mod.F90 index 232d702f55e1..9151ecbc050b 100644 --- a/components/homme/src/theta-l/prim_advance_mod.F90 +++ b/components/homme/src/theta-l/prim_advance_mod.F90 @@ -23,7 +23,7 @@ module prim_advance_mod use element_mod, only: element_t use element_state, only: max_itercnt_perstep,avg_itercnt,max_itererr_perstep, nu_scale_top use element_ops, only: get_temperature, set_theta_ref, state0, get_R_star - use eos, only: get_pnh_and_exner,get_phinh,get_dirk_jacobian + use eos, only: get_pnh_and_exner,get_theta_from_T,get_phinh,get_dirk_jacobian use hybrid_mod, only: hybrid_t use hybvcoord_mod, only: hvcoord_t use kinds, only: iulog, real_kind @@ -409,17 +409,11 @@ subroutine applyCAMforcing_ps(elem,hvcoord,np1,np1_qdp,dt,nets,nete) ! apply forcing to temperature call get_temperature(elem(ie),temperature,hvcoord,np1) -#if (defined COLUMN_OPENMP) -!$omp parallel do private(k) -#endif do k=1,nlev temperature(:,:,k) = temperature(:,:,k) + dt*elem(ie)%derived%FT(:,:,k) enddo - -#if (defined COLUMN_OPENMP) -!$omp parallel do private(q,k,i,j,v1) -#endif + do q=1,qsize do k=1,nlev do j=1,np @@ -453,16 +447,10 @@ subroutine applyCAMforcing_ps(elem,hvcoord,np1,np1_qdp,dt,nets,nete) ! Qdp(np1) and ps_v(np1) were updated by forcing - update Q(np1) -#if (defined COLUMN_OPENMP) -!$omp parallel do private(q,k) -#endif do k=1,nlev dp(:,:,k) = ( hvcoord%hyai(k+1) - hvcoord%hyai(k) )*hvcoord%ps0 + & ( hvcoord%hybi(k+1) - hvcoord%hybi(k) )*elem(ie)%state%ps_v(:,:,np1) enddo -#if (defined COLUMN_OPENMP) -!$omp parallel do private(q,k) -#endif do q=1,qsize do k=1,nlev elem(ie)%state%Q(:,:,k,q) = elem(ie)%state%Qdp(:,:,k,q,np1_qdp)/dp(:,:,k) @@ -470,13 +458,10 @@ subroutine applyCAMforcing_ps(elem,hvcoord,np1,np1_qdp,dt,nets,nete) enddo ! now that we have updated Qdp and dp, compute vtheta_dp from temperature - call get_pnh_and_exner(hvcoord,elem(ie)%state%vtheta_dp(:,:,:,np1),dp,& - elem(ie)%state%phinh_i(:,:,:,np1),pnh,exner,dpnh_dp_i) - call get_R_star(Rstar,elem(ie)%state%Q(:,:,:,1)) - elem(ie)%state%vtheta_dp(:,:,:,np1) = & - (Rstar(:,:,:)/Rgas)*temperature(:,:,:)*& - dp(:,:,:)/exner(:,:,:) + call get_theta_from_T(hvcoord,Rstar,temperature,dp,& + elem(ie)%state%phinh_i(:,:,:,np1),elem(ie)%state%vtheta_dp(:,:,:,np1)) + enddo call applyCAMforcing_dynamics(elem,hvcoord,np1,np1_qdp,dt,nets,nete) From e75f39b1d88b93b142423e51301cb089b8f02b2b Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sat, 20 Oct 2018 20:50:29 -0500 Subject: [PATCH 37/41] update dcmip2016 TC and baroclinic tests to use R&J physics R&J physics avoids grid point storms seen with Kessler physics, allows most test cases to run with rsplit=6 --- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl | 2 +- .../theta-l/namelist-ne1024.nl | 4 ++-- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl | 4 ++-- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl | 4 ++-- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl | 4 ++-- .../theta-l/namelist-r100-h.nl | 4 ++-- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl | 4 ++-- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl | 2 +- .../dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl | 4 ++-- .../dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh | 4 ++-- .../dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl | 2 +- .../dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl | 2 +- .../dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl | 2 +- .../dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl | 2 +- .../dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl | 2 +- .../theta-l/namelist-r50-h.nl | 4 ++-- .../dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl | 4 ++-- .../dcmip2016_test3_supercell/theta-l/namelist-r200.nl | 2 +- 20 files changed, 30 insertions(+), 30 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl index 56871271186c..5b8a96876226 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r100.nl @@ -26,7 +26,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .true. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl index 36e23e480e50..3fbb0705dbcc 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r400.nl @@ -25,7 +25,7 @@ hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl index 625c53671a93..7811f5d32d90 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/preqx/namelist-r50.nl @@ -26,7 +26,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .true. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl index 808dee0b2e9e..76cc7239edc3 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl @@ -16,7 +16,7 @@ tstep = 9 ! 20 was unstable integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e10 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1e10 @@ -27,7 +27,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .false. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl index f397bd3f3b1f..fa7d8070900d 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne120.nl @@ -16,7 +16,7 @@ tstep = 75 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e13 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1e13 @@ -27,7 +27,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .false. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl index 77a431689039..36f4bc76962b 100755 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne256.nl @@ -16,7 +16,7 @@ tstep = 40 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e12 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1e12 @@ -27,7 +27,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .false. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl index 422b23c1da45..fbdc14cc07bc 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl @@ -16,7 +16,7 @@ tstep = 18 ! 20 was unstable integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e11 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1e11 @@ -27,7 +27,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .false. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl index 3700d48ab4ee..1377c56afa49 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100-h.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 @@ -26,7 +26,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .true. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl index 779572a8f032..74064688c787 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r100.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 ! may require ftype=4 to get to rsplit=6? + rsplit = 6 ! may require ftype=4 to get to rsplit=6? qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 @@ -26,7 +26,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .false. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl index b782a9aa18a0..afb4c6ad714a 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r400.nl @@ -26,7 +26,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .false. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl index 5cb4238ec6a8..d2686a3590c1 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-r50.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 1 ! r50 crashes with rsplit>1 + rsplit = 6 ! r50 crashes with rsplit>1 qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 @@ -26,7 +26,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' theta_hydrostatic_mode = .false. - dcmip16_prec_type = 0 ! 0=kessler physics + dcmip16_prec_type = 1 ! 0=kessler physics dcmip16_pbl_type = -1 ! 0=reed-jablonowski pbl, -1 = none / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh index dd46d76d7b75..909968183782 100755 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/jobscript-snl.sh @@ -56,5 +56,5 @@ ncl plot-horiz-ps.ncl prefix=r400 ; run $(($NCPU>384?384:NCPU)) -#prefix=r100 ; run $NCPU -#prefix=r50 ; run $NCPU +prefix=r100 ; run $NCPU +prefix=r50 ; run $NCPU diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl index 77652d2ba998..35b0fd1067d8 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl @@ -25,7 +25,7 @@ hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_prec_type = 1 ! 0=kessler, 1= reed-jablonowski dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl index 1d3fed1e75c9..e70a4dc45be4 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r400.nl @@ -25,7 +25,7 @@ hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_prec_type = 1 ! 0=kessler, 1= reed-jablonowski dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl index c317b499ffbf..c96cc3cc98a0 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl @@ -25,7 +25,7 @@ hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_prec_type = 1 ! 0=kessler, 1= reed-jablonowski dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl index 465441f64eda..2a104227f372 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r100.nl @@ -25,7 +25,7 @@ hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_prec_type = 1 ! 0=kessler, 1= reed-jablonowski dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl index 73d1b121152d..5889ca59de43 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r400.nl @@ -25,7 +25,7 @@ hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_prec_type = 1 ! 0=kessler, 1= reed-jablonowski dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl index 46055c501374..b60cee19b5a4 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 3 + rsplit = 3 ! 6 with ftype=4??? qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 @@ -26,7 +26,7 @@ hypervis_subcycle = 1 ! 1 = no hyperviz subcycling theta_hydrostatic_mode = .true. moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_prec_type = 1 ! 0=kessler, 1= reed-jablonowski dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl index 7f0808172837..b67510cd7213 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 7 - rsplit = 3 ! aborts with rsplit=6. needs ftype=4? + rsplit = 6 ! qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 @@ -25,7 +25,7 @@ hypervis_order = 2 ! 2 = hyperviscosity hypervis_subcycle = 1 ! 1 = no hyperviz subcycling moisture = 'wet' - dcmip16_prec_type = 0 ! 0=kessler, 1= reed-jablonowski + dcmip16_prec_type = 1 ! 0=kessler, 1= reed-jablonowski dcmip16_pbl_type = 0 ! 0=basic pbl, 1= bryan pbl / &vert_nl diff --git a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl index 2defe82045ca..13f81eeeda83 100644 --- a/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl +++ b/components/homme/dcmip_tests/dcmip2016_test3_supercell/theta-l/namelist-r200.nl @@ -9,7 +9,7 @@ ne = 15 ! number of elements per cube face qsize = 3 ! num tracer fields nmax = 1800 ! 7200s(120min)/tstep - statefreq = 100 ! number of steps between screen dumps + statefreq = 120 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run tstep = 4. ! largest timestep in seconds From f63679adfa29996c3616bd17e4d37cd10a82f568 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 21 Oct 2018 05:48:12 -0700 Subject: [PATCH 38/41] tweaks to ultra high-res namelists ne512 and ne1024 namelists which run for 2h, with large rsplit and R&J physics --- .../theta-l/namelist-ne1024.nl | 6 +++--- .../theta-l/namelist-ne512.nl | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl index 76cc7239edc3..9a9517b988ac 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne1024.nl @@ -9,11 +9,11 @@ ne = 1024 ! number of elements per cube face qsize = 6 ! num tracer fields ! ndays = 1 - nmax = 270 ! - statefreq = 18 ! number of steps between screen dumps + nmax = 810 ! run for 7200s (2h) + statefreq = 30 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 9 ! 20 was unstable + tstep = 9 integration = 'explicit' ! explicit time integration tstep_type = 7 rsplit = 6 diff --git a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl index fbdc14cc07bc..981965c03ad3 100644 --- a/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl +++ b/components/homme/dcmip_tests/dcmip2016_test1_baroclinic_wave/theta-l/namelist-ne512.nl @@ -9,11 +9,11 @@ ne = 512 ! number of elements per cube face qsize = 6 ! num tracer fields ! ndays = 1 - nmax = 402 ! run for 7200s (2h) - statefreq = 18 ! number of steps between screen dumps + nmax = 360 ! run for 7200s (2h) + statefreq = 30 ! number of steps between screen dumps restartfreq = -1 ! don't write restart files if < 0 runtype = 0 ! 0 => new run - tstep = 18 ! 20 was unstable + tstep = 20 ! 20 was unstable integration = 'explicit' ! explicit time integration tstep_type = 7 rsplit = 6 From 4cdbc1d9c4586fbb6a6130da13f7615be9840198 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 21 Oct 2018 15:14:16 -0500 Subject: [PATCH 39/41] updates to dcmip2016 forcing for hydrostatic case now all tests run with rsplit=6 --- .../preqx/namelist-r100.nl | 2 +- .../preqx/namelist-r50.nl | 2 +- .../theta-l/namelist-r50-h.nl | 2 +- components/homme/src/share/control_mod.F90 | 5 ++- components/homme/src/share/namelist_mod.F90 | 5 +++ .../homme/src/test_src/dcmip16_wrapper.F90 | 44 +++++++++++++------ 6 files changed, 41 insertions(+), 19 deletions(-) diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl index 35b0fd1067d8..5e20cdee430d 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r100.nl @@ -15,7 +15,7 @@ tstep = 300 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1e15 ! default= 1e15*(ne30/ne30)**3.2 = 1e15 nu_s = 1e15 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl index c96cc3cc98a0..ad698c27872b 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/preqx/namelist-r50.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 3 + rsplit = 6 qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 diff --git a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl index b60cee19b5a4..ddfed89e24fe 100644 --- a/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl +++ b/components/homme/dcmip_tests/dcmip2016_test2_tropical_cyclone/theta-l/namelist-r50-h.nl @@ -15,7 +15,7 @@ tstep = 150 ! largest timestep in seconds integration = 'explicit' ! explicit time integration tstep_type = 5 - rsplit = 3 ! 6 with ftype=4??? + rsplit = 6 ! 6 with ftype=4??? qsplit = 1 nu = 1.1e14 ! default= 1e15*(ne30/ne60)**3.2 = 1.1e14 nu_s = 1.1e14 diff --git a/components/homme/src/share/control_mod.F90 b/components/homme/src/share/control_mod.F90 index 6be27860e0a6..926b0f44fcbd 100644 --- a/components/homme/src/share/control_mod.F90 +++ b/components/homme/src/share/control_mod.F90 @@ -17,8 +17,9 @@ module control_mod logical, public :: use_semi_lagrange_transport = .false. logical, public :: use_semi_lagrange_transport_local_conservation = .false. -! set to .true. to run the theta nonydrostatic model in hydrostatic mode - logical, public :: theta_hydrostatic_mode = .false. +! flag used by preqx, theta-l and theta-c models +! should be renamed to "hydrostatic_mode" + logical, public :: theta_hydrostatic_mode integer, public :: tstep_type= 5 ! preqx timestepping options diff --git a/components/homme/src/share/namelist_mod.F90 b/components/homme/src/share/namelist_mod.F90 index 81b7d821b05f..a16b799fcfdf 100644 --- a/components/homme/src/share/namelist_mod.F90 +++ b/components/homme/src/share/namelist_mod.F90 @@ -360,6 +360,11 @@ subroutine readnl(par) use_semi_lagrange_transport_local_conservation = .false. disable_diagnostics = .false. + theta_hydrostatic_mode = .true. ! for preqx, this must be .true. +#if ( defined MODEL_THETA_C || defined MODEL_THETA_L ) + theta_hydrostatic_mode = .false. ! default NH +#endif + ! ======================= ! Read namelist variables diff --git a/components/homme/src/test_src/dcmip16_wrapper.F90 b/components/homme/src/test_src/dcmip16_wrapper.F90 index df5c8ba94e32..a66d4b66a608 100644 --- a/components/homme/src/test_src/dcmip16_wrapper.F90 +++ b/components/homme/src/test_src/dcmip16_wrapper.F90 @@ -485,11 +485,16 @@ subroutine dcmip2016_test1_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) enddo; enddo; - - rho_new = rho_dry*(1+qv) - Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) - p_pk = rho_new*Rstar*theta_kess - exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + if (theta_hydrostatic_mode) then + ! hydrostatic model assumes physics does not change pressure + ! so assume T,PHI change, with P held fixed + ! ps_v will be adjusted after physics to conserve dry mass + else + rho_new = rho_dry*(1+qv) + Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) + p_pk = rho_new*Rstar*theta_kess + exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + endif T = exner_kess*theta_kess ! set dynamics forcing @@ -609,10 +614,16 @@ subroutine dcmip2016_test2_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl, t theta_kess(i,j,:) = th_c(nlev:1:-1) enddo; enddo; - rho_new = rho_dry*(1+qv) - Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) - p_pk = rho_new*Rstar*theta_kess - exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + if (theta_hydrostatic_mode) then + ! hydrostatic model assumes physics does not change pressure + ! so assume T,PHI change, with P held fixed + ! ps_v will be adjusted after physics to conserve dry mass + else + rho_new = rho_dry*(1+qv) + Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) + p_pk = rho_new*Rstar*theta_kess + exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + endif T = exner_kess*theta_kess @@ -731,11 +742,16 @@ subroutine dcmip2016_test3_forcing(elem,hybrid,hvcoord,nets,nete,nt,ntQ,dt,tl) qr = qr_inv (:,:,nlev:1:-1) - - rho_new = rho_dry*(1+qv) - Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) - p_pk = rho_new*Rstar*theta_kess - exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + if (theta_hydrostatic_mode) then + ! hydrostatic model assumes physics does not change pressure + ! so assume T,PHI change, with P held fixed + ! ps_v will be adjusted after physics to conserve dry mass + else + rho_new = rho_dry*(1+qv) + Rstar = (Rgas+(Rwater_vapor-Rgas)*qv*rho_dry/rho_new) + p_pk = rho_new*Rstar*theta_kess + exner_kess = ( p_pk / p0)**( (Rgas/Cp) / ( 1 - (Rgas/Cp))) + endif T = exner_kess*theta_kess ! set dynamics forcing From 237c96ba2746604f20a21ea1cfca89d29051df59 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Sun, 28 Oct 2018 11:50:29 -0500 Subject: [PATCH 40/41] revert init serialization code since better solution now on master --- components/homme/src/share/parallel_mod.F90 | 13 +- .../homme/src/share/prim_driver_base.F90 | 314 +++++++++--------- 2 files changed, 171 insertions(+), 156 deletions(-) diff --git a/components/homme/src/share/parallel_mod.F90 b/components/homme/src/share/parallel_mod.F90 index ad6e2a6ae1fb..f4a998a319f4 100644 --- a/components/homme/src/share/parallel_mod.F90 +++ b/components/homme/src/share/parallel_mod.F90 @@ -48,9 +48,9 @@ module parallel_mod integer :: root ! local root integer :: nprocs ! number of processes in group integer :: comm ! local communicator - integer :: node_comm ! local communicator of all procs per node - integer :: node_rank ! local rank in node_comm - integer :: node_nprocs ! local rank in node_comm +! integer :: node_comm ! local communicator of all procs per node +! integer :: node_rank ! local rank in node_comm +! integer :: node_nprocs ! local rank in node_comm logical :: masterproc logical :: dynproc ! Designation of a dynamics processor - AaronDonahue end type @@ -217,9 +217,10 @@ function initmp(npes_in,npes_stride) result(par) if (node_color==0) call abortmp("initmp: Errror computing procs per node") ! create a communicator of all procs per node - call mpi_comm_split(par%comm, node_color, par%rank, par%node_comm, ierr) - call MPI_comm_rank(par%node_comm,par%node_rank,ierr) - call MPI_comm_size(par%node_comm,par%node_nprocs,ierr) + ! currently not used, so commenting out +! call mpi_comm_split(par%comm, node_color, par%rank, par%node_comm, ierr) +! call MPI_comm_rank(par%node_comm,par%node_rank,ierr) +! call MPI_comm_size(par%node_comm,par%node_nprocs,ierr) ! ======================================================================= ! Verify that everybody agrees on this number otherwise do not do diff --git a/components/homme/src/share/prim_driver_base.F90 b/components/homme/src/share/prim_driver_base.F90 index 78b92b082d74..96e958b2c896 100644 --- a/components/homme/src/share/prim_driver_base.F90 +++ b/components/homme/src/share/prim_driver_base.F90 @@ -5,7 +5,6 @@ ! 08/2016: O. Guba Inserting code for "espilon bubble" reference element map ! 03/2018: M. Taylor fix memory leak ! 06/2018: O. Guba code for new ftypes -! 10/2018: M. Taylor serialize domain decmposition and schedule setup ! #ifdef HAVE_CONFIG_H #include "config.h" @@ -23,11 +22,6 @@ module prim_driver_base use quadrature_mod, only: quadrature_t, test_gauss, test_gausslobatto, gausslobatto use reduction_mod, only: reductionbuffer_ordered_1d_t, red_min, red_max, red_max_int, & red_sum, red_sum_int, red_flops, initreductionbuffer - use parallel_mod, only : iam, parallel_t, syncmp, syncmp_comm, abortmp, global_shared_buf, nrepro_vars -#ifdef _MPI - use parallel_mod, only : mpiinteger_t, mpi_max -#endif - #ifndef CAM use prim_restart_mod, only : initrestartfile use restart_io_mod , only : RestFile,readrestart @@ -71,12 +65,11 @@ subroutine prim_init1(elem, par, dom_mt, Tl) use mesh_mod, only : MeshSetCoordinates, MeshUseMeshFile, MeshCubeTopology, & MeshCubeElemCount, MeshCubeEdgeCount, MeshCubeTopologyCoords ! -------------------------------- - use metagraph_mod, only : metavertex_t, metaedge_t, localelemcount, initmetagraph, & - deallocate_metavertex_data, printmetavertex + use metagraph_mod, only : metavertex_t, metaedge_t, localelemcount, initmetagraph, printmetavertex ! -------------------------------- use gridgraph_mod, only : gridvertex_t, gridedge_t, allocate_gridvertex_nbrs, deallocate_gridvertex_nbrs ! -------------------------------- - use schedtype_mod, only : schedule, schedule_t + use schedtype_mod, only : schedule ! -------------------------------- use schedule_mod, only : genEdgeSched, PrintSchedule ! -------------------------------- @@ -86,6 +79,11 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! -------------------------------- #ifdef TRILINOS use prim_implicit_mod, only : prim_implicit_init +#endif + ! -------------------------------- + use parallel_mod, only : iam, parallel_t, syncmp, abortmp, global_shared_buf, nrepro_vars +#ifdef _MPI + use parallel_mod, only : mpiinteger_t, mpireal_t, mpi_max, mpi_sum, haltmp #endif ! -------------------------------- use metis_mod, only : genmetispart @@ -126,17 +124,21 @@ subroutine prim_init1(elem, par, dom_mt, Tl) type (GridVertex_t), target,allocatable :: GridVertex(:) type (GridEdge_t), target,allocatable :: Gridedge(:) - type (MetaVertex_t) :: MetaVertex + type (MetaVertex_t), target,allocatable :: MetaVertex(:) - integer :: i,ii,ie, ith + integer :: ii,ie, ith integer :: nets, nete integer :: nelem_edge,nedge integer :: nstep integer :: nlyr integer :: iMv - integer :: err, ierr, l, j, task_id, task_id_stride + integer :: err, ierr, l, j logical, parameter :: Debug = .FALSE. + integer :: i + integer,allocatable :: TailPartition(:) + integer,allocatable :: HeadPartition(:) + integer total_nelem real(kind=real_kind) :: approx_elements_per_task type (quadrature_t) :: gp ! element GLL points @@ -205,156 +207,145 @@ subroutine prim_init1(elem, par, dom_mt, Tl) ! ================================== call derivinit(deriv1) - if (MeshUseMeshFile) then - nelem = MeshCubeElemCount() - nelem_edge = MeshCubeEdgeCount() - else - nelem = CubeElemCount() - nelem_edge = CubeEdgeCount() - end if - - ! we want to exit elegantly when we are using too many processors. - if (nelem < par%nprocs) then - call abortmp('Error: too many MPI tasks. set dyn_npes <= nelem') - end if - - - ! ==================================================== - ! start domain decomposition - ! non-scalable global arrays: GridVertex, GridEdge - ! OOM errors with 64 MPI tasks per node at ne512 on 96GB/node system - ! cost breakdown (Anvil, ne1024): 2x higher on KNL - ! compute GridVertex, GridEDge: 22s - ! compute MetaVertex for 1 task: 8s - ! compute Schedule for 1 task: .1s - ! - ! in order to not run out of memory, only allow task_id_stride MPI tasks to be - ! active per node, and active tasks must deallocate GridVertex and GridEdge data - ! before the next tasks start ! =============================================================== - call t_startf('MeshSetup') - allocate(Schedule(1)) - ! Anvil (64GB/node). for ne=1024, keep nelem*task_id_stide < 20M - ! cori-KNL: 60M works. 80M OOM. set default to 50M - ! if OOM errors during initialization, decrease this number - ! if initialization time is too slow, and memory per node > 64GB, increase this number - task_id_stride=max(51000000/nelem,1) - if (task_id_stride0) then + allocate(elem(nelemd)) + call setup_element_pointers(elem) + call allocate_element_desc(elem) + endif + + ! ==================================================== + ! Generate the communication schedule + ! ==================================================== + + call genEdgeSched(elem,iam,Schedule(1),MetaVertex(1)) + allocate(global_shared_buf(nelemd,nrepro_vars)) global_shared_buf=0.0_real_kind + ! nlyr=edge3p1%nlyr + ! call MessageStats(nlyr) + ! call testchecksum(par,GridEdge) + + ! ======================================================== + ! load graph information into local element descriptors + ! ======================================================== + ! do ii=1,nelemd + ! elem(ii)%vertex = MetaVertex(iam)%members(ii) + ! enddo + + call syncmp(par) ! ================================================================= ! Set number of domains (for 'decompose') equal to number of threads @@ -490,6 +481,27 @@ subroutine prim_init1(elem, par, dom_mt, Tl) #endif !DBG write(iulog,*) 'prim_init: after call to initRestartFile' + deallocate(GridEdge) + do j =1,nelem + call deallocate_gridvertex_nbrs(GridVertex(j)) + end do + deallocate(GridVertex) + + do j = 1, MetaVertex(1)%nmembers + call deallocate_gridvertex_nbrs(MetaVertex(1)%members(j)) + end do + do j = 1, MetaVertex(1)%nedges + deallocate(MetaVertex(1)%edges(j)%members) + deallocate(MetaVertex(1)%edges(j)%edgeptrP) + deallocate(MetaVertex(1)%edges(j)%edgeptrS) + deallocate(MetaVertex(1)%edges(j)%edgeptrP_ghost) + end do + deallocate(MetaVertex(1)%edges) + deallocate(MetaVertex(1)%members) + deallocate(MetaVertex) + deallocate(TailPartition) + deallocate(HeadPartition) + ! single global edge buffer for all models: ! hydrostatic 4*nlev NH: 6*nlev+1 ! SL tracers: (qsize+1)*nlev @@ -524,12 +536,14 @@ end subroutine prim_init1 !_____________________________________________________________________ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) - use control_mod, only: runtype, integration, & + use control_mod, only: runtype, integration, test_case, & + debug_level, vfile_int, vform, vfile_mid, & topology,rsplit, qsplit, rk_stage_user,& sub_case, limiter_option, nu, nu_q, nu_div, tstep_type, hypervis_subcycle, & hypervis_subcycle_q, moisture, use_moisture use global_norms_mod, only: test_global_integral, print_cfl use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: parallel_t, haltmp, syncmp, abortmp use prim_state_mod, only: prim_printstate, prim_diag_scalars use prim_si_mod, only: prim_set_mass use prim_advection_mod, only: prim_advec_init2 @@ -601,7 +615,6 @@ subroutine prim_init2(elem, hybrid, nets, nete, tl, hvcoord) integer :: qn0 real (kind=real_kind) :: eta_ave_w - interface subroutine noxinit(vectorSize,vector,comm,v_container,p_container,j_container) & bind(C,name='noxinit') @@ -807,7 +820,6 @@ end subroutine noxinit ! timesteps to use for advective stability: tstep*qsplit and tstep call print_cfl(elem,hybrid,nets,nete,dtnu) - if (hybrid%masterthread) then ! CAM has set tstep based on dtime before calling prim_init2(), ! so only now does HOMME learn the timstep. print them out: @@ -859,6 +871,7 @@ subroutine prim_run_subcycle(elem, hybrid,nets,nete, dt, single_column, tl, hvco use control_mod, only: statefreq, ftype, qsplit, rsplit, disable_diagnostics use hybvcoord_mod, only: hvcoord_t + use parallel_mod, only: abortmp use prim_state_mod, only: prim_printstate, prim_diag_scalars, prim_energy_halftimes use vertremap_mod, only: vertical_remap use reduction_mod, only: parallelmax @@ -1072,6 +1085,7 @@ subroutine prim_step(elem, hybrid,nets,nete, dt, tl, hvcoord, compute_diagnostic use control_mod, only: statefreq, integration, ftype, qsplit, nu_p, rsplit use control_mod, only: use_semi_lagrange_transport use hybvcoord_mod, only : hvcoord_t + use parallel_mod, only: abortmp use prim_advance_mod, only: prim_advance_exp use prim_advection_mod, only: prim_advec_tracers_remap use reduction_mod, only: parallelmax From d0b943447dc249cb63b0ad26e4ab5bb32bec20bb Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Mon, 29 Oct 2018 09:36:02 -0500 Subject: [PATCH 41/41] remove unused subroutine --- components/homme/src/share/metagraph_mod.F90 | 24 -------------------- 1 file changed, 24 deletions(-) diff --git a/components/homme/src/share/metagraph_mod.F90 b/components/homme/src/share/metagraph_mod.F90 index adc3f57f41c5..72613fe5add7 100644 --- a/components/homme/src/share/metagraph_mod.F90 +++ b/components/homme/src/share/metagraph_mod.F90 @@ -48,7 +48,6 @@ module metagraph_mod public :: edge_uses_vertex public :: PrintMetaEdge, PrintMetaVertex public :: LocalElemCount - public :: deallocate_metavertex_data !public :: MetaEdgeCount public :: initMetaGraph @@ -179,29 +178,6 @@ function LocalElemCount(Vertex) result(nelemd) end function LocalElemCount - subroutine deallocate_metavertex_data(Vertex) - implicit none - - type (MetaVertex_t),intent(in) :: Vertex - integer :: j - - do j = 1, Vertex%nmembers - call deallocate_gridvertex_nbrs(Vertex%members(j)) - end do - do j = 1, Vertex%nedges - deallocate(Vertex%edges(j)%members) - deallocate(Vertex%edges(j)%edgeptrP) - deallocate(Vertex%edges(j)%edgeptrS) - deallocate(Vertex%edges(j)%edgeptrP_ghost) - end do - deallocate(Vertex%edges) - deallocate(Vertex%members) - - end subroutine - - - - function edge_uses_vertex(Vertex,Edge) result(log) implicit none