Skip to content

Commit

Permalink
case optimization of wenoX
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChrisZYJ
ChrisZYJ committed Jun 21, 2024
1 parent 43e4a07 commit 0829f92
Show file tree
Hide file tree
Showing 6 changed files with 78 additions and 54 deletions.
41 changes: 29 additions & 12 deletions src/simulation/m_global_parameters.fpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,20 +116,25 @@ module m_global_parameters
logical :: prim_vars_wrt

#:if MFC_CASE_OPTIMIZATION
integer, parameter :: weno_polyn = ${weno_polyn}$ !< Degree of the WENO polynomials (polyn)
integer, parameter :: weno_order = ${weno_order}$ !< Order of the WENO reconstruction
integer, parameter :: num_fluids = ${num_fluids}$ !< number of fluids in the simulation
integer, parameter :: weno_polyn = ${weno_polyn}$ !< Degree of the WENO polynomials (polyn)
integer, parameter :: weno_order = ${weno_order}$ !< Order of the WENO reconstruction
integer, parameter :: num_fluids = ${num_fluids}$ !< number of fluids in the simulation
logical, parameter :: wenojs = ${wenojs}$ !< WENO-JS (default)
logical, parameter :: mapped_weno = ${mapped_weno}$ !< WENO-M (WENO with mapping of nonlinear weights)
logical, parameter :: wenoz = ${wenoz}$ !< WENO-Z
logical, parameter :: teno = ${teno}$ !< TENO (Targeted ENO)
#:else
integer :: weno_polyn !< Degree of the WENO polynomials (polyn)
integer :: weno_order !< Order of the WENO reconstruction
integer :: num_fluids !< number of fluids in the simulation
logical :: wenojs !< WENO-JS (default)
logical :: mapped_weno !< WENO-M (WENO with mapping of nonlinear weights)
logical :: wenoz !< WENO-Z
logical :: teno !< TENO (Targeted ENO)
#:endif

real(kind(0d0)) :: weno_eps !< Binding for the WENO nonlinear weights
real(kind(0d0)) :: teno_CT !< Smoothness threshold for TENO
logical :: mapped_weno !< WENO-M (WENO with mapping of nonlinear weights)
logical :: wenoz !< WENO-Z
logical :: teno !< TENO (Targeted ENO)
logical :: mp_weno !< Monotonicity preserving (MP) WENO
logical :: weno_avg ! Average left/right cell-boundary states
logical :: weno_Re_flux !< WENO reconstruct velocity gradients for viscous stress tensor
Expand All @@ -156,10 +161,10 @@ module m_global_parameters
integer :: cpu_start, cpu_end, cpu_rate

#:if not MFC_CASE_OPTIMIZATION
!$acc declare create(num_dims, weno_polyn, weno_order, num_fluids)
!$acc declare create(num_dims, weno_polyn, weno_order, num_fluids, wenojs, mapped_weno, wenoz, teno)
#:endif

!$acc declare create(mpp_lim, model_eqns, mixture_err, alt_soundspeed, avg_state, mapped_weno, wenoz, teno, mp_weno, weno_eps, teno_CT, hypoelasticity)
!$acc declare create(mpp_lim, model_eqns, mixture_err, alt_soundspeed, avg_state, mp_weno, weno_eps, teno_CT, hypoelasticity)

logical :: relax !< activate phase change
integer :: relax_model !< Relaxation model
Expand Down Expand Up @@ -491,9 +496,6 @@ contains
time_stepper = dflt_int
weno_eps = dflt_real
teno_CT = 1d-6
mapped_weno = .false.
wenoz = .false.
teno = .false.
mp_weno = .false.
weno_avg = .false.
weno_Re_flux = .false.
Expand All @@ -515,6 +517,12 @@ contains
riemann_flat = .true.
rdma_mpi = .false.

#:if not MFC_CASE_OPTIMIZATION
mapped_weno = .false.
wenoz = .false.
teno = .false.
#:endif

bc_x%beg = dflt_int; bc_x%end = dflt_int
bc_y%beg = dflt_int; bc_y%end = dflt_int
bc_z%beg = dflt_int; bc_z%end = dflt_int
Expand Down Expand Up @@ -957,6 +965,11 @@ contains
wa_flg = 0d0; if (weno_avg) wa_flg = 1d0
!$acc update device(wa_flg)

! Resort to default WENO-JS if no other WENO scheme is selected
#:if not MFC_CASE_OPTIMIZATION
wenojs = .not. (mapped_weno .or. wenoz .or. teno)
#:endif

if (ib) allocate (MPI_IO_IB_DATA%var%sf(0:m, 0:n, 0:p))
Np = 0

Expand Down Expand Up @@ -1030,7 +1043,11 @@ contains
!$acc update device(m, n, p)

!$acc update device(alt_soundspeed, monopole, num_mono)
!$acc update device(dt, sys_size, buff_size, pref, rhoref, gamma_idx, pi_inf_idx, E_idx, alf_idx, stress_idx, mpp_lim, bubbles, hypoelasticity, alt_soundspeed, avg_state, num_fluids, model_eqns, num_dims, mixture_err, grid_geometry, cyl_coord, mapped_weno, wenoz, teno, mp_weno, weno_eps, teno_CT)
!$acc update device(dt, sys_size, buff_size, pref, rhoref, gamma_idx, pi_inf_idx, E_idx, alf_idx, stress_idx, mpp_lim, bubbles, hypoelasticity, alt_soundspeed, avg_state, num_fluids, model_eqns, num_dims, mixture_err, grid_geometry, cyl_coord, mp_weno, weno_eps, teno_CT)

#:if not MFC_CASE_OPTIMIZATION
!$acc update device(wenojs, mapped_weno, wenoz, teno)
#:endif

!$acc enter data copyin(nb, R0ref, Ca, Web, Re_inv, weight, R0, V0, bubbles, polytropic, polydisperse, qbmm, R0_type, ptil, bubble_model, thermal, poly_sigma)
!$acc enter data copyin(R_n, R_v, phi_vn, phi_nv, Pe_c, Tw, pv, M_n, M_v, k_n, k_v, pb0, mass_n0, mass_v0, Pe_T, Re_trans_T, Re_trans_c, Im_trans_T, Im_trans_c, omegaN , mul0, ss, gamma_v, mu_v, gamma_m, gamma_n, mu_n, gam)
Expand Down
5 changes: 4 additions & 1 deletion src/simulation/m_mpi_proxy.fpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ contains
#:endfor
#:for VAR in [ 'run_time_info','cyl_coord', 'adv_alphan', 'mpp_lim', &
& 'mapped_weno', 'mp_weno', 'rdma_mpi', 'weno_flat', 'riemann_flat', &
& 'mp_weno', 'rdma_mpi', 'weno_flat', 'riemann_flat', &
& 'weno_Re_flux', 'alt_soundspeed', 'null_weights', 'mixture_err', &
& 'parallel_io', 'hypoelasticity', 'bubbles', 'polytropic', &
& 'polydisperse', 'qbmm', 'monopole', 'probe_wrt', 'integral_wrt', &
Expand All @@ -215,6 +215,9 @@ contains
#:endfor
#:if not MFC_CASE_OPTIMIZATION
call MPI_BCAST(mapped_weno, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(wenoz, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(teno, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(weno_order, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(nb, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr)
call MPI_BCAST(num_fluids, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr)
Expand Down
4 changes: 2 additions & 2 deletions src/simulation/m_start_up.fpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ contains
model_eqns, adv_alphan, &
mpp_lim, time_stepper, weno_eps, weno_flat, &
riemann_flat, rdma_mpi, cu_tensor, &
mapped_weno, wenoz, teno, teno_CT, mp_weno, weno_avg, &
teno_CT, mp_weno, weno_avg, &
riemann_solver, wave_speeds, avg_state, &
bc_x, bc_y, bc_z, &
x_domain, y_domain, z_domain, &
Expand All @@ -146,7 +146,7 @@ contains
rhoref, pref, bubbles, bubble_model, &
R0ref, &
#:if not MFC_CASE_OPTIMIZATION
nb, weno_order, num_fluids, &
nb, mapped_weno, wenoz, teno, weno_order, num_fluids, &
#:endif
Ca, Web, Re_inv, &
monopole, mono, num_mono, &
Expand Down
71 changes: 33 additions & 38 deletions src/simulation/m_weno.fpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -604,22 +604,21 @@ contains
beta(1) = beta_coef_${XYZ}$ (j, 1, 0)*dvd(-1)*dvd(-1) &
+ weno_eps

if (mapped_weno) then
if (wenojs) then
alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta)
end if

if (mapped_weno) then
alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta)
omega = alpha/sum(alpha)
alpha = (d_cbL_${XYZ}$ (:, j)*(1d0 + d_cbL_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) &
*(omega/(d_cbL_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbL_${XYZ}$ (:, j))))
end if

elseif (wenoz) then

! Borges, et al. (2008)
if (wenoz) then
tau5 = abs(beta(1) - beta(0))
alpha = d_cbL_${XYZ}$ (:, j)*(1d0 + tau5/beta)

else ! WENO-JS by default

alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta)

end if

omega = alpha/sum(alpha)
Expand All @@ -633,21 +632,19 @@ contains
poly(1) = v_rs_ws_${XYZ}$ (j, k, l, i) &
+ poly_coef_cbR_${XYZ}$ (j, 1, 0)*dvd(-1)

if (mapped_weno) then
if (wenojs) then
alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta)
end if

if (mapped_weno) then
alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta)
omega = alpha/sum(alpha)
alpha = (d_cbR_${XYZ}$ (:, j)*(1d0 + d_cbR_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) &
*(omega/(d_cbR_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbR_${XYZ}$ (:, j))))
end if

elseif (wenoz) then

if (wenoz) then
alpha = d_cbR_${XYZ}$ (:, j)*(1d0 + tau5/beta)

else ! WENO-JS by default

alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta)

end if

omega = alpha/sum(alpha)
Expand All @@ -664,7 +661,7 @@ contains
elseif (weno_order == 5) then
#:for WENO_DIR, XYZ in [(1, 'x'), (2, 'y'), (3, 'z')]
if (weno_dir == ${WENO_DIR}$) then
!$acc parallel loop vector gang collapse(3) default(present) private(dvd, poly, beta, alpha, omega, delta, tau5)
!$acc parallel loop vector gang collapse(3) default(present) private(dvd, poly, beta, alpha, omega, tau5, delta)
do l = is3_weno%beg, is3_weno%end
do k = is2_weno%beg, is2_weno%end
do j = is1_weno%beg, is1_weno%end
Expand Down Expand Up @@ -704,31 +701,31 @@ contains
+ beta_coef_${XYZ}$ (j, 2, 2)*dvd(-2)*dvd(-2) &
+ weno_eps

if (mapped_weno) then
if (wenojs) then
alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta)
end if

if (mapped_weno) then
alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta)
omega = alpha/sum(alpha)
alpha = (d_cbL_${XYZ}$ (:, j)*(1d0 + d_cbL_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) &
*(omega/(d_cbL_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbL_${XYZ}$ (:, j))))
end if

elseif (wenoz) then ! Borges, et al. (2008)

! Borges, et al. (2008)
if (wenoz) then
tau5 = abs(beta(2) - beta(0)) ! Equation 25
alpha = d_cbL_${XYZ}$ (:, j)*(1d0 + tau5/beta) ! Equation 28 (note: weno_eps was already added to beta)
end if

elseif (teno) then ! Fu, et al. (2016)
! Fu's code: https://dx.doi.org/10.13140/RG.2.2.36250.34247
! Fu, et al. (2016)
! Fu's code: https://dx.doi.org/10.13140/RG.2.2.36250.34247
if (teno) then
tau5 = abs(beta(2) - beta(0))
alpha = (1d0 + tau5/beta)**6d0 ! Equation 22 (reuse alpha as gamma; pick C=1 & q=6)
omega = alpha/sum(alpha) ! Equation 25 (reuse omega as xi)
delta = merge(0d0, 1d0, omega < teno_CT) ! Equation 26
alpha = delta*d_cbL_${XYZ}$ (:, j) ! Equation 27
else ! WENO-JS by default
alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta)
end if
omega = alpha/sum(alpha)
Expand All @@ -747,25 +744,23 @@ contains
+ poly_coef_cbR_${XYZ}$ (j, 2, 0)*dvd(-1) &
+ poly_coef_cbR_${XYZ}$ (j, 2, 1)*dvd(-2)
if (mapped_weno) then
if (wenojs) then
alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta)
end if
if (mapped_weno) then
alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta)
omega = alpha/sum(alpha)
alpha = (d_cbR_${XYZ}$ (:, j)*(1d0 + d_cbR_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) &
*(omega/(d_cbR_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbR_${XYZ}$ (:, j))))
end if
elseif (wenoz) then
if (wenoz) then
alpha = d_cbR_${XYZ}$ (:, j)*(1d0 + tau5/beta)
end if
elseif (teno) then
if (teno) then
alpha = delta*d_cbR_${XYZ}$ (:, j)
else ! WENO-JS by default
alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta)
end if
omega = alpha/sum(alpha)
Expand Down
9 changes: 9 additions & 0 deletions toolchain/mfc/case.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,11 @@ def __get_sim_fpp(self, print: bool) -> str:
elif bubble_model == 3:
nterms = 7

mapped_weno = 1 if self.params.get("mapped_weno", 'F') == 'T' else 0
wenoz = 1 if self.params.get("wenoz", 'F') == 'T' else 0
teno = 1 if self.params.get("teno", 'F') == 'T' else 0
wenojs = 0 if (mapped_weno or wenoz or teno) else 1

return f"""\
#:set MFC_CASE_OPTIMIZATION = {ARG("case_optimization")}
#:set weno_order = {int(self.params["weno_order"])}
Expand All @@ -193,6 +198,10 @@ def __get_sim_fpp(self, print: bool) -> str:
#:set num_dims = {1 + min(int(self.params.get("n", 0)), 1) + min(int(self.params.get("p", 0)), 1)}
#:set nterms = {nterms}
#:set num_fluids = {int(self.params["num_fluids"])}
#:set wenojs = {wenojs}
#:set mapped_weno = {mapped_weno}
#:set wenoz = {wenoz}
#:set teno = {teno}
"""

return """\
Expand Down
2 changes: 1 addition & 1 deletion toolchain/mfc/run/case_dicts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ def analytic(self):
ALL.update(SIMULATION)
ALL.update(POST_PROCESS)

CASE_OPTIMIZATION = [ "nb", "weno_order", "num_fluids" ]
CASE_OPTIMIZATION = [ "mapped_weno", "wenoz", "teno", "nb", "weno_order", "num_fluids" ]

_properties = { k: v.value for k, v in ALL.items() }

Expand Down

0 comments on commit 0829f92

Please sign in to comment.