From 0829f924c366e1a1baace8c8dfffb62b3851eae5 Mon Sep 17 00:00:00 2001 From: ChrisZYJ Date: Fri, 21 Jun 2024 02:51:46 -0700 Subject: [PATCH] case optimization of wenoX --- src/simulation/m_global_parameters.fpp | 41 ++++++++++----- src/simulation/m_mpi_proxy.fpp | 5 +- src/simulation/m_start_up.fpp | 4 +- src/simulation/m_weno.fpp | 71 ++++++++++++-------------- toolchain/mfc/case.py | 9 ++++ toolchain/mfc/run/case_dicts.py | 2 +- 6 files changed, 78 insertions(+), 54 deletions(-) diff --git a/src/simulation/m_global_parameters.fpp b/src/simulation/m_global_parameters.fpp index 09897a51c4..f541b8dcea 100644 --- a/src/simulation/m_global_parameters.fpp +++ b/src/simulation/m_global_parameters.fpp @@ -116,20 +116,25 @@ module m_global_parameters logical :: prim_vars_wrt #:if MFC_CASE_OPTIMIZATION - integer, parameter :: weno_polyn = ${weno_polyn}$ !< Degree of the WENO polynomials (polyn) - integer, parameter :: weno_order = ${weno_order}$ !< Order of the WENO reconstruction - integer, parameter :: num_fluids = ${num_fluids}$ !< number of fluids in the simulation + integer, parameter :: weno_polyn = ${weno_polyn}$ !< Degree of the WENO polynomials (polyn) + integer, parameter :: weno_order = ${weno_order}$ !< Order of the WENO reconstruction + integer, parameter :: num_fluids = ${num_fluids}$ !< number of fluids in the simulation + logical, parameter :: wenojs = ${wenojs}$ !< WENO-JS (default) + logical, parameter :: mapped_weno = ${mapped_weno}$ !< WENO-M (WENO with mapping of nonlinear weights) + logical, parameter :: wenoz = ${wenoz}$ !< WENO-Z + logical, parameter :: teno = ${teno}$ !< TENO (Targeted ENO) #:else integer :: weno_polyn !< Degree of the WENO polynomials (polyn) integer :: weno_order !< Order of the WENO reconstruction integer :: num_fluids !< number of fluids in the simulation + logical :: wenojs !< WENO-JS (default) + logical :: mapped_weno !< WENO-M (WENO with mapping of nonlinear weights) + logical :: wenoz !< WENO-Z + logical :: teno !< TENO (Targeted ENO) #:endif real(kind(0d0)) :: weno_eps !< Binding for the WENO nonlinear weights real(kind(0d0)) :: teno_CT !< Smoothness threshold for TENO - logical :: mapped_weno !< WENO-M (WENO with mapping of nonlinear weights) - logical :: wenoz !< WENO-Z - logical :: teno !< TENO (Targeted ENO) logical :: mp_weno !< Monotonicity preserving (MP) WENO logical :: weno_avg ! Average left/right cell-boundary states logical :: weno_Re_flux !< WENO reconstruct velocity gradients for viscous stress tensor @@ -156,10 +161,10 @@ module m_global_parameters integer :: cpu_start, cpu_end, cpu_rate #:if not MFC_CASE_OPTIMIZATION - !$acc declare create(num_dims, weno_polyn, weno_order, num_fluids) + !$acc declare create(num_dims, weno_polyn, weno_order, num_fluids, wenojs, mapped_weno, wenoz, teno) #:endif - !$acc declare create(mpp_lim, model_eqns, mixture_err, alt_soundspeed, avg_state, mapped_weno, wenoz, teno, mp_weno, weno_eps, teno_CT, hypoelasticity) + !$acc declare create(mpp_lim, model_eqns, mixture_err, alt_soundspeed, avg_state, mp_weno, weno_eps, teno_CT, hypoelasticity) logical :: relax !< activate phase change integer :: relax_model !< Relaxation model @@ -491,9 +496,6 @@ contains time_stepper = dflt_int weno_eps = dflt_real teno_CT = 1d-6 - mapped_weno = .false. - wenoz = .false. - teno = .false. mp_weno = .false. weno_avg = .false. weno_Re_flux = .false. @@ -515,6 +517,12 @@ contains riemann_flat = .true. rdma_mpi = .false. + #:if not MFC_CASE_OPTIMIZATION + mapped_weno = .false. + wenoz = .false. + teno = .false. + #:endif + bc_x%beg = dflt_int; bc_x%end = dflt_int bc_y%beg = dflt_int; bc_y%end = dflt_int bc_z%beg = dflt_int; bc_z%end = dflt_int @@ -957,6 +965,11 @@ contains wa_flg = 0d0; if (weno_avg) wa_flg = 1d0 !$acc update device(wa_flg) + ! Resort to default WENO-JS if no other WENO scheme is selected + #:if not MFC_CASE_OPTIMIZATION + wenojs = .not. (mapped_weno .or. wenoz .or. teno) + #:endif + if (ib) allocate (MPI_IO_IB_DATA%var%sf(0:m, 0:n, 0:p)) Np = 0 @@ -1030,7 +1043,11 @@ contains !$acc update device(m, n, p) !$acc update device(alt_soundspeed, monopole, num_mono) - !$acc update device(dt, sys_size, buff_size, pref, rhoref, gamma_idx, pi_inf_idx, E_idx, alf_idx, stress_idx, mpp_lim, bubbles, hypoelasticity, alt_soundspeed, avg_state, num_fluids, model_eqns, num_dims, mixture_err, grid_geometry, cyl_coord, mapped_weno, wenoz, teno, mp_weno, weno_eps, teno_CT) + !$acc update device(dt, sys_size, buff_size, pref, rhoref, gamma_idx, pi_inf_idx, E_idx, alf_idx, stress_idx, mpp_lim, bubbles, hypoelasticity, alt_soundspeed, avg_state, num_fluids, model_eqns, num_dims, mixture_err, grid_geometry, cyl_coord, mp_weno, weno_eps, teno_CT) + + #:if not MFC_CASE_OPTIMIZATION + !$acc update device(wenojs, mapped_weno, wenoz, teno) + #:endif !$acc enter data copyin(nb, R0ref, Ca, Web, Re_inv, weight, R0, V0, bubbles, polytropic, polydisperse, qbmm, R0_type, ptil, bubble_model, thermal, poly_sigma) !$acc enter data copyin(R_n, R_v, phi_vn, phi_nv, Pe_c, Tw, pv, M_n, M_v, k_n, k_v, pb0, mass_n0, mass_v0, Pe_T, Re_trans_T, Re_trans_c, Im_trans_T, Im_trans_c, omegaN , mul0, ss, gamma_v, mu_v, gamma_m, gamma_n, mu_n, gam) diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index 403adf1c0f..847743b1ad 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -195,7 +195,7 @@ contains #:endfor #:for VAR in [ 'run_time_info','cyl_coord', 'adv_alphan', 'mpp_lim', & - & 'mapped_weno', 'mp_weno', 'rdma_mpi', 'weno_flat', 'riemann_flat', & + & 'mp_weno', 'rdma_mpi', 'weno_flat', 'riemann_flat', & & 'weno_Re_flux', 'alt_soundspeed', 'null_weights', 'mixture_err', & & 'parallel_io', 'hypoelasticity', 'bubbles', 'polytropic', & & 'polydisperse', 'qbmm', 'monopole', 'probe_wrt', 'integral_wrt', & @@ -215,6 +215,9 @@ contains #:endfor #:if not MFC_CASE_OPTIMIZATION + call MPI_BCAST(mapped_weno, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(wenoz, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) + call MPI_BCAST(teno, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(weno_order, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(nb, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) call MPI_BCAST(num_fluids, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr) diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 150433cb9e..582b7231ae 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -133,7 +133,7 @@ contains model_eqns, adv_alphan, & mpp_lim, time_stepper, weno_eps, weno_flat, & riemann_flat, rdma_mpi, cu_tensor, & - mapped_weno, wenoz, teno, teno_CT, mp_weno, weno_avg, & + teno_CT, mp_weno, weno_avg, & riemann_solver, wave_speeds, avg_state, & bc_x, bc_y, bc_z, & x_domain, y_domain, z_domain, & @@ -146,7 +146,7 @@ contains rhoref, pref, bubbles, bubble_model, & R0ref, & #:if not MFC_CASE_OPTIMIZATION - nb, weno_order, num_fluids, & + nb, mapped_weno, wenoz, teno, weno_order, num_fluids, & #:endif Ca, Web, Re_inv, & monopole, mono, num_mono, & diff --git a/src/simulation/m_weno.fpp b/src/simulation/m_weno.fpp index 95333fd218..5d6fd814f2 100644 --- a/src/simulation/m_weno.fpp +++ b/src/simulation/m_weno.fpp @@ -604,22 +604,21 @@ contains beta(1) = beta_coef_${XYZ}$ (j, 1, 0)*dvd(-1)*dvd(-1) & + weno_eps - if (mapped_weno) then + if (wenojs) then + alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta) + end if + if (mapped_weno) then alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta) omega = alpha/sum(alpha) alpha = (d_cbL_${XYZ}$ (:, j)*(1d0 + d_cbL_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) & *(omega/(d_cbL_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbL_${XYZ}$ (:, j)))) + end if - elseif (wenoz) then - + ! Borges, et al. (2008) + if (wenoz) then tau5 = abs(beta(1) - beta(0)) alpha = d_cbL_${XYZ}$ (:, j)*(1d0 + tau5/beta) - - else ! WENO-JS by default - - alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta) - end if omega = alpha/sum(alpha) @@ -633,21 +632,19 @@ contains poly(1) = v_rs_ws_${XYZ}$ (j, k, l, i) & + poly_coef_cbR_${XYZ}$ (j, 1, 0)*dvd(-1) - if (mapped_weno) then + if (wenojs) then + alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta) + end if + if (mapped_weno) then alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta) omega = alpha/sum(alpha) alpha = (d_cbR_${XYZ}$ (:, j)*(1d0 + d_cbR_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) & *(omega/(d_cbR_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbR_${XYZ}$ (:, j)))) + end if - elseif (wenoz) then - + if (wenoz) then alpha = d_cbR_${XYZ}$ (:, j)*(1d0 + tau5/beta) - - else ! WENO-JS by default - - alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta) - end if omega = alpha/sum(alpha) @@ -664,7 +661,7 @@ contains elseif (weno_order == 5) then #:for WENO_DIR, XYZ in [(1, 'x'), (2, 'y'), (3, 'z')] if (weno_dir == ${WENO_DIR}$) then - !$acc parallel loop vector gang collapse(3) default(present) private(dvd, poly, beta, alpha, omega, delta, tau5) + !$acc parallel loop vector gang collapse(3) default(present) private(dvd, poly, beta, alpha, omega, tau5, delta) do l = is3_weno%beg, is3_weno%end do k = is2_weno%beg, is2_weno%end do j = is1_weno%beg, is1_weno%end @@ -704,31 +701,31 @@ contains + beta_coef_${XYZ}$ (j, 2, 2)*dvd(-2)*dvd(-2) & + weno_eps - if (mapped_weno) then + if (wenojs) then + alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta) + end if + if (mapped_weno) then alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta) omega = alpha/sum(alpha) alpha = (d_cbL_${XYZ}$ (:, j)*(1d0 + d_cbL_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) & *(omega/(d_cbL_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbL_${XYZ}$ (:, j)))) + end if - elseif (wenoz) then ! Borges, et al. (2008) - + ! Borges, et al. (2008) + if (wenoz) then tau5 = abs(beta(2) - beta(0)) ! Equation 25 alpha = d_cbL_${XYZ}$ (:, j)*(1d0 + tau5/beta) ! Equation 28 (note: weno_eps was already added to beta) + end if - elseif (teno) then ! Fu, et al. (2016) - ! Fu's code: https://dx.doi.org/10.13140/RG.2.2.36250.34247 - + ! Fu, et al. (2016) + ! Fu's code: https://dx.doi.org/10.13140/RG.2.2.36250.34247 + if (teno) then tau5 = abs(beta(2) - beta(0)) alpha = (1d0 + tau5/beta)**6d0 ! Equation 22 (reuse alpha as gamma; pick C=1 & q=6) omega = alpha/sum(alpha) ! Equation 25 (reuse omega as xi) delta = merge(0d0, 1d0, omega < teno_CT) ! Equation 26 alpha = delta*d_cbL_${XYZ}$ (:, j) ! Equation 27 - - else ! WENO-JS by default - - alpha = d_cbL_${XYZ}$ (:, j)/(beta*beta) - end if omega = alpha/sum(alpha) @@ -747,25 +744,23 @@ contains + poly_coef_cbR_${XYZ}$ (j, 2, 0)*dvd(-1) & + poly_coef_cbR_${XYZ}$ (j, 2, 1)*dvd(-2) - if (mapped_weno) then + if (wenojs) then + alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta) + end if + if (mapped_weno) then alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta) omega = alpha/sum(alpha) alpha = (d_cbR_${XYZ}$ (:, j)*(1d0 + d_cbR_${XYZ}$ (:, j) - 3d0*omega) + omega**2d0) & *(omega/(d_cbR_${XYZ}$ (:, j)**2d0 + omega*(1d0 - 2d0*d_cbR_${XYZ}$ (:, j)))) + end if - elseif (wenoz) then - + if (wenoz) then alpha = d_cbR_${XYZ}$ (:, j)*(1d0 + tau5/beta) + end if - elseif (teno) then - + if (teno) then alpha = delta*d_cbR_${XYZ}$ (:, j) - - else ! WENO-JS by default - - alpha = d_cbR_${XYZ}$ (:, j)/(beta*beta) - end if omega = alpha/sum(alpha) diff --git a/toolchain/mfc/case.py b/toolchain/mfc/case.py index 3117af577f..2d05be3e08 100644 --- a/toolchain/mfc/case.py +++ b/toolchain/mfc/case.py @@ -185,6 +185,11 @@ def __get_sim_fpp(self, print: bool) -> str: elif bubble_model == 3: nterms = 7 + mapped_weno = 1 if self.params.get("mapped_weno", 'F') == 'T' else 0 + wenoz = 1 if self.params.get("wenoz", 'F') == 'T' else 0 + teno = 1 if self.params.get("teno", 'F') == 'T' else 0 + wenojs = 0 if (mapped_weno or wenoz or teno) else 1 + return f"""\ #:set MFC_CASE_OPTIMIZATION = {ARG("case_optimization")} #:set weno_order = {int(self.params["weno_order"])} @@ -193,6 +198,10 @@ def __get_sim_fpp(self, print: bool) -> str: #:set num_dims = {1 + min(int(self.params.get("n", 0)), 1) + min(int(self.params.get("p", 0)), 1)} #:set nterms = {nterms} #:set num_fluids = {int(self.params["num_fluids"])} +#:set wenojs = {wenojs} +#:set mapped_weno = {mapped_weno} +#:set wenoz = {wenoz} +#:set teno = {teno} """ return """\ diff --git a/toolchain/mfc/run/case_dicts.py b/toolchain/mfc/run/case_dicts.py index cb8956da7d..542eb7767b 100644 --- a/toolchain/mfc/run/case_dicts.py +++ b/toolchain/mfc/run/case_dicts.py @@ -340,7 +340,7 @@ def analytic(self): ALL.update(SIMULATION) ALL.update(POST_PROCESS) -CASE_OPTIMIZATION = [ "nb", "weno_order", "num_fluids" ] +CASE_OPTIMIZATION = [ "mapped_weno", "wenoz", "teno", "nb", "weno_order", "num_fluids" ] _properties = { k: v.value for k, v in ALL.items() }