From a2ae832e034fed1813682600aa9f0a70c3ad63b1 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Tue, 30 Aug 2022 07:48:54 -0700 Subject: [PATCH 01/33] cell center BTD functors for RZ with openpmd --- Source/Diagnostics/BTDiagnostics.cpp | 102 +++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 90b00a24a08..8b1a79ce2b7 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -523,6 +523,12 @@ BTDiagnostics::InitializeFieldFunctors (int lev) nvars, m_num_buffers, m_varnames); } + // For RZ, initialize field functors RZ for openpmd + // This is a specialized call for intializing cell-center functors + // such that, all modes of a field component are stored contiguously + // For examply, Er0, Er1_real, Er1_imag, etc + InitializeFieldFunctorsRZopenPMD(lev); + // Define all cell-centered functors required to compute cell-centere data // Fill vector of cell-center functors for all field-components, namely, // Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, and rho are included in the @@ -695,6 +701,102 @@ BTDiagnostics::AddRZModesToOutputNames (const std::string& field, const int ncom #endif } +void +BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) +{ +#ifdef WARPX_DIM_RZ + auto & warpx = WarpX::GetInstance(); + int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); + int ncomp = ncomp_multimodefab; + // This function may be called multiple times, for different values of `lev` + // but the `varnames` need only be updated once. + bool update_cellcenter_varnames = (lev == 0); + if (update_cellcenter_varnames) { + m_cellcenter_varnames.clear(); + const int n_rz = ncomp * m_cellcenter_varnames.size(); + m_cellcenter_varnames.reserve(n_rz); + } + + // Reset field functors + m_cell_center_functors[lev].clear(); + m_cell_center_functors[lev].resize(m_cellcenter_varnames_fields.size()); + + for (int comp=0, n=m_cell_center_functors.at(lev).size(); comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("Er"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("Et"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("Ez"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("Br"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("Bt"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("Bz"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("jr"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("jt"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ + m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("jz"), ncomp, true); + } + } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ + m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, -1, false, ncomp); + if (update_cellcenter_varnames) { + AddRZModesToOutputNames(std::string("rho"), ncomp, true); + } + } + } + +#endif + amrex::ignore_unused(lev); +} + +void +BTDiagnostics::AddRZModesToOutputNames (const std::string& field, const int ncomp, bool cellcenter_data) +{ +#ifdef WARPX_DIM_RZ + // In cylindrical geometry, real and imag part of each mode are also + // dumped to file separately, so they need to be added to m_varnames + if (cellcenter_data) { + m_cellcenter_varnames.push_back( field + "_0_real" ); + for (int ic=1 ; ic < (ncomp+1)/2 ; ic += 1) { + m_cellcenter_varnames.push_back( field + "_" + std::to_string(ic) + "_real" ); + m_cellcenter_varnames.push_back( field + "_" + std::to_string(ic) + "_imag" ); + } + } +#else + amrex::ignore_unused(field, ncomp, cellcenter_data); +#endif +} + void BTDiagnostics::PrepareBufferData () { From 11b6dd71b53cfd1e11e09945f43a25a392d01891 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 19 Sep 2022 10:29:38 -0700 Subject: [PATCH 02/33] add RZ modes to output varnames too --- Source/Diagnostics/BTDiagnostics.cpp | 55 +++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 8b1a79ce2b7..e8b92d43e53 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -482,6 +482,7 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) amrex::DistributionMapping dmap = warpx.DistributionMap(lev); int ngrow = 1; int ncomps = static_cast(m_cellcenter_varnames.size()); + amrex::Print() << " ncomps for CC mf: " << ncomps << "\n"; m_cell_centered_data[lev] = std::make_unique(ba, dmap, ncomps, ngrow); } @@ -523,11 +524,6 @@ BTDiagnostics::InitializeFieldFunctors (int lev) nvars, m_num_buffers, m_varnames); } - // For RZ, initialize field functors RZ for openpmd - // This is a specialized call for intializing cell-center functors - // such that, all modes of a field component are stored contiguously - // For examply, Er0, Er1_real, Er1_imag, etc - InitializeFieldFunctorsRZopenPMD(lev); // Define all cell-centered functors required to compute cell-centere data // Fill vector of cell-center functors for all field-components, namely, @@ -708,6 +704,47 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) auto & warpx = WarpX::GetInstance(); int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); int ncomp = ncomp_multimodefab; + + + bool update_varnames = (lev==0); + if (update_varnames) { + m_varnames.clear(); + const int n_rz = ncomp * m_varnames.size(); + m_varnames.reserve(n_rz); + } + // AddRZ modes to output names for the back-transformed data + if (update_varnames) { + for (int comp=0, n=m_varnames.size(); comp(m_varnames.size()); + m_all_field_functors[lev][i] = std::make_unique( + m_cell_centered_data[lev].get(), lev, + nvars, m_num_buffers, m_varnames); + } + // This function may be called multiple times, for different values of `lev` // but the `varnames` need only be updated once. bool update_cellcenter_varnames = (lev == 0); @@ -717,7 +754,7 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) m_cellcenter_varnames.reserve(n_rz); } - // Reset field functors + // Reset field functors for cell-center multifab m_cell_center_functors[lev].clear(); m_cell_center_functors[lev].resize(m_cellcenter_varnames_fields.size()); @@ -791,6 +828,12 @@ BTDiagnostics::AddRZModesToOutputNames (const std::string& field, const int ncom m_cellcenter_varnames.push_back( field + "_" + std::to_string(ic) + "_real" ); m_cellcenter_varnames.push_back( field + "_" + std::to_string(ic) + "_imag" ); } + } else { + m_varnames.push_back(field + "_0_real"); + for (int ic=1 ; ic < (ncomp+1)/2 ; ic += 1) { + m_varnames.push_back( field + "_" + std::to_string(ic) + "_real" ); + m_varnames.push_back( field + "_" + std::to_string(ic) + "_imag" ); + } } #else amrex::ignore_unused(field, ncomp, cellcenter_data); From 4bd1e384a9ba9065aa69213ca58c4b171c097f3a Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 19 Sep 2022 17:56:43 -0700 Subject: [PATCH 03/33] update varnames once and set map for RZ fields in BTfunctor --- Source/Diagnostics/BTDiagnostics.cpp | 146 +-------------------------- 1 file changed, 2 insertions(+), 144 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index e8b92d43e53..3ddd2b13ba5 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -237,10 +237,12 @@ BTDiagnostics::ReadParameters () if(m_max_box_size < m_buffer_size) m_max_box_size = m_buffer_size; } #ifdef WARPX_DIM_RZ + pp_diag_name.query("dump_rz_modes", m_dump_rz_modes); amrex::Vector< std::string > BTD_varnames_supported = {"Er", "Et", "Ez", "Br", "Bt", "Bz", "jr", "jt", "jz", "rho"}; #else + amrex::ignore_unused(m_dump_rz_modes); amrex::Vector< std::string > BTD_varnames_supported = {"Ex", "Ey", "Ez", "Bx", "By", "Bz", "jx", "jy", "jz", "rho"}; @@ -482,7 +484,6 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) amrex::DistributionMapping dmap = warpx.DistributionMap(lev); int ngrow = 1; int ncomps = static_cast(m_cellcenter_varnames.size()); - amrex::Print() << " ncomps for CC mf: " << ncomps << "\n"; m_cell_centered_data[lev] = std::make_unique(ba, dmap, ncomps, ngrow); } @@ -697,149 +698,6 @@ BTDiagnostics::AddRZModesToOutputNames (const std::string& field, const int ncom #endif } -void -BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) -{ -#ifdef WARPX_DIM_RZ - auto & warpx = WarpX::GetInstance(); - int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); - int ncomp = ncomp_multimodefab; - - - bool update_varnames = (lev==0); - if (update_varnames) { - m_varnames.clear(); - const int n_rz = ncomp * m_varnames.size(); - m_varnames.reserve(n_rz); - } - // AddRZ modes to output names for the back-transformed data - if (update_varnames) { - for (int comp=0, n=m_varnames.size(); comp(m_varnames.size()); - m_all_field_functors[lev][i] = std::make_unique( - m_cell_centered_data[lev].get(), lev, - nvars, m_num_buffers, m_varnames); - } - - // This function may be called multiple times, for different values of `lev` - // but the `varnames` need only be updated once. - bool update_cellcenter_varnames = (lev == 0); - if (update_cellcenter_varnames) { - m_cellcenter_varnames.clear(); - const int n_rz = ncomp * m_cellcenter_varnames.size(); - m_cellcenter_varnames.reserve(n_rz); - } - - // Reset field functors for cell-center multifab - m_cell_center_functors[lev].clear(); - m_cell_center_functors[lev].resize(m_cellcenter_varnames_fields.size()); - - for (int comp=0, n=m_cell_center_functors.at(lev).size(); comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("Er"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("Et"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("Ez"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("Br"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("Bt"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("Bz"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("jr"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("jt"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("jz"), ncomp, true); - } - } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ - m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, -1, false, ncomp); - if (update_cellcenter_varnames) { - AddRZModesToOutputNames(std::string("rho"), ncomp, true); - } - } - } - -#endif - amrex::ignore_unused(lev); -} - -void -BTDiagnostics::AddRZModesToOutputNames (const std::string& field, const int ncomp, bool cellcenter_data) -{ -#ifdef WARPX_DIM_RZ - // In cylindrical geometry, real and imag part of each mode are also - // dumped to file separately, so they need to be added to m_varnames - if (cellcenter_data) { - m_cellcenter_varnames.push_back( field + "_0_real" ); - for (int ic=1 ; ic < (ncomp+1)/2 ; ic += 1) { - m_cellcenter_varnames.push_back( field + "_" + std::to_string(ic) + "_real" ); - m_cellcenter_varnames.push_back( field + "_" + std::to_string(ic) + "_imag" ); - } - } else { - m_varnames.push_back(field + "_0_real"); - for (int ic=1 ; ic < (ncomp+1)/2 ; ic += 1) { - m_varnames.push_back( field + "_" + std::to_string(ic) + "_real" ); - m_varnames.push_back( field + "_" + std::to_string(ic) + "_imag" ); - } - } -#else - amrex::ignore_unused(field, ncomp, cellcenter_data); -#endif -} - void BTDiagnostics::PrepareBufferData () { From 9fffba7e661b92099bc155af2cdd365582bae87e Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 24 Oct 2022 13:27:12 -0700 Subject: [PATCH 04/33] clean commented line --- Source/Diagnostics/BTDiagnostics.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 3ddd2b13ba5..c2338c84e48 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -270,6 +270,7 @@ bool BTDiagnostics::DoDump (int step, int i_buffer, bool force_flush) { // timestep < 0, i.e., at initialization time when step == -1 + amrex::Print() << " buffer " << i_buffer << " full ? " << buffer_full(i_buffer) << " snapshot full " << m_snapshot_full[i_buffer] << " empty ? " << buffer_empty(i_buffer) << "\n"; if (step < 0 ) return false; // Do not call dump if the snapshot is already full and the files are closed. @@ -954,6 +955,7 @@ BTDiagnostics::GetZSliceInDomainFlag (const int i_buffer, const int lev) void BTDiagnostics::Flush (int i_buffer) { + amrex::Print() << " in flush buffer " << i_buffer << "\n"; auto & warpx = WarpX::GetInstance(); std::string file_name = m_file_prefix; if (m_format=="plotfile") { @@ -1013,6 +1015,7 @@ BTDiagnostics::Flush (int i_buffer) } } } + amrex::Print() << " writing data " << i_buffer <<"\n"; m_flush_format->WriteToFile( m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), labtime, m_output_species[i_buffer], nlev_output, file_name, m_file_min_digits, From 96c4f92dd94fbea54d4cfc6b807cb7a957ba3376 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Fri, 28 Oct 2022 09:44:42 -0700 Subject: [PATCH 05/33] Apply suggestions from code review From Axels' review Co-authored-by: Axel Huebl --- Source/Diagnostics/BTDiagnostics.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index c2338c84e48..e79cb1206d5 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -270,7 +270,7 @@ bool BTDiagnostics::DoDump (int step, int i_buffer, bool force_flush) { // timestep < 0, i.e., at initialization time when step == -1 - amrex::Print() << " buffer " << i_buffer << " full ? " << buffer_full(i_buffer) << " snapshot full " << m_snapshot_full[i_buffer] << " empty ? " << buffer_empty(i_buffer) << "\n"; + // amrex::AllPrint() << " buffer " << i_buffer << " full ? " << buffer_full(i_buffer) << " snapshot full " << m_snapshot_full[i_buffer] << " empty ? " << buffer_empty(i_buffer) << "\n"; if (step < 0 ) return false; // Do not call dump if the snapshot is already full and the files are closed. @@ -526,7 +526,6 @@ BTDiagnostics::InitializeFieldFunctors (int lev) nvars, m_num_buffers, m_varnames); } - // Define all cell-centered functors required to compute cell-centere data // Fill vector of cell-center functors for all field-components, namely, // Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, and rho are included in the @@ -955,7 +954,7 @@ BTDiagnostics::GetZSliceInDomainFlag (const int i_buffer, const int lev) void BTDiagnostics::Flush (int i_buffer) { - amrex::Print() << " in flush buffer " << i_buffer << "\n"; + // amrex::AllPrint() << " in flush buffer " << i_buffer << "\n"; auto & warpx = WarpX::GetInstance(); std::string file_name = m_file_prefix; if (m_format=="plotfile") { @@ -1015,7 +1014,7 @@ BTDiagnostics::Flush (int i_buffer) } } } - amrex::Print() << " writing data " << i_buffer <<"\n"; + // amrex::AllPrint() << " writing data " << i_buffer <<"\n"; m_flush_format->WriteToFile( m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), labtime, m_output_species[i_buffer], nlev_output, file_name, m_file_min_digits, From bb9329ed94d58c32ec0ff9c8c07089c77c6f529b Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Fri, 28 Oct 2022 10:53:11 -0700 Subject: [PATCH 06/33] adding comments, doxygen, and clean-up --- Source/Diagnostics/BTDiagnostics.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index e79cb1206d5..90b00a24a08 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -237,12 +237,10 @@ BTDiagnostics::ReadParameters () if(m_max_box_size < m_buffer_size) m_max_box_size = m_buffer_size; } #ifdef WARPX_DIM_RZ - pp_diag_name.query("dump_rz_modes", m_dump_rz_modes); amrex::Vector< std::string > BTD_varnames_supported = {"Er", "Et", "Ez", "Br", "Bt", "Bz", "jr", "jt", "jz", "rho"}; #else - amrex::ignore_unused(m_dump_rz_modes); amrex::Vector< std::string > BTD_varnames_supported = {"Ex", "Ey", "Ez", "Bx", "By", "Bz", "jx", "jy", "jz", "rho"}; @@ -270,7 +268,6 @@ bool BTDiagnostics::DoDump (int step, int i_buffer, bool force_flush) { // timestep < 0, i.e., at initialization time when step == -1 - // amrex::AllPrint() << " buffer " << i_buffer << " full ? " << buffer_full(i_buffer) << " snapshot full " << m_snapshot_full[i_buffer] << " empty ? " << buffer_empty(i_buffer) << "\n"; if (step < 0 ) return false; // Do not call dump if the snapshot is already full and the files are closed. @@ -954,7 +951,6 @@ BTDiagnostics::GetZSliceInDomainFlag (const int i_buffer, const int lev) void BTDiagnostics::Flush (int i_buffer) { - // amrex::AllPrint() << " in flush buffer " << i_buffer << "\n"; auto & warpx = WarpX::GetInstance(); std::string file_name = m_file_prefix; if (m_format=="plotfile") { @@ -1014,7 +1010,6 @@ BTDiagnostics::Flush (int i_buffer) } } } - // amrex::AllPrint() << " writing data " << i_buffer <<"\n"; m_flush_format->WriteToFile( m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), labtime, m_output_species[i_buffer], nlev_output, file_name, m_file_min_digits, From c8db53a480a6328ab6ce3fb8d1e473821ff32961 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Tue, 25 Oct 2022 13:19:20 -0700 Subject: [PATCH 07/33] adding mulitple modes to RZ BTD --- Source/Diagnostics/BTDiagnostics.cpp | 10 ++- .../BackTransformFunctor.H | 3 + .../BackTransformFunctor.cpp | 71 ++++++++++++++++++- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 90b00a24a08..150257c8aec 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -198,7 +198,6 @@ BTDiagnostics::ReadParameters () m_crse_ratio == amrex::IntVect(1), "Only support for coarsening ratio of 1 in all directions is included for BTD\n" ); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(WarpX::n_rz_azimuthal_modes==1, "Currently only one mode is supported for BTD"); // Read list of back-transform diag parameters requested by the user // amrex::ParmParse pp_diag_name(m_diag_name); @@ -481,7 +480,11 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) ba.coarsen(m_crse_ratio); amrex::DistributionMapping dmap = warpx.DistributionMap(lev); int ngrow = 1; +#ifdef WARPX_DIM_RZ + int ncomps = (2*WarpX::n_rz_azimuthal_modes - 1) * static_cast(m_cellcenter_varnames.size()); +#else int ncomps = static_cast(m_cellcenter_varnames.size()); +#endif m_cell_centered_data[lev] = std::make_unique(ba, dmap, ncomps, ngrow); } @@ -520,7 +523,7 @@ BTDiagnostics::InitializeFieldFunctors (int lev) int nvars = static_cast(m_varnames.size()); m_all_field_functors[lev][i] = std::make_unique( m_cell_centered_data[lev].get(), lev, - nvars, m_num_buffers, m_varnames); + nvars, m_num_buffers, m_varnames, m_varnames_fields); } // Define all cell-centered functors required to compute cell-centere data @@ -632,7 +635,8 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) int nvars = static_cast(m_varnames.size()); m_all_field_functors[lev][i] = std::make_unique( m_cell_centered_data[lev].get(), lev, - nvars, m_num_buffers, m_varnames); + nvars, m_num_buffers, m_varnames, + m_varnames_fields); } // Reset field functors for cell-center multifab diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H index 35c22908cb3..142bb098da8 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.H @@ -47,6 +47,7 @@ public: BackTransformFunctor ( const amrex::MultiFab * const mf_src, const int lev, const int ncomp, const int num_buffers, amrex::Vector< std::string > varnames, + amrex::Vector< std::string > varnames_fields, const amrex::IntVect crse_ratio= amrex::IntVect(1)); /** \brief Lorentz-transform mf_src for the ith buffer and write the result in mf_dst. @@ -118,6 +119,8 @@ private: amrex::Vector m_k_index_zlab; /** Vector of user-defined field names to be stored in the output multifab */ amrex::Vector< std::string > m_varnames; + /** Vector of user-defined field names without modifications for rz modes */ + amrex::Vector< std::string > m_varnames_fields; /** max grid size used to generate BoxArray to define output MultiFabs */ int m_max_box_size = 256; diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index 10a9ff12a7c..9cf6b48ee53 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -34,8 +34,10 @@ using namespace amrex; BackTransformFunctor::BackTransformFunctor (amrex::MultiFab const * mf_src, int lev, const int ncomp, const int num_buffers, amrex::Vector< std::string > varnames, - const amrex::IntVect crse_ratio) - : ComputeDiagFunctor(ncomp, crse_ratio), m_mf_src(mf_src), m_lev(lev), m_num_buffers(num_buffers), m_varnames(varnames) + amrex::Vector< std::string > varnames_fields, + const amrex::IntVect crse_ratio + ) + : ComputeDiagFunctor(ncomp, crse_ratio), m_mf_src(mf_src), m_lev(lev), m_num_buffers(num_buffers), m_varnames(varnames), m_varnames_fields(varnames_fields) { InitData(); } @@ -112,14 +114,20 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const const Box& tbx = mfi.tilebox(); amrex::Array4 src_arr = tmp[mfi].array(); amrex::Array4 dst_arr = mf_dst[mfi].array(); +#ifdef WARPX_DIM_RZ + const int nrz = WarpX::n_rz_azimuthal_modes; +#endif amrex::ParallelFor( tbx, ncomp_dst, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) { const int icomp = field_map_ptr[n]; #if defined(WARPX_DIM_3D) dst_arr(i, j, k_lab, n) = src_arr(i, j, k, icomp); -#else +#elif defined(WARPX_DIM_2D) dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp); +#elif defined(WARPX_DIM_RZ) + const int rzcomp = n - static_cast( floor(n/(2.0*nrz-1.0)) ) * (2*nrz-1); + dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp*(2*nrz-1)+rzcomp); #endif } ); } @@ -185,7 +193,12 @@ BackTransformFunctor::InitData () for (int i = 0; i < m_varnames.size(); ++i) { +#ifdef WARPX_DIM_RZ + const int field_id = i / (WarpX::n_rz_azimuthal_modes*2-1); + m_map_varnames[i] = m_possible_fields_to_dump[ m_varnames_fields[field_id] ]; +#else m_map_varnames[i] = m_possible_fields_to_dump[ m_varnames[i] ] ; +#endif } } @@ -202,6 +215,57 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm amrex::Array4< amrex::Real > arr = data[mfi].array(); amrex::Real clight = PhysConst::c; amrex::Real inv_clight = 1.0_rt/clight; +#ifdef WARPX_DIM_RZ + const int n_rz = WarpX::n_rz_azimuthal_modes; + amrex::ParallelFor( tbx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + // arr(x,y,z,comp) has (2*num_rz_modes-1)*10 components + // If there are 2 components, then Er_0_real, Er_1_real, Er_2_real are the first the components + // followed by three components each for Et, Ez, Br, Bt, Bz, jr, jt, jz, and rho + for (int imode = 0; imode < n_rz; ++imode) { + int offset = 1; + int ncomp = 2; + if (imode == 0) { + offset = 0; + ncomp = 1; + } + for (int icomp = 0; icomp < ncomp; ++icomp) { + // Back-transform the transverse electric and magnetic fields. + // Note that the z-components, Ez, Bz, are not changed by the transform. + amrex::Real e_lab, b_lab, j_lab, rho_lab; + // Transform Ex_boost (ncomp=0) & By_boost (ncomp=4) to lab-frame + e_lab = gamma_boost * ( arr(i, j, k, n_rz*0 + 2*imode + offset*(icomp-1)) + + beta_boost * clight * arr(i, j, k, n_rz*4+ 2*imode + offset*(icomp-1)) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rz*4 + 2*imode + offset*(icomp-1)) + + beta_boost * inv_clight * arr(i, j, k, n_rz*0 + 2*imode + offset*(icomp-1)) ); + // Store lab-frame data in-place + arr(i, j, k, n_rz*0 + 2*imode + offset*(icomp-1)) = e_lab; + arr(i, j, k, n_rz*4 + 2*imode + offset*(icomp-1)) = b_lab; + + // Transform Ey_boost (ncomp=1) & Bx_boost (ncomp=3) to lab-frame + e_lab = gamma_boost * ( arr(i, j, k, n_rz*1 + 2*imode + offset*(icomp-1)) + - beta_boost * clight * arr(i, j, k, n_rz*3 + 2*imode + offset*(icomp-1)) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rz*3 + 2*imode + offset*(icomp-1)) + - beta_boost * inv_clight * arr(i, j, k, n_rz*1 + 2*imode + offset*(icomp-1)) ); + // Store lab-frame data in-place + arr(i, j, k, n_rz*1 + 2*imode + offset*(icomp-1)) = e_lab; + arr(i, j, k, n_rz*3 + 2*imode + offset*(icomp-1)) = b_lab; + + // Transform charge density (ncomp=9) + // and z-component of current density (ncomp=8) + j_lab = gamma_boost * ( arr(i, j, k, n_rz*8 + 2*imode + offset*(icomp-1)) + + beta_boost * clight * arr(i, j, k, n_rz*9 + 2*imode + offset*(icomp-1)) ); + rho_lab = gamma_boost * ( arr(i, j, k, n_rz*9 + 2*imode + offset*(icomp-1)) + + beta_boost * inv_clight * arr(i, j, k, n_rz*8 + 2*imode + offset*(icomp-1)) ); + // Store lab-frame jz and rho in-place + arr(i, j, k, n_rz*8 + 2*imode + offset*(icomp-1)) = j_lab; + arr(i, j, k, n_rz*9 + 2*imode + offset*(icomp-1)) = rho_lab; + } + } + } + ); +#else // arr(x,y,z,comp) has ten-components namely, // Ex Ey Ez Bx By Bz jx jy jz rho in that order. amrex::ParallelFor( tbx, @@ -239,6 +303,7 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm arr(i, j, k, 9) = rho_lab; } ); +#endif } } From 63c36e0661176b4b5ab9144bc90ef5e414377cb0 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 26 Oct 2022 10:16:15 -0700 Subject: [PATCH 08/33] fix comment --- .../Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index 9cf6b48ee53..317a6e0ff39 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -221,13 +221,14 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm [=] AMREX_GPU_DEVICE (int i, int j, int k) { // arr(x,y,z,comp) has (2*num_rz_modes-1)*10 components - // If there are 2 components, then Er_0_real, Er_1_real, Er_2_real are the first the components + // If there are 2 modes, then Er_0_real, Er_1_real, Er_2_real are the first three components // followed by three components each for Et, Ez, Br, Bt, Bz, jr, jt, jz, and rho for (int imode = 0; imode < n_rz; ++imode) { int offset = 1; int ncomp = 2; if (imode == 0) { offset = 0; + // Mode 0 has only real part ncomp = 1; } for (int icomp = 0; icomp < ncomp; ++icomp) { From 9c93329fc28d2e892682c3641224087791e604e2 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 26 Oct 2022 21:37:47 -0700 Subject: [PATCH 09/33] fix bug using 2*nrz-1 , instead of nrz --- .../BackTransformFunctor.cpp | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index 317a6e0ff39..a6ec98d98d2 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -217,6 +217,7 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm amrex::Real inv_clight = 1.0_rt/clight; #ifdef WARPX_DIM_RZ const int n_rz = WarpX::n_rz_azimuthal_modes; + const int n_rcomps = 2*n_rz - 1; amrex::ParallelFor( tbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { @@ -236,32 +237,32 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm // Note that the z-components, Ez, Bz, are not changed by the transform. amrex::Real e_lab, b_lab, j_lab, rho_lab; // Transform Ex_boost (ncomp=0) & By_boost (ncomp=4) to lab-frame - e_lab = gamma_boost * ( arr(i, j, k, n_rz*0 + 2*imode + offset*(icomp-1)) - + beta_boost * clight * arr(i, j, k, n_rz*4+ 2*imode + offset*(icomp-1)) ); - b_lab = gamma_boost * ( arr(i, j, k, n_rz*4 + 2*imode + offset*(icomp-1)) - + beta_boost * inv_clight * arr(i, j, k, n_rz*0 + 2*imode + offset*(icomp-1)) ); + e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*0 + 2*imode + offset*(icomp-1)) + + beta_boost * clight * arr(i, j, k, n_rcomps*4+ 2*imode + offset*(icomp-1)) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*4 + 2*imode + offset*(icomp-1)) + + beta_boost * inv_clight * arr(i, j, k, n_rcomps*0 + 2*imode + offset*(icomp-1)) ); // Store lab-frame data in-place - arr(i, j, k, n_rz*0 + 2*imode + offset*(icomp-1)) = e_lab; - arr(i, j, k, n_rz*4 + 2*imode + offset*(icomp-1)) = b_lab; + arr(i, j, k, n_rcomps*0 + 2*imode + offset*(icomp-1)) = e_lab; + arr(i, j, k, n_rcomps*4 + 2*imode + offset*(icomp-1)) = b_lab; // Transform Ey_boost (ncomp=1) & Bx_boost (ncomp=3) to lab-frame - e_lab = gamma_boost * ( arr(i, j, k, n_rz*1 + 2*imode + offset*(icomp-1)) - - beta_boost * clight * arr(i, j, k, n_rz*3 + 2*imode + offset*(icomp-1)) ); - b_lab = gamma_boost * ( arr(i, j, k, n_rz*3 + 2*imode + offset*(icomp-1)) - - beta_boost * inv_clight * arr(i, j, k, n_rz*1 + 2*imode + offset*(icomp-1)) ); + e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*1 + 2*imode + offset*(icomp-1)) + - beta_boost * clight * arr(i, j, k, n_rcomps*3 + 2*imode + offset*(icomp-1)) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*3 + 2*imode + offset*(icomp-1)) + - beta_boost * inv_clight * arr(i, j, k, n_rcomps*1 + 2*imode + offset*(icomp-1)) ); // Store lab-frame data in-place - arr(i, j, k, n_rz*1 + 2*imode + offset*(icomp-1)) = e_lab; - arr(i, j, k, n_rz*3 + 2*imode + offset*(icomp-1)) = b_lab; + arr(i, j, k, n_rcomps*1 + 2*imode + offset*(icomp-1)) = e_lab; + arr(i, j, k, n_rcomps*3 + 2*imode + offset*(icomp-1)) = b_lab; // Transform charge density (ncomp=9) // and z-component of current density (ncomp=8) - j_lab = gamma_boost * ( arr(i, j, k, n_rz*8 + 2*imode + offset*(icomp-1)) - + beta_boost * clight * arr(i, j, k, n_rz*9 + 2*imode + offset*(icomp-1)) ); - rho_lab = gamma_boost * ( arr(i, j, k, n_rz*9 + 2*imode + offset*(icomp-1)) - + beta_boost * inv_clight * arr(i, j, k, n_rz*8 + 2*imode + offset*(icomp-1)) ); + j_lab = gamma_boost * ( arr(i, j, k, n_rcomps*8 + 2*imode + offset*(icomp-1)) + + beta_boost * clight * arr(i, j, k, n_rcomps*9 + 2*imode + offset*(icomp-1)) ); + rho_lab = gamma_boost * ( arr(i, j, k, n_rcomps*9 + 2*imode + offset*(icomp-1)) + + beta_boost * inv_clight * arr(i, j, k, n_rcomps*8 + 2*imode + offset*(icomp-1)) ); // Store lab-frame jz and rho in-place - arr(i, j, k, n_rz*8 + 2*imode + offset*(icomp-1)) = j_lab; - arr(i, j, k, n_rz*9 + 2*imode + offset*(icomp-1)) = rho_lab; + arr(i, j, k, n_rcomps*8 + 2*imode + offset*(icomp-1)) = j_lab; + arr(i, j, k, n_rcomps*9 + 2*imode + offset*(icomp-1)) = rho_lab; } } } From 491defbe16039065c5edb00be92933b5fdd05630 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Fri, 28 Oct 2022 11:33:04 -0700 Subject: [PATCH 10/33] add comments and clean up --- .../BackTransformFunctor.cpp | 56 ++++++++++++------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index a6ec98d98d2..cf52966e5c1 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -116,18 +116,29 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const amrex::Array4 dst_arr = mf_dst[mfi].array(); #ifdef WARPX_DIM_RZ const int nrz = WarpX::n_rz_azimuthal_modes; + const int n_rz_comp = 2*nrz - 1; #endif amrex::ParallelFor( tbx, ncomp_dst, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) { + // Field id that corresponds to the nth user-requested component const int icomp = field_map_ptr[n]; #if defined(WARPX_DIM_3D) dst_arr(i, j, k_lab, n) = src_arr(i, j, k, icomp); #elif defined(WARPX_DIM_2D) dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp); #elif defined(WARPX_DIM_RZ) - const int rzcomp = n - static_cast( floor(n/(2.0*nrz-1.0)) ) * (2*nrz-1); - dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp*(2*nrz-1)+rzcomp); + // rzcomp below gives the component id, 0 to (n_rz_comp-1) for a given field + const int rzcomp = n - static_cast( floor(n/(2.0*nrz-1.0)) ) * n_rz_comp; + // Accessing the correct rz component from the cell-centered multifab + // that has back-transformed fields and storing it for the appropriate user-requested field, icomp + // For example, for 2 rz modes, we have three components (n_rz_comp=3) for each field + // If n = 4 gives icomp = 1 (for Et) obtained from field_map_ptr, + // rzcomp = 4 - int(floor(4/3))*3 = 4 - 3 = 1 + // Thus we are accessing real component of mode 1 of Et (note that modes go from 0 to 1) + // Since the fields are stored contiguously in src_arr, icomp*n_rz_comp + rz_comp accesses + // real part of mode 1 for Et (1*3+1) = 4 + dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp*n_rz_comp+rzcomp); #endif } ); } @@ -226,43 +237,48 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm // followed by three components each for Et, Ez, Br, Bt, Bz, jr, jt, jz, and rho for (int imode = 0; imode < n_rz; ++imode) { int offset = 1; + // number of components for a given mode int ncomp = 2; if (imode == 0) { offset = 0; - // Mode 0 has only real part + // Mode 0 has only real part, thus ncomp = 1 ncomp = 1; } for (int icomp = 0; icomp < ncomp; ++icomp) { // Back-transform the transverse electric and magnetic fields. // Note that the z-components, Ez, Bz, are not changed by the transform. amrex::Real e_lab, b_lab, j_lab, rho_lab; + + // Offset for a given mode, imode, and component, icomp (0-real, 1-imag) + const int mode_comp_offset = 2*imode + offset*(icomp-1); + // Transform Ex_boost (ncomp=0) & By_boost (ncomp=4) to lab-frame - e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*0 + 2*imode + offset*(icomp-1)) - + beta_boost * clight * arr(i, j, k, n_rcomps*4+ 2*imode + offset*(icomp-1)) ); - b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*4 + 2*imode + offset*(icomp-1)) - + beta_boost * inv_clight * arr(i, j, k, n_rcomps*0 + 2*imode + offset*(icomp-1)) ); + e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*0 + mode_comp_offset) + + beta_boost * clight * arr(i, j, k, n_rcomps*4+ mode_comp_offset) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*4 + mode_comp_offset) + + beta_boost * inv_clight * arr(i, j, k, n_rcomps*0 + mode_comp_offset) ); // Store lab-frame data in-place - arr(i, j, k, n_rcomps*0 + 2*imode + offset*(icomp-1)) = e_lab; - arr(i, j, k, n_rcomps*4 + 2*imode + offset*(icomp-1)) = b_lab; + arr(i, j, k, n_rcomps*0 + mode_component_offset) = e_lab; + arr(i, j, k, n_rcomps*4 + mode_component_offset) = b_lab; // Transform Ey_boost (ncomp=1) & Bx_boost (ncomp=3) to lab-frame - e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*1 + 2*imode + offset*(icomp-1)) - - beta_boost * clight * arr(i, j, k, n_rcomps*3 + 2*imode + offset*(icomp-1)) ); - b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*3 + 2*imode + offset*(icomp-1)) - - beta_boost * inv_clight * arr(i, j, k, n_rcomps*1 + 2*imode + offset*(icomp-1)) ); + e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*1 + mode_comp_offset) + - beta_boost * clight * arr(i, j, k, n_rcomps*3 + mode_comp_offset) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*3 + mode_comp_offset) + - beta_boost * inv_clight * arr(i, j, k, n_rcomps*1 + mode_comp_offset) ); // Store lab-frame data in-place - arr(i, j, k, n_rcomps*1 + 2*imode + offset*(icomp-1)) = e_lab; - arr(i, j, k, n_rcomps*3 + 2*imode + offset*(icomp-1)) = b_lab; + arr(i, j, k, n_rcomps*1 + mode_comp_offset) = e_lab; + arr(i, j, k, n_rcomps*3 + mode_comp_offset) = b_lab; // Transform charge density (ncomp=9) // and z-component of current density (ncomp=8) - j_lab = gamma_boost * ( arr(i, j, k, n_rcomps*8 + 2*imode + offset*(icomp-1)) - + beta_boost * clight * arr(i, j, k, n_rcomps*9 + 2*imode + offset*(icomp-1)) ); + j_lab = gamma_boost * ( arr(i, j, k, n_rcomps*8 + mode_comp_offset) + + beta_boost * clight * arr(i, j, k, n_rcomps*9 + mode_comp_offset) ); rho_lab = gamma_boost * ( arr(i, j, k, n_rcomps*9 + 2*imode + offset*(icomp-1)) - + beta_boost * inv_clight * arr(i, j, k, n_rcomps*8 + 2*imode + offset*(icomp-1)) ); + + beta_boost * inv_clight * arr(i, j, k, n_rcomps*8 + mode_comp_offset) ); // Store lab-frame jz and rho in-place - arr(i, j, k, n_rcomps*8 + 2*imode + offset*(icomp-1)) = j_lab; - arr(i, j, k, n_rcomps*9 + 2*imode + offset*(icomp-1)) = rho_lab; + arr(i, j, k, n_rcomps*8 + mode_comp_offset) = j_lab; + arr(i, j, k, n_rcomps*9 + mode_comp_offset) = rho_lab; } } } From 0a5eb916994d869661758fb10424c87a0ea3459d Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Fri, 28 Oct 2022 11:39:45 -0700 Subject: [PATCH 11/33] fix typo --- .../Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index cf52966e5c1..c14e3e2ed51 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -258,8 +258,8 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*4 + mode_comp_offset) + beta_boost * inv_clight * arr(i, j, k, n_rcomps*0 + mode_comp_offset) ); // Store lab-frame data in-place - arr(i, j, k, n_rcomps*0 + mode_component_offset) = e_lab; - arr(i, j, k, n_rcomps*4 + mode_component_offset) = b_lab; + arr(i, j, k, n_rcomps*0 + mode_comp_offset) = e_lab; + arr(i, j, k, n_rcomps*4 + mode_comp_offset) = b_lab; // Transform Ey_boost (ncomp=1) & Bx_boost (ncomp=3) to lab-frame e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*1 + mode_comp_offset) From 80a8165d657472726f901ef4fdd80d653fb8a55a Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 31 Oct 2022 15:23:02 -0700 Subject: [PATCH 12/33] 1D --- Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index c14e3e2ed51..dc6f0efb0e8 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -139,6 +139,8 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const // Since the fields are stored contiguously in src_arr, icomp*n_rz_comp + rz_comp accesses // real part of mode 1 for Et (1*3+1) = 4 dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp*n_rz_comp+rzcomp); +#else + dst_arr(k_lab, j, k, n) = src_arr(i, j, k, icomp); #endif } ); } From 033154c98925d858f382683b6b003208fea84d77 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 31 Oct 2022 17:18:19 -0700 Subject: [PATCH 13/33] WARPX_DIM_XZ instead of 2D --- Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index dc6f0efb0e8..8522c74d541 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -125,7 +125,7 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const const int icomp = field_map_ptr[n]; #if defined(WARPX_DIM_3D) dst_arr(i, j, k_lab, n) = src_arr(i, j, k, icomp); -#elif defined(WARPX_DIM_2D) +#elif defined(WARPX_DIM_XZ) dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp); #elif defined(WARPX_DIM_RZ) // rzcomp below gives the component id, 0 to (n_rz_comp-1) for a given field From 55930043442c897e030ff4bd8bdda22e2240bbb3 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Tue, 1 Nov 2022 10:09:49 -0700 Subject: [PATCH 14/33] Apply suggestions from code review suggestion from Edoardo Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> --- Source/Diagnostics/BTDiagnostics.cpp | 2 +- .../ComputeDiagFunctors/BackTransformFunctor.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 150257c8aec..0d86d65835f 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -481,7 +481,7 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) amrex::DistributionMapping dmap = warpx.DistributionMap(lev); int ngrow = 1; #ifdef WARPX_DIM_RZ - int ncomps = (2*WarpX::n_rz_azimuthal_modes - 1) * static_cast(m_cellcenter_varnames.size()); + int ncomps = WarpX::ncomps * static_cast(m_cellcenter_varnames.size()); #else int ncomps = static_cast(m_cellcenter_varnames.size()); #endif diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index 8522c74d541..ea52d109569 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -116,7 +116,7 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const amrex::Array4 dst_arr = mf_dst[mfi].array(); #ifdef WARPX_DIM_RZ const int nrz = WarpX::n_rz_azimuthal_modes; - const int n_rz_comp = 2*nrz - 1; + const int n_rz_comp = WarpX::ncomps; #endif amrex::ParallelFor( tbx, ncomp_dst, [=] AMREX_GPU_DEVICE(int i, int j, int k, int n) @@ -207,7 +207,7 @@ BackTransformFunctor::InitData () for (int i = 0; i < m_varnames.size(); ++i) { #ifdef WARPX_DIM_RZ - const int field_id = i / (WarpX::n_rz_azimuthal_modes*2-1); + const int field_id = i / WarpX::ncomps; m_map_varnames[i] = m_possible_fields_to_dump[ m_varnames_fields[field_id] ]; #else m_map_varnames[i] = m_possible_fields_to_dump[ m_varnames[i] ] ; @@ -230,7 +230,7 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm amrex::Real inv_clight = 1.0_rt/clight; #ifdef WARPX_DIM_RZ const int n_rz = WarpX::n_rz_azimuthal_modes; - const int n_rcomps = 2*n_rz - 1; + const int n_rcomps = WarpX::ncomps; amrex::ParallelFor( tbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { From 4575edb78d1e408213e47fc7473fe4b9fdb082df Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Sun, 6 Nov 2022 14:55:44 -0800 Subject: [PATCH 15/33] remove BTD RZ field warning that does not apply anymore --- Source/Diagnostics/MultiDiagnostics.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/Diagnostics/MultiDiagnostics.cpp b/Source/Diagnostics/MultiDiagnostics.cpp index 62218a17568..1b9f5615ae7 100644 --- a/Source/Diagnostics/MultiDiagnostics.cpp +++ b/Source/Diagnostics/MultiDiagnostics.cpp @@ -25,9 +25,6 @@ MultiDiagnostics::MultiDiagnostics () alldiags[i] = std::make_unique(i, diags_names[i]); } else if ( diags_types[i] == DiagTypes::BackTransformed ){ alldiags[i] = std::make_unique(i, diags_names[i]); -#ifdef WARPX_DIM_RZ - ablastr::warn_manager::WMRecordWarning("MultiDiagnostics", "BackTransformed diagnostics for fields is not yet fully implemented in RZ. Field output might be incorrect."); -#endif } else if ( diags_types[i] == DiagTypes::BoundaryScraping ){ alldiags[i] = std::make_unique(i, diags_names[i]); } else { From 4ed469c1245a5e1a3d4eb184f32fe16fd4b6198d Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 7 Nov 2022 13:10:09 -0800 Subject: [PATCH 16/33] BTD rz test for field using laser antenna --- Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD | 64 +++++++++++++++++++ .../Tests/BTD_rz/laser_antenna_compare.py | 52 +++++++++++++++ Regression/WarpX-tests.ini | 16 +++++ 3 files changed, 132 insertions(+) create mode 100644 Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD create mode 100644 Examples/Tests/BTD_rz/laser_antenna_compare.py diff --git a/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD b/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD new file mode 100644 index 00000000000..d2aa789ebcd --- /dev/null +++ b/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD @@ -0,0 +1,64 @@ +# Maximum number of time steps +warpx.zmax_plasma_to_compute_max_step = 160e-6 +# number of grid points +amr.n_cell = 32 256 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 128 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = RZ +geometry.prob_lo = 0.e-6 -20.e-6 +geometry.prob_hi = 40.e-6 0.e-6 + +boundary.field_lo = none absorbing_silver_mueller +boundary.field_hi = absorbing_silver_mueller absorbing_silver_mueller + +# Boosted frame and moving window +warpx.do_moving_window = 1 +warpx.moving_window_dir = z +warpx.moving_window_v = 1.0 # in units of the speed of light +warpx.gamma_boost = 10. +warpx.boost_direction = z + + +# Verbosity +warpx.verbose = 1 +warpx.n_rz_azimuthal_modes = 2 + +# Algorithms +warpx.cfl = 1.0 +warpx.use_filter = 0 + + +# Order of particle shape factors +algo.particle_shape = 1 + +# Laser +lasers.names = laser1 +laser1.profile = Gaussian +laser1.position = 0. 0. -1.e-6 # This point is on the laser plane +laser1.direction = 0. 0. 1. # The plane normal direction +laser1.polarization = 1. 0. 0. # The main polarization vector +laser1.a0 = 1.5e1 # Maximum amplitude of the laser field (in V/m) +laser1.profile_waist = 10.e-6 # The waist of the laser (in meters) +laser1.profile_duration = 10.e-15 # The duration of the laser (in seconds) +laser1.profile_t_peak = 20.e-15 # The time at which the laser reaches its peak (in seconds) +laser1.profile_focal_distance = 1.e-6 # Focal distance from the antenna (in meters) +laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) + +# Diagnostics +diagnostics.diags_names = diag1 back_rz +diag1.intervals = 50 +diag1.diag_type = Full + +back_rz.diag_type = BackTransformed +back_rz.dt_snapshots_lab = 80.e-15 +back_rz.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz rho +back_rz.format = openpmd +back_rz.buffer_size = 32 +back_rz.num_snapshots_lab = 2 diff --git a/Examples/Tests/BTD_rz/laser_antenna_compare.py b/Examples/Tests/BTD_rz/laser_antenna_compare.py new file mode 100644 index 00000000000..3ea0f498cbd --- /dev/null +++ b/Examples/Tests/BTD_rz/laser_antenna_compare.py @@ -0,0 +1,52 @@ +# Copyright 2022 +# Authors: Revathi Jambunathan, Remi Lehe +# +# This file tests if the back-transformed signal is a gaussian +# +# +import os +import sys + +import numpy as np +from scipy.optimize import curve_fit +from scipy.constants import c, m_e, e +from openpmd_viewer import OpenPMDTimeSeries +import matplotlib.pyplot as plt + +def gaussian_laser( z, a0, z0_phase, z0_prop, ctau, lambda0 ): + """ + Returns a Gaussian laser profile + """ + k0 = 2*np.pi/lambda0 + E0 = a0*m_e*c**2*k0/e + return( E0*np.exp( - (z-z0_prop)**2/ctau**2 ) \ + *np.cos( k0*(z-z0_phase) ) ) + +# Fit the on-axis profile to extract a0 +def fit_function(z, z0_phase): + return( gaussian_laser( z, a0, z0_phase, + z0_b+Lprop_b, ctau0, lambda0 ) ) + +# The values must be consistent with the values provided in the simulation input +t_current = 80e-15 # Time of the snapshot1 +c = 299792458; +z0_antenna = -1.e-6 # position of laser +lambda0 = 0.8e-6 # wavelength of the signal +tau0 = 10e-15 # duration of the signal +ctau0 = tau0 * c +a0 = 15 # amplitude +t_peak = 20e-15 # Time at which laser reaches its peak +Lprop_b = c*t_current +z0_b = z0_antenna - c * t_peak + +ts = OpenPMDTimeSeries('./diags/back_rz') +Ex, info = ts.get_field('E', 'x', iteration=1, slice_across='r') + +fit_result = curve_fit( fit_function, info.z, Ex, + p0=np.array([z0_b+Lprop_b]) ) +z0_fit = fit_result[0] + +Ex_fit = gaussian_laser( info.z, a0, z0_fit, z0_b+Lprop_b, ctau0, lambda0) + +## Check that the a0 agrees within 5% of the predicted value +assert np.allclose( Ex, Ex_fit, atol=0.18*Ex.max() ) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index cb4cacf00a0..bbcbce4c471 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3715,3 +3715,19 @@ doVis = 0 compareParticles = 1 particleTypes = electron ion analysisRoutine = Examples/Tests/VayDeposition/analysis.py + +[BTD_rz] +buildDir = . +inputFile = Examples/Tests/BTZ_rz/inputs_rz_z_boosted_BTD +runtime_params = +dim = 2 +addToCompileString = USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +analysisRoutine = Examples/Tests/SilverMueller/laser_antenna_compare.py From 25d1e8ccd185de656eff32791792e7493dd1377f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 21:12:14 +0000 Subject: [PATCH 17/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Examples/Tests/BTD_rz/laser_antenna_compare.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Examples/Tests/BTD_rz/laser_antenna_compare.py b/Examples/Tests/BTD_rz/laser_antenna_compare.py index 3ea0f498cbd..2646b77c19d 100644 --- a/Examples/Tests/BTD_rz/laser_antenna_compare.py +++ b/Examples/Tests/BTD_rz/laser_antenna_compare.py @@ -4,14 +4,13 @@ # This file tests if the back-transformed signal is a gaussian # # -import os -import sys +import matplotlib.pyplot as plt import numpy as np -from scipy.optimize import curve_fit -from scipy.constants import c, m_e, e from openpmd_viewer import OpenPMDTimeSeries -import matplotlib.pyplot as plt +from scipy.constants import c, e, m_e +from scipy.optimize import curve_fit + def gaussian_laser( z, a0, z0_phase, z0_prop, ctau, lambda0 ): """ @@ -28,12 +27,12 @@ def fit_function(z, z0_phase): z0_b+Lprop_b, ctau0, lambda0 ) ) # The values must be consistent with the values provided in the simulation input -t_current = 80e-15 # Time of the snapshot1 +t_current = 80e-15 # Time of the snapshot1 c = 299792458; z0_antenna = -1.e-6 # position of laser lambda0 = 0.8e-6 # wavelength of the signal tau0 = 10e-15 # duration of the signal -ctau0 = tau0 * c +ctau0 = tau0 * c a0 = 15 # amplitude t_peak = 20e-15 # Time at which laser reaches its peak Lprop_b = c*t_current From f18d5b3f7f32679f72ff6a2f5d9789eed8d2667a Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 7 Nov 2022 13:19:39 -0800 Subject: [PATCH 18/33] file starts with analysis --- .../{laser_antenna_compare.py => analysis_BTD_laser_antenna.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Examples/Tests/BTD_rz/{laser_antenna_compare.py => analysis_BTD_laser_antenna.py} (100%) diff --git a/Examples/Tests/BTD_rz/laser_antenna_compare.py b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py similarity index 100% rename from Examples/Tests/BTD_rz/laser_antenna_compare.py rename to Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py From 440ad31ecdabd328a735ce3c0cda7a4db661a2d7 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 7 Nov 2022 13:21:15 -0800 Subject: [PATCH 19/33] change analysis --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index bbcbce4c471..54166fc00f5 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3730,4 +3730,4 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -analysisRoutine = Examples/Tests/SilverMueller/laser_antenna_compare.py +analysisRoutine = Examples/Tests/SilverMueller/analysis_BTD_laser_antenna.py From 954513abd707185e00d8841191d8b6012847fced Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 7 Nov 2022 13:26:00 -0800 Subject: [PATCH 20/33] fix typo --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 54166fc00f5..25fb765d573 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3718,7 +3718,7 @@ analysisRoutine = Examples/Tests/VayDeposition/analysis.py [BTD_rz] buildDir = . -inputFile = Examples/Tests/BTZ_rz/inputs_rz_z_boosted_BTD +inputFile = Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD runtime_params = dim = 2 addToCompileString = USE_RZ=TRUE From ad6ebe9859a7a5b80fefa6aa51827c2529e5496f Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 9 Nov 2022 16:10:48 -0800 Subject: [PATCH 21/33] fix rz input so all snapshots are filled --- Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD b/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD index d2aa789ebcd..edcc51ed5b0 100644 --- a/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD +++ b/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD @@ -1,5 +1,5 @@ # Maximum number of time steps -warpx.zmax_plasma_to_compute_max_step = 160e-6 +warpx.zmax_plasma_to_compute_max_step = 500e-6 # number of grid points amr.n_cell = 32 256 From 305d5cddeb5378d845ed52a190911f4ae9d5404a Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 9 Nov 2022 16:12:10 -0800 Subject: [PATCH 22/33] remove plt from analysis script --- Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py index 2646b77c19d..801073fd6cf 100644 --- a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py +++ b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py @@ -5,7 +5,6 @@ # # -import matplotlib.pyplot as plt import numpy as np from openpmd_viewer import OpenPMDTimeSeries from scipy.constants import c, e, m_e From fb9c65c7d04405a46291c01d7993fc734b33c829 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 11:16:25 -0800 Subject: [PATCH 23/33] initialize cell-centered data to 0 so that guard cells are initialized --- Source/Diagnostics/BTDiagnostics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 0d86d65835f..f641b493c21 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -486,6 +486,7 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) int ncomps = static_cast(m_cellcenter_varnames.size()); #endif m_cell_centered_data[lev] = std::make_unique(ba, dmap, ncomps, ngrow); + m_cell_centered_data[lev]->setVal(0.); } From 049610424e21e99616045ddd913bd7b9b49643e6 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Wed, 16 Nov 2022 11:28:58 -0800 Subject: [PATCH 24/33] Remi's suggestions Co-authored-by: Remi Lehe --- Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py | 7 +++++-- Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD | 2 +- .../ComputeDiagFunctors/BackTransformFunctor.cpp | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py index 801073fd6cf..8c914b5c0b2 100644 --- a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py +++ b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py @@ -1,7 +1,10 @@ # Copyright 2022 # Authors: Revathi Jambunathan, Remi Lehe # -# This file tests if the back-transformed signal is a gaussian +# This tests checks the backtransformed diagnostics by emitting a laser +# (with the antenna) in the boosted-frame and then checking that the +# fields recorded by the backtransformed diagnostics have the right amplitude, +# wavelength, and envelope (i.e. gaussian envelope with the right duration. # # @@ -20,7 +23,7 @@ def gaussian_laser( z, a0, z0_phase, z0_prop, ctau, lambda0 ): return( E0*np.exp( - (z-z0_prop)**2/ctau**2 ) \ *np.cos( k0*(z-z0_phase) ) ) -# Fit the on-axis profile to extract a0 +# Fit the on-axis profile to extract the phase (a.k.a. CEP) def fit_function(z, z0_phase): return( gaussian_laser( z, a0, z0_phase, z0_b+Lprop_b, ctau0, lambda0 ) ) diff --git a/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD b/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD index edcc51ed5b0..a2bdc8089c1 100644 --- a/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD +++ b/Examples/Tests/BTD_rz/inputs_rz_z_boosted_BTD @@ -44,7 +44,7 @@ laser1.profile = Gaussian laser1.position = 0. 0. -1.e-6 # This point is on the laser plane laser1.direction = 0. 0. 1. # The plane normal direction laser1.polarization = 1. 0. 0. # The main polarization vector -laser1.a0 = 1.5e1 # Maximum amplitude of the laser field (in V/m) +laser1.a0 = 1.5e1 # Maximum amplitude of the laser field laser1.profile_waist = 10.e-6 # The waist of the laser (in meters) laser1.profile_duration = 10.e-15 # The duration of the laser (in seconds) laser1.profile_t_peak = 20.e-15 # The time at which the laser reaches its peak (in seconds) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index ea52d109569..ca23ce85a93 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -234,8 +234,8 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm amrex::ParallelFor( tbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // arr(x,y,z,comp) has (2*num_rz_modes-1)*10 components - // If there are 2 modes, then Er_0_real, Er_1_real, Er_2_real are the first three components + // arr(x,y,z,comp) has n_rcomps*10 components + // If there are 2 modes, then Er_0_real, Er_1_real, Er_1_imag are the first three components // followed by three components each for Et, Ez, Br, Bt, Bz, jr, jt, jz, and rho for (int imode = 0; imode < n_rz; ++imode) { int offset = 1; From 123eea6c6b290007223b2a318b1768da0a26baa3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Nov 2022 19:29:20 +0000 Subject: [PATCH 25/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py index 8c914b5c0b2..e9b4dfab817 100644 --- a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py +++ b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py @@ -1,8 +1,8 @@ # Copyright 2022 # Authors: Revathi Jambunathan, Remi Lehe # -# This tests checks the backtransformed diagnostics by emitting a laser -# (with the antenna) in the boosted-frame and then checking that the +# This tests checks the backtransformed diagnostics by emitting a laser +# (with the antenna) in the boosted-frame and then checking that the # fields recorded by the backtransformed diagnostics have the right amplitude, # wavelength, and envelope (i.e. gaussian envelope with the right duration. # From fd1102668301a7d96951b3a80718b6b5793e8b1b Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 12:03:09 -0800 Subject: [PATCH 26/33] AllocInitMultifab to add mf to maps of mfs --- Source/Diagnostics/BTDiagnostics.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index f641b493c21..2b4efcb5ed8 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -485,8 +485,7 @@ BTDiagnostics::DefineCellCenteredMultiFab(int lev) #else int ncomps = static_cast(m_cellcenter_varnames.size()); #endif - m_cell_centered_data[lev] = std::make_unique(ba, dmap, ncomps, ngrow); - m_cell_centered_data[lev]->setVal(0.); + WarpX::AllocInitMultiFab(m_cell_centered_data[lev], ba, dmap, ncomps, amrex::IntVect(ngrow), "cellcentered_BTD",0._rt); } From fed982be2309f18d13ddfec268d761d7b8ef1219 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 14:31:42 -0800 Subject: [PATCH 27/33] fix path to analysis script --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 25fb765d573..9934fb64c4a 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3730,4 +3730,4 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -analysisRoutine = Examples/Tests/SilverMueller/analysis_BTD_laser_antenna.py +analysisRoutine = Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py From 5c5b2e5b1d023c3aa210ac050406c42ae74df370 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 14:55:28 -0800 Subject: [PATCH 28/33] analysis script executable --- Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py diff --git a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py old mode 100644 new mode 100755 From 5e31a4ddfae1f5003e68f829ff6740432dd8c646 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 15:01:42 -0800 Subject: [PATCH 29/33] a better and succint for loop --- .../BackTransformFunctor.cpp | 73 +++++++------------ 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index ca23ce85a93..59647ae9270 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -234,54 +234,37 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm amrex::ParallelFor( tbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { - // arr(x,y,z,comp) has n_rcomps*10 components - // If there are 2 modes, then Er_0_real, Er_1_real, Er_1_imag are the first three components - // followed by three components each for Et, Ez, Br, Bt, Bz, jr, jt, jz, and rho - for (int imode = 0; imode < n_rz; ++imode) { - int offset = 1; - // number of components for a given mode - int ncomp = 2; - if (imode == 0) { - offset = 0; - // Mode 0 has only real part, thus ncomp = 1 - ncomp = 1; - } - for (int icomp = 0; icomp < ncomp; ++icomp) { - // Back-transform the transverse electric and magnetic fields. - // Note that the z-components, Ez, Bz, are not changed by the transform. - amrex::Real e_lab, b_lab, j_lab, rho_lab; + for (int mode_comp = 0; mode_comp < n_rcomps; ++mode_comp) { + // Back-transform the transverse electric and magnetic fields. + // Note that the z-components, Ez, Bz, are not changed by the transform. + amrex::Real e_lab, b_lab, j_lab, rho_lab; - // Offset for a given mode, imode, and component, icomp (0-real, 1-imag) - const int mode_comp_offset = 2*imode + offset*(icomp-1); + // Transform Er_boost & Bt_boost to lab-frame for corresponding mode (mode_comp) + e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*0 + mode_comp) + + beta_boost * clight * arr(i, j, k, n_rcomps*4+ mode_comp) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*4 + mode_comp) + + beta_boost * inv_clight * arr(i, j, k, n_rcomps*0 + mode_comp) ); + // Store lab-frame data in-place + arr(i, j, k, n_rcomps*0 + mode_comp) = e_lab; + arr(i, j, k, n_rcomps*4 + mode_comp) = b_lab; - // Transform Ex_boost (ncomp=0) & By_boost (ncomp=4) to lab-frame - e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*0 + mode_comp_offset) - + beta_boost * clight * arr(i, j, k, n_rcomps*4+ mode_comp_offset) ); - b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*4 + mode_comp_offset) - + beta_boost * inv_clight * arr(i, j, k, n_rcomps*0 + mode_comp_offset) ); - // Store lab-frame data in-place - arr(i, j, k, n_rcomps*0 + mode_comp_offset) = e_lab; - arr(i, j, k, n_rcomps*4 + mode_comp_offset) = b_lab; + // Transform Et_boost & Br_boost to lab-frame for corresponding mode (mode_comp) + e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*1 + mode_comp) + - beta_boost * clight * arr(i, j, k, n_rcomps*3 + mode_comp) ); + b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*3 + mode_comp) + - beta_boost * inv_clight * arr(i, j, k, n_rcomps*1 + mode_comp) ); + // Store lab-frame data in-place + arr(i, j, k, n_rcomps*1 + mode_comp) = e_lab; + arr(i, j, k, n_rcomps*3 + mode_comp) = b_lab; - // Transform Ey_boost (ncomp=1) & Bx_boost (ncomp=3) to lab-frame - e_lab = gamma_boost * ( arr(i, j, k, n_rcomps*1 + mode_comp_offset) - - beta_boost * clight * arr(i, j, k, n_rcomps*3 + mode_comp_offset) ); - b_lab = gamma_boost * ( arr(i, j, k, n_rcomps*3 + mode_comp_offset) - - beta_boost * inv_clight * arr(i, j, k, n_rcomps*1 + mode_comp_offset) ); - // Store lab-frame data in-place - arr(i, j, k, n_rcomps*1 + mode_comp_offset) = e_lab; - arr(i, j, k, n_rcomps*3 + mode_comp_offset) = b_lab; - - // Transform charge density (ncomp=9) - // and z-component of current density (ncomp=8) - j_lab = gamma_boost * ( arr(i, j, k, n_rcomps*8 + mode_comp_offset) - + beta_boost * clight * arr(i, j, k, n_rcomps*9 + mode_comp_offset) ); - rho_lab = gamma_boost * ( arr(i, j, k, n_rcomps*9 + 2*imode + offset*(icomp-1)) - + beta_boost * inv_clight * arr(i, j, k, n_rcomps*8 + mode_comp_offset) ); - // Store lab-frame jz and rho in-place - arr(i, j, k, n_rcomps*8 + mode_comp_offset) = j_lab; - arr(i, j, k, n_rcomps*9 + mode_comp_offset) = rho_lab; - } + // Transform charge density z-component of current density + j_lab = gamma_boost * ( arr(i, j, k, n_rcomps*8 + mode_comp) + + beta_boost * clight * arr(i, j, k, n_rcomps*9 + mode_comp) ); + rho_lab = gamma_boost * ( arr(i, j, k, n_rcomps*9 + mode_comp) + + beta_boost * inv_clight * arr(i, j, k, n_rcomps*8 + mode_comp) ); + // Store lab-frame jz and rho in-place + arr(i, j, k, n_rcomps*8 + mode_comp) = j_lab; + arr(i, j, k, n_rcomps*9 + mode_comp) = rho_lab; } } ); From dec5a1788330015ed8b4598026239b46e3fc3d3c Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 15:09:15 -0800 Subject: [PATCH 30/33] unused var --- Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index 59647ae9270..a57b61b666f 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -229,7 +229,6 @@ BackTransformFunctor::LorentzTransformZ (amrex::MultiFab& data, amrex::Real gamm amrex::Real clight = PhysConst::c; amrex::Real inv_clight = 1.0_rt/clight; #ifdef WARPX_DIM_RZ - const int n_rz = WarpX::n_rz_azimuthal_modes; const int n_rcomps = WarpX::ncomps; amrex::ParallelFor( tbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) From 8e14707e533c45ae51b744031a6cdc04e6b931b3 Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Wed, 16 Nov 2022 15:24:56 -0800 Subject: [PATCH 31/33] Update Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp Co-authored-by: Remi Lehe --- Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index a57b61b666f..0cc1530266c 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -129,7 +129,7 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const dst_arr(i, k_lab, k, n) = src_arr(i, j, k, icomp); #elif defined(WARPX_DIM_RZ) // rzcomp below gives the component id, 0 to (n_rz_comp-1) for a given field - const int rzcomp = n - static_cast( floor(n/(2.0*nrz-1.0)) ) * n_rz_comp; + const int rzcomp = n % n_rz_comp; // Accessing the correct rz component from the cell-centered multifab // that has back-transformed fields and storing it for the appropriate user-requested field, icomp // For example, for 2 rz modes, we have three components (n_rz_comp=3) for each field From bc10eed4699e0780eac5a7e677866865f5caf34f Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 15:45:34 -0800 Subject: [PATCH 32/33] fix unused var --- Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index 0cc1530266c..3678e03f0c2 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -115,7 +115,6 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const amrex::Array4 src_arr = tmp[mfi].array(); amrex::Array4 dst_arr = mf_dst[mfi].array(); #ifdef WARPX_DIM_RZ - const int nrz = WarpX::n_rz_azimuthal_modes; const int n_rz_comp = WarpX::ncomps; #endif amrex::ParallelFor( tbx, ncomp_dst, From 37c9587c41944eea09c58a3143272dcf240adac6 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Wed, 16 Nov 2022 16:36:26 -0800 Subject: [PATCH 33/33] add python path --- Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py index e9b4dfab817..5b5f8c94bf3 100755 --- a/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py +++ b/Examples/Tests/BTD_rz/analysis_BTD_laser_antenna.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # Copyright 2022 # Authors: Revathi Jambunathan, Remi Lehe # @@ -5,8 +7,6 @@ # (with the antenna) in the boosted-frame and then checking that the # fields recorded by the backtransformed diagnostics have the right amplitude, # wavelength, and envelope (i.e. gaussian envelope with the right duration. -# -# import numpy as np from openpmd_viewer import OpenPMDTimeSeries