Latest upstream pull (#2)

* pin intel-oneapi version in CI (AMReX-Combustion#541) * Bump Submodules/amrex from `8349789` to `00e6f75` (AMReX-Combustion#540) Bumps [Submodules/amrex](https://github.com/AMReX-Codes/amrex) from `8349789` to `00e6f75`. - [Release notes](https://github.com/AMReX-Codes/amrex/releases) - [Commits](AMReX-Codes/amrex@8349789...00e6f75) --- updated-dependencies: - dependency-name: Submodules/amrex dependency-type: direct:production ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Bruce Perry <[email protected]> * Make EOS functions templates to allow for CellData as an argument (AMReX-Combustion#536) * template for celldata in EOS * static asserts in templates * templating for RY2 functions for all EOS * Fix clang-tidy (AMReX-Combustion#535) * fix unused variable * free shared ptr for manfunc parm --------- Co-authored-by: Marc T. Henry de Frahan <[email protected]> * Unit conversions (AMReX-Combustion#539) * Utilities and unit conversions * Formatting * 2 structs and renamed functions to avoid macros issue * Templating and namespaces * Added Utilities to Docs * Overload instead of templating * Fixed nested namespaces for clang-tidy error in PeleLMeX, removed Utilities.H from PelePhysics.H.wq (AMReX-Combustion#545) * Update CI to ubuntu-24.04 (AMReX-Combustion#546) --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: Bruce Perry <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marc T. Henry de Frahan <[email protected]> Co-authored-by: Dave Montgomery <[email protected]> Co-authored-by: Jon Rood <[email protected]>
ITV-RWTH · Nov 14, 2024 · edbb570 · edbb570
1 parent 5761481
commit edbb570
Show file tree

Hide file tree

Showing 20 changed files with 404 additions and 89 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   Formatting:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Clone
         uses: actions/checkout@v4
@@ -27,7 +27,7 @@ jobs:
 
   Codespell:
     needs: Formatting
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Clone
         uses: actions/checkout@v4
@@ -46,7 +46,7 @@ jobs:
         run: codespell
 
   CEPTR:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -81,7 +81,7 @@ jobs:
 
   Transport-EOS-Reactions:
     needs: Formatting
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         comp: [gnu, llvm, cuda, hip, sycl]
@@ -131,9 +131,9 @@ jobs:
           echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV
           echo "CCACHE_MAXSIZE=1G" >> $GITHUB_ENV
           if [ "${{matrix.comp}}" == 'cuda' ]; then \
-            echo "CUDA_HOME=/usr/local/cuda-11.2" >> $GITHUB_ENV; \
-            echo "LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV; \
-            echo "/usr/local/cuda-11.2/bin" >> $GITHUB_PATH; \
+            echo "CUDA_HOME=/usr/local/cuda-12.6" >> $GITHUB_ENV; \
+            echo "LD_LIBRARY_PATH=/usr/local/cuda-12.6/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV; \
+            echo "/usr/local/cuda-12.6/bin" >> $GITHUB_PATH; \
           fi
       - name: Install Ccache
         run: |
@@ -150,6 +150,7 @@ jobs:
       - name: Dependencies
         working-directory: ${{env.TRANSPORT_WORKING_DIRECTORY}}
         run: |
+          ${{github.workspace}}/PelePhysics-${{matrix.comp}}/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh
           if [ "${{matrix.comp}}" == 'cuda' ]; then \
             ${{github.workspace}}/PelePhysics-${{matrix.comp}}/${{matrix.dependency_cmds}}; \
             echo "CUDA_HOME=${CUDA_HOME}"; \

diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml
@@ -9,7 +9,7 @@ on:
 jobs:
   CleanUpCcacheCachePostPR:
     name: Clean Up Ccahe Cache Post PR
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       actions: write
       contents: read

diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml
@@ -9,7 +9,7 @@ on:
 jobs:
   CleanUpCcacheCache:
     name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       actions: write
       contents: read

diff --git a/.github/workflows/dependencies/dependencies_cuda.sh b/.github/workflows/dependencies/dependencies_cuda.sh
@@ -1,21 +1,11 @@
 #!/usr/bin/env bash
-#
-# Copyright 2020 Axel Huebl
-#
-# License: BSD-3-Clause-LBNL
-
-# search recursive inside a folder if a file contains tabs
-#
-# @result 0 if no files are found, else 1
-#
 
 set -eu -o pipefail
 
-CUDA_VERSION=11-2
+CUDA_VERSION=12-6
 
-sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
-echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" \
-    | sudo tee /etc/apt/sources.list.d/cuda.list
+wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
+sudo dpkg -i cuda-keyring_1.1-1_all.deb
 sudo apt-get update
 sudo apt-get install -y \
     cuda-command-line-tools-${CUDA_VERSION} \
@@ -28,8 +18,9 @@ sudo apt-get install -y \
     libcusolver-dev-${CUDA_VERSION}         \
     libcusparse-dev-${CUDA_VERSION}         \
     libcublas-dev-${CUDA_VERSION}           \
-    libcurand-dev-${CUDA_VERSION}
+    libcurand-dev-${CUDA_VERSION}           \
+    libnvjitlink-${CUDA_VERSION}
 
-export PATH=/usr/local/cuda-11.2/bin:${PATH}
+export PATH=/usr/local/nvidia/bin:/usr/local/cuda-12.6/bin:${PATH}
 which nvcc
 nvcc --version
diff --git a/.github/workflows/dependencies/dependencies_dpcpp.sh b/.github/workflows/dependencies/dependencies_dpcpp.sh
@@ -1,18 +1,9 @@
 #!/usr/bin/env bash
-#
-# Copyright 2020 The AMReX Community
-#
-# License: BSD-3-Clause-LBNL
-# Authors: Axel Huebl
 
 set -eu -o pipefail
 
 sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
-echo "deb https://apt.repos.intel.com/oneapi all main" \
-  | sudo tee /etc/apt/sources.list.d/oneAPI.list
-
+echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
 sudo apt-get update
-sudo apt-get install -y \
-    intel-oneapi-compiler-dpcpp-cpp \
-    intel-oneapi-mkl-devel
+sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 intel-oneapi-mkl-devel=2024.2.2-15
diff --git a/.github/workflows/dependencies/dependencies_hip.sh b/.github/workflows/dependencies/dependencies_hip.sh
@@ -1,30 +1,19 @@
 #!/usr/bin/env bash
-#
-# Copyright 2020 The AMReX Community
-#
-# License: BSD-3-Clause-LBNL
-# Authors: Axel Huebl
-
-# search recursive inside a folder if a file contains tabs
-#
-# @result 0 if no files are found, else 1
-#
 
 set -eu -o pipefail
 
-sudo wget https://repo.radeon.com/rocm/rocm.gpg.key
-sudo apt-key add rocm.gpg.key
-echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/5.6.1 ubuntu main' \
-  | sudo tee /etc/apt/sources.list.d/rocm.list
-echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
-  | sudo tee -a /etc/profile.d/rocm.sh
-
+sudo mkdir --parents --mode=0755 /etc/apt/keyrings
+wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
+echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.2.1 noble main" | sudo tee --append /etc/apt/sources.list.d/rocm.list
+echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
+echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh
 sudo apt-get update
-sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev
+sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev hiprand-dev
 
 source /etc/profile.d/rocm.sh
 which hipcc
 which clang
 which clang++
 which flang
 hipcc --version
+hipconfig --full
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -16,7 +16,7 @@ on:
 
 jobs:
   Docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Clone
         uses: actions/checkout@v4

diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml
@@ -6,7 +6,7 @@ on:
 
 jobs:
   cleanup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Save PR number
         env:

diff --git a/Docs/sphinx/Utility.rst b/Docs/sphinx/Utility.rst
@@ -12,6 +12,7 @@ In addition to routines for evaluating chemical reactions, transport properties,
 * Turbulent inflows (``TurbInflow``) on domain boundaries from saved turbulence data
 * Plt file management (``PltFileManager``)
 * Output of runtime ``Diagnostics``
+* Basic ``Utilities``, including unit conversions
 
 This section provides relevant notes on using these utilities across the Pele family of codes. There may be additional subtleties based on the code-specific implementation of these capabilities, in which case it will be necessary to refer to the documentation of the code of interest.
 
@@ -146,3 +147,26 @@ discussed in PeleC milestone reports. An example call to the filtering operation
    les_filter.apply_filter(bxtmp, flux[i], filtered_flux[i], Density, NUM_STATE);
 
 The user must ensure that the correct number of grow cells is present in the Fab or MultiFab.
+
+Utilities
+=============================
+The utilities namespace contains unit conversions, which are particularly useful for PeleLM(eX) users working with mixed unit systems. The following aliases, defined in a file header, enable straightforward conversions between MKS and CGS units for use in equation of state (``eos``) function calls:
+
+::
+
+   namespace m2c = pele::physics::utilities::mks2cgs;
+   namespace c2m = pele::physics::utilities::cgs2mks;
+
+For example, users can call ``eos.PYT2R`` as follows:
+
+::
+
+   auto eos = pele::physics::PhysicsType::eos();
+   amrex::Real P_mean = 101325.0_rt; // Pressure in Pa (MKS)
+   amrex::Real massfrac[NUM_SPECIES]; // Mass fractions tracked by PeleLM(eX)
+   amrex::Real Temp = 300.0_rt; // Temp in K
+   amrex::Real rho_cgs = 0.0_rt; // Density in g/cm^3 (CGS)
+   
+   // Calculate density in CGS units, then convert to MKS
+   eos.PYT2R(m2c::P(P_mean), massfrac, Temp, rho_cgs);
+   amrex::Real rho = c2m::Rho(rho_cgs); // Convert eos density to MKS
diff --git a/Source/Eos/EosParams.H b/Source/Eos/EosParams.H
@@ -166,6 +166,7 @@ struct InitParm<eos::EosParm<eos::Manifold>>
   static void host_deallocate(PeleParams<eos::EosParm<eos::Manifold>>* parm_in)
   {
     parm_in->m_host_only_parm.manfunc_par->deallocate();
+    parm_in->m_host_only_parm.manfunc_par = nullptr;
   }
 };
 #endif

diff --git a/Source/Eos/Fuego.H b/Source/Eos/Fuego.H
@@ -16,20 +16,26 @@ struct Fuego
 
   static std::string identifier() { return "Fuego"; }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2R(const amrex::Real RY[], amrex::Real& R)
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void
+  RY2R(const RealArrayLike& RY, amrex::Real& R, int first = 0)
   {
     R = 0.0;
     for (int i = 0; i < NUM_SPECIES; i++) {
-      R += RY[i];
+      R += RY[first + i];
     }
   }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2RRinvY(
-    const amrex::Real RY[], amrex::Real& R, amrex::Real& Rinv, amrex::Real Y[])
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike1, typename RealArrayLike2>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void RY2RRinvY(
+    const RealArrayLike1& RY,
+    amrex::Real& R,
+    amrex::Real& Rinv,
+    RealArrayLike2&& Y)
   {
     RY2R(RY, R);
     Rinv = 1.0 / R;

diff --git a/Source/Eos/GammaLaw.H b/Source/Eos/GammaLaw.H
@@ -18,18 +18,32 @@ struct GammaLaw
 
   static std::string identifier() { return "GammaLaw"; }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2R(const amrex::Real RY[], amrex::Real& R) { R = RY[0]; }
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void
+  RY2R(const RealArrayLike& RY, amrex::Real& R, int first = 0)
+  {
+    R = 0.0;
+    for (int i = 0; i < NUM_SPECIES; i++) {
+      R += RY[first + i];
+    }
+  }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2RRinvY(
-    const amrex::Real RY[], amrex::Real& R, amrex::Real& Rinv, amrex::Real Y[])
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike1, typename RealArrayLike2>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void RY2RRinvY(
+    const RealArrayLike1& RY,
+    amrex::Real& R,
+    amrex::Real& Rinv,
+    RealArrayLike2&& Y)
   {
-    R = RY[0];
+    RY2R(RY, R);
     Rinv = 1.0 / R;
-    Y[0] = 1.0;
+    for (int i = 0; i < NUM_SPECIES; i++) {
+      Y[i] = RY[i] * Rinv;
+    }
   }
 
   AMREX_GPU_HOST_DEVICE

diff --git a/Source/Eos/Manifold.H b/Source/Eos/Manifold.H
@@ -33,17 +33,23 @@ struct Manifold
 
   static std::string identifier() { return "Manifold"; }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2R(const amrex::Real RY[], amrex::Real& R)
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void
+  RY2R(const RealArrayLike& RY, amrex::Real& R, int first = 0)
   {
-    R = RY[NUM_SPECIES - 1];
+    R = RY[first + NUM_SPECIES - 1];
   }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2RRinvY(
-    const amrex::Real RY[], amrex::Real& R, amrex::Real& Rinv, amrex::Real Y[])
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike1, typename RealArrayLike2>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void RY2RRinvY(
+    const RealArrayLike1& RY,
+    amrex::Real& R,
+    amrex::Real& Rinv,
+    RealArrayLike2&& Y)
   {
     RY2R(RY, R);
     Rinv = 1.0 / R;

diff --git a/Source/Eos/SRK.H b/Source/Eos/SRK.H
@@ -16,20 +16,26 @@ struct SRK
 
   static std::string identifier() { return "SRK"; }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2R(const amrex::Real RY[], amrex::Real& R)
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void
+  RY2R(const RealArrayLike& RY, amrex::Real& R, int first = 0)
   {
     R = 0.0;
     for (int i = 0; i < NUM_SPECIES; i++) {
-      R += RY[i];
+      R += RY[first + i];
     }
   }
 
-  AMREX_GPU_HOST_DEVICE
-  AMREX_FORCE_INLINE
-  static void RY2RRinvY(
-    const amrex::Real RY[], amrex::Real& R, amrex::Real& Rinv, amrex::Real Y[])
+  // ReallArrayLike can be anything with an [] operator that returns
+  // amrex::Real&
+  template <typename RealArrayLike1, typename RealArrayLike2>
+  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE static void RY2RRinvY(
+    const RealArrayLike1& RY,
+    amrex::Real& R,
+    amrex::Real& Rinv,
+    RealArrayLike2&& Y)
   {
     RY2R(RY, R);
     Rinv = 1.0 / R;

diff --git a/Source/Transport/TransportParams.H b/Source/Transport/TransportParams.H
@@ -222,6 +222,7 @@ struct InitParm<transport::TransParm<EOSType, transport::ManifoldTransport>>
     if (!parm_in->m_host_only_parm.use_eos_manifold) {
       parm_in->m_host_only_parm.manfunc_par->deallocate();
     }
+    parm_in->m_host_only_parm.manfunc_par = nullptr;
   }
 };
 

diff --git a/Source/Utility/Utilities/Make.package b/Source/Utility/Utilities/Make.package
@@ -0,0 +1,4 @@
+CEXE_headers += Utilities.H UnitConversions.H
+
+VPATH_LOCATIONS += $(PELE_PHYSICS_HOME)/Source/Utility/Utilities
+INCLUDE_LOCATIONS += $(PELE_PHYSICS_HOME)/Source/Utility/Utilities