From 6db755b07f216d1a969a6aa0e2750e640c6c655d Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Sun, 10 Nov 2024 10:12:41 -0700 Subject: [PATCH 01/17] Update CI to ubuntu-24.04. --- .github/workflows/ci.yml | 121 ++++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47958f42e..ad3fb13a9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ concurrency: jobs: Formatting: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -26,7 +26,7 @@ jobs: clangFormatVersion: 18 CPU-GNUmake: needs: Formatting - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -72,7 +72,7 @@ jobs: - name: PMF debug report working-directory: ./Exec/RegTests/PMF run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -97,7 +97,7 @@ jobs: - name: EB debug report working-directory: ./Exec/RegTests/EB-C10 run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -122,7 +122,7 @@ jobs: - name: EB OMP debug report working-directory: ./Exec/RegTests/EB-C10 run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -147,7 +147,7 @@ jobs: - name: TG debug single precision report working-directory: ./Exec/RegTests/TG run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt @@ -169,9 +169,9 @@ jobs: - name: Soot debug report working-directory: ./Exec/RegTests/Soot-Flame run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ - | egrep -v "Submodules/PelePhysics/Mechanisms" | sort | uniq \ + | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt cat warnings.txt export return=$(tail -n 1 warnings.txt | awk '{print $2}') @@ -191,9 +191,9 @@ jobs: - name: Spray debug report working-directory: ./Exec/RegTests/Spray-Conv run: | - egrep "warning:|error:" build-output.txt \ + egrep "Warning:|Error:|warning:|error:" build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials|GNUmakefile|ThirdParty" \ - | egrep -v "Submodules/PelePhysics/Mechanisms" | sort | uniq \ + | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > warnings.txt cat warnings.txt export return=$(tail -n 1 warnings.txt | awk '{print $2}') @@ -203,7 +203,7 @@ jobs: runs-on: ${{matrix.os}} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-24.04, macos-latest] build_type: [Release, Debug] include: - os: macos-latest @@ -211,7 +211,7 @@ jobs: comp: llvm procs: $(sysctl -n hw.ncpu) ccache_cache: /Users/runner/Library/Caches/ccache - - os: ubuntu-latest + - os: ubuntu-24.04 install_deps: sudo apt-get update && sudo apt-get install mpich libmpich-dev libtool-bin comp: gnu procs: $(nproc) @@ -320,7 +320,7 @@ jobs: - name: Report run: | echo "::add-matcher::.github/problem-matchers/gcc.json" - egrep "warning:|error:" ${{runner.workspace}}/build-output.txt \ + egrep "Warning:|Error:|warning:|error:" ${{runner.workspace}}/build-output.txt \ | egrep -v "Submodules/amrex|Submodules/sundials" \ | egrep -v "ld: warning:" | egrep -v "lto-wrapper: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > ${{runner.workspace}}/build-output-warnings.txt @@ -338,13 +338,7 @@ jobs: GPU-Nvidia: name: GPU-CUDA needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-20.04 - strategy: - matrix: - cuda_ver: [11.4] - include: - - cuda_ver: "11.4" - cuda_pkg: 11-4 + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -360,16 +354,15 @@ jobs: - name: Dependencies run: | ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh - curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-keyring_1.0-1_all.deb - sudo dpkg -i cuda-keyring_1.0-1_all.deb + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb + sudo dpkg -i cuda-keyring_1.1-1_all.deb sudo apt-get update - sudo apt-get install -y \ - cuda-command-line-tools-${{matrix.cuda_pkg}} \ - cuda-compiler-${{matrix.cuda_pkg}} cuda-minimal-build-${{matrix.cuda_pkg}} \ - cuda-nvml-dev-${{matrix.cuda_pkg}} cuda-nvtx-${{matrix.cuda_pkg}} \ - libcurand-dev-${{matrix.cuda_pkg}} cuda-cupti-dev-${{matrix.cuda_pkg}} \ - libcusolver-dev-${{matrix.cuda_pkg}} libcusparse-dev-${{matrix.cuda_pkg}} \ - libcublas-dev-${{matrix.cuda_pkg}} + sudo apt-get install -y cuda-command-line-tools-12-6 \ + cuda-compiler-12-6 cuda-minimal-build-12-6 \ + cuda-nvml-dev-12-6 cuda-nvtx-12-6 \ + libcurand-dev-12-6 cuda-cupti-dev-12-6 \ + libcusolver-dev-12-6 libcusparse-dev-12-6 \ + libcublas-dev-12-6 libcurand-dev-12-6 libnvjitlink-12-6 - name: Install Ccache run: | wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz @@ -384,23 +377,23 @@ jobs: ccache-${{github.workflow}}-${{github.job}}-git- - name: Configure run: | - export PATH=/usr/local/nvidia/bin:/usr/local/cuda-${{matrix.cuda_ver}}/bin:${PATH} - export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-${{matrix.cuda_ver}}/lib:${LD_LIBRARY_PATH} - cmake -Bbuild-${{matrix.cuda_pkg}} \ + export PATH=/usr/local/nvidia/bin:/usr/local/cuda-12.6/bin:${PATH} + export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-12.6/lib:${LD_LIBRARY_PATH} + cmake -Bbuild-cuda \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DPELE_ENABLE_MPI:BOOL=OFF \ -DPELE_ENABLE_CUDA:BOOL=ON \ -DPELE_EXCLUDE_BUILD_IN_CI:BOOL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS:BOOL=ON \ - -DCMAKE_CUDA_ARCHITECTURES:STRING=70 \ + -DCMAKE_CUDA_ARCHITECTURES:STRING=80 \ -DCMAKE_CUDA_COMPILER_LAUNCHER:STRING=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \ ${{github.workspace}} - name: Build run: | ccache -z - cmake --build build-${{matrix.cuda_pkg}} --parallel ${{env.NPROCS}} + cmake --build build-cuda --parallel ${{env.NPROCS}} - name: Ccache Report run: | ccache -s @@ -408,7 +401,7 @@ jobs: GPU-AMD: name: GPU-HIP needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -424,14 +417,13 @@ jobs: - name: Dependencies run: | ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh - curl -O https://repo.radeon.com/rocm/rocm.gpg.key - sudo apt-key add rocm.gpg.key - echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/5.6.1 ubuntu main' \ - | sudo tee /etc/apt/sources.list.d/rocm.list - echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ - | sudo tee -a /etc/profile.d/rocm.sh + sudo mkdir --parents --mode=0755 /etc/apt/keyrings + wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.2.1 noble main" | sudo tee --append /etc/apt/sources.list.d/rocm.list + echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 + echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update - sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev + sudo apt-get install -y rocm-dev rocrand-dev rocprim-dev hiprand-dev - name: Install Ccache run: | wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz @@ -448,7 +440,10 @@ jobs: run: | source /etc/profile.d/rocm.sh which clang + which clang++ which hipcc + hipcc --version + hipconfig --full cmake -B${{runner.workspace}}/build-hip \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ @@ -480,7 +475,7 @@ jobs: GPU-Intel: name: GPU-SYCL needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -499,8 +494,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - echo "deb https://apt.repos.intel.com/oneapi all main" \ - | sudo tee /etc/apt/sources.list.d/oneAPI.list + echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp=2024.2.1-1079 intel-oneapi-mkl-devel=2024.2.2-15 - name: Install Ccache @@ -581,7 +575,7 @@ jobs: # exit ${return} Lint-clang-tidy: needs: Formatting - runs-on: macos-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -589,25 +583,30 @@ jobs: submodules: 'recursive' - name: Setup run: | - echo "NPROCS=$(sysctl -n hw.ncpu)" >> $GITHUB_ENV + echo "NPROCS=$(nproc)" >> $GITHUB_ENV echo "CCACHE_COMPRESS=1" >> $GITHUB_ENV echo "CCACHE_COMPRESSLEVEL=1" >> $GITHUB_ENV echo "CCACHE_LOGFILE=${{github.workspace}}/ccache.log.txt" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=500M" >> $GITHUB_ENV - - name: Dependencies + echo "CCACHE_EXTRAFILES=${{github.workspace}}/.clang-tidy" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=50M" >> $GITHUB_ENV + echo "CTCACHE_DIR=~/.cache/ctcache" >> $GITHUB_ENV + - name: Install Ccache run: | - brew install ccache - brew install llvm + wget https://github.com/ccache/ccache/releases/download/v4.8/ccache-4.8-linux-x86_64.tar.xz + sudo curl https://raw.githubusercontent.com/matus-chochlik/ctcache/7fd516e91c17779cbc6fc18bd119313d9532dd90/clang-tidy-cache -Lo /usr/bin/clang-tidy-cache + tar xvf ccache-4.8-linux-x86_64.tar.xz + sudo cp -f ccache-4.8-linux-x86_64/ccache /usr/local/bin/ + sudo chmod +x /usr/bin/clang-tidy-cache + mkdir -p ~/.cache/ctcache - name: Set Up Ccache uses: actions/cache@v4 with: - path: /Users/runner/Library/Caches/ccache + path: ~/.cache key: ccache-${{github.workflow}}-${{github.job}}-git-${{github.sha}} restore-keys: | ccache-${{github.workflow}}-${{github.job}}-git- - name: Configure run: | - export PATH=$(brew --prefix llvm)/bin:${PATH} cmake -B${{runner.workspace}}/build-clang-tidy \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ @@ -623,16 +622,19 @@ jobs: - name: Check working-directory: ${{runner.workspace}}/build-clang-tidy run: | - export PATH=$(brew --prefix llvm)/bin:${PATH} cmake --build . --parallel ${{env.NPROCS}} 2>&1 | tee -a clang-tidy-full-report.txt - egrep "warning:|error:" clang-tidy-full-report.txt \ + egrep "Warning:|Error:|warning:|error:" clang-tidy-full-report.txt \ | egrep -v "Submodules/amrex|Submodules/sundials" \ | egrep -v "ld: warning:" | sort | uniq \ | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > clang-tidy-warnings.txt - name: Ccache Report run: | + ls ~/.cache + ls ~/.cache/ccache + du -hs ~/.cache/ccache + ls ~/.cache/ctcache + du -hs ~/.cache/ctcache ccache -s - du -hs /Users/runner/Library/Caches/ccache - name: Full report working-directory: ${{runner.workspace}}/build-clang-tidy run: cat clang-tidy-full-report.txt @@ -645,7 +647,7 @@ jobs: exit ${return} Lint-codeql: needs: Formatting - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -693,7 +695,7 @@ jobs: sarif_file: sarif-results/cpp.sarif Lint-codespell: needs: Formatting - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -705,14 +707,13 @@ jobs: python-version: '3.12' - name: Dependencies run: | - # Install Python packages python -m pip install --upgrade pip pip install codespell - name: Run codespell run: codespell Save-PR-Number: if: github.event_name == 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Save PR number env: From e3af179b728bce3ffde86f064266902907a8f46b Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Sun, 10 Nov 2024 10:21:33 -0700 Subject: [PATCH 02/17] Update more. --- .github/workflows/cleanup-cache-postpr.yml | 2 +- .github/workflows/cleanup-cache.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/post-pr.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 5dc32a685..6a30a57d9 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -9,7 +9,7 @@ on: jobs: CleanUpCcacheCachePostPR: name: Clean Up Ccahe Cache Post PR - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: actions: write contents: read diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index eca78d4cb..952d95097 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -9,7 +9,7 @@ on: jobs: CleanUpCcacheCache: name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: actions: write contents: read diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6d0dcff45..01162d1cc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,7 +16,7 @@ on: jobs: Docs: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml index 2768ef376..8dbeb26f1 100644 --- a/.github/workflows/post-pr.yml +++ b/.github/workflows/post-pr.yml @@ -6,7 +6,7 @@ on: jobs: cleanup: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Save PR number env: From 3111a277f6cc47d00f65c7554f58eca2dce8be73 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Sun, 10 Nov 2024 11:06:55 -0700 Subject: [PATCH 03/17] Try this. --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad3fb13a9..945d25fdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -447,9 +447,8 @@ jobs: cmake -B${{runner.workspace}}/build-hip \ -DCMAKE_BUILD_TYPE:STRING=Release \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ - -DCMAKE_CXX_COMPILER:STRING=$(which hipcc) \ + -DCMAKE_CXX_COMPILER:STRING=$(which clang++) \ -DCMAKE_C_COMPILER:STRING=$(which clang) \ - -DCMAKE_CXX_STANDARD:STRING=17 \ -DPELE_ENABLE_MPI:BOOL=OFF \ -DPELE_ENABLE_HIP:BOOL=ON \ -DPELE_EXCLUDE_BUILD_IN_CI:BOOL=ON \ From 9dec9f59b5187c9706f82fe4574c2115e9c57aeb Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Tue, 12 Nov 2024 11:59:28 -0700 Subject: [PATCH 04/17] Try adding cxx standard 17 back. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 945d25fdc..66615e7ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -449,6 +449,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DCMAKE_CXX_COMPILER:STRING=$(which clang++) \ -DCMAKE_C_COMPILER:STRING=$(which clang) \ + -DCMAKE_CXX_STANDARD:STRING=17 \ -DPELE_ENABLE_MPI:BOOL=OFF \ -DPELE_ENABLE_HIP:BOOL=ON \ -DPELE_EXCLUDE_BUILD_IN_CI:BOOL=ON \ From 2046534cdf65a89340423f09180cae9ab2380a6f Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Tue, 12 Nov 2024 15:33:54 -0700 Subject: [PATCH 05/17] User older ubuntu for HIP. --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66615e7ce..8c6d98284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -401,7 +401,7 @@ jobs: GPU-AMD: name: GPU-HIP needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - name: Clone uses: actions/checkout@v4 @@ -419,7 +419,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo mkdir --parents --mode=0755 /etc/apt/keyrings wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.2.1 noble main" | sudo tee --append /etc/apt/sources.list.d/rocm.list + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.4/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update @@ -449,7 +449,6 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DCMAKE_CXX_COMPILER:STRING=$(which clang++) \ -DCMAKE_C_COMPILER:STRING=$(which clang) \ - -DCMAKE_CXX_STANDARD:STRING=17 \ -DPELE_ENABLE_MPI:BOOL=OFF \ -DPELE_ENABLE_HIP:BOOL=ON \ -DPELE_EXCLUDE_BUILD_IN_CI:BOOL=ON \ From 3b788ddc8e2dd29699c5b4dd886266a0df4fc61d Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Wed, 13 Nov 2024 09:28:07 -0700 Subject: [PATCH 06/17] Try this. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c6d98284..0f31da8a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -419,7 +419,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo mkdir --parents --mode=0755 /etc/apt/keyrings wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.4/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.2/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update From 7484d79dae91ff4b6e14750af1fbd74065bfde7a Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Wed, 13 Nov 2024 10:01:49 -0700 Subject: [PATCH 07/17] Try this. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f31da8a6..ed2d079ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -401,7 +401,7 @@ jobs: GPU-AMD: name: GPU-HIP needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Clone uses: actions/checkout@v4 @@ -419,7 +419,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo mkdir --parents --mode=0755 /etc/apt/keyrings wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.2/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/amdgpu.list + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.4/ubuntu noble main" | sudo tee /etc/apt/sources.list.d/amdgpu.list echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update From 2fe27bff961cc06a534868c350974865bd738590 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Wed, 13 Nov 2024 10:32:11 -0700 Subject: [PATCH 08/17] Try this. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed2d079ec..1aa5e9bf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -419,7 +419,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo mkdir --parents --mode=0755 /etc/apt/keyrings wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.4/ubuntu noble main" | sudo tee /etc/apt/sources.list.d/amdgpu.list + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.2/ubuntu noble main" | sudo tee /etc/apt/sources.list.d/amdgpu.list echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update From 05de1e1eca125e115a7b21244c528f5c5dc2d4e4 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Wed, 13 Nov 2024 11:49:02 -0700 Subject: [PATCH 09/17] Try this. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa5e9bf4..f7de0b06e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -419,7 +419,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo mkdir --parents --mode=0755 /etc/apt/keyrings wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.2/ubuntu noble main" | sudo tee /etc/apt/sources.list.d/amdgpu.list + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.1/ubuntu noble main" | sudo tee /etc/apt/sources.list.d/amdgpu.list echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update From 0c1a6e1f37e71d12bcaba8d1c435892b5a840ce4 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Wed, 13 Nov 2024 14:44:21 -0700 Subject: [PATCH 10/17] Try this. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7de0b06e..f4fe08238 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -401,7 +401,7 @@ jobs: GPU-AMD: name: GPU-HIP needs: [Formatting, CPU-GNUmake] - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - name: Clone uses: actions/checkout@v4 @@ -419,7 +419,7 @@ jobs: ${{github.workspace}}/Submodules/PelePhysics/Submodules/amrex/.github/workflows/dependencies/ubuntu_free_disk_space.sh sudo mkdir --parents --mode=0755 /etc/apt/keyrings wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/amdgpu/6.2.1/ubuntu noble main" | sudo tee /etc/apt/sources.list.d/amdgpu.list + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.2.4 jammy main" | sudo tee --append /etc/apt/sources.list.d/rocm.list echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' | sudo tee -a /etc/profile.d/rocm.sh sudo apt-get update From 0e69a5eee31b262e108d587be317f290367294db Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Thu, 14 Nov 2024 12:43:13 -0700 Subject: [PATCH 11/17] Try non parallel ctest. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4fe08238..59b6892fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -332,7 +332,7 @@ jobs: (for DIM in 2 3; do \ printf "\n-------- Testing ${DIM}D --------\n"; \ cd ${{runner.workspace}}/build-${DIM}d-${{matrix.os}}-${{matrix.build_type}}; \ - ctest -j ${{env.NPROCS}} ${{matrix.ctest_args}} --output-on-failure; \ + ctest ${{matrix.ctest_args}} --output-on-failure; \ if [ $? -ne 0 ]; then exit 1; fi \ done) GPU-Nvidia: From 22c208a3deeac516177d6f5ed96d0668509deda9 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Thu, 14 Nov 2024 14:40:53 -0700 Subject: [PATCH 12/17] Don't write plots or checkpoints if not using fcompare. --- Tests/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 0106d18e3..2f23bac5e 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -56,7 +56,13 @@ macro(setup_test) file(COPY ${TEST_FILES} DESTINATION "${CURRENT_TEST_BINARY_DIR}/") # Set some default runtime options for all tests - set(RUNTIME_OPTIONS "amr.plot_file=plt amr.checkpoint_files_output=0 amr.plot_files_output=1 amrex.the_arena_is_managed=0 amrex.abort_on_unused_inputs=1") + set(RUNTIME_OPTIONS "amrex.the_arena_is_managed=0 amrex.abort_on_unused_inputs=1 amr.plot_int=-1 amr.check_int=-1 amr.checkpoint_files_output=0") + if(PELE_ENABLE_FCOMPARE_FOR_TESTS) + set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_file=plt amr.plot_files_output=1") + else() + set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=0") + endif() + if(PELE_ENABLE_FPE_TRAP_FOR_TESTS AND (NOT APPLE)) set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amrex.signal_handling=1 amrex.fpe_trap_invalid=1 amrex.fpe_trap_zero=1 amrex.fpe_trap_overflow=1") else() @@ -64,7 +70,7 @@ macro(setup_test) endif() if(PELE_ENABLE_MPI) if(PELE_ENABLE_CUDA) - set(PELE_NP 2) # 1 rank per GPU on Eagle + set(PELE_NP 2) else() set(PELE_NP 4) endif() From 9a7dceac61d81ed657c0543391041cd916919687 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Thu, 14 Nov 2024 17:22:55 -0700 Subject: [PATCH 13/17] Don't abort on unused inputs. --- Tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 2f23bac5e..1b4967948 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -56,7 +56,7 @@ macro(setup_test) file(COPY ${TEST_FILES} DESTINATION "${CURRENT_TEST_BINARY_DIR}/") # Set some default runtime options for all tests - set(RUNTIME_OPTIONS "amrex.the_arena_is_managed=0 amrex.abort_on_unused_inputs=1 amr.plot_int=-1 amr.check_int=-1 amr.checkpoint_files_output=0") + set(RUNTIME_OPTIONS "amrex.the_arena_is_managed=0 amr.plot_int=-1 amr.check_int=-1 amr.checkpoint_files_output=0") if(PELE_ENABLE_FCOMPARE_FOR_TESTS) set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_file=plt amr.plot_files_output=1") else() From fa7a0cba4c0d116a3604363cb08dda3738243f6f Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Fri, 15 Nov 2024 11:39:25 -0700 Subject: [PATCH 14/17] Try this. --- Tests/test_masscons.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Tests/test_masscons.py b/Tests/test_masscons.py index 767ed16a2..015a2c3eb 100644 --- a/Tests/test_masscons.py +++ b/Tests/test_masscons.py @@ -24,8 +24,10 @@ def test_conservation(self): fdir = os.path.abspath(".") fname = os.path.join(fdir, "datlog") df = pd.read_csv(fname, sep="\\s+") - npt.assert_allclose(df.mass, df.mass[0], rtol=1e-13) - npt.assert_allclose(df.rho_E, df.rho_E[0], rtol=1e-13) + init_mass = df.mass[0] + npt.assert_allclose(df.mass, init_mass, rtol=1e-13) + init_rho_E = df.rho_E[0] + npt.assert_allclose(df.rho_E, init_rho_E, rtol=1e-13) # ======================================================================== From 6fc1bc2753614481157c4884b615b838eea3a64f Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Fri, 15 Nov 2024 11:45:34 -0700 Subject: [PATCH 15/17] Add unused inputs. --- Tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 1b4967948..a66217fe1 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -56,9 +56,9 @@ macro(setup_test) file(COPY ${TEST_FILES} DESTINATION "${CURRENT_TEST_BINARY_DIR}/") # Set some default runtime options for all tests - set(RUNTIME_OPTIONS "amrex.the_arena_is_managed=0 amr.plot_int=-1 amr.check_int=-1 amr.checkpoint_files_output=0") + set(RUNTIME_OPTIONS "amr.plot_file=plt amr.checkpoint_files_output=0 amrex.the_arena_is_managed=0 amrex.abort_on_unused_inputs=1") if(PELE_ENABLE_FCOMPARE_FOR_TESTS) - set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_file=plt amr.plot_files_output=1") + set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=1") else() set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=0") endif() From bd6177bd182007d34bc2dbefd86aebc78e1ddb10 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Fri, 15 Nov 2024 11:59:36 -0700 Subject: [PATCH 16/17] Removed unused. --- Tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index a66217fe1..58a9fe3af 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -56,7 +56,7 @@ macro(setup_test) file(COPY ${TEST_FILES} DESTINATION "${CURRENT_TEST_BINARY_DIR}/") # Set some default runtime options for all tests - set(RUNTIME_OPTIONS "amr.plot_file=plt amr.checkpoint_files_output=0 amrex.the_arena_is_managed=0 amrex.abort_on_unused_inputs=1") + set(RUNTIME_OPTIONS "amr.plot_file=plt amr.checkpoint_files_output=0 amrex.the_arena_is_managed=0") if(PELE_ENABLE_FCOMPARE_FOR_TESTS) set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=1") else() From dd162ff6b4941bceb68fc6edb3ea4c26c3464f68 Mon Sep 17 00:00:00 2001 From: Jon Rood Date: Fri, 15 Nov 2024 12:24:33 -0700 Subject: [PATCH 17/17] Fix plot stuff. --- Tests/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index 58a9fe3af..769f6b00a 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -56,9 +56,9 @@ macro(setup_test) file(COPY ${TEST_FILES} DESTINATION "${CURRENT_TEST_BINARY_DIR}/") # Set some default runtime options for all tests - set(RUNTIME_OPTIONS "amr.plot_file=plt amr.checkpoint_files_output=0 amrex.the_arena_is_managed=0") + set(RUNTIME_OPTIONS "amr.checkpoint_files_output=0 amrex.the_arena_is_managed=0") if(PELE_ENABLE_FCOMPARE_FOR_TESTS) - set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=1") + set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=1 amr.plot_int=10 amr.plot_file=plt") else() set(RUNTIME_OPTIONS "${RUNTIME_OPTIONS} amr.plot_files_output=0") endif()