Skip to content
🐧 Linux

[WIP] Unify Random-Access API and Streaming API into Series::snapshots() #3741

Sign in to view logs

[WIP] Unify Random-Access API and Streaming API into Series::snapshots()

[WIP] Unify Random-Access API and Streaming API into Series::snapshots() #3741

Workflow file for this run

.github/workflows/linux.yml at a73c4c6
name: 🐧 Linux
on: [push, pull_request]
concurrency:
group: ${{ github.ref }}-${{ github.head_ref }}-linux
cancel-in-progress: true
jobs:
clang7_nopy_nompi_h5_libcpp:
runs-on: ubuntu-20.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- name: Spack Cache
uses: actions/cache@v3
with: {path: /opt/spack, key: clang7_nopy_nompi_h5_libcpp_v2}
- name: Install
run: |
sudo apt-get update
sudo apt-get install clang-7 libc++-7-dev libc++abi-7-dev
sudo .github/workflows/dependencies/install_spack
- name: Build
env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable}
# -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a
# false positive on src/auxiliary/Filesystem.cpp
# [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED;
run: |
eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_nompi_h5_libcpp/)
spack install
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DopenPMD_USE_PYTHON=OFF \
-DopenPMD_USE_MPI=OFF \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON
cmake --build build --parallel 2
cd build
ctest --output-on-failure
clang7_nopy_ompi_h5_ad2_libcpp:
runs-on: ubuntu-20.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- name: Spack Cache
uses: actions/cache@v3
with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad2_libcpp_v2}
- name: Install
run: |
sudo apt-get update
sudo apt-get install clang-7 libc++-7-dev libc++abi-7-dev gfortran libopenmpi-dev python3
sudo .github/workflows/dependencies/install_spack
- name: Build
env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -stdlib=libc++ -Werror -Wno-ignored-attributes -Wno-unused-const-variable}
# -Wno-ignored-attributes -Wno-unused-const-variable: clang-6 has a
# false positive on src/auxiliary/Filesystem.cpp
# [[maybe_unused]] MPI_Datatype const MPI_Types< unsigned >::value = MPI_UNSIGNED;
run: |
cmake --version
mpiexec --version
perl --version
python --version
eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/)
spack install
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DopenPMD_USE_PYTHON=OFF \
-DopenPMD_USE_MPI=ON \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_ADIOS2=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON
cmake --build build --parallel 2
cd build
ctest --output-on-failure
find . -name *.bp | xargs -n1 -P1 -I {} rm -rf {}
find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {}
ctest --output-on-failure
clang7_nopy_ompi_h5_ad2:
runs-on: ubuntu-20.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- name: Spack Cache
uses: actions/cache@v3
with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad2_v2 }
- name: Install
run: |
sudo apt-get update
sudo apt-get install clang-7 gfortran libopenmpi-dev python3
sudo .github/workflows/dependencies/install_spack
- name: Build
env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror}
run: |
eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/)
spack install
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DopenPMD_USE_PYTHON=OFF \
-DopenPMD_USE_MPI=ON \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_ADIOS2=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON
cmake --build build --parallel 2
cd build
ctest --output-on-failure
# TODO
# clang7_py36_nompi_h5_ad2_libstdc++
clang14_py311_nompi_h5_ad2:
runs-on: ubuntu-22.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- name: Spack Cache
uses: actions/cache@v3
with: {path: /opt/spack, key: clang14_py311_nompi_h5_ad2_v2 }
- name: Install
run: |
sudo apt update
sudo apt install clang-14 cmake gfortran libhdf5-dev python3.11 python3.11-dev wget python3.11-venv
sudo .github/workflows/dependencies/install_spack
python3.11 -m venv /opt/python_venv
. /opt/python_venv/bin/activate
python -m pip install numpy pandas
git clone -b v4.0.3 https://github.com/ToruNiina/toml11
cmake -S toml11 -B build_toml11 \
-DCMAKE_INSTALL_PREFIX=toml11_install \
-DCMAKE_CXX_STANDARD_REQUIRED=OFF \
-DCMAKE_CXX_STANDARD=11
cmake --build build_toml11 -j 2 --target install
- name: Build
env: {CC: clang-14, CXX: clang++-14, CXXFLAGS: -Werror}
run: |
# Build Spack packages against system Python
# and activate the virtual environment just for the openPMD build.
# Spack does not play nice with venv.
eval $(spack env activate --sh .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/)
spack install
. /opt/python_venv/bin/activate
share/openPMD/download_samples.sh build
export CMAKE_PREFIX_PATH="$(realpath toml11_install):$CMAKE_PREFIX_PATH"
cmake -S . -B build \
-DopenPMD_USE_PYTHON=ON \
-DopenPMD_USE_MPI=OFF \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_ADIOS2=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON \
-DopenPMD_USE_INTERNAL_TOML11=OFF \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DPython_EXECUTABLE="$(which python)"
cmake --build build --parallel 2
ctest --test-dir build --output-on-failure
# ADIOS2 v2.7.1
clang8_py38_mpich_h5_ad2:
runs-on: ubuntu-20.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- name: Spack Cache
uses: actions/cache@v3
with: {path: /opt/spack, key: clang8_py38_mpich_h5_ad2_newLayout_v2 }
- name: Install
run: |
sudo apt-get update
sudo apt-get install clang-8 gfortran libmpich-dev python3
sudo .github/workflows/dependencies/install_spack
- name: Build
env: {CC: clang-8, CXX: clang++-8, CXXFLAGS: -Werror}
run: |
cmake --version
mpiexec --version
perl --version
python --version
eval $(spack env activate --sh .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/)
spack install
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DopenPMD_USE_PYTHON=OFF \
-DopenPMD_USE_MPI=ON \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_ADIOS2=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON
cmake --build build --parallel 2
cd build
ctest --output-on-failure
# TODO: (old Travis-CI coverage)
# clang10_py38_ompi_h5_1-10-6_ad2_release
# ..._h5coll with OPENPMD_HDF5_INDEPENDENT: OFF
# TODO: (old Travis-CI coverage)
# gcc-4.9.4_nopy_nompi_h5_ad2
# gcc-4.9.4_nopy_ompi_h5_ad2
# gcc-9.3.0_nopy_nompi_h5_ad2-2.6.0
# gcc-7.4.0_nopy_ompi_h5_h5coll
# gcc-6.5.0_py35_nompi_h5-1.13.1
# gcc-8.1.0_py37_nompi_h5_static
# gcc-6.5.0_py36_nompi_h5-1.8.13
# gcc-4.8.5_py35_nompi_h5
# gcc-7.4.0_py_ompi_h5_ad2_coveralls
gcc7_py36_pd_dd_ompi_h5_ad2:
runs-on: ubuntu-20.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- name: Spack Cache
uses: actions/cache@v3
with: {path: /opt/spack, key: gcc7_py36_ompi_h5_ad2_v2 }
- name: Install
run: |
sudo apt-get update
sudo apt-get install g++-7 gfortran libopenmpi-dev python3 python3-setuptools
sudo .github/workflows/dependencies/install_spack
python3 -m pip install -U pip
python3 -m pip install -U numpy
python3 -m pip install -U mpi4py
python3 -m pip install -U pandas
python3 -m pip install -U dask
python3 -m pip install -U pyarrow
- name: Build
env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror}
run: |
eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/)
spack install
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DopenPMD_USE_PYTHON=ON \
-DopenPMD_USE_MPI=ON \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_ADIOS2=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON
cmake --build build --parallel 2
cd build
ctest --output-on-failure
gcc9_py38_pd_nompi_h5_ad2_libcpp:
runs-on: ubuntu-20.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- name: Install
run: |
sudo apt-get update
sudo apt-get install g++ libopenmpi-dev libhdf5-openmpi-dev python3 python3-numpy python3-mpi4py python3-pandas
# TODO ADIOS2
- name: Build
env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig}
run: |
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DopenPMD_USE_PYTHON=ON \
-DopenPMD_USE_MPI=ON \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON
cmake --build build --parallel 2
cd build
ctest --output-on-failure
musllinux_py10:
runs-on: ubuntu-20.04
if: github.event.pull_request.draft == false
container:
image: quay.io/pypa/musllinux_1_1_x86_64
steps:
- uses: actions/checkout@v4
- name: Install
run: |
apk update
apk add hdf5-dev
# Use a virtual environment in order to avoid compatibility issues
# between the various Python installations in this image.
python3.10 -m venv /opt/python_env
. /opt/python_env/bin/activate
python -m pip install numpy
- name: Build
env: {CXXFLAGS: -Werror}
run: |
. /opt/python_env/bin/activate
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DopenPMD_USE_PYTHON=ON \
-DopenPMD_USE_MPI=OFF \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON \
-DPython_EXECUTABLE=$(which python)
cmake --build build --parallel 2
cd build
ctest --output-on-failure
conda_ompi_all:
runs-on: ubuntu-22.04
if: github.event.pull_request.draft == false
steps:
- uses: actions/checkout@v4
- uses: conda-incubator/setup-miniconda@v3
name: Setup conda
with:
auto-update-conda: true
activate-environment: openpmd-api-dev
environment-file: conda.yml
channels: conda-forge,defaults
channel-priority: true
- name: Build
shell: bash -eo pipefail -l {0}
env: {CXXFLAGS: -Werror}
run: |
share/openPMD/download_samples.sh build
cmake -S . -B build \
-DCMAKE_CXX_FLAGS="-Wno-error=stringop-overread" \
-DCMAKE_C_FLAGS="-Wno-error=stringop-overread" \
-DopenPMD_USE_PYTHON=ON \
-DopenPMD_USE_MPI=ON \
-DopenPMD_USE_HDF5=ON \
-DopenPMD_USE_ADIOS2=ON \
-DopenPMD_USE_INVASIVE_TESTS=ON \
-DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh"
cmake --build build --parallel 2
cd build
ctest --output-on-failure