Skip to content

Commit

Permalink
switch to building HDF5 with CMake in the Actions (#46)
Browse files Browse the repository at this point in the history 
* switch to building HDF5 with CMake for 2.0+ versions, added tests for 1.14 series.
* added codeql to actions
  • Loading branch information
@brtnfld
brtnfld authored Dec 6, 2024
1 parent c1363df commit a54d4cb
Show file tree
Hide file tree
Showing 3 changed files with 136 additions and 96 deletions.
5 changes: 5 additions & 0 deletions .github/codeql-config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
query-filters:
- exclude:
id: cpp/toctou-race-condition
- exclude:
id: cpp/short-global-name
127 changes: 78 additions & 49 deletions .github/workflows/codeql.yml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,3 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL Advanced"

on:
Expand All @@ -21,13 +10,14 @@ on:

jobs:
analyze:
name: Analyze (${{ matrix.language }})
name: Analyze
# Runner size impacts CodeQL analysis time. To learn more, please see:
# - https://gh.io/recommended-hardware-resources-for-running-codeql
# - https://gh.io/supported-runners-and-hardware-resources
# - https://gh.io/using-larger-runners (GitHub.com only)
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
runs-on: ubuntu-latest
#runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
permissions:
# required for all workflows
security-events: write
Expand All @@ -36,17 +26,15 @@ jobs:
packages: read

# only required for workflows in private repositories
actions: read
contents: read
#actions: read
#contents: read

strategy:
fail-fast: false
matrix:
include:
- language: c-cpp
build-mode: autobuild
- language: python
build-mode: none
build-mode: manual
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
# Use `c-cpp` to analyze code written in C, C++ or both
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
Expand All @@ -56,39 +44,80 @@ jobs:
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Checkout submodule Argobots
uses: actions/[email protected]
with:
submodules: 'true'

- name: Checkout HDF5 latest
uses: actions/[email protected]
with:
repository: HDFGroup/hdf5
path: hdf5

- name: Dependencies
run: |
sudo apt-get update
sudo apt-get install libtool libopenmpi-dev zlib1g-dev python3 cmake
- name: Configure Async VOL (install dependencies)
run: |
mydir="$PWD"
export HDF5_DIR=$mydir/hdf5/install
export ABT_DIR=$mydir/argobots
export VOL_DIR=$mydir/vol-async/install
mkdir $HDF5_DIR
# Compile HDF5
cd hdf5
mkdir build
cd build
cmake -DCMAKE_INSTALL_PREFIX=$HDF5_DIR -DHDF5_ENABLE_PARALLEL=ON -DHDF5_ENABLE_THREADSAFE=ON \
-DALLOW_UNSUPPORTED=ON -DBUILD_TESTING=OFF -DHDF5_BUILD_HL_LIB=OFF \
-DHDF5_BUILD_EXAMPLES=OFF -DHDF5_BUILD_FORTRAN=OFF -DCMAKE_C_COMPILER=mpicc ..
make -j && make install
# Compile Argobots
cd $ABT_DIR
./autogen.sh
./configure --prefix=$ABT_DIR
make && make install
# Configure Asynchronous VOL connector
pwd
cd ../
mkdir build
cd build
cmake -DCMAKE_INSTALL_PREFIX=$VOL_DIR -DCMAKE_C_COMPILER=mpicc ..
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
config-file: ./.github/codeql-config.yml
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.

# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
queries: +security-extended,security-and-quality

# If the analyze step fails for one of the languages you are analyzing with
# "We were unable to automatically build your code", modify the matrix above
# to set the build mode to "manual" for that language. Then modify this step
# to build your code.
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
- if: matrix.build-mode == 'manual'
shell: bash
run: |
echo 'If you are using a "manual" build mode for one or more of the' \
'languages you are analyzing, replace this with the commands to build' \
'your code, for example:'
echo ' make bootstrap'
echo ' make release'
exit 1
# If the analyze step fails for one of the languages you are analyzing with
# "We were unable to automatically build your code", modify the matrix above
# to set the build mode to "manual" for that language. Then modify this step
# to build your code.
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
- name: Async VOL,build and test
shell: bash
run: |
cd build
make && make install
ctest -V -E async_test_parallel5 .
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"
100 changes: 53 additions & 47 deletions .github/workflows/linux.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ on:
# Allows to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
vol-async-hdf5-latest-Makefile:

###############################################
# Testing last version of the HDF5 1.14 series
###############################################

vol-async-hdf5-1_14-Makefile:
runs-on: ubuntu-latest
timeout-minutes: 60

Expand All @@ -31,9 +36,9 @@ jobs:
- name: Checkout latest HDF5 release
run: |
wget https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz
tar xzf hdf5.tar.gz
ln -sf hdf5-* hdf5
wget https://github.com/HDFGroup/hdf5/releases/download/hdf5_1.14.5/hdf5-1.14.5.tar.gz
tar xzf hdf5-1.14.5.tar.gz
mv hdf5-1.14.5 hdf5
- name: Installation
run: |
Expand Down Expand Up @@ -101,7 +106,7 @@ jobs:
make check
date
vol-async-hdf5-develop-Makefile:
vol-async-hdf5-1_14-cmake:
runs-on: ubuntu-latest
timeout-minutes: 30

Expand All @@ -110,33 +115,34 @@ jobs:
uses: actions/[email protected]
with:
submodules: 'true'

- name: Checkout HDF5 latest
uses: actions/[email protected]
with:
repository: HDFGroup/hdf5
path: hdf5


- name: Dependencies
run: |
sudo apt-get update
sudo apt-get install libtool libopenmpi-dev zlib1g-dev python3
sudo apt-get install libtool libopenmpi-dev zlib1g-dev python3 cmake
- name: Checkout latest HDF5 release
run: |
wget https://github.com/HDFGroup/hdf5/releases/download/hdf5_1.14.5/hdf5-1.14.5.tar.gz
tar xzf hdf5-1.14.5.tar.gz
mv hdf5-1.14.5 hdf5
- name: Installation
run: |
mydir="$PWD"
export HDF5_DIR=$mydir/hdf5/install
export ABT_DIR=$mydir/argobots
export VOL_DIR=$mydir
export VOL_DIR=$mydir/vol-async/install
mkdir $HDF5_DIR
# Compile HDF5
cd hdf5
export HDF5_LIBTOOL=/usr/bin/libtoolize
./autogen.sh
./configure --prefix=$HDF5_DIR --enable-parallel --enable-threadsafe --enable-unsupported \
--disable-tests --disable-hl --disable-fortran --disable-tools
make && make install
mkdir build
cd build
cmake -DCMAKE_INSTALL_PREFIX=$HDF5_DIR -DHDF5_ENABLE_PARALLEL=ON -DHDF5_ENABLE_THREADSAFE=ON \
-DALLOW_UNSUPPORTED=ON -DBUILD_TESTING=OFF -DHDF5_BUILD_HL_LIB=OFF \
-DHDF5_BUILD_EXAMPLES=OFF -DHDF5_BUILD_FORTRAN=OFF -DCMAKE_C_COMPILER=mpicc ..
make -j && make install
# Compile Argobots
cd $ABT_DIR
Expand All @@ -145,51 +151,47 @@ jobs:
make && make install
# Compile Asynchronous VOL connector
cd $VOL_DIR/src
tail -n 48 Makefile.summit > Makefile
make
pwd
cd ../
mkdir build
cd build
cmake -DCMAKE_INSTALL_PREFIX=$VOL_DIR -DCMAKE_C_COMPILER=mpicc ..
make && make install
- name: Compile and Test Vol-Async
run: |
# Compile vol-async tests
mydir="$PWD"
cd $mydir
cd test
echo $PWD
cd build
# Set Environmental Variables
export H5_DIR=$mydir/hdf5
export VOL_DIR=$mydir/
export VOL_DIR=$mydir/vol-async/install
export ABT_DIR=$mydir/argobots
export ASYNC_DIR=$mydir/src
export HDF5_DIR=$mydir/hdf5/install
export LD_LIBRARY_PATH=$VOL_DIR/src:$H5_DIR/install/lib:$ABT_DIR/lib:$LD_LIBRARY_PATH
export HDF5_PLUGIN_PATH="$VOL_DIR/src"
export LD_LIBRARY_PATH=$VOL_DIR/lib:$HDF5_DIR/install/lib:$ABT_DIR/lib:$LD_LIBRARY_PATH
export HDF5_PLUGIN_PATH="$VOL_DIR/lib"
export HDF5_VOL_CONNECTOR="async under_vol=0;under_info={}"
# Makefile
tail -n 47 Makefile.summit > Makefile
export LD_PRELOAD=$mydir/argobots/lib/libabt.so
make
date
# Run serial and parallel tests
make check
date
ctest -V
date
# Test with async VOL double buffering
cd ../src
sed -i "s/CFLAGS =/CFLAGS = -DENABLE_WRITE_MEMCPY=1 /g" ./Makefile
cat Makefile
# Test with async VOL write memcopy (double buffering)
cmake -DCMAKE_INSTALL_PREFIX=$VOL_DIR -DCMAKE_C_COMPILER=mpicc -DENABLE_WRITE_MEMCPY=ON ..
make clean
make
cd ../test
make && make install
date
make clean
make
make check
ctest -V
date
#################################################
# Testing with the HDF5 2.0+ series
# * Note: autools support was dropped in HDF5
#################################################

vol-async-hdf5-latest-cmake:
runs-on: ubuntu-latest
timeout-minutes: 30
Expand Down Expand Up @@ -271,6 +273,10 @@ jobs:
ctest -V
date
#######################################
# Testing using HDF5's "develop" branch
#######################################

vol-async-hdf5-develop-cmake:
runs-on: ubuntu-latest
timeout-minutes: 30
Expand Down

0 comments on commit a54d4cb

Please sign in to comment.