Merge pull request #648 from lukemartinlogan/dev

Improve queueing times

.github/workflows/main.yml

@@ -57,24 +57,16 @@ jobs:
       - name: Install APT Dependencies
         run: |
           sudo apt update
-          sudo apt-get install -y autoconf
-          sudo apt-get install -y automake
-          sudo apt-get install -y libtool
-          sudo apt-get install -y libtool-bin
-          sudo apt-get install -y mpich
-          sudo apt-get install -y lcov
-          sudo apt-get install -y zlib1g-dev
-          sudo apt-get install -y libsdl2-dev
-          sudo apt-get install -y hdf5-tools
+          sudo apt-get install -y docker
 
       - name: Build And Install Dependencies
-        if: steps.spack-cache.outputs.cache-hit != 'true'
+#        if: steps.spack-cache.outputs.cache-hit != 'true'
         run: ci/install_deps.sh
 
-#      - name: Build
+      - name: Build and Test
 #        if: steps.hermes-cache.outputs.cache-hit != 'true'
-#        run: ci/build_hermes.sh
-#
+        run: docker exec /hermes_deps_c bash hermes/ci/build_hermes.sh
+
 #      - name: Test
 #        run: bash ci/test_hermes.sh
 #
@@ -86,8 +78,8 @@ jobs:
 #      - name: Multi-node Test
 #        run: pushd ci/cluster && ./multi_node_ci_test.sh
 
-#      - name: Generate coverage file
-#        run: bash ci/coverage.sh "${GITHUB_WORKSPACE}/coverage" "${GITHUB_WORKSPACE}/build"
+      - name: Generate coverage file
+        run: docker exec /hermes_deps_c bash hermes/ci/coverage.sh "hermes/coverage" "hermes/build"
 
 #      - name: Coveralls
 #        uses: coverallsapp/github-action@master

CMakeLists.txt

@@ -74,6 +74,8 @@ add_custom_target(coverage COMMAND bash ${CMAKE_SOURCE_DIR}/ci/coverage.sh
 #-----------------------------------------------------------------------------
 # Find Packages
 #-----------------------------------------------------------------------------
+# This is for compatability with SPACK
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
 # HermesShm
 find_package(HermesShm CONFIG REQUIRED)

README.md

@@ -53,3 +53,34 @@ make install
 ## Contributing
 
 We follow the [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html). You can run `make lint` to ensure that your code conforms to the style. This requires the `cpplint` Python module (`pip install cpplint`). Alternatively, you can let the CI build inform you of required style changes.
+
+## Docker
+
+Build container with Hermes dependencies:
+```
+docker build -t lukemartinlogan/hermes_deps . -f docker/deps.Dockerfile
+```
+
+Run the container with the Hermes source mounted:
+```
+docker run -it --mount src=${PWD},target=/hermes,type=bind \
+--name hermes_deps_c \
+--network host \
+--memory=4G \
+--shm-size=4G \
+-p 4000:4000 \
+-p 4001:4001 \
+lukemartinlogan/hermes_deps
+```
+
+Build Hermes + Jarvis (in container):
+```
+bash /hermes/ci/build_hermes.sh
+```
+
+```
+docker commit hermes_deps_c lukemartinlogan/hermes_deps
+docker push lukemartinlogan/hermes_deps
+docker stop /hermes_deps_c
+docker rm /hermes_deps_c
+```

benchmark/hermes_api_bench.cc

@@ -147,7 +147,6 @@ void CreateBucketTest(int nprocs, int rank,
   MpiTimer t(MPI_COMM_WORLD);
   t.Resume();
   hapi::Context ctx;
-  std::unordered_map<std::string, std::string> mdm_;
   for (size_t i = 0; i < bkts_per_rank; ++i) {
     int bkt_name_int = rank * bkts_per_rank + i;
     std::string bkt_name = std::to_string(bkt_name_int);
@@ -242,8 +241,7 @@ int main(int argc, char **argv) {
   MPI_Init(&argc, &argv);
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
   MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-  TRANSPARENT_HRUN();
-  HERMES->ClientInit();
+  TRANSPARENT_HERMES();
 
   // Get mode
   REQUIRE_ARGC_GE(2)

benchmark/test_latency.cc

@@ -270,8 +270,8 @@ TEST_CASE("TestRoundTripLatency") {
   size_t ops = (1 << 20);
   // size_t ops = 1024;
   for (size_t i = 0; i < ops; ++i) {
-    client.MdRoot(hrun::DomainId::GetLocal());
-    // client.MdPushRoot(hrun::DomainId::GetLocal());
+    // client.MdRoot(hrun::DomainId::GetLocal());
+    client.MdPushRoot(hrun::DomainId::GetLocal());
   }
   t.Pause();
 

ci/build_hermes.sh

@@ -1,42 +1,75 @@
 #!/bin/bash
 
-# CD into git workspace
-cd ${GITHUB_WORKSPACE}
+# ARGS:
+# SPACK_DIR: the path to spack
 
+# THIS SCRIPT IS EXECUTED BY CONTAINER!!!
 set -x
 set -e
 set -o pipefail
 
-# Set spack env
-INSTALL_DIR="${HOME}"
-SPACK_DIR=${INSTALL_DIR}/spack
-. ${SPACK_DIR}/share/spack/setup-env.sh
+# Update jarvis-cd
+pushd jarvis-cd
+git pull
+pip install -e . -r requirements.txt
+popd
 
-mkdir -p "${HOME}/install"
-mkdir build
-cd build
+# Update scspkg
+pushd scspkg
+git pull
+pip install -e . -r requirements.txt
+popd
+
+# Load scspkg environment
+. /module_load.sh
+module use "$(scspkg module dir)"
+
+# Load hermes_shm
+. "${SPACK_DIR}/share/spack/setup-env.sh"
+spack module tcl refresh --delete-tree -y
 spack load hermes_shm
+# module use "${SPACK_DIR}/share/spack/modules/linux-ubuntu22.04-zen2"
+
+# Create Hermes module
+scspkg create hermes
+scspkg env prepend hermes PATH /hermes/build/bin
+scspkg env prepend hermes LIBRARY_PATH /hermes/build/bin
+scspkg env prepend hermes LD_LIBRARY_PATH /hermes/build/bin
+module load hermes
+
+# Initialize the Jarvis testing Hermes environment
+jarvis init \
+"${HOME}/jarvis-config" \
+"${HOME}/jarvis-priv" \
+"${HOME}/jarvis-shared"
+cp /hermes/ci/resource_graph.yaml /jarvis-cd/config/resource_graph.yaml
+jarvis env build hermes
+
+# CD into Hermes directory in container
+cd /hermes
+git config --global --add safe.directory '*'
+git submodule update --init
+
+# Build Hermes
+mkdir -p build
+cd build
 cmake ../ \
 -DCMAKE_BUILD_TYPE=Debug \
--DCMAKE_INSTALL_PREFIX="${HOME}/install"
+-DCMAKE_INSTALL_PREFIX="$(scspkg pkg root hermes)" \
+-DHERMES_ENABLE_MPIIO_ADAPTER=ON \
+-DHERMES_MPICH=ON \
+-DHERMES_ENABLE_STDIO_ADAPTER=ON \
+-DHERMES_ENABLE_POSIX_ADAPTER=ON \
+-DHERMES_ENABLE_COVERAGE=ON
 make -j8
 make install
 
+# Test Hermes
 export CXXFLAGS=-Wall
 ctest -VV
 
-# Set proper flags for cmake to find Hermes
-INSTALL_PREFIX="${HOME}/install"
-export LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LIBRARY_PATH}"
-export LD_LIBRARY_PATH="${INSTALL_PREFIX}/lib:${LD_LIBRARY_PATH}"
-export LDFLAGS="-L${INSTALL_PREFIX}/lib:${LDFLAGS}"
-export CFLAGS="-I${INSTALL_PREFIX}/include:${CFLAGS}"
-export CPATH="${INSTALL_PREFIX}/include:${CPATH}"
-export CMAKE_PREFIX_PATH="${INSTALL_PREFIX}:${CMAKE_PREFIX_PATH}"
-export CXXFLAGS="-I${INSTALL_PREFIX}/include:${CXXFLAGS}"
-
 # Run make install unit test
-cd test/unit/external
+cd /hermes/test/unit/external
 mkdir build
 cd build
 cmake ../

ci/hermes/packages/hermes/package.py

@@ -45,14 +45,15 @@ class Hermes(CMakePackage):
     depends_on('cereal')
     depends_on('yaml-cpp')
     depends_on('libaio')
-    depends_on('[email protected]')
+    depends_on('doxygen')  # @1.9.3
     depends_on('[email protected]: +context +fiber +filesystem +system +atomic +chrono +serialization +signals +pic +regex')
     depends_on('libfabric fabrics=sockets,tcp,udp,verbs',
                when='+ares')
     depends_on('libfabric fabrics=verbs',
                when='+only_verbs')
     depends_on('libzmq', '+zmq')
     depends_on('[email protected]', when='+vfd')
+    depends_on('adios2', when='+adios')
 
     def cmake_args(self):
         args = []

ci/hermes/packages/hermes_shm/package.py

@@ -24,7 +24,7 @@ class HermesShm(CMakePackage):
     depends_on('cereal')
     depends_on('yaml-cpp')
     depends_on('libaio')
-    depends_on('[email protected]')
+    depends_on('doxygen')  # @1.9.3
     depends_on('[email protected]: +context +fiber +filesystem +system +atomic +chrono +serialization +signals +pic +regex')
     depends_on('libfabric fabrics=sockets,tcp,udp,verbs',
                when='+ares')

ci/install_deps.sh

@@ -1,46 +1,18 @@
 #!/bin/bash
 
-# CD into git workspace
-cd ${GITHUB_WORKSPACE}
-
-# This script will build and install them via Spack from source
-# because Hermes requires a very specific version and configuration options
-# for each package.
-
 set -x
 set -e
 set -o pipefail
 
-# Change this especially when your $HOME doesn't have enough disk space. 
-INSTALL_DIR="${HOME}"
-SPACK_DIR=${INSTALL_DIR}/spack
-SPACK_VERSION=0.20.2
-
-echo "Installing dependencies at ${INSTALL_DIR}"
-mkdir -p ${INSTALL_DIR}
-
-# Load Spack
-git clone https://github.com/spack/spack ${SPACK_DIR}
-cd ${SPACK_DIR}
-git checkout v${SPACK_VERSION}
-
-# Set spack env
-set +x
-. ${SPACK_DIR}/share/spack/setup-env.sh
-set -x
-
-# Install jarvis-cd
-git clone https://github.com/grc-iit/jarvis-cd.git
-cd jarvis-cd
-pip install -e . -r requirements.txt
-
-# This will allow Spack to skip building some packages that are directly
-# available from the system. For example, autoconf, cmake, m4, etc.
-# Modify ci/pckages.yaml to skip building compilers or build tools via Spack.
-cd ${GITHUB_WORKSPACE}
-cp ci/packages.yaml ${SPACK_DIR}/etc/spack/packages.yaml
-
-# Install hermes_shm (needed for dependencies)
-#
-spack repo add ci/hermes
-spack install hermes_shm@master+vfd+mpiio^[email protected]
+# Pull the Hermes dependencies image
+docker pull lukemartinlogan/hermes_deps:latest
+docker run -d \
+--mount src=${PWD},target=/hermes,type=bind \
+--name hermes_deps_c \
+--network host \
+--memory=4G \
+--shm-size=4G \
+-p 4000:4000 \
+-p 4001:4001 \
+lukemartinlogan/hermes_deps \
+tail -f /dev/null

ci/module_load.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+if ! shopt -q login_shell; then
+  if [ -d /etc/profile.d ]; then
+    for i in /etc/profile.d/*.sh; do
+      if [ -r $i ]; then
+        . $i
+      fi
+    done
+  fi
+fi