From 118a2e4c68607b5d6ddf47261593f3bcee37c047 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 6 Feb 2024 13:10:35 -0600 Subject: [PATCH 1/8] User dockerfile --- docker/user.Dockerfile | 63 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 docker/user.Dockerfile diff --git a/docker/user.Dockerfile b/docker/user.Dockerfile new file mode 100644 index 000000000..aa077beb9 --- /dev/null +++ b/docker/user.Dockerfile @@ -0,0 +1,63 @@ +# Install ubuntu 20.04 +FROM ubuntu:20.04 +LABEL maintainer="llogan@hawk.iit.edu" +LABEL version="0.0" +LABEL description="Hermes Docker image with CI" + +# Disable Prompt During Packages Installation +ARG DEBIAN_FRONTEND=noninteractive + +# Update ubuntu +SHELL ["/bin/bash", "-c"] +RUN apt update && apt install + +# Install some basic packages +RUN apt install -y \ + openssh-server \ + sudo \ + git \ + gcc g++ gfortran make binutils gpg \ + tar zip xz-utils bzip2 \ + perl m4 libncurses5-dev libxml2-dev diffutils \ + pkg-config cmake pkg-config \ + python3 python3-pip doxygen \ + lcov zlib1g-dev hdf5-tools \ + build-essential ca-certificates \ + coreutils curl environment-modules \ + gfortran git gpg lsb-release python3 python3-distutils \ + python3-venv unzip zip \ + bash jq python gdbserver gdb + +# Setup basic environment +ENV USER="root" +ENV HOME="/root" +ENV SPACK_DIR="${HOME}/spack" +ENV SPACK_VERSION="v0.20.2" +ENV HERMES_DEPS_DIR="${HOME}/hermes_deps" +ENV HERMES_DIR="${HOME}/hermes" +COPY ci/module_load.sh /module_load.sh + +# Install Spack +RUN . /module_load.sh && \ + git clone -b ${SPACK_VERSION} https://github.com/spack/spack ${SPACK_DIR} && \ + . "${SPACK_DIR}/share/spack/setup-env.sh" && \ + git clone -b dev https://github.com/lukemartinlogan/hermes.git ${HERMES_DEPS_DIR} && \ + # git clone -b dev https://github.com/HDFGroup/hermes.git ${HERMES_DEPS_DIR} && \ + spack repo add ${HERMES_DEPS_DIR}/ci/hermes && \ + mkdir -p ${HERMES_DIR} && \ + spack external find + +# Install hermes +RUN . /module_load.sh && \ + . "${SPACK_DIR}/share/spack/setup-env.sh" && \ + spack install hermes@master+vfd+mpiio^mpich@3.3.2 + +# Install jarvis-cd +RUN git clone https://github.com/grc-iit/jarvis-cd.git && \ + cd jarvis-cd && \ + pip install -e . -r requirements.txt + +# Install scspkg +RUN git clone https://github.com/grc-iit/scspkg.git && \ + cd scspkg && \ + pip install -e . -r requirements.txt From 81632bf9529ed3ab2db036c2d0caaffbdf6a4508 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 6 Feb 2024 13:48:14 -0600 Subject: [PATCH 2/8] Remove extra adapter_types.h --- include/hermes/adapter_types.h | 85 ---------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 include/hermes/adapter_types.h diff --git a/include/hermes/adapter_types.h b/include/hermes/adapter_types.h deleted file mode 100644 index faf3486da..000000000 --- a/include/hermes/adapter_types.h +++ /dev/null @@ -1,85 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ADAPTER_ADAPTER_TYPES_H_ -#define HERMES_ADAPTER_ADAPTER_TYPES_H_ - -namespace hermes::adapter { - -/** Adapter types */ -enum class AdapterType { - kNone, - kPosix, - kStdio, - kMpiio, - kPubsub, - kVfd -}; - -/** Adapter modes */ -enum class AdapterMode { - kNone, - kDefault, - kBypass, - kScratch, - kWorkflow -}; - -/** - * Per-Object Adapter Settings. - * An object may be a file, for example. - * */ -struct AdapterObjectConfig { - AdapterMode mode_; - size_t page_size_; -}; - -/** Adapter Mode converter */ -class AdapterModeConv { - public: - static std::string str(AdapterMode mode) { - switch (mode) { - case AdapterMode::kDefault: { - return "AdapterMode::kDefault"; - } - case AdapterMode::kBypass: { - return "AdapterMode::kBypass"; - } - case AdapterMode::kScratch: { - return "AdapterMode::kScratch"; - } - case AdapterMode::kWorkflow: { - return "AdapterMode::kWorkflow"; - } - default: { - return "Unkown adapter mode"; - } - } - } - - static AdapterMode to_enum(const std::string &mode) { - if (mode.find("kDefault") != std::string::npos) { - return AdapterMode::kDefault; - } else if (mode.find("kBypass") != std::string::npos) { - return AdapterMode::kBypass; - } else if (mode.find("kScratch") != std::string::npos) { - return AdapterMode::kScratch; - } else if (mode.find("kWorkflow") != std::string::npos) { - return AdapterMode::kWorkflow; - } - return AdapterMode::kDefault; - } -}; - -} // namespace hermes::adapter - -#endif // HERMES_ADAPTER_ADAPTER_TYPES_H_ From 3a7310c849f66d34f6fe0bf678d01f194249b2fb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 6 Feb 2024 16:50:05 -0600 Subject: [PATCH 3/8] Add transparent hermes to open --- hermes_adapters/vfd/H5FDhermes.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/hermes_adapters/vfd/H5FDhermes.cc b/hermes_adapters/vfd/H5FDhermes.cc index 8734f8c45..b53bd7e8f 100644 --- a/hermes_adapters/vfd/H5FDhermes.cc +++ b/hermes_adapters/vfd/H5FDhermes.cc @@ -229,6 +229,7 @@ H5FD__hermes_term(void) { static H5FD_t * H5FD__hermes_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { + TRANSPARENT_HERMES(); H5FD_hermes_t *file = NULL; /* hermes VFD info */ int fd = -1; int o_flags = 0; From 69988bb1d8ec4a259c51daa72d820af4a34907d9 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 6 Feb 2024 16:50:05 -0600 Subject: [PATCH 4/8] Add transparent hermes to open --- hermes_adapters/vfd/H5FDhermes.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/hermes_adapters/vfd/H5FDhermes.cc b/hermes_adapters/vfd/H5FDhermes.cc index 8734f8c45..b53bd7e8f 100644 --- a/hermes_adapters/vfd/H5FDhermes.cc +++ b/hermes_adapters/vfd/H5FDhermes.cc @@ -229,6 +229,7 @@ H5FD__hermes_term(void) { static H5FD_t * H5FD__hermes_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { + TRANSPARENT_HERMES(); H5FD_hermes_t *file = NULL; /* hermes VFD info */ int fd = -1; int o_flags = 0; From 0e23f0e4359038f7b934d389dfca62e1e9446cdc Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 6 Feb 2024 16:52:58 -0600 Subject: [PATCH 5/8] Remove load from pluing type --- hermes_adapters/vfd/H5FDhermes.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/hermes_adapters/vfd/H5FDhermes.cc b/hermes_adapters/vfd/H5FDhermes.cc index b53bd7e8f..fd9acde47 100644 --- a/hermes_adapters/vfd/H5FDhermes.cc +++ b/hermes_adapters/vfd/H5FDhermes.cc @@ -517,13 +517,11 @@ static herr_t H5FD__hermes_write(H5FD_t *_file, H5FD_mem_t type, */ H5PL_type_t H5PLget_plugin_type(void) { - TRANSPARENT_HERMES(); return H5PL_TYPE_VFD; } const void* H5PLget_plugin_info(void) { - TRANSPARENT_HERMES(); return &H5FD_hermes_g; } From 63c04d76205f0c39c18c3f12a16024df4b3201b6 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 7 Feb 2024 13:42:56 -0600 Subject: [PATCH 6/8] Add configuration test --- hermes_adapters/filesystem/filesystem.h | 2 +- include/hermes/config_client.h | 21 +++- test/unit/CMakeLists.txt | 1 + test/unit/config/CMakeLists.txt | 45 ++++++++ test/unit/config/test_config.cc | 42 ++++++++ test/unit/config/test_init.cc | 22 ++++ test/unit/config/test_init.h | 19 ++++ test/unit/hermes/config/hermes_server.yaml | 116 --------------------- 8 files changed, 150 insertions(+), 118 deletions(-) create mode 100644 test/unit/config/CMakeLists.txt create mode 100644 test/unit/config/test_config.cc create mode 100644 test/unit/config/test_init.cc create mode 100644 test/unit/config/test_init.h delete mode 100644 test/unit/hermes/config/hermes_server.yaml diff --git a/hermes_adapters/filesystem/filesystem.h b/hermes_adapters/filesystem/filesystem.h index 6d3261245..ea6a0c433 100644 --- a/hermes_adapters/filesystem/filesystem.h +++ b/hermes_adapters/filesystem/filesystem.h @@ -692,7 +692,7 @@ class Filesystem : public FilesystemIoClient { auto &paths = HERMES_CLIENT_CONF.path_list_; // Check if path is included or excluded for (config::UserPathInfo &pth : paths) { - if (abs_path.rfind(pth.path_) != std::string::npos) { + if (pth.Match(abs_path)) { if (abs_path == pth.path_ && pth.is_directory_) { // Do not include if path is a tracked directory return false; diff --git a/include/hermes/config_client.h b/include/hermes/config_client.h index 9dcf043d4..28d2f617a 100644 --- a/include/hermes/config_client.h +++ b/include/hermes/config_client.h @@ -34,6 +34,7 @@ static inline const bool do_exclude = false; /** Stores information about path inclusions and exclusions */ struct UserPathInfo { + std::regex regex_; /**< The regex to match the path */ std::string path_; /**< The path the user specified */ bool include_; /**< Whether to track path. */ bool is_directory_; /**< Whether the path is a file or directory */ @@ -43,7 +44,25 @@ struct UserPathInfo { /** Emplace Constructor */ UserPathInfo(const std::string &path, bool include, bool is_directory) - : path_(path), include_(include), is_directory_(is_directory) {} + : path_(path), include_(include), is_directory_(is_directory) { + std::string regexPattern = "^"; // Start of line anchor + for (char c : path) { + if (c == '.') { + regexPattern += "\\."; // Escape period + } else if (c == '/') { + regexPattern += "\\/"; // Escape forward slash + } else { + regexPattern += c; + } + } + regex_ = std::regex(regexPattern); + } + + /** Detect if a path matches the input path */ + bool Match(const std::string &abs_path) { + return std::regex_match(abs_path, regex_); + // return abs_path.rfind(path_) != std::string::npos; + } }; /** diff --git a/test/unit/CMakeLists.txt b/test/unit/CMakeLists.txt index 559ffe832..53311ce4c 100644 --- a/test/unit/CMakeLists.txt +++ b/test/unit/CMakeLists.txt @@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD 17) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${CMAKE_SOURCE_DIR}/tasks/hrun_admin/include) add_subdirectory(ipc) +add_subdirectory(config) add_subdirectory(hermes) add_subdirectory(hermes_adapters) add_subdirectory(boost) diff --git a/test/unit/config/CMakeLists.txt b/test/unit/config/CMakeLists.txt new file mode 100644 index 000000000..e2326e1ee --- /dev/null +++ b/test/unit/config/CMakeLists.txt @@ -0,0 +1,45 @@ +cmake_minimum_required(VERSION 3.10) +project(hermes) + +set(CMAKE_CXX_STANDARD 17) + +#------------------------------------------------------------------------------ +# Build Tests +#------------------------------------------------------------------------------ + +add_executable(test_config_exec + ${TEST_MAIN}/main_mpi.cc + test_init.cc + test_config.cc +) +add_dependencies(test_config_exec + ${Hermes_CLIENT_DEPS} hermes) +target_link_libraries(test_config_exec + ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX) +jarvis_test(hermes test_hermes) + +#------------------------------------------------------------------------------ +# Test Cases +#------------------------------------------------------------------------------ + +# STRING TESTS +#add_test(NAME test_ipc COMMAND +# ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc") + +#------------------------------------------------------------------------------ +# Install Targets +#------------------------------------------------------------------------------ +install(TARGETS + test_config_exec + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR}) + +#----------------------------------------------------------------------------- +# Coverage +#----------------------------------------------------------------------------- +if(HERMES_ENABLE_COVERAGE) + set_coverage_flags(test_config_exec) +endif() diff --git a/test/unit/config/test_config.cc b/test/unit/config/test_config.cc new file mode 100644 index 000000000..57ae762ae --- /dev/null +++ b/test/unit/config/test_config.cc @@ -0,0 +1,42 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "basic_test.h" +#include "hrun/api/hrun_client.h" +#include "hrun_admin/hrun_admin.h" +#include "hermes/hermes.h" +#include "hermes/bucket.h" +#include "data_stager/factory/binary_stager.h" +#include + +TEST_CASE("TestHermesPaths") { + PAGE_DIVIDE("Directory path") { + hermes::config::UserPathInfo info("/home/hello", true, true); + REQUIRE(info.Match("/home") == false); + REQUIRE(info.Match("/home/hello") == true); + REQUIRE(info.Match("/home/hello/hi.txt") == true); + } + + PAGE_DIVIDE("Simple path") { + hermes::config::UserPathInfo info("/home/hello.txt", true, false); + REQUIRE(info.Match("/home/hello") == false); + REQUIRE(info.Match("/home/hello.txt") == true); + } + + PAGE_DIVIDE("Wildcard path") { + hermes::config::UserPathInfo info("/home/hello/*.json", true, false); + REQUIRE(info.Match("/home/hello") == false); + REQUIRE(info.Match("/home/hello/hi.json") == true); + REQUIRE(info.Match("/home/hello/.json") == true); + } +} + diff --git a/test/unit/config/test_init.cc b/test/unit/config/test_init.cc new file mode 100644 index 000000000..1757fe387 --- /dev/null +++ b/test/unit/config/test_init.cc @@ -0,0 +1,22 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#include "hrun/api/hrun_client.h" +#include "basic_test.h" +#include "test_init.h" + +void MainPretest() { +} + +void MainPosttest() { +} diff --git a/test/unit/config/test_init.h b/test/unit/config/test_init.h new file mode 100644 index 000000000..cb3b36df7 --- /dev/null +++ b/test/unit/config/test_init.h @@ -0,0 +1,19 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +#ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_ +#define HRUN_TEST_UNIT_IPC_TEST_INIT_H_ + +#include "hrun/hrun_types.h" + +#endif // HRUN_TEST_UNIT_IPC_TEST_INIT_H_ diff --git a/test/unit/hermes/config/hermes_server.yaml b/test/unit/hermes/config/hermes_server.yaml deleted file mode 100644 index 14cf03b81..000000000 --- a/test/unit/hermes/config/hermes_server.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# Example Hermes configuration file - -### Define properties of the storage devices -devices: - ram: - mount_point: "" - capacity: 4GB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 6000MBps - latency: 15us - is_shared_device: false - borg_capacity_thresh: [0.0, 1.0] - - nvme: - mount_point: "./" - capacity: 100MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 1GBps - latency: 600us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - ssd: - mount_point: "./" - capacity: 100MB - block_size: 4KB - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 500MBps - latency: 1200us - is_shared_device: false - borg_capacity_thresh: [ 0.0, 1.0 ] - - pfs: - mount_point: "./" - capacity: 100MB - block_size: 64KB # The stripe size of PFS - slab_sizes: [ 4KB, 16KB, 64KB, 1MB ] - bandwidth: 100MBps # Per-device bandwidth - latency: 200ms - is_shared_device: true - borg_capacity_thresh: [ 0.0, 1.0 ] - -### Define properties of RPCs -rpc: - host_file: "" - host_names: ["localhost"] - protocol: "ofi+sockets" - domain: "" - port: 8080 - num_threads: 32 - -### Define properties of the BORG -buffer_organizer: - num_threads: 1 - flush_period: 1024 - blob_reorg_period: 1024 - recency_min: 0 - recency_max: 60 - freq_max: 15 - freq_min: 0 - -### Define the default data placement policy -dpe: - default_placement_policy: "MinimizeIoTime" - default_rr_split: 0 - -### Define I/O tracing properties -tracing: - enabled: false - output: "" - -### Define prefetcher properties -prefetch: - enabled: false - io_trace_path: "" - apriori_schema_path: "" - epoch_ms: 50 - is_mpi: false - -### Define mdm properties -mdm: - est_blob_count: 100000 - est_bucket_count: 100000 - est_num_traits: 256 - -# The interval in milliseconds at which to update the global system view. -system_view_state_update_interval_ms: 1000 - -### Runtime orchestration settings -work_orchestrator: - max_dworkers: 4 - max_oworkers: 32 - owork_per_core: 32 - -### Queue Manager settings -queue_manager: - queue_depth: 100000 - max_lanes: 16 - max_queues: 1024 - shm_allocator: kScalablePageAllocator - shm_name: "hrun_shm" - shm_size: 0g - data_shm_size: 4g - -### Task Registry -task_registry: [ - 'hermes_mdm', - 'hermes_blob_mdm', - 'hermes_bucket_mdm', - 'hermes_data_op', - 'data_stager', - 'posix_bdev', - 'ram_bdev' -] \ No newline at end of file From 6546b624dbd02078ee4eefd8cc5193688ff1490f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 7 Feb 2024 13:56:12 -0600 Subject: [PATCH 7/8] Add path regexing --- include/hermes/config_client.h | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/include/hermes/config_client.h b/include/hermes/config_client.h index 28d2f617a..a10008580 100644 --- a/include/hermes/config_client.h +++ b/include/hermes/config_client.h @@ -42,20 +42,30 @@ struct UserPathInfo { /** Default constructor */ UserPathInfo() = default; - /** Emplace Constructor */ - UserPathInfo(const std::string &path, bool include, bool is_directory) - : path_(path), include_(include), is_directory_(is_directory) { - std::string regexPattern = "^"; // Start of line anchor + static std::string ToRegex(const std::string &path) { + std::string regex_pattern = "^"; // Start of line anchor for (char c : path) { if (c == '.') { - regexPattern += "\\."; // Escape period + regex_pattern += "\\."; // Escape period } else if (c == '/') { - regexPattern += "\\/"; // Escape forward slash + regex_pattern += "\\/"; // Escape forward slash + } else if (c == '*') { + regex_pattern += ".*"; // Match any character } else { - regexPattern += c; + regex_pattern += c; } } - regex_ = std::regex(regexPattern); + return regex_pattern; + } + + /** Emplace Constructor */ + UserPathInfo(const std::string &path, bool include, bool is_directory) + : path_(path), include_(include), is_directory_(is_directory) { + std::string regex_pattern = ToRegex(path); + if (is_directory) { + regex_pattern += ".*"; + } + regex_ = std::regex(regex_pattern); } /** Detect if a path matches the input path */ From f193f7476c7dc0a1035d9576cddfaf9c4cedef52 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Wed, 7 Feb 2024 14:00:29 -0600 Subject: [PATCH 8/8] Fix lint issues --- include/hermes/config_client.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/hermes/config_client.h b/include/hermes/config_client.h index a10008580..0dddf7e89 100644 --- a/include/hermes/config_client.h +++ b/include/hermes/config_client.h @@ -43,14 +43,14 @@ struct UserPathInfo { UserPathInfo() = default; static std::string ToRegex(const std::string &path) { - std::string regex_pattern = "^"; // Start of line anchor + std::string regex_pattern = "^"; for (char c : path) { if (c == '.') { - regex_pattern += "\\."; // Escape period + regex_pattern += "\\."; } else if (c == '/') { - regex_pattern += "\\/"; // Escape forward slash + regex_pattern += "\\/"; } else if (c == '*') { - regex_pattern += ".*"; // Match any character + regex_pattern += ".*"; } else { regex_pattern += c; }