Extension the support for platforms consisting of arbitrary numbers of GRID storages and workers

.gitignore

@@ -33,9 +33,16 @@ wrench*/*
 
 
 # outputs
+data/sgbatch/*
 tmp
 *.pdf
+*.png
 *.csv
+*.txt
+
+# logs and tests
+logs
+test
 
 # test/profiling outputs
 *.ipynb

.vscode/settings.json

@@ -62,8 +62,12 @@
         "typeindex": "cpp",
         "typeinfo": "cpp",
         "valarray": "cpp",
-        "variant": "cpp"
+        "variant": "cpp",
+        "codecvt": "cpp",
+        "iomanip": "cpp",
+        "strstream": "cpp",
+        "cfenv": "cpp"
     },
     "cmake.configureOnOpen": false,
-    "cmake.sourceDirectory": "${workspaceFolder}/sgbatch"
+    "cmake.sourceDirectory": "${workspaceFolder}"
 }

sgbatch/CMakeLists.txt → CMakeLists.txt

@@ -10,7 +10,7 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMakeModules/")
 
 find_package(SimGrid REQUIRED)
-find_package(Boost COMPONENTS program_options REQUIRED)
+find_package(Boost COMPONENTS program_options regex REQUIRED)
 
 
 # include directories for dependencies and WRENCH libraries
@@ -20,21 +20,28 @@ include_directories(src/ ${SimGrid_INCLUDE_DIR}/include /usr/local/include /opt/
 set(SOURCE_FILES
         src/SimpleExecutionController.h
         src/SimpleExecutionController.cpp
+        src/SimpleSimulator.h
         src/SimpleSimulator.cpp
         src/JobSpecification.h
+        src/LRU_FileList.h
+        src/MonitorAction.h
+        src/MonitorAction.cpp
+        src/util/DefaultValues.h
+        src/util/Enums.h
         src/computation/CacheComputation.h
         src/computation/CacheComputation.cpp
         src/computation/StreamedComputation.h
         src/computation/StreamedComputation.cpp
         src/computation/CopyComputation.h
         src/computation/CopyComputation.cpp
-        src/SimpleSimulator.h
-        src/LRU_FileList.h
         )
 
 # test files
 set(TEST_FILES
         src/JobSpecification.h
+        src/MonitorAction.h
+        src/util/DefaultValues.h
+        src/util/Enums.h
         src/computation/CacheComputation.h
         src/computation/StreamedComputation.h
         src/computation/CopyComputation.h
@@ -44,32 +51,27 @@ set(TEST_FILES
 
 # wrench library and dependencies
 find_library(WRENCH_LIBRARY NAMES wrench)
-find_library(PUGIXML_LIBRARY NAMES pugixml)
 find_library(GTEST_LIBRARY NAMES gtest)
 find_library(SimGrid_LIBRARY NAMES simgrid)
 
 # generating the executable
-add_executable(sgbatch-sim ${SOURCE_FILES})
+add_executable(dc-sim ${SOURCE_FILES})
 
 if (ENABLE_BATSCHED)
-target_link_libraries(sgbatch-sim
+target_link_libraries(dc-sim
                        ${WRENCH_LIBRARY}
-                       ${WRENCH_PEGASUS_WORKFLOW_PARSER_LIBRARY}
                        ${SimGrid_LIBRARY}
                        ${Boost_LIBRARIES}
-                       ${PUGIXML_LIBRARY}
                       -lzmq )
 else()
-target_link_libraries(sgbatch-sim
+target_link_libraries(dc-sim
                        ${WRENCH_LIBRARY}
-                       ${WRENCH_PEGASUS_WORKFLOW_PARSER_LIBRARY}
                        ${SimGrid_LIBRARY}
                        ${Boost_LIBRARIES}
-                       ${PUGIXML_LIBRARY}
                       )
 endif()
 
-install(TARGETS sgbatch-sim DESTINATION bin)
+install(TARGETS dc-sim DESTINATION bin)
 
 # generating unit tests
 add_executable(unit_tests EXCLUDE_FROM_ALL

README.md

@@ -2,24 +2,91 @@
 
 Simulator for the simulation of high energy physics workflows on distributed computing systems with caching.
 
+
 ## Install instructions
-To install `git clone` this repository and execute the checkout script inside:
+
+### Option 1
+To get a fresh installation on your own system, either `git clone` this repository and execute the checkout script inside:
 ```bash
-source checkout.sh
+source checkout_scripts/checkout_with_sudo.sh
 ```
 Mind that you will need super-user rights to do so, as well as `cmake`, `git`, `clang` and `boost` installed on your system.
-This will install the executable `sgbatch-sim` for this simulator and all its software dependencies.
+This will install the executable `dc-sim` for this simulator and all its software dependencies.
+
+### Option 2
+Create a `conda` environment using the provided script
+```bash
+checkout_scripts/install_conda_environment.sh
+```
+This will automatically take care of all the dependencies needed and include them into the environment.
+
+To work with this environment interactively, you first have to initialize conda on your system. This can be done via:
+
+```bash
+</path/to/your/conda/installation>/bin/conda init
+```
+
+This would adapt your `~/.bashrc` to be able to call `conda` directly. So please re-open your shell or `source ~/.bashrc`.
+
+To activate the environment, execute
+```bash
+conda activate dcsim-env
+```
+and deactivate it accordingly with 
+```bash
+conda deactivate
+```
+
+### Tips for Conda
+
+With a `conda` environment, you would be able to install the full software setup without super-user rights.
+More information on how to work with and develop in a `conda` environment can be found in the [Conda Documentation](https://docs.anaconda.com/)
+
+Furthermore, it is possible to put a complete conda environment into a tarball to be able to export it to a different machine, e.g. a batch system node. To do that execute:
+
+```bash
+conda activate dcsim-env # in case you don't have it activated yet
+conda-pack
+```
+
+The created tarball `dcsim-env.tar.gz` can then be uploaded to a storage element and copied from there to a different machine.
+
 
 ## Usage
-When you have successfully installed the simulator you can run
+When you have successfully installed the simulator or activated the conda environment you can run
 ```bash
-sgbatch-sim --help
+dc-sim --help
 ```
-to see possible options. 
+to see all possible execution options. 
 
-Obligatory parameters are a platform file and a path and name for the resulting simulation output CSV-file:
+Mandatory parameters are a platform file and a path and name for the resulting simulation output CSV-file:
 ```bash
-sgbatch-sim -p <platform-file> -o <output-path>
+dc-sim -p <platform-file> -o <output-path>
 ```
 The platform file has to follow the [SimGrid-defined DTD](https://simgrid.org/doc/latest/Platform.html).
+Example files can be found in `data/platform-files`.
 The output-path can be any relative or absolute path of your file-system where you are allowed to write to.
+Instead of manually setting up all workflow parameters via command line options, 
+there is also the option to provide a JSON file, which contains all necessary information about a workflow by adding the option:
+```bash
+--workflow-configurations <path_to_workflow_json>
+```
+The workflow should contain the full information as it would be set via the command line, e.g.:
+```json
+{
+    "name":"stream_and_compute_workflow",
+    "num_jobs": 60,
+    "infiles_per_job":10,
+    "average_flops":2164428000000,
+    "sigma_flops":216442800000,
+    "average_memory":2000000000,
+    "sigma_memory":200000000,
+    "average_infile_size":3600000000,
+    "sigma_infile_size":360000000,
+    "average_outfile_size":18000000000,
+    "sigma_outfile_size":1800000000,
+    "workflow_type":"streaming"
+}
+```
+It is also possible to give a list of workflow configuration files, which enables to simulate the execution of multiple workflows.
+Example configuration files covering different workflow-types are given in `data/workflow-configs`.

checkout.sh → checkout_scripts/checkout_with_sudo.sh

@@ -15,74 +15,93 @@ set -e
 
 this_file="$( [ ! -z "$ZSH_VERSION" ] && echo "${(%):-%x}" || echo "${BASH_SOURCE[0]}" )"
 this_dir="$( cd "$( dirname "$this_file" )" && pwd )"
+work_dir="$PWD"
 
 
 # checking out packages from git as prerequisites for WRENCH:
 #
 # 1) pugixml, docu: https://pugixml.org/docs/manual.html, git: https://github.com/zeux/pugixml
 echo "Installing C++ XML processing library pugixml..."
-wget http://github.com/zeux/pugixml/releases/download/v1.11/pugixml-1.11.tar.gz
-tar -xf pugixml-1.11.tar.gz
-rm pugixml-1.11.tar.gz
-pushd pugixml-1.11
+if [ ! -d "$work_dir/pugixml-1.12.1" ]; then
+    wget http://github.com/zeux/pugixml/releases/download/v1.12.1/pugixml-1.12.1.tar.gz
+    tar -xf pugixml-1.12.1.tar.gz
+    rm pugixml-1.12.1.tar.gz
+fi
+pushd pugixml-1.12
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Debug ..
+cmake ..
+# cmake -DCMAKE_BUILD_TYPE=Debug ..
 make -j 6; sudo make install
 popd
 
 # 2) nlohmann json, docu: https://json.nlohmann.me/, git: https://github.com/nlohmann/json
 echo "Installing C++ JSON library..."
-wget https://github.com/nlohmann/json/archive/refs/tags/v3.10.4.tar.gz
-tar -xf v3.10.4.tar.gz
-rm v3.10.4.tar.gz
-pushd json-3.10.4
+if [ ! -d "$work_dir/json-3.11.2" ]; then
+    wget https://github.com/nlohmann/json/archive/refs/tags/v3.11.2.tar.gz
+    tar -xf v3.11.2.tar.gz
+    rm v3.11.2.tar.gz
+fi
+pushd json-3.11.2
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Debug ..
+cmake ..
+# cmake -DCMAKE_BUILD_TYPE=Debug ..
 make -j 6; sudo make install
 popd
 
 # 3) googletest, docu & git: https://github.com/google/googletest
 echo "Installing C++ code testing library googletest..."
-wget https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz
-tar -xf release-1.11.0.tar.gz
-rm release-1.11.0.tar.gz
-pushd googletest-release-1.11.0
+if [ ! -d "$work_dir/googletest-release-1.12.1" ]; then
+    wget https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz
+    tar -xf release-1.12.1.tar.gz
+    rm release-1.12.1.tar.gz
+fi
+pushd googletest-release-1.12.1
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Debug ..
+cmake ..
+# cmake -DCMAKE_BUILD_TYPE=Debug ..
 make -j 6; sudo make install
 popd
 
 # 4) simgrid, docu: https://simgrid.org/doc/latest/, git: https://framagit.org/simgrid/simgrid
 echo "Installing SimGrid..."
-wget https://framagit.org/simgrid/simgrid/-/archive/v3.30/simgrid-v3.30.tar.gz
-tar -xf simgrid-v3.30.tar.gz
-rm simgrid-v3.30.tar.gz
-pushd simgrid-v3.30
+if [ ! -d "$work_dir/simgrid-v3.32" ]; then
+    wget https://framagit.org/simgrid/simgrid/-/archive/v3.32/simgrid-v3.32.tar.gz
+    tar -xf simgrid-v3.32.tar.gz
+    rm simgrid-v3.32.tar.gz
+fi
+pushd simgrid-v3.32
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Debug ..
+cmake ..
+# cmake -DCMAKE_BUILD_TYPE=Debug ..
 make -j 6; sudo make install
 popd
 
 # installing WRENCH 2.0:
 echo "Installing WRENCH..."
-git clone --branch wrench-2.0 [email protected]:wrench-project/wrench.git
-pushd wrench
+if [ ! -d "$work_dir/wrench-2.1" ]; then
+    wget https://github.com/wrench-project/wrench/archive/refs/tags/v2.1.tar.gz
+    tar -xf v2.1.tar.gz
+    rm v2.1.tar.gz
+fi
+pushd wrench-2.1
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Debug ..
+cmake ..
+# cmake -DCMAKE_BUILD_TYPE=Debug ..
 make -j 6; sudo make install
 # make -j 6 examples; sudo make install examples # needed additionally, since not done by default
 popd
 
 # install the sgbatch simulator
 echo "Installing the DistCacheSim simulator..."
-pushd $this_dir/sgbatch
+pushd $this_dir/../
 mkdir -p build
 cd build
-cmake -DCMAKE_BUILD_TYPE=Debug ..
+cmake ..
+# cmake -DCMAKE_BUILD_TYPE=Debug ..
 make -j 6; sudo make install
 popd