feat: make SILO Docker image by default read data from /data

.dockerignore

@@ -1,5 +1,6 @@
 *
 !docker_default_preprocessing_config.yaml
+!docker_runtime_config.yaml
 !CMakeLists.txt
 !build_with_conan.py
 !conanprofile.docker

Dockerfile

@@ -36,6 +36,7 @@ FROM alpine:3.17.0 AS server
 
 WORKDIR /app
 COPY docker_default_preprocessing_config.yaml ./default_preprocessing_config.yaml
+COPY docker_runtime_config.yaml ./runtime_config.yaml
 COPY --from=builder /src/siloApi ./
 
 RUN apk update && apk add libtbb=2021.7.0-r0 curl jq
@@ -46,4 +47,4 @@ HEALTHCHECK --start-period=20s CMD curl --fail --silent localhost:8081/info | jq
 EXPOSE 8081
 ENV SPDLOG_LEVEL="off,file_logger=debug"
 
-ENTRYPOINT ["./siloApi"]
+ENTRYPOINT ["./siloApi"]

docker-compose-for-tests-api.yml

@@ -3,9 +3,9 @@ services:
   silo:
     image: ${SILO_IMAGE}
     volumes:
-      - ./testBaseData/exampleDataset:/data
+      - ./testBaseData/output:/data
+      - ./logs:/app/logs
     ports:
       - "8080:8081"
     command:
       - "--api"
-      - "--preprocessingConfig=./preprocessing_config.yaml"

docker_runtime_config.yaml

@@ -0,0 +1 @@
+dataDirectory: /data/

include/silo_api/runtime_config.h

@@ -0,0 +1,17 @@
+#ifndef SILO_RUNTIME_CONFIG_H
+#define SILO_RUNTIME_CONFIG_H
+
+#include <filesystem>
+#include <optional>
+
+namespace silo_api {
+
+struct RuntimeConfig {
+   std::optional<std::filesystem::path> data_directory;
+
+   static RuntimeConfig readFromFile(const std::filesystem::path& config_path);
+};
+
+}  // namespace silo_api
+
+#endif  // SILO_RUNTIME_CONFIG_H

src/silo_api/api.cpp

@@ -21,6 +21,7 @@
 #include "silo_api/info_handler.h"
 #include "silo_api/logging.h"
 #include "silo_api/request_handler_factory.h"
+#include "silo_api/runtime_config.h"
 
 silo::preprocessing::PreprocessingConfig preprocessingConfig(
    const Poco::Util::AbstractConfiguration& config
@@ -38,9 +39,8 @@ silo::preprocessing::PreprocessingConfig preprocessingConfig(
          config.getString("preprocessingConfig")
       );
    } else if (std::filesystem::exists("./preprocessing_config.yaml")) {
-      user_preprocessing_config = silo::preprocessing::PreprocessingConfigReader().readConfig(
-         "./preprocessing_config.yaml"
-      );
+      user_preprocessing_config =
+         silo::preprocessing::PreprocessingConfigReader().readConfig("./preprocessing_config.yaml");
    }
 
    const auto preprocessing_config =
@@ -59,12 +59,27 @@ silo::config::DatabaseConfig databaseConfig(const Poco::Util::AbstractConfigurat
    return silo::config::ConfigRepository().getValidatedConfig("database_config.yaml");
 }
 
-std::filesystem::path dataFolder(const Poco::Util::AbstractConfiguration& config) {
-   if (config.hasProperty("data_folder")) {
-      return config.getString("data_folder");
+std::filesystem::path dataDirectory(
+   const Poco::Util::AbstractConfiguration& config,
+   const silo_api::RuntimeConfig& runtime_config
+) {
+   if (config.hasProperty("dataDirectory")) {
+      SPDLOG_DEBUG(
+         "Using dataDirectory passed via command line argument: {}",
+         config.getString("dataDirectory")
+      );
+      return config.getString("dataDirectory");
    }
+   if (runtime_config.data_directory.has_value()) {
+      SPDLOG_DEBUG(
+         "Using dataDirectory from runtime config file: {}",
+         runtime_config.data_directory.value().string()
+      );
+      return runtime_config.data_directory.value();
+   }
+
    SPDLOG_DEBUG(
-      "data_folder not found in config file. Using default value: {}",
+      "dataDirectory not found in specified. Using default value: {}",
       silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory
    );
    return silo::preprocessing::DEFAULT_OUTPUT_DIRECTORY.directory;
@@ -98,11 +113,11 @@ class SiloServer : public Poco::Util::ServerApplication {
             .binding("databaseConfig")
       );
 
-      options.addOption(Poco::Util::Option("data_folder", "df", "path to the preprocessed data")
+      options.addOption(Poco::Util::Option("dataDirectory", "d", "path to the preprocessed data")
                            .required(false)
                            .repeatable(false)
                            .argument("PATH")
-                           .binding("data_folder"));
+                           .binding("dataDirectory"));
 
       options.addOption(
          Poco::Util::Option("api", "a", "Execution mode: start the SILO web interface")
@@ -151,9 +166,14 @@ class SiloServer : public Poco::Util::ServerApplication {
       SPDLOG_INFO("Starting SILO API");
       const int port = 8081;
 
-      const auto data_folder = dataFolder(config());
+      silo_api::RuntimeConfig runtime_config;
+      if (std::filesystem::exists("./runtime_config.yaml")) {
+         runtime_config = silo_api::RuntimeConfig::readFromFile("./runtime_config.yaml");
+      }
+
+      const auto data_directory = dataDirectory(config(), runtime_config);
 
-      auto database = silo::Database::loadDatabaseState(data_folder);
+      auto database = silo::Database::loadDatabaseState(data_directory);
 
       const Poco::Net::ServerSocket server_socket(port);
       const silo::query_engine::QueryEngine query_engine(database);
@@ -177,10 +197,10 @@ class SiloServer : public Poco::Util::ServerApplication {
       const auto preprocessing_config = preprocessingConfig(config());
       auto database_config = databaseConfig(config());
 
-      auto database_preprocessing =
+      auto database =
          silo::Database::preprocessing(preprocessing_config, database_config);
 
-      database_preprocessing.saveDatabaseState(preprocessing_config.getOutputDirectory());
+      database.saveDatabaseState(preprocessing_config.getOutputDirectory());
 
       return Application::EXIT_OK;
    };

src/silo_api/runtime_config.cpp

@@ -0,0 +1,30 @@
+#include "silo_api/runtime_config.h"
+
+#include <spdlog/spdlog.h>
+#include <yaml-cpp/yaml.h>
+
+namespace YAML {
+
+template <>
+struct convert<silo_api::RuntimeConfig> {
+   static bool decode(const Node& node, silo_api::RuntimeConfig& config) {
+      config = silo_api::RuntimeConfig(
+         node["dataDirectory"]
+            ? std::optional<std::filesystem::path>(node["dataDirectory"].as<std::string>())
+            : std::nullopt
+      );
+
+      return true;
+   }
+};
+
+}  // namespace YAML
+
+namespace silo_api {
+
+RuntimeConfig RuntimeConfig::readFromFile(const std::filesystem::path& config_path) {
+   SPDLOG_INFO("Reading runtime config from {}", config_path.string());
+   return YAML::LoadFile(config_path.string()).as<RuntimeConfig>();
+}
+
+}  // namespace silo_api

src/silo_api/runtime_config.test.cpp

@@ -0,0 +1,10 @@
+#include "silo_api/runtime_config.h"
+
+#include <gtest/gtest.h>
+
+TEST(RuntimeConfig, shouldReadConfig) {
+   const auto result =
+      silo_api::RuntimeConfig::readFromFile("./testBaseData/test_runtime_config.yaml");
+
+   ASSERT_EQ(result.data_directory, std::filesystem::path("test/directory"));
+}

testBaseData/test_runtime_config.yaml

@@ -0,0 +1 @@
+dataDirectory: test/directory