Easier runtime config for mapping mocks and remove unnecessary mock cloud adapter config

.accepted_words.txt

@@ -170,6 +170,7 @@ toolchain
 URI
 uri
 url
+USERPROFILE
 westus
 www
 xamarin

Cargo.toml

@@ -6,6 +6,7 @@
 # The default resolver for workspaces is different than for regular packages, so use v2 to avoid warnings
 resolver = "2"
 members = [
+  "build_common",
   "cloud_adapters/azure_cloud_connector_adapter",
   "cloud_adapters/in_memory_mock_cloud_adapter",
   "cloud_connectors/azure/mqtt_connector",
@@ -28,6 +29,7 @@ members = [
 [workspace.dependencies]
 # Freyja dependencies
 azure-cloud-connector-proto = { path = "cloud_connectors/azure/proto-build" }
+freyja-build-common = { path = "build_common" }
 freyja-common = { path = "common" }
 freyja-contracts = { path = "contracts" }
 mock-digital-twin = { path = "mocks/mock_digital_twin" }
@@ -42,10 +44,12 @@ service_discovery_proto = { git = "https://github.com/eclipse-chariott/chariott"
 # crates.io dependencies
 async-trait = "0.1.64"
 axum = "0.6.12"
+config = "0.13.3"
 convert_case = "0.6.0"
 crossbeam = "0.8.2"
 env_logger = "0.10.0"
 futures = "0.3.28"
+home = "0.5.5"
 httptest = "0.15.4"
 log = "^0.4"
 mockall = "0.11.4"

build_common/Cargo.toml

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+# SPDX-License-Identifier: MIT
+
+[package]
+name = "freyja-build-common"
+version = "0.1.0"
+edition = "2021"
+license = "MIT"
+
+[dependencies]

build_common/src/lib.rs

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+
+use std::{env, fs, path::Path};
+
+const OUT_DIR: &str = "OUT_DIR";
+
+/// Copies a file to the build output in `OUT_DIR`.
+/// Includes a `cargo:rerun-if-changed` instruction for use in `build.rs` scripts.
+/// This will likely panic outside of a build script and is not recommended for use in services.
+///
+/// # Arguments:
+/// - `source_path`: The source file to copy
+/// - `dest_filename`: The filename for the destination
+pub fn copy_to_build_out_dir<P: AsRef<Path>>(source_path: P, dest_filename: &str) {
+    let target_dir = env::var(OUT_DIR).unwrap();
+    let destination = Path::new(&target_dir).join(dest_filename);
+
+    fs::copy(&source_path, destination).unwrap();
+
+    println!(
+        "cargo:rerun-if-changed={}",
+        source_path.as_ref().to_str().unwrap()
+    );
+}

cloud_adapters/azure_cloud_connector_adapter/src/azure_cloud_connector_adapter.rs

@@ -13,7 +13,7 @@ use serde::{Deserialize, Serialize};
 use tonic::transport::Channel;
 
 use crate::azure_cloud_connector_adapter_config::{Config, CONFIG_FILE};
-use freyja_common::utils::execute_with_retry;
+use freyja_common::retry_utils::execute_with_retry;
 use freyja_contracts::cloud_adapter::{
     CloudAdapter, CloudAdapterError, CloudMessageRequest, CloudMessageResponse,
 };

cloud_adapters/in_memory_mock_cloud_adapter/README.md

@@ -4,6 +4,4 @@ The In-Memory Mock Cloud Adapter mocks the behavior of the cloud connector from
 
 ## Config
 
-The adapter's config is located at `res/config.json` and will be copied to the build output automatically. This file contains the following properties:
-
-- `cloud_service_name` and `host_connection_string`: these are fake values and serve no functional purpose. They are only logged in the output and changing them does not affect the fundamental behavior of the adapter.
+This adapter requires no configuration.

cloud_adapters/in_memory_mock_cloud_adapter/src/in_memory_mock_cloud_adapter.rs

@@ -2,53 +2,21 @@
 // Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-use std::{env, fs, path::Path};
-
 use async_trait::async_trait;
 use log::{debug, info};
 
-use crate::config_item::ConfigItem;
 use freyja_contracts::cloud_adapter::{
     CloudAdapter, CloudAdapterError, CloudMessageRequest, CloudMessageResponse,
 };
 
-const CONFIG_FILE: &str = "config.json";
-
 /// Mocks a cloud adapter in memory
-pub struct InMemoryMockCloudAdapter {
-    /// The mock's config
-    pub config: ConfigItem,
-}
-
-impl InMemoryMockCloudAdapter {
-    /// Creates a new InMemoryMockCloudAdapter with config from the specified file
-    ///
-    /// # Arguments
-    ///
-    /// - `config_path`: the path to the config to use
-    pub fn from_config_file<P: AsRef<Path>>(config_path: P) -> Result<Self, CloudAdapterError> {
-        let config_contents = fs::read_to_string(config_path).map_err(CloudAdapterError::io)?;
-        let config: ConfigItem = serde_json::from_str(config_contents.as_str())
-            .map_err(CloudAdapterError::deserialize)?;
-
-        Self::from_config(config)
-    }
-
-    /// Creates a new InMemoryMockCloudAdapter with the specified config
-    ///
-    /// # Arguments
-    ///
-    /// - `config_path`: the config to use
-    pub fn from_config(config: ConfigItem) -> Result<Self, CloudAdapterError> {
-        Ok(Self { config })
-    }
-}
+pub struct InMemoryMockCloudAdapter {}
 
 #[async_trait]
 impl CloudAdapter for InMemoryMockCloudAdapter {
     /// Creates a new instance of a CloudAdapter with default settings
     fn create_new() -> Result<Self, CloudAdapterError> {
-        Self::from_config_file(Path::new(env!("OUT_DIR")).join(CONFIG_FILE))
+        Ok(Self {})
     }
 
     /// Sends the signal to the cloud
@@ -63,15 +31,6 @@ impl CloudAdapter for InMemoryMockCloudAdapter {
         debug!("Received a request to send to the cloud");
         let cloud_message_json =
             serde_json::to_string_pretty(&cloud_message).map_err(CloudAdapterError::serialize)?;
-        let instance_end_point = format!(
-            "https://{}/{:?}",
-            self.config.host_connection_string, cloud_message.cloud_signal
-        );
-
-        debug!(
-            "Sending signal update to the {} endpoint:\n{}",
-            self.config.cloud_service_name, instance_end_point,
-        );
 
         info!("Cloud canonical value:\n{cloud_message_json}");
 
@@ -88,13 +47,7 @@ mod in_memory_mock_cloud_adapter_tests {
     use time::OffsetDateTime;
 
     #[test]
-    fn from_config_file_returns_err_on_nonexistent_file() {
-        let result = InMemoryMockCloudAdapter::from_config_file("fake_file.foo");
-        assert!(result.is_err());
-    }
-
-    #[test]
-    fn can_get_default_config() {
+    fn can_get_new() {
         let result = InMemoryMockCloudAdapter::create_new();
         assert!(result.is_ok());
     }

cloud_adapters/in_memory_mock_cloud_adapter/src/lib.rs

@@ -2,5 +2,4 @@
 // Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-pub mod config_item;
 pub mod in_memory_mock_cloud_adapter;

cloud_connectors/azure/mqtt_connector/src/mqtt_connector.rs

@@ -14,7 +14,7 @@ use azure_cloud_connector_proto::azure_cloud_connector::azure_cloud_connector_se
 use azure_cloud_connector_proto::azure_cloud_connector::{
     UpdateDigitalTwinRequest, UpdateDigitalTwinResponse,
 };
-use freyja_common::utils::execute_with_retry;
+use freyja_common::retry_utils::execute_with_retry;
 
 /// Implementation of the MQTTConnector gRPC trait
 pub struct MQTTConnector {

common/Cargo.toml

@@ -9,6 +9,10 @@ edition = "2021"
 license = "MIT"
 
 [dependencies]
+config = { workspace = true }
 freyja-contracts = { workspace = true }
+home = { workspace = true }
 log = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 tokio = { workspace = true }

common/src/config_utils.rs

@@ -0,0 +1,77 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+// SPDX-License-Identifier: MIT
+
+use std::{env, path::Path};
+
+use config::{ConfigError, File};
+use home::home_dir;
+use serde::Deserialize;
+
+const CONFIG_DIR: &str = "config";
+const DOT_FREYJA_DIR: &str = ".freyja";
+const FREYJA_HOME: &str = "FREYJA_HOME";
+
+/// Read config from layered configuration files.
+/// Uses `{config_file_name}.default.{config_file_ext}` as the base configuration,
+/// then searches for overrides named `{config_file_name}.{config_file_ext}` in the current directory and `$FREYJA_HOME`.
+/// If `$FREYJA_HOME` is not set, it defaults to `$HOME/.freyja`.
+///
+/// # Arguments
+/// - `config_file_name`: The config file name without an extension. This is used to construct the file names to search for
+/// - `config_file_ext`: The config file extension. This is used to construct the file names to search for
+/// - `default_config_path`: The path to the directory containing the default configuration
+/// - `io_error_handler`: The error handler for `std::io::Error` errors
+/// - `config_error_handler`: The error handler for errors from the config library
+pub fn read_from_files<TConfig, TError, TPath, TIoErrorHandler, TConfigErrorHandler>(
+    config_file_name: &str,
+    config_file_ext: &str,
+    default_config_path: TPath,
+    io_error_handler: TIoErrorHandler,
+    config_error_handler: TConfigErrorHandler,
+) -> Result<TConfig, TError>
+where
+    TConfig: for<'a> Deserialize<'a>,
+    TPath: AsRef<Path>,
+    TIoErrorHandler: Fn(std::io::Error) -> TError,
+    TConfigErrorHandler: Fn(ConfigError) -> TError,
+{
+    let default_config_filename = format!("{config_file_name}.default.{config_file_ext}");
+    let default_config_file = default_config_path.as_ref().join(default_config_filename);
+
+    let overrides_filename = format!("{config_file_name}.{config_file_ext}");
+
+    // The path below resolves to {current_dir}/{overrides_filename}
+    let current_dir_config_path = env::current_dir()
+        .map_err(&io_error_handler)?
+        .join(overrides_filename.clone());
+
+    let freyja_dir_config_path = match env::var(FREYJA_HOME) {
+        Ok(freyja_home) => {
+            // The path below resolves to $FREYJA_HOME/config/{overrides_filename}
+            Path::new(&freyja_home)
+                .join(CONFIG_DIR)
+                .join(overrides_filename)
+        }
+        Err(_) => {
+            // The path below resolves to $HOME/.freyja/config/{overrides_filename}
+            home_dir()
+                .ok_or(io_error_handler(std::io::Error::new(
+                    std::io::ErrorKind::Other,
+                    "Could not retrieve home directory",
+                )))?
+                .join(DOT_FREYJA_DIR)
+                .join(CONFIG_DIR)
+                .join(overrides_filename)
+        }
+    };
+
+    let config_store = config::Config::builder()
+        .add_source(File::from(default_config_file))
+        .add_source(File::from(current_dir_config_path).required(false))
+        .add_source(File::from(freyja_dir_config_path).required(false))
+        .build()
+        .map_err(&config_error_handler)?;
+
+    config_store.try_deserialize().map_err(config_error_handler)
+}

common/src/lib.rs

@@ -2,5 +2,16 @@
 // Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
+pub mod config_utils;
+pub mod retry_utils;
 pub mod signal_store;
-pub mod utils;
+
+/// Expands to `env!("OUT_DIR")`.
+/// Since we cannot use a constant in the `env!` macro,
+/// this is the next best option to avoid duplicating the `"OUT_DIR"` literal.
+#[macro_export]
+macro_rules! out_dir {
+    () => {
+        env!("OUT_DIR")
+    };
+}

common/src/signal_store.rs

@@ -409,7 +409,7 @@ mod signal_store_tests {
         // - emission.next_emission_ms
         // - emission.last_emitted_value
         assert_eq!(updated_signal.value, Default::default());
-        assert_eq!(updated_signal.emission.next_emission_ms, Default::default());
+        assert_eq!(updated_signal.emission.next_emission_ms, u64::default());
         assert_eq!(
             updated_signal.emission.last_emitted_value,
             Default::default()

digital_twin_adapters/ibeji_adapter/src/ibeji_adapter.rs

@@ -14,7 +14,7 @@ use service_discovery_proto::service_registry::v1::DiscoverRequest;
 use tonic::{transport::Channel, Request};
 
 use crate::config::{IbejiDiscoveryMetadata, Settings, CONFIG_FILE};
-use freyja_common::utils::execute_with_retry;
+use freyja_common::retry_utils::execute_with_retry;
 use freyja_contracts::{
     digital_twin_adapter::{
         DigitalTwinAdapter, DigitalTwinAdapterError, GetDigitalTwinProviderRequest,

mapping_clients/in_memory_mock_mapping_client/Cargo.toml

@@ -10,7 +10,11 @@ license = "MIT"
 
 [dependencies]
 async-trait = { workspace = true }
+freyja-common = { workspace = true }
 freyja-contracts = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-tokio = { workspace = true }
+tokio = { workspace = true }
+
+[build-dependencies]
+freyja-build-common = { workspace = true }

mapping_clients/in_memory_mock_mapping_client/README.md

@@ -4,7 +4,7 @@ The In-Memory Mock Mapping Client mocks the behavior of a mapping service from w
 
 ## Configuration
 
-The adapter's config is located at `res/config.json` and will be copied to the build output automatically. This file is a list of config objects with the following properties:
+The adapter's default config is located at `res/mock_mapping_config.default.json` and will be copied to the build output automatically. This file is a list of config objects with the following properties:
 
 - `begin`: an integer indicating when to enable the mapping value below
 - `end`: an optional integer indicating when to disable the mapping value below. Set to `null` if you never want the value to "turn off"
@@ -15,6 +15,14 @@ The adapter's config is located at `res/config.json` and will be copied to the b
   - `emit_on_change`: a value indicating whether data emission should be skipped if the value hasn't changed since the last emission. Set to true to enable this behavior.
   - `conversion`: a conversion that should be applied. Set to null if no conversion is needed. Otherwise the conversion is configured with the `mul` and `offset` properties, and the value `y` that is emitted is calculated as `y = mul * x + offset`. Note that conversions are only supported for signal values which can be parsed as `f64`.
 
+You can override the default values by defining your own `mock_mapping_config.json`. The adapter will probe for and unify config in this order, with values near the end of the list taking higher precedence:
+
+- The default config
+- A `mock_mapping_config.json` file in the working directory of the executable (for example, the directory you were in when you ran the `cargo run` command)
+- `$FREYJA_HOME/config/mock_mapping_config.json`. If you have not set a `$FREYJA_HOME` directory, this defaults to:
+  - Unix: `$HOME/.freyja/config/mock_mapping_config.json`
+  - Windows: `%USERPROFILE%\.freyja\config\mock_mapping_config.json` (note that Windows support is not guaranteed by Freyja or this adapter)
+
 ## Behavior
 
 The client maintains an internal count, and only mappings satisfying the condition `begin <= count [< end]` will be returned in the `get_mapping` API. This count is incremented every time the `check_for_work` API is invoked. This will also affect the `check_for_work` API, which returns true if the set of mappings has changed since the last time it was called. This effectively means that the state can potentially change with each loop of the cartographer.