From ad2209f7d382b40c84ce635fa2892c4d67989b0d Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Mon, 16 Dec 2024 15:39:39 +0000 Subject: [PATCH 1/4] Hide cloud specific device configurations Signed-off-by: James Rhodes --- .../tedge/src/cli/config/commands/list.rs | 13 +++++++- crates/core/tedge/src/cli/connect/command.rs | 31 ++++++++++--------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/crates/core/tedge/src/cli/config/commands/list.rs b/crates/core/tedge/src/cli/config/commands/list.rs index e0f76db71cb..b82bd60c6d2 100644 --- a/crates/core/tedge/src/cli/config/commands/list.rs +++ b/crates/core/tedge/src/cli/config/commands/list.rs @@ -30,13 +30,21 @@ impl Command for ListConfigCommand { } } +fn should_hide(key: &str) -> bool { + (key.starts_with("c8y") || key.starts_with("az") || key.starts_with("aws")) + && key.contains(".device.") +} + fn print_config_list( config: &TEdgeConfig, all: bool, filter: Option<&str>, ) -> Result<(), anyhow::Error> { let mut keys_without_values = Vec::new(); - for config_key in config.readable_keys() { + for config_key in config + .readable_keys() + .filter(|key| !should_hide(&key.to_cow_str())) + { if !key_matches_filter(&config_key.to_cow_str(), filter) { continue; } @@ -67,6 +75,9 @@ fn print_config_doc(filter: Option<&str>) { .unwrap_or_default(); for (key, ty) in READABLE_KEYS.iter() { + if should_hide(key) { + continue; + } if !key_matches_filter(key, filter) { continue; } diff --git a/crates/core/tedge/src/cli/connect/command.rs b/crates/core/tedge/src/cli/connect/command.rs index 1d00c85d06c..1a9d79858d4 100644 --- a/crates/core/tedge/src/cli/connect/command.rs +++ b/crates/core/tedge/src/cli/connect/command.rs @@ -362,19 +362,21 @@ fn disallow_matching_url_device_id( .map(|&(k, _)| format!("{}", url(k.clone()).yellow().bold())) .collect::>() .join(", "); - let device_id_keys: String = matches - .iter() - .map(|(_, key)| format!("{}", key.yellow().bold())) - .collect::>() - .join(", "); - bail!( - "You have matching URLs and device IDs for different profiles. - -{url_keys} are set to the same value, but so are {device_id_keys}. - -Each cloud profile requires either a unique URL or unique device ID, \ -so it corresponds to a unique device in the associated cloud." - ); + // TODO re-enable this logic once multiple device IDs are properly supported + // let device_id_keys: String = matches + // .iter() + // .map(|(_, key)| format!("{}", key.yellow().bold())) + // .collect::>() + // .join(", "); + // bail!( + // "You have matching URLs and device IDs for different profiles. + + // {url_keys} are set to the same value, but so are {device_id_keys}. + + // Each cloud profile requires either a unique URL or unique device ID, \ + // so it corresponds to a unique device in the associated cloud." + // ); + bail!("The configurations: {url_keys} should be set to different values before connecting, but are currently set to the same value"); } } Ok(()) @@ -1292,7 +1294,8 @@ mod tests { let config = loc.load().unwrap(); let err = validate_config(&config, &cloud).unwrap_err(); - assert_eq!(err.to_string(), "You have matching URLs and device IDs for different profiles. + // TODO change me to assert eq once device IDs are properly supported + assert_ne!(err.to_string(), "You have matching URLs and device IDs for different profiles. c8y.url, c8y.profiles.new.url are set to the same value, but so are c8y.device.id, c8y.profiles.new.device.id. From 7af07723cbe2827766ef358e3e34d7c7e9bd206b Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Mon, 16 Dec 2024 15:41:13 +0000 Subject: [PATCH 2/4] Document cloud profiles Signed-off-by: James Rhodes --- docs/src/operate/c8y/cloud-profiles.md | 176 +++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 docs/src/operate/c8y/cloud-profiles.md diff --git a/docs/src/operate/c8y/cloud-profiles.md b/docs/src/operate/c8y/cloud-profiles.md new file mode 100644 index 00000000000..e2136773e5f --- /dev/null +++ b/docs/src/operate/c8y/cloud-profiles.md @@ -0,0 +1,176 @@ +--- +title: Cloud Profiles +tags: [Operate, Cumulocity, Cloud Profile] +description: Connecting %%te%% to multiple Cumulocity tenants +--- + +import UserContext from '@site/src/components/UserContext'; +import UserContextForm from '@site/src/components/UserContextForm'; + +Starting with version 1.4.0, **%%te%%** supports multiple Cumulocity connections +using cloud profiles. This can be useful for migrating from one Cumulocity +tenant to another, where Cloud profiles allow you to configure and run multiple +`tedge-mapper c8y` instances from a single `tedge.toml` configuration file. + +:::tip +#### User Context {#user-context} + +You can customize the documentation and commands shown on this page by providing +relevant settings which will be reflected in the instructions. It makes it even +easier to explore and use %%te%%. + + + +The user context will be persisted in your web browser's local storage. +::: + +## Configuration +There are a few values that need to be configured before we are able to connect +to a second Cumulocity tenant. + +### URL +To connect to a second tenant, start by configuring the URL of the new tenant: + +```sh +sudo tedge config set c8y.url other.cumulocity.com --profile second +``` + +The profile name can be any combination of letters and numbers, and is used only +to identify the cloud profile within thin-edge. The names are case insensitive, +so `--profile second` and `--profile SECOND` are equivalent. + +You can now see the configuration has been applied to `tedge.toml`: + +```sh +tedge config list url +``` + + + +```sh +c8y.url=$C8Y_URL +c8y.profiles.second.url=other.cumulocity.com +``` + + + +In addition to the URL there are a couple of other configurations that need to +be set for the second mapper: +- the MQTT bridge topic prefix +- the Cumulocity proxy bind port + +### MQTT bridge topic prefix +```sh +sudo tedge config set c8y.bridge.topic_prefix c8y-second --profile second +``` + +Setting `c8y.bridge.topic_prefix` will change the MQTT topics that the +Cumulocity bridge publishes to/listens to in mosquitto. The default value is +`c8y`, so the mappper publishes measurements to `c8y/s/us`, and this is +forwarded to Cumulocity on the `s/us` topic. In the example above, we set the +topic prefix to `c8y-second`, so the equivalent local topic would +`c8y-second/s/us`. It is recommended, but not required, to include `c8y` in the +topic prefix, to make it clear that the relevant topics are bridge topics that +will forward to and from Cumulocity. + +### Cumulocity proxy bind port +```sh +sudo tedge config set c8y.proxy.bind.port 8002 --profile second +``` + +Since the Cumulocity mapper hosts a [proxy server for +Cumulocity](../../references/cumulocity-proxy.md) and there will be a second +mapper instance running, this configuration also needs to be unique per profile. + +### Optional per-profile configurations +All the Cumulocity-specific configurations (those listed by `tedge config list +--doc c8y`) can be specified per-profile to match tenant-specific constraints. + +## Connecting +Once the second cloud profile has been configured, you can finally connect the +second mapper using: +```sh +sudo tedge connect --profile second +``` + +``` +Connecting to Cumulocity with config: + device id: $DEVICE_ID + cloud profile: second + cloud host: other.cumulocity.com:8883 + certificate file: /etc/tedge/device-certs/tedge-certificate.pem + bridge: mosquitto + service manager: systemd + mosquitto version: 2.0.11 +Creating device in Cumulocity cloud... ✓ +Restarting mosquitto... ✓ +Waiting for mosquitto to be listening for connections... ✓ +Verifying device is connected to cloud... ✓ +Enabling tedge-mapper-c8y@second... ✓ +Checking Cumulocity is connected to intended tenant... ✓ +Enabling tedge-agent... ✓ +``` + + +Once the mapper is running, you can restart it by running: + +``` +sudo systemctl restart tedge-mapper-c8y@second +``` + +This uses a systemd service template to create the `tedge-mapper-c8y@second` +service. If you are not using systemd, you will need to create a service +definition for `tedge-mapper-c8y@second` before attempting to connect your +device to a second Cumulocity instance. + +## Environment variables +For easy configuration of profiles in shell scripts, you can set the profile +name using the environment variable `TEDGE_CLOUD_PROFILE`. + + +```sh +sudo tedge config set c8y.url $C8Y_URL +sudo tedge connect c8y + +sudo tedge config set c8y.url other.cumulocity.com --profile second +sudo tedge config set c8y.bridge.topic_prefix c8y-second --profile second +sudo tedge config set c8y.proxy.bind.port 8002 --profile second +sudo tedge connect c8y --profile second +``` + + + +```sh title="With environment variable" +# You can set the profile name to an empty string to use the default profile +export TEDGE_CLOUD_PROFILE= +sudo tedge config set c8y.url $C8Y_URL +sudo tedge connect c8y + +export TEDGE_CLOUD_PROFILE=second +sudo tedge config set c8y.url other.cumulocity.com +sudo tedge config set c8y.bridge.topic_prefix c8y-second +sudo tedge config set c8y.proxy.bind.port 8002 +sudo tedge connect c8y + +export TEDGE_CLOUD_PROFILE= +sudo tedge config get c8y.url #=> $C8Y_URL + +export TEDGE_CLOUD_PROFILE=$C8Y_PROFILE_NAME +sudo tedge config get c8y.url #=> $C8Y_PROFILE_URL +``` + + +If you need to temporarily override a profiled configuration, you can use +environment variables of the form `TEDGE_C8Y_PROFILES__`. +For example: + +```sh +$ TEDGE_C8Y_PROFILES_SECOND_URL=different.example.com tedge config get c8y.url --profile second +different.example.com +$ TEDGE_C8Y_PROFILES_SECOND_PROXY_BIND_PORT=1234 tedge config get c8y.proxy.bind.port --profile second +1234 +``` + +If you are configuring %%te%% entirely with environment variables, e.g. in a +containerised deployment, you probably don't need to make use of cloud profiles +as you can set the relevant configurations directly on each mapper instance. \ No newline at end of file From 6821390d391d458aa333fd4533aba68f2015fb74 Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Mon, 16 Dec 2024 15:45:14 +0000 Subject: [PATCH 3/4] Lowercase profile names in config keys Signed-off-by: James Rhodes --- .../common/tedge_config_macros/src/multi.rs | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/common/tedge_config_macros/src/multi.rs b/crates/common/tedge_config_macros/src/multi.rs index 1c381ac0991..ac1f5579e1b 100644 --- a/crates/common/tedge_config_macros/src/multi.rs +++ b/crates/common/tedge_config_macros/src/multi.rs @@ -125,22 +125,15 @@ impl From for String { #[derive(Debug, thiserror::Error)] pub enum MultiError { - #[error( - "You are trying to access a profile `{1}` of {0}, but profiles are not enabled for {0}" - )] - SingleNotMulti(String, String), - #[error("A profile is required for the multi-profile property {0}")] - MultiNotSingle(String), #[error("Unknown profile `{1}` for the multi-profile property {0}")] MultiKeyNotFound(String, String), #[error("Invalid profile name `{1}` for the multi-profile property {0}")] InvalidProfileName(String, String, #[source] anyhow::Error), } -fn try_profile_name<'a>(key: &'a str, parent: &str) -> Result<&'a str, MultiError> { - validate_profile_name(key) - .map_err(|e| MultiError::InvalidProfileName(parent.to_owned(), key.to_owned(), e))?; - Ok(key) +fn parse_profile_name(name: &str, parent: &str) -> Result { + name.parse() + .map_err(|e| MultiError::InvalidProfileName(parent.to_owned(), name.to_owned(), e)) } impl MultiDto { @@ -149,7 +142,7 @@ impl MultiDto { None => Ok(&self.non_profile), Some(key) => self .profiles - .get(try_profile_name(key, parent)?) + .get(&parse_profile_name(key, parent)?) .ok_or_else(|| MultiError::MultiKeyNotFound(parent.to_owned(), key.to_owned())), } } @@ -159,9 +152,7 @@ impl MultiDto { None => Ok(&mut self.non_profile), Some(key) => Ok(self .profiles - .entry(key.parse().map_err(|e| { - MultiError::InvalidProfileName(parent.to_owned(), key.to_owned(), e) - })?) + .entry(parse_profile_name(key, parent)?) .or_default()), } } @@ -177,7 +168,7 @@ impl MultiReader { None => Ok(&self.non_profile), Some(key) => self .profiles - .get(try_profile_name(key, self.parent)?) + .get(&parse_profile_name(key, self.parent)?) .ok_or_else(|| MultiError::MultiKeyNotFound((*self.parent).into(), key.into())), } } From 5ff261b6e01a18bdfc77b7d54a1981e769d6cc8c Mon Sep 17 00:00:00 2001 From: James Rhodes Date: Tue, 17 Dec 2024 09:25:29 +0000 Subject: [PATCH 4/4] Add cloud profile variables Signed-off-by: James Rhodes --- docs/src/operate/c8y/cloud-profiles.md | 60 ++++++++++++++++---------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/src/operate/c8y/cloud-profiles.md b/docs/src/operate/c8y/cloud-profiles.md index e2136773e5f..a05377a4f8d 100644 --- a/docs/src/operate/c8y/cloud-profiles.md +++ b/docs/src/operate/c8y/cloud-profiles.md @@ -19,7 +19,7 @@ You can customize the documentation and commands shown on this page by providing relevant settings which will be reflected in the instructions. It makes it even easier to explore and use %%te%%. - + The user context will be persisted in your web browser's local storage. ::: @@ -31,9 +31,11 @@ to a second Cumulocity tenant. ### URL To connect to a second tenant, start by configuring the URL of the new tenant: + ```sh -sudo tedge config set c8y.url other.cumulocity.com --profile second +sudo tedge config set c8y.url $C8Y_PROFILE_URL --profile $C8Y_PROFILE_NAME ``` + The profile name can be any combination of letters and numbers, and is used only to identify the cloud profile within thin-edge. The names are case insensitive, @@ -49,7 +51,7 @@ tedge config list url ```sh c8y.url=$C8Y_URL -c8y.profiles.second.url=other.cumulocity.com +c8y.profiles.$C8Y_PROFILE_NAME.url=$C8Y_PROFILE_URL ``` @@ -60,9 +62,11 @@ be set for the second mapper: - the Cumulocity proxy bind port ### MQTT bridge topic prefix + ```sh -sudo tedge config set c8y.bridge.topic_prefix c8y-second --profile second +sudo tedge config set c8y.bridge.topic_prefix c8y-$C8Y_PROFILE_NAME --profile $C8Y_PROFILE_NAME ``` + Setting `c8y.bridge.topic_prefix` will change the MQTT topics that the Cumulocity bridge publishes to/listens to in mosquitto. The default value is @@ -74,9 +78,11 @@ topic prefix, to make it clear that the relevant topics are bridge topics that will forward to and from Cumulocity. ### Cumulocity proxy bind port -```sh -sudo tedge config set c8y.proxy.bind.port 8002 --profile second + ``` +sudo tedge config set c8y.proxy.bind.port 8002 --profile $C8Y_PROFILE_NAME +``` + Since the Cumulocity mapper hosts a [proxy server for Cumulocity](../../references/cumulocity-proxy.md) and there will be a second @@ -89,15 +95,19 @@ All the Cumulocity-specific configurations (those listed by `tedge config list ## Connecting Once the second cloud profile has been configured, you can finally connect the second mapper using: -```sh -sudo tedge connect --profile second + + ``` +sudo tedge connect --profile $C8Y_PROFILE_NAME +``` + + ``` Connecting to Cumulocity with config: device id: $DEVICE_ID - cloud profile: second - cloud host: other.cumulocity.com:8883 + cloud profile: $C8Y_PROFILE_NAME + cloud host: $C8Y_PROFILE_URL:8883 certificate file: /etc/tedge/device-certs/tedge-certificate.pem bridge: mosquitto service manager: systemd @@ -106,7 +116,7 @@ Creating device in Cumulocity cloud... ✓ Restarting mosquitto... ✓ Waiting for mosquitto to be listening for connections... ✓ Verifying device is connected to cloud... ✓ -Enabling tedge-mapper-c8y@second... ✓ +Enabling tedge-mapper-c8y@$C8Y_PROFILE_NAME... ✓ Checking Cumulocity is connected to intended tenant... ✓ Enabling tedge-agent... ✓ ``` @@ -114,9 +124,11 @@ Enabling tedge-agent... ✓ Once the mapper is running, you can restart it by running: + ``` -sudo systemctl restart tedge-mapper-c8y@second +sudo systemctl restart tedge-mapper-c8y@$C8Y_PROFILE_NAME ``` + This uses a systemd service template to create the `tedge-mapper-c8y@second` service. If you are not using systemd, you will need to create a service @@ -128,14 +140,14 @@ For easy configuration of profiles in shell scripts, you can set the profile name using the environment variable `TEDGE_CLOUD_PROFILE`. -```sh +``` sudo tedge config set c8y.url $C8Y_URL sudo tedge connect c8y -sudo tedge config set c8y.url other.cumulocity.com --profile second -sudo tedge config set c8y.bridge.topic_prefix c8y-second --profile second -sudo tedge config set c8y.proxy.bind.port 8002 --profile second -sudo tedge connect c8y --profile second +sudo tedge config set c8y.url $C8Y_PROFILE_URL --profile $C8Y_PROFILE_NAME +sudo tedge config set c8y.bridge.topic_prefix c8y-$C8Y_PROFILE_NAME --profile $C8Y_PROFILE_NAME +sudo tedge config set c8y.proxy.bind.port 8002 --profile $C8Y_PROFILE_NAME +sudo tedge connect c8y --profile $C8Y_PROFILE_NAME ``` @@ -146,9 +158,9 @@ export TEDGE_CLOUD_PROFILE= sudo tedge config set c8y.url $C8Y_URL sudo tedge connect c8y -export TEDGE_CLOUD_PROFILE=second -sudo tedge config set c8y.url other.cumulocity.com -sudo tedge config set c8y.bridge.topic_prefix c8y-second +export TEDGE_CLOUD_PROFILE=$C8Y_PROFILE_NAME +sudo tedge config set c8y.url $C8Y_PROFILE_URL +sudo tedge config set c8y.bridge.topic_prefix c8y-$C8Y_PROFILE_NAME sudo tedge config set c8y.proxy.bind.port 8002 sudo tedge connect c8y @@ -164,12 +176,14 @@ If you need to temporarily override a profiled configuration, you can use environment variables of the form `TEDGE_C8Y_PROFILES__`. For example: -```sh -$ TEDGE_C8Y_PROFILES_SECOND_URL=different.example.com tedge config get c8y.url --profile second + +``` +$ TEDGE_C8Y_PROFILES_$C8Y_PROFILE_NAME_URL=different.example.com tedge config get c8y.url --profile $C8Y_PROFILE_NAME different.example.com -$ TEDGE_C8Y_PROFILES_SECOND_PROXY_BIND_PORT=1234 tedge config get c8y.proxy.bind.port --profile second +$ TEDGE_C8Y_PROFILES_$C8Y_PROFILE_NAME_PROXY_BIND_PORT=1234 tedge config get c8y.proxy.bind.port --profile $C8Y_PROFILE_NAME 1234 ``` + If you are configuring %%te%% entirely with environment variables, e.g. in a containerised deployment, you probably don't need to make use of cloud profiles