[sled-agent] Oximeter self-assembling zone (#4534)

Related #1898 Closes: #2883
oxidecomputer · Jan 25, 2024 · 9e08978 · 9e08978
1 parent 97318e9
commit 9e08978
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 22 deletions.
diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh
@@ -117,7 +117,7 @@ zones=(
   out/internal-dns.tar.gz
   out/omicron-nexus.tar.gz
   out/omicron-nexus-single-sled.tar.gz
-  out/oximeter-collector.tar.gz
+  out/oximeter.tar.gz
   out/propolis-server.tar.gz
   out/switch-*.tar.gz
   out/ntp.tar.gz

diff --git a/oximeter/db/schema/README.md b/oximeter/db/schema/README.md
@@ -32,7 +32,7 @@ To run this program:
 - Run this tool, pointing it at the desired schema directory, e.g.:
 
 ```bash
-# /opt/oxide/oximeter/bin/clickhouse-schema-updater \
+# /opt/oxide/oximeter-collector/bin/clickhouse-schema-updater \
     --host <ADDR_FROM_ABOVE> \
     --schema-dir /opt/oxide/oximeter/sql
     up VERSION

diff --git a/package-manifest.toml b/package-manifest.toml
@@ -116,9 +116,16 @@ setup_hint = """
 - Run `pkg install library/postgresql-13` to download Postgres libraries
 """
 
-[package.oximeter-collector]
+[package.oximeter]
 service_name = "oximeter"
 only_for_targets.image = "standard"
+source.type = "composite"
+source.packages = [ "oximeter-collector.tar.gz", "zone-network-setup.tar.gz" ]
+output.type = "zone"
+
+[package.oximeter-collector]
+service_name = "oximeter-collector"
+only_for_targets.image = "standard"
 source.type = "local"
 source.rust.binary_names = ["oximeter", "clickhouse-schema-updater"]
 source.rust.release = true
@@ -127,6 +134,7 @@ source.paths = [
     { from = "oximeter/db/schema", to = "/opt/oxide/oximeter/schema" },
 ]
 output.type = "zone"
+output.intermediate_only = true
 
 [package.clickhouse]
 service_name = "clickhouse"

diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs
@@ -61,7 +61,6 @@ use illumos_utils::zone::Zones;
 use illumos_utils::{execute, PFEXEC};
 use internal_dns::resolver::Resolver;
 use itertools::Itertools;
-use omicron_common::address::AZ_PREFIX;
 use omicron_common::address::BOOTSTRAP_ARTIFACT_PORT;
 use omicron_common::address::CLICKHOUSE_KEEPER_PORT;
 use omicron_common::address::CLICKHOUSE_PORT;
@@ -75,6 +74,7 @@ use omicron_common::address::SLED_PREFIX;
 use omicron_common::address::WICKETD_NEXUS_PROXY_PORT;
 use omicron_common::address::WICKETD_PORT;
 use omicron_common::address::{Ipv6Subnet, NEXUS_TECHPORT_EXTERNAL_PORT};
+use omicron_common::address::{AZ_PREFIX, OXIMETER_PORT};
 use omicron_common::api::external::Generation;
 use omicron_common::api::internal::shared::{
     HostPortConfig, RackNetworkConfig,
@@ -1798,7 +1798,55 @@ impl ServiceManager {
                 let running_zone = RunningZone::boot(installed_zone).await?;
                 return Ok(running_zone);
             }
+            ZoneArgs::Omicron(OmicronZoneConfigLocal {
+                zone:
+                    OmicronZoneConfig {
+                        id,
+                        zone_type: OmicronZoneType::Oximeter { .. },
+                        underlay_address,
+                        ..
+                    },
+                ..
+            }) => {
+                let Some(info) = self.inner.sled_info.get() else {
+                    return Err(Error::SledAgentNotReady);
+                };
+
+                // Configure the Oximeter service.
+                let address = SocketAddr::new(
+                    IpAddr::V6(*underlay_address),
+                    OXIMETER_PORT,
+                );
+
+                let listen_addr = &address.ip().to_string();
+
+                let nw_setup_service = Self::zone_network_setup_install(
+                    info,
+                    &installed_zone,
+                    listen_addr,
+                )?;
+
+                let oximeter_config = PropertyGroupBuilder::new("config")
+                    .add_property("id", "astring", &id.to_string())
+                    .add_property("address", "astring", &address.to_string());
+                let oximeter_service = ServiceBuilder::new("oxide/oximeter")
+                    .add_instance(
+                        ServiceInstanceBuilder::new("default")
+                            .add_property_group(oximeter_config),
+                    );
 
+                let profile = ProfileBuilder::new("omicron")
+                    .add_service(nw_setup_service)
+                    .add_service(disabled_ssh_service)
+                    .add_service(oximeter_service);
+                profile
+                    .add_to_zone(&self.inner.log, &installed_zone)
+                    .await
+                    .map_err(|err| {
+                        Error::io("Failed to setup Oximeter profile", err)
+                    })?;
+                return Ok(RunningZone::boot(installed_zone).await?);
+            }
             _ => {}
         }
 
@@ -2154,14 +2202,6 @@ impl ServiceManager {
                         // service is enabled.
                         smfh.refresh()?;
                     }
-
-                    OmicronZoneType::Oximeter { address } => {
-                        info!(self.inner.log, "Setting up oximeter service");
-                        smfh.setprop("config/id", zone_config.zone.id)?;
-                        smfh.setprop("config/address", address.to_string())?;
-                        smfh.refresh()?;
-                    }
-
                     OmicronZoneType::BoundaryNtp {
                         ntp_servers,
                         dns_servers,
@@ -2227,7 +2267,8 @@ impl ServiceManager {
                     | OmicronZoneType::ClickhouseKeeper { .. }
                     | OmicronZoneType::CockroachDb { .. }
                     | OmicronZoneType::Crucible { .. }
-                    | OmicronZoneType::CruciblePantry { .. } => {
+                    | OmicronZoneType::CruciblePantry { .. }
+                    | OmicronZoneType::Oximeter { .. } => {
                         panic!(
                             "{} is a service which exists as part of a \
                             self-assembling zone",
@@ -3729,7 +3770,7 @@ mod test {
     const GLOBAL_ZONE_BOOTSTRAP_IP: Ipv6Addr = Ipv6Addr::LOCALHOST;
     const SWITCH_ZONE_BOOTSTRAP_IP: Ipv6Addr = Ipv6Addr::LOCALHOST;
 
-    const EXPECTED_ZONE_NAME_PREFIX: &str = "oxz_oximeter";
+    const EXPECTED_ZONE_NAME_PREFIX: &str = "oxz_ntp";
     const EXPECTED_PORT: u16 = 12223;
 
     fn make_bootstrap_networking_config() -> BootstrapNetworking {
@@ -3906,7 +3947,12 @@ mod test {
             mgr,
             id,
             generation,
-            OmicronZoneType::Oximeter { address },
+            OmicronZoneType::InternalNtp {
+                address,
+                ntp_servers: vec![],
+                dns_servers: vec![],
+                domain: None,
+            },
         )
         .await
         .expect("Could not create service");
@@ -3945,7 +3991,12 @@ mod test {
             zones: vec![OmicronZoneConfig {
                 id,
                 underlay_address: Ipv6Addr::LOCALHOST,
-                zone_type: OmicronZoneType::Oximeter { address },
+                zone_type: OmicronZoneType::InternalNtp {
+                    address,
+                    ntp_servers: vec![],
+                    dns_servers: vec![],
+                    domain: None,
+                },
             }],
         })
         .await
@@ -4314,7 +4365,12 @@ mod test {
         let mut zones = vec![OmicronZoneConfig {
             id: id1,
             underlay_address: Ipv6Addr::LOCALHOST,
-            zone_type: OmicronZoneType::Oximeter { address },
+            zone_type: OmicronZoneType::InternalNtp {
+                address,
+                ntp_servers: vec![],
+                dns_servers: vec![],
+                domain: None,
+            },
         }];
         mgr.ensure_all_omicron_zones_persistent(OmicronZonesConfig {
             generation: v2,
@@ -4335,7 +4391,12 @@ mod test {
         zones.push(OmicronZoneConfig {
             id: id2,
             underlay_address: Ipv6Addr::LOCALHOST,
-            zone_type: OmicronZoneType::Oximeter { address },
+            zone_type: OmicronZoneType::InternalNtp {
+                address,
+                ntp_servers: vec![],
+                dns_servers: vec![],
+                domain: None,
+            },
         });
 
         // Now try to apply that list with an older generation number.  This
@@ -4508,7 +4569,12 @@ mod test {
         zones.push(OmicronZoneConfig {
             id,
             underlay_address: Ipv6Addr::LOCALHOST,
-            zone_type: OmicronZoneType::Oximeter { address },
+            zone_type: OmicronZoneType::InternalNtp {
+                address,
+                ntp_servers: vec![],
+                dns_servers: vec![],
+                domain: None,
+            },
         });
         mgr.ensure_all_omicron_zones_persistent(OmicronZonesConfig {
             generation: vv,

diff --git a/smf/oximeter/manifest.xml b/smf/oximeter/manifest.xml
@@ -4,21 +4,28 @@
 <service_bundle type='manifest' name='oximeter'>
 
 <service name='oxide/oximeter' type='service' version='1'>
-  <create_default_instance enabled='false' />
+  <create_default_instance enabled='true' />
 
   <dependency name='multi_user' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/milestone/multi-user:default' />
   </dependency>
 
+  <dependency name='zone_network_setup' grouping='require_all' restart_on='none'
+    type='service'>
+  <service_fmri value='svc:/oxide/zone-network-setup:default' />
+  </dependency>
+
   <method_context>
     <method_environment>
       <envvar name="LD_LIBRARY_PATH" value="/opt/ooce/pgsql-13/lib/amd64" />
     </method_environment>
   </method_context>
   <exec_method type='method' name='start'
-    exec='ctrun -l child -o noorphan,regent /opt/oxide/oximeter/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address %{config/address} --id %{config/id} &amp;'
-    timeout_seconds='0' />
+    exec='ctrun -l child -o noorphan,regent /opt/oxide/oximeter-collector/bin/oximeter run /var/svc/manifest/site/oximeter/config.toml --address %{config/address} --id %{config/id} &amp;'
+    timeout_seconds='0'>
+  </exec_method>
+
   <exec_method type='method' name='stop' exec=':kill' timeout_seconds='0' />
 
   <property_group name='config' type='application'>