[nexus] Split Nexus configuration (package vs deployment)

common/Cargo.toml

@@ -29,6 +29,7 @@ steno = { git = "https://github.com/oxidecomputer/steno", branch = "main" }
 thiserror = "1.0"
 tokio = { version = "1.18", features = [ "full" ] }
 tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] }
+toml = "0.5.9"
 uuid = { version = "1.1.0", features = [ "serde", "v4" ] }
 parse-display = "0.5.4"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }

common/src/lib.rs

@@ -24,7 +24,8 @@ pub mod address;
 pub mod api;
 pub mod backoff;
 pub mod cmd;
-pub mod config;
+pub mod nexus_config;
+pub mod postgres_config;
 
 #[macro_export]
 macro_rules! generate_logging_api {

common/src/nexus_config.rs

@@ -0,0 +1,128 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Configuration parameters to Nexus that are usually only known
+//! at deployment time.
+
+use super::address::{Ipv6Subnet, RACK_PREFIX};
+use super::postgres_config::PostgresConfigWithUrl;
+use dropshot::ConfigDropshot;
+use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
+use serde_with::DisplayFromStr;
+use std::fmt;
+use std::path::{Path, PathBuf};
+use uuid::Uuid;
+
+#[derive(Debug)]
+pub struct LoadError {
+    pub path: PathBuf,
+    pub kind: LoadErrorKind,
+}
+
+#[derive(Debug)]
+pub struct InvalidTunable {
+    pub tunable: String,
+    pub message: String,
+}
+
+impl std::fmt::Display for InvalidTunable {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message)
+    }
+}
+impl std::error::Error for InvalidTunable {}
+
+#[derive(Debug)]
+pub enum LoadErrorKind {
+    Io(std::io::Error),
+    Parse(toml::de::Error),
+    InvalidTunable(InvalidTunable),
+}
+
+impl From<(PathBuf, std::io::Error)> for LoadError {
+    fn from((path, err): (PathBuf, std::io::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Io(err) }
+    }
+}
+
+impl From<(PathBuf, toml::de::Error)> for LoadError {
+    fn from((path, err): (PathBuf, toml::de::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Parse(err) }
+    }
+}
+
+impl std::error::Error for LoadError {}
+
+impl fmt::Display for LoadError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.kind {
+            LoadErrorKind::Io(e) => {
+                write!(f, "read \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::Parse(e) => {
+                write!(f, "parse \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::InvalidTunable(inner) => {
+                write!(
+                    f,
+                    "invalid tunable \"{}\": {}",
+                    self.path.display(),
+                    inner,
+                )
+            }
+        }
+    }
+}
+
+impl std::cmp::PartialEq<std::io::Error> for LoadError {
+    fn eq(&self, other: &std::io::Error) -> bool {
+        if let LoadErrorKind::Io(e) = &self.kind {
+            e.kind() == other.kind()
+        } else {
+            false
+        }
+    }
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+#[allow(clippy::large_enum_variant)]
+pub enum Database {
+    FromDns,
+    FromUrl {
+        #[serde_as(as = "DisplayFromStr")]
+        url: PostgresConfigWithUrl,
+    },
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+pub struct DeploymentConfig {
+    /// Uuid of the Nexus instance
+    pub id: Uuid,
+    /// Dropshot configuration for external API server
+    pub dropshot_external: ConfigDropshot,
+    /// Dropshot configuration for internal API server
+    pub dropshot_internal: ConfigDropshot,
+    /// Portion of the IP space to be managed by the Rack.
+    pub subnet: Ipv6Subnet<RACK_PREFIX>,
+    /// DB configuration.
+    pub database: Database,
+}
+
+impl DeploymentConfig {
+    /// Load a `DeploymentConfig` from the given TOML file
+    ///
+    /// This config object can then be used to create a new `Nexus`.
+    /// The format is described in the README.
+    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self, LoadError> {
+        let path = path.as_ref();
+        let file_contents = std::fs::read_to_string(path)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        let config_parsed: Self = toml::from_str(&file_contents)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        Ok(config_parsed)
+    }
+}

nexus/benches/setup_benchmark.rs

@@ -19,7 +19,7 @@ async fn do_full_setup() {
 // Wraps exclusively the CockroachDB portion of setup/teardown.
 async fn do_crdb_setup() {
     let cfg = nexus_test_utils::load_test_config();
-    let logctx = LogContext::new("crdb_setup", &cfg.log);
+    let logctx = LogContext::new("crdb_setup", &cfg.pkg.log);
     let mut db = test_setup_database(&logctx.log).await;
     db.cleanup().await.unwrap();
 }

nexus/examples/config.toml

@@ -2,9 +2,6 @@
 # Oxide API: example configuration file
 #
 
-# Identifier for this instance of Nexus
-id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
-
 [console]
 # Directory for static assets. Absolute path or relative to CWD.
 static_dir = "nexus/static" # TODO: figure out value
@@ -20,21 +17,6 @@ session_absolute_timeout_minutes = 480
 # TODO(https://github.com/oxidecomputer/omicron/issues/372): Remove "spoof".
 schemes_external = ["spoof", "session_cookie"]
 
-[database]
-# URL for connecting to the database
-url = "postgresql://[email protected]:32221/omicron?sslmode=disable"
-
-[dropshot_external]
-# IP address and TCP port on which to listen for the external API
-bind_address = "127.0.0.1:12220"
-# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one
-# rule is ~500 bytes)
-request_body_max_bytes = 1048576
-
-[dropshot_internal]
-# IP address and TCP port on which to listen for the internal API
-bind_address = "127.0.0.1:12221"
-
 [log]
 # Show log messages of this level and more severe
 level = "info"
@@ -51,6 +33,29 @@ mode = "stderr-terminal"
 [timeseries_db]
 address = "[::1]:8123"
 
+[deployment]
+# Identifier for this instance of Nexus
+id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
+
+[deployment.dropshot_external]
+# IP address and TCP port on which to listen for the external API
+bind_address = "127.0.0.1:12220"
+# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one
+# rule is ~500 bytes)
+request_body_max_bytes = 1048576
+
+[deployment.dropshot_internal]
+# IP address and TCP port on which to listen for the internal API
+bind_address = "127.0.0.1:12221"
+
+[deployment.subnet]
+net = "fd00:1122:3344:0100::/56"
+
+[deployment.database]
+# URL for connecting to the database
+type = "from_url"
+url = "postgresql://[email protected]:32221/omicron?sslmode=disable"
+
 # Tunable configuration parameters, for testing or experimentation
 [tunables]
 

nexus/src/app/mod.rs

@@ -112,7 +112,7 @@ impl Nexus {
         authz: Arc<authz::Authz>,
     ) -> Arc<Nexus> {
         let pool = Arc::new(pool);
-        let my_sec_id = db::SecId::from(config.id);
+        let my_sec_id = db::SecId::from(config.deployment.id);
         let db_datastore = Arc::new(db::DataStore::new(Arc::clone(&pool)));
         let sec_store = Arc::new(db::CockroachDbSecStore::new(
             my_sec_id,
@@ -127,7 +127,7 @@ impl Nexus {
             sec_store,
         ));
         let timeseries_client =
-            oximeter_db::Client::new(config.timeseries_db.address, &log);
+            oximeter_db::Client::new(config.pkg.timeseries_db.address, &log);
 
         // TODO-cleanup We may want a first-class subsystem for managing startup
         // background tasks.  It could use a Future for each one, a status enum
@@ -143,7 +143,7 @@ impl Nexus {
             populate_start(populate_ctx, Arc::clone(&db_datastore));
 
         let nexus = Nexus {
-            id: config.id,
+            id: config.deployment.id,
             rack_id,
             log: log.new(o!()),
             api_rack_identity: db::model::RackIdentity::new(rack_id),
@@ -153,8 +153,8 @@ impl Nexus {
             recovery_task: std::sync::Mutex::new(None),
             populate_status,
             timeseries_client,
-            updates_config: config.updates.clone(),
-            tunables: config.tunables.clone(),
+            updates_config: config.pkg.updates.clone(),
+            tunables: config.pkg.tunables.clone(),
             opctx_alloc: OpContext::for_background(
                 log.new(o!("component" => "InstanceAllocator")),
                 Arc::clone(&authz),