[nexus] Add HTTPS support, plumbing x509 certificates

README.adoc

@@ -77,35 +77,15 @@ Supported config properties include:
 |Yes
 |URL identifying the CockroachDB instance(s) to connect to.  CockroachDB is used for all persistent data.
 
-|`dropshot_external`
-|
-|Yes
-|Dropshot configuration for the external server (i.e., the one that operators and developers using the Oxide rack will use).  Specific properties are documented below, but see the Dropshot README for details.
-
-|`dropshot_external.bind_address`
-|`"127.0.0.1:12220"`
-|Yes
-|Specifies that the server should bind to the given IP address and TCP port for the **external** API (i.e., the one that operators and developers using the Oxide rack will use).  In general, servers can bind to more than one IP address and port, but this is not (yet?) supported.
-
-|`dropshot_external.request_body_max_bytes`
-|`1000`
-|Yes
-|Specifies the maximum request body size for the **external** API.
-
-|`dropshot_internal`
-|
-|Yes
-|Dropshot configuration for the internal server (i.e., the one used by the sled agent).  Specific properties are documented below, but see the Dropshot README for details.
-
-|`dropshot_internal.bind_address`
-|`"127.0.0.1:12220"`
+|`external_ip`
+|`"127.0.0.1"`
 |Yes
-|Specifies that the server should bind to the given IP address and TCP port for the **internal** API (i.e., the one used by the sled agent).  In general, servers can bind to more than one IP address and port, but this is not (yet?) supported.
+|Specifies that the server should bind to the given IP address for the **external** API (i.e., the one that operators and developers using the Oxide rack will use).
 
-|`dropshot_internal.request_body_max_bytes`
-|`1000`
+|`internal_ip`
+|`"127.0.0.1"`
 |Yes
-|Specifies the maximum request body size for the **internal** API.
+|Dropshot configuration for the internal server (i.e., the one used by the sled agent).
 
 |`id`
 |`"e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"`

common/src/nexus_config.rs

@@ -7,11 +7,11 @@
 
 use super::address::{Ipv6Subnet, RACK_PREFIX};
 use super::postgres_config::PostgresConfigWithUrl;
-use dropshot::ConfigDropshot;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 use serde_with::DisplayFromStr;
 use std::fmt;
+use std::net::IpAddr;
 use std::path::{Path, PathBuf};
 use uuid::Uuid;
 
@@ -98,16 +98,36 @@ pub enum Database {
     },
 }
 
+/// Describes how ports are selected for dropshot's HTTP servers.
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum PortPicker {
+    /// Use default values for ports, defined by Nexus.
+    NexusChoice,
+    /// Use port zero - this is avoids conflicts during tests,
+    /// by letting the OS pick free ports.
+    Zero,
+}
+
+impl Default for PortPicker {
+    fn default() -> Self {
+        PortPicker::NexusChoice
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct DeploymentConfig {
     /// Uuid of the Nexus instance
     pub id: Uuid,
     /// Uuid of the Rack where Nexus is executing.
     pub rack_id: Uuid,
-    /// Dropshot configuration for external API server
-    pub dropshot_external: ConfigDropshot,
-    /// Dropshot configuration for internal API server
-    pub dropshot_internal: ConfigDropshot,
+    /// External address of Nexus.
+    pub external_ip: IpAddr,
+    /// Internal address of Nexus.
+    pub internal_ip: IpAddr,
+    /// Decides how ports are selected
+    #[serde(default)]
+    pub port_picker: PortPicker,
     /// Portion of the IP space to be managed by the Rack.
     pub subnet: Ipv6Subnet<RACK_PREFIX>,
     /// DB configuration.

docs/how-to-run.adoc

@@ -53,6 +53,13 @@ This script requires Omicron be uninstalled, e.g., with `pfexec
 that is not the case. The script will then remove the file-based vdevs and the
 VNICs created by `create_virtual_hardware.sh`.
 
+=== Make me a certificate!
+
+Nexus's external interface will typically be served using public-facing x.509
+certificate. While we are still configuring the mechanism to integrate this real
+certificate into the package system, `./tools/create_self_signed_cert.sh` can be
+used to generate an equivalent self-signed certificate.
+
 == Deploying Omicron
 
 The control plane repository contains a packaging tool which bundles binaries

nexus/examples/config.toml

@@ -37,17 +37,8 @@ address = "[::1]:8123"
 # Identifier for this instance of Nexus
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
 rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
-
-[deployment.dropshot_external]
-# IP address and TCP port on which to listen for the external API
-bind_address = "127.0.0.1:12220"
-# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one
-# rule is ~500 bytes)
-request_body_max_bytes = 1048576
-
-[deployment.dropshot_internal]
-# IP address and TCP port on which to listen for the internal API
-bind_address = "127.0.0.1:12221"
+external_ip = "127.0.0.1"
+internal_ip = "127.0.0.1"
 
 [deployment.subnet]
 net = "fd00:1122:3344:0100::/56"

nexus/src/config.rs

@@ -215,17 +215,16 @@ mod test {
         AuthnConfig, Config, ConsoleConfig, LoadError, PackageConfig,
         SchemeName, TimeseriesDbConfig, UpdatesConfig,
     };
-    use dropshot::ConfigDropshot;
     use dropshot::ConfigLogging;
     use dropshot::ConfigLoggingIfExists;
     use dropshot::ConfigLoggingLevel;
     use libc;
     use omicron_common::address::{Ipv6Subnet, RACK_PREFIX};
     use omicron_common::nexus_config::{
-        Database, DeploymentConfig, LoadErrorKind,
+        Database, DeploymentConfig, LoadErrorKind, PortPicker,
     };
     use std::fs;
-    use std::net::{Ipv6Addr, SocketAddr};
+    use std::net::{IpAddr, Ipv6Addr};
     use std::path::Path;
     use std::path::PathBuf;
 
@@ -336,12 +335,8 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
-            bind_address = "10.1.2.3:4567"
-            request_body_max_bytes = 1024
-            [deployment.dropshot_internal]
-            bind_address = "10.1.2.3:4568"
-            request_body_max_bytes = 1024
+            external_ip = "10.1.2.3"
+            internal_ip = "10.1.2.4"
             [deployment.subnet]
             net = "::/56"
             [deployment.database]
@@ -358,18 +353,9 @@ mod test {
                     rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f"
                         .parse()
                         .unwrap(),
-                    dropshot_external: ConfigDropshot {
-                        bind_address: "10.1.2.3:4567"
-                            .parse::<SocketAddr>()
-                            .unwrap(),
-                        ..Default::default()
-                    },
-                    dropshot_internal: ConfigDropshot {
-                        bind_address: "10.1.2.3:4568"
-                            .parse::<SocketAddr>()
-                            .unwrap(),
-                        ..Default::default()
-                    },
+                    external_ip: "10.1.2.3".parse::<IpAddr>().unwrap(),
+                    internal_ip: "10.1.2.4".parse::<IpAddr>().unwrap(),
+                    port_picker: PortPicker::default(),
                     subnet: Ipv6Subnet::<RACK_PREFIX>::new(Ipv6Addr::LOCALHOST),
                     database: Database::FromDns,
                 },
@@ -418,12 +404,8 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
-            bind_address = "10.1.2.3:4567"
-            request_body_max_bytes = 1024
-            [deployment.dropshot_internal]
-            bind_address = "10.1.2.3:4568"
-            request_body_max_bytes = 1024
+            external_ip = "10.1.2.3"
+            internal_ip = "10.1.2.4"
             [deployment.subnet]
             net = "::/56"
             [deployment.database]
@@ -460,12 +442,8 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
-            bind_address = "10.1.2.3:4567"
-            request_body_max_bytes = 1024
-            [deployment.dropshot_internal]
-            bind_address = "10.1.2.3:4568"
-            request_body_max_bytes = 1024
+            external_ip = "10.1.2.3"
+            internal_ip = "10.1.2.4"
             [deployment.subnet]
             net = "::/56"
             [deployment.database]
@@ -516,12 +494,8 @@ mod test {
             [deployment]
             id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
             rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
-            [deployment.dropshot_external]
-            bind_address = "10.1.2.3:4567"
-            request_body_max_bytes = 1024
-            [deployment.dropshot_internal]
-            bind_address = "10.1.2.3:4568"
-            request_body_max_bytes = 1024
+            external_ip = "10.1.2.3"
+            internal_ip = "10.1.2.4"
             [deployment.subnet]
             net = "::/56"
             [deployment.database]

nexus/src/lib.rs

@@ -34,6 +34,8 @@ pub use crucible_agent_client;
 use external_api::http_entrypoints::external_api;
 use internal_api::http_entrypoints::internal_api;
 use slog::Logger;
+use std::net::SocketAddr;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 #[macro_use]
@@ -71,7 +73,9 @@ pub fn run_openapi_internal() -> Result<(), String> {
 pub struct Server {
     /// shared state used by API request handlers
     pub apictx: Arc<ServerContext>,
-    /// dropshot server for external API
+    /// dropshot server for external API (encrypted)
+    pub https_server_external: Option<dropshot::HttpServer<Arc<ServerContext>>>,
+    /// dropshot server for external API (unencrypted)
     pub http_server_external: dropshot::HttpServer<Arc<ServerContext>>,
     /// dropshot server for internal API
     pub http_server_internal: dropshot::HttpServer<Arc<ServerContext>>,
@@ -92,26 +96,98 @@ impl Server {
             ServerContext::new(config.deployment.rack_id, ctxlog, &config)
                 .await?;
 
-        let http_server_starter_external = dropshot::HttpServerStarter::new(
-            &config.deployment.dropshot_external,
-            external_api(),
-            Arc::clone(&apictx),
-            &log.new(o!("component" => "dropshot_external")),
-        )
-        .map_err(|error| format!("initializing external server: {}", error))?;
-
+        // Determine port choices
+
+        let (external_http_port, external_https_port, internal_http_port) =
+            match config.deployment.port_picker {
+                omicron_common::nexus_config::PortPicker::NexusChoice => {
+                    (80, 443, omicron_common::address::NEXUS_INTERNAL_PORT)
+                }
+                omicron_common::nexus_config::PortPicker::Zero => (0, 0, 0),
+            };
+
+        // Launch the internal server.
+
+        let dropshot_internal_config = dropshot::ConfigDropshot {
+            bind_address: SocketAddr::new(
+                config.deployment.internal_ip,
+                internal_http_port,
+            ),
+            request_body_max_bytes: 1048576,
+            ..Default::default()
+        };
         let http_server_starter_internal = dropshot::HttpServerStarter::new(
-            &config.deployment.dropshot_internal,
+            &dropshot_internal_config,
             internal_api(),
             Arc::clone(&apictx),
             &log.new(o!("component" => "dropshot_internal")),
         )
         .map_err(|error| format!("initializing internal server: {}", error))?;
+        let http_server_internal = http_server_starter_internal.start();
 
+        // Launch the external server(s).
+        //
+        // - The HTTP server is unconditionally started.
+        // - The HTTPS server is started if the necessary certificate files
+        // exist.
+        //
+        // TODO: Consider changing this disposition, making "HTTPS" the default,
+        // and returning an error if the certificates don't exist. Doing so
+        // would be the more secure long-term plan, but would make gradual
+        // deployment of this feature more difficult.
+
+        let cert_file = PathBuf::from("/var/nexus/certs/cert.pem");
+        let key_file = PathBuf::from("/var/nexus/certs/key.pem");
+
+        let https_server_external = if cert_file.exists() && key_file.exists() {
+            let dropshot_external_https_config = dropshot::ConfigDropshot {
+                bind_address: SocketAddr::new(
+                    config.deployment.external_ip,
+                    external_https_port,
+                ),
+                request_body_max_bytes: 1048576,
+                tls: Some(dropshot::ConfigTls { cert_file, key_file }),
+            };
+            let https_server_starter_external =
+                dropshot::HttpServerStarter::new(
+                    &dropshot_external_https_config,
+                    external_api(),
+                    Arc::clone(&apictx),
+                    &log.new(
+                        o!("component" => "dropshot_external (encrypted)"),
+                    ),
+                )
+                .map_err(|error| {
+                    format!("initializing external server: {}", error)
+                })?;
+            Some(https_server_starter_external.start())
+        } else {
+            None
+        };
+
+        let dropshot_external_http_config = dropshot::ConfigDropshot {
+            bind_address: SocketAddr::new(
+                config.deployment.external_ip,
+                external_http_port,
+            ),
+            request_body_max_bytes: 1048576,
+            tls: None,
+        };
+        let http_server_starter_external = dropshot::HttpServerStarter::new(
+            &dropshot_external_http_config,
+            external_api(),
+            Arc::clone(&apictx),
+            &log.new(o!("component" => "dropshot_external (unencrypted)")),
+        )
+        .map_err(|error| format!("initializing external server: {}", error))?;
         let http_server_external = http_server_starter_external.start();
-        let http_server_internal = http_server_starter_internal.start();
 
-        Ok(Server { apictx, http_server_external, http_server_internal })
+        Ok(Server {
+            apictx,
+            https_server_external,
+            http_server_external,
+            http_server_internal,
+        })
     }
 
     /// Wait for the given server to shut down

nexus/tests/config.test.toml

@@ -41,19 +41,12 @@ max_vpc_ipv4_subnet_prefix = 29
 id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
 rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"
 
-#
+external_ip = "127.0.0.1"
+internal_ip = "127.0.0.1"
 # NOTE: for the test suite, the port MUST be 0 (in order to bind to any
 # available port) because the test suite will be running many servers
 # concurrently.
-#
-[deployment.dropshot_external]
-bind_address = "127.0.0.1:0"
-request_body_max_bytes = 1048576
-
-# port must be 0. see above
-[deployment.dropshot_internal]
-bind_address = "127.0.0.1:0"
-request_body_max_bytes = 1048576
+port_picker = "zero"
 
 [deployment.subnet]
 net = "fd00:1122:3344:0100::/56"