From 920ad131c0c4b5ed6388d4223b4bf9046b2658d0 Mon Sep 17 00:00:00 2001 From: Sean Klein Date: Tue, 26 Jul 2022 16:54:20 -0400 Subject: [PATCH] Launch Nexus with both and HTTP and HTTPS server --- common/src/nexus_config.rs | 10 ++--- nexus/src/lib.rs | 68 ++++++++++++++++++++++++++++++---- sled-agent/src/params.rs | 18 +++++---- sled-agent/src/services.rs | 26 +++---------- smf/sled-agent/config-rss.toml | 4 +- 5 files changed, 85 insertions(+), 41 deletions(-) diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index a18454e02d..a159dea7fe 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -7,11 +7,11 @@ use super::address::{Ipv6Subnet, RACK_PREFIX}; use super::postgres_config::PostgresConfigWithUrl; -use dropshot::ConfigDropshot; use serde::{Deserialize, Serialize}; use serde_with::serde_as; use serde_with::DisplayFromStr; use std::fmt; +use std::net::IpAddr; use std::path::{Path, PathBuf}; use uuid::Uuid; @@ -104,10 +104,10 @@ pub struct DeploymentConfig { pub id: Uuid, /// Uuid of the Rack where Nexus is executing. pub rack_id: Uuid, - /// Dropshot configuration for external API server - pub dropshot_external: ConfigDropshot, - /// Dropshot configuration for internal API server - pub dropshot_internal: ConfigDropshot, + /// External address of Nexus. + pub external_ip: IpAddr, + /// Internal address of Nexus. + pub internal_ip: IpAddr, /// Portion of the IP space to be managed by the Rack. pub subnet: Ipv6Subnet, /// DB configuration. diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index fd7b50802e..7d44e85bd2 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -34,6 +34,8 @@ pub use crucible_agent_client; use external_api::http_entrypoints::external_api; use internal_api::http_entrypoints::internal_api; use slog::Logger; +use std::net::SocketAddr; +use std::path::PathBuf; use std::sync::Arc; #[macro_use] @@ -71,7 +73,9 @@ pub fn run_openapi_internal() -> Result<(), String> { pub struct Server { /// shared state used by API request handlers pub apictx: Arc, - /// dropshot server for external API + /// dropshot server for external API (encrypted) + pub https_server_external: dropshot::HttpServer>, + /// dropshot server for external API (unencrypted) pub http_server_external: dropshot::HttpServer>, /// dropshot server for internal API pub http_server_internal: dropshot::HttpServer>, @@ -92,26 +96,76 @@ impl Server { ServerContext::new(config.deployment.rack_id, ctxlog, &config) .await?; + // We launch separate dropshot servers for the "encrypted" and + // "unencrypted" ports. + + const HTTPS_PORT: u16 = 443; + const HTTP_PORT: u16 = 80; + + let dropshot_external_https_config = dropshot::ConfigDropshot { + bind_address: SocketAddr::new( + config.deployment.external_ip, + HTTPS_PORT, + ), + request_body_max_bytes: 1048576, + tls: Some(dropshot::ConfigTls { + cert_file: PathBuf::from("/var/nexus/certs/cert.pem"), + key_file: PathBuf::from("/var/nexus/certs/key.pem"), + }), + }; + // TODO: Consider removing this interface when all clients are using + // https? + let dropshot_external_http_config = dropshot::ConfigDropshot { + bind_address: SocketAddr::new( + config.deployment.external_ip, + HTTP_PORT, + ), + request_body_max_bytes: 1048576, + tls: None, + }; + + let dropshot_internal_config = dropshot::ConfigDropshot { + bind_address: SocketAddr::new( + config.deployment.internal_ip, + omicron_common::address::NEXUS_INTERNAL_PORT, + ), + request_body_max_bytes: 1048576, + ..Default::default() + }; + + let https_server_starter_external = dropshot::HttpServerStarter::new( + &dropshot_external_https_config, + external_api(), + Arc::clone(&apictx), + &log.new(o!("component" => "dropshot_external (encrypted)")), + ) + .map_err(|error| format!("initializing external server: {}", error))?; + let https_server_external = https_server_starter_external.start(); + let http_server_starter_external = dropshot::HttpServerStarter::new( - &config.deployment.dropshot_external, + &dropshot_external_http_config, external_api(), Arc::clone(&apictx), - &log.new(o!("component" => "dropshot_external")), + &log.new(o!("component" => "dropshot_external (unencrypted)")), ) .map_err(|error| format!("initializing external server: {}", error))?; + let http_server_external = http_server_starter_external.start(); let http_server_starter_internal = dropshot::HttpServerStarter::new( - &config.deployment.dropshot_internal, + &dropshot_internal_config, internal_api(), Arc::clone(&apictx), &log.new(o!("component" => "dropshot_internal")), ) .map_err(|error| format!("initializing internal server: {}", error))?; - - let http_server_external = http_server_starter_external.start(); let http_server_internal = http_server_starter_internal.start(); - Ok(Server { apictx, http_server_external, http_server_internal }) + Ok(Server { + apictx, + https_server_external, + http_server_external, + http_server_internal, + }) } /// Wait for the given server to shut down diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 494a901be5..731a0ecae0 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -3,7 +3,9 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use internal_dns_client::names::{BackendName, ServiceName, AAAA, SRV}; -use omicron_common::address::{DENDRITE_PORT, OXIMETER_PORT}; +use omicron_common::address::{ + DENDRITE_PORT, NEXUS_INTERNAL_PORT, OXIMETER_PORT, +}; use omicron_common::api::external; use omicron_common::api::internal::nexus::{ DiskRuntimeState, InstanceRuntimeState, @@ -342,7 +344,7 @@ impl From for sled_agent_client::types::DatasetEnsureBody { )] #[serde(tag = "type", rename_all = "snake_case")] pub enum ServiceType { - Nexus { internal_address: SocketAddrV6, external_address: SocketAddr }, + Nexus { internal_ip: Ipv6Addr, external_ip: IpAddr }, InternalDns { server_address: SocketAddrV6, dns_address: SocketAddrV6 }, Oximeter, Dendrite { asic: DendriteAsic }, @@ -354,9 +356,9 @@ impl From for sled_agent_client::types::ServiceType { use ServiceType as St; match s { - St::Nexus { internal_address, external_address } => AutoSt::Nexus { - internal_address: internal_address.to_string(), - external_address: external_address.to_string(), + St::Nexus { internal_ip, external_ip } => AutoSt::Nexus { + internal_address: internal_ip.to_string(), + external_address: external_ip.to_string(), }, St::InternalDns { server_address, dns_address } => { AutoSt::InternalDns { @@ -370,7 +372,7 @@ impl From for sled_agent_client::types::ServiceType { } } -/// Describes a request to create a service. This information +/// Describes a request to create a service. This informatios tn /// should be sufficient for a Sled Agent to start a zone /// containing the requested service. #[derive( @@ -415,7 +417,9 @@ impl ServiceRequest { pub fn address(&self) -> SocketAddrV6 { match self.service_type { ServiceType::InternalDns { server_address, .. } => server_address, - ServiceType::Nexus { internal_address, .. } => internal_address, + ServiceType::Nexus { internal_ip, .. } => { + SocketAddrV6::new(internal_ip, NEXUS_INTERNAL_PORT, 0, 0) + } ServiceType::Oximeter => { SocketAddrV6::new(self.addresses[0], OXIMETER_PORT, 0, 0) } diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 8b2a281de9..609130e290 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -13,7 +13,6 @@ use crate::illumos::zfs::ZONE_ZFS_DATASET_MOUNTPOINT; use crate::illumos::zone::AddressRequest; use crate::params::{ServiceEnsureBody, ServiceRequest, ServiceType}; use crate::zone::Zones; -use dropshot::ConfigDropshot; use omicron_common::address::Ipv6Subnet; use omicron_common::address::OXIMETER_PORT; use omicron_common::address::RACK_PREFIX; @@ -25,7 +24,7 @@ use omicron_common::postgres_config::PostgresConfigWithUrl; use slog::Logger; use std::collections::HashSet; use std::iter::FromIterator; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::path::{Path, PathBuf}; use std::str::FromStr; use tokio::io::AsyncWriteExt; @@ -373,13 +372,13 @@ impl ServiceManager { let default_smf_name = format!("{}:default", smf_name); match service.service_type { - ServiceType::Nexus { internal_address, external_address } => { + ServiceType::Nexus { internal_ip, external_ip } => { info!(self.log, "Setting up Nexus service"); // The address of Nexus' external interface is a special // case; it may be an IPv4 address. let addr_request = - AddressRequest::new_static(external_address.ip(), None); + AddressRequest::new_static(external_ip, None); running_zone .ensure_external_address_with_name( addr_request, @@ -387,7 +386,7 @@ impl ServiceManager { ) .await?; - if let IpAddr::V4(_public_addr4) = external_address.ip() { + if let IpAddr::V4(_public_addr4) = external_ip { // If requested, create a default route back through // the internet gateway. if let Some(ref gateway) = self.config.gateway_address { @@ -406,21 +405,8 @@ impl ServiceManager { let deployment_config = NexusDeploymentConfig { id: service.id, rack_id: self.rack_id, - dropshot_external: ConfigDropshot { - bind_address: external_address, - request_body_max_bytes: 1048576, - tls: Some( - dropshot::ConfigTls { - cert_file: PathBuf::from("/var/nexus/certs/cert.pem"), - key_file: PathBuf::from("/var/nexus/certs/key.pem"), - } - ), - }, - dropshot_internal: ConfigDropshot { - bind_address: SocketAddr::V6(internal_address), - request_body_max_bytes: 1048576, - ..Default::default() - }, + external_ip, + internal_ip: IpAddr::V6(internal_ip), subnet: Ipv6Subnet::::new( self.underlay_address, ), diff --git a/smf/sled-agent/config-rss.toml b/smf/sled-agent/config-rss.toml index fbb5214c55..4774b0c435 100644 --- a/smf/sled-agent/config-rss.toml +++ b/smf/sled-agent/config-rss.toml @@ -55,9 +55,9 @@ addresses = [ "fd00:1122:3344:0101::3" ] gz_addresses = [] [request.service.service_type] type = "nexus" -internal_address = "[fd00:1122:3344:0101::3]:12221" +internal_address = "fd00:1122:3344:0101::3" # NOTE: In the lab, use "172.20.15.226" -external_address = "192.168.1.20:80" +external_address = "192.168.1.20" # TODO(https://github.com/oxidecomputer/omicron/issues/732): Nexus # should allocate Oximeter services.