Skip to content

Commit

Permalink
Add port picker, to let http servers run even with tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@smklein
smklein committed Jul 27, 2022
1 parent 920ad13 commit 494a6f1
Show file tree
Hide file tree
Showing 9 changed files with 123 additions and 131 deletions.
32 changes: 6 additions & 26 deletions README.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,35 +77,15 @@ Supported config properties include:
|Yes
|URL identifying the CockroachDB instance(s) to connect to. CockroachDB is used for all persistent data.

|`dropshot_external`
|
|Yes
|Dropshot configuration for the external server (i.e., the one that operators and developers using the Oxide rack will use). Specific properties are documented below, but see the Dropshot README for details.

|`dropshot_external.bind_address`
|`"127.0.0.1:12220"`
|Yes
|Specifies that the server should bind to the given IP address and TCP port for the **external** API (i.e., the one that operators and developers using the Oxide rack will use). In general, servers can bind to more than one IP address and port, but this is not (yet?) supported.

|`dropshot_external.request_body_max_bytes`
|`1000`
|Yes
|Specifies the maximum request body size for the **external** API.

|`dropshot_internal`
|
|Yes
|Dropshot configuration for the internal server (i.e., the one used by the sled agent). Specific properties are documented below, but see the Dropshot README for details.

|`dropshot_internal.bind_address`
|`"127.0.0.1:12220"`
|`external_ip`
|`"127.0.0.1"`
|Yes
|Specifies that the server should bind to the given IP address and TCP port for the **internal** API (i.e., the one used by the sled agent). In general, servers can bind to more than one IP address and port, but this is not (yet?) supported.
|Specifies that the server should bind to the given IP address for the **external** API (i.e., the one that operators and developers using the Oxide rack will use).

|`dropshot_internal.request_body_max_bytes`
|`1000`
|`internal_ip`
|`"127.0.0.1"`
|Yes
|Specifies the maximum request body size for the **internal** API.
|Dropshot configuration for the internal server (i.e., the one used by the sled agent).

|`id`
|`"e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"`
Expand Down
20 changes: 20 additions & 0 deletions common/src/nexus_config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,23 @@ pub enum Database {
},
}

/// Describes how ports are selected for dropshot's HTTP servers.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum PortPicker {
/// Use default values for ports, defined by Nexus.
NexusChoice,
/// Use port zero - this is avoids conflicts during tests,
/// by letting the OS pick free ports.
Zero,
}

impl Default for PortPicker {
fn default() -> Self {
PortPicker::NexusChoice
}
}

#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct DeploymentConfig {
/// Uuid of the Nexus instance
Expand All @@ -108,6 +125,9 @@ pub struct DeploymentConfig {
pub external_ip: IpAddr,
/// Internal address of Nexus.
pub internal_ip: IpAddr,
/// Decides how ports are selected
#[serde(default)]
pub port_picker: PortPicker,
/// Portion of the IP space to be managed by the Rack.
pub subnet: Ipv6Subnet<RACK_PREFIX>,
/// DB configuration.
Expand Down
13 changes: 2 additions & 11 deletions nexus/examples/config.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,8 @@ address = "[::1]:8123"
# Identifier for this instance of Nexus
id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"

[deployment.dropshot_external]
# IP address and TCP port on which to listen for the external API
bind_address = "127.0.0.1:12220"
# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one
# rule is ~500 bytes)
request_body_max_bytes = 1048576

[deployment.dropshot_internal]
# IP address and TCP port on which to listen for the internal API
bind_address = "127.0.0.1:12221"
external_ip = "127.0.0.1"
internal_ip = "127.0.0.1"

[deployment.subnet]
net = "fd00:1122:3344:0100::/56"
Expand Down
52 changes: 13 additions & 39 deletions nexus/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,17 +215,16 @@ mod test {
AuthnConfig, Config, ConsoleConfig, LoadError, PackageConfig,
SchemeName, TimeseriesDbConfig, UpdatesConfig,
};
use dropshot::ConfigDropshot;
use dropshot::ConfigLogging;
use dropshot::ConfigLoggingIfExists;
use dropshot::ConfigLoggingLevel;
use libc;
use omicron_common::address::{Ipv6Subnet, RACK_PREFIX};
use omicron_common::nexus_config::{
Database, DeploymentConfig, LoadErrorKind,
Database, DeploymentConfig, LoadErrorKind, PortPicker,
};
use std::fs;
use std::net::{Ipv6Addr, SocketAddr};
use std::net::{IpAddr, Ipv6Addr};
use std::path::Path;
use std::path::PathBuf;

Expand Down Expand Up @@ -336,12 +335,8 @@ mod test {
[deployment]
id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
[deployment.dropshot_external]
bind_address = "10.1.2.3:4567"
request_body_max_bytes = 1024
[deployment.dropshot_internal]
bind_address = "10.1.2.3:4568"
request_body_max_bytes = 1024
external_ip = "10.1.2.3"
internal_ip = "10.1.2.4"
[deployment.subnet]
net = "::/56"
[deployment.database]
Expand All @@ -358,18 +353,9 @@ mod test {
rack_id: "38b90dc4-c22a-65ba-f49a-f051fe01208f"
.parse()
.unwrap(),
dropshot_external: ConfigDropshot {
bind_address: "10.1.2.3:4567"
.parse::<SocketAddr>()
.unwrap(),
..Default::default()
},
dropshot_internal: ConfigDropshot {
bind_address: "10.1.2.3:4568"
.parse::<SocketAddr>()
.unwrap(),
..Default::default()
},
external_ip: "10.1.2.3".parse::<IpAddr>().unwrap(),
internal_ip: "10.1.2.4".parse::<IpAddr>().unwrap(),
port_picker: PortPicker::default(),
subnet: Ipv6Subnet::<RACK_PREFIX>::new(Ipv6Addr::LOCALHOST),
database: Database::FromDns,
},
Expand Down Expand Up @@ -418,12 +404,8 @@ mod test {
[deployment]
id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
[deployment.dropshot_external]
bind_address = "10.1.2.3:4567"
request_body_max_bytes = 1024
[deployment.dropshot_internal]
bind_address = "10.1.2.3:4568"
request_body_max_bytes = 1024
external_ip = "10.1.2.3"
internal_ip = "10.1.2.4"
[deployment.subnet]
net = "::/56"
[deployment.database]
Expand Down Expand Up @@ -460,12 +442,8 @@ mod test {
[deployment]
id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
[deployment.dropshot_external]
bind_address = "10.1.2.3:4567"
request_body_max_bytes = 1024
[deployment.dropshot_internal]
bind_address = "10.1.2.3:4568"
request_body_max_bytes = 1024
external_ip = "10.1.2.3"
internal_ip = "10.1.2.4"
[deployment.subnet]
net = "::/56"
[deployment.database]
Expand Down Expand Up @@ -516,12 +494,8 @@ mod test {
[deployment]
id = "28b90dc4-c22a-65ba-f49a-f051fe01208f"
rack_id = "38b90dc4-c22a-65ba-f49a-f051fe01208f"
[deployment.dropshot_external]
bind_address = "10.1.2.3:4567"
request_body_max_bytes = 1024
[deployment.dropshot_internal]
bind_address = "10.1.2.3:4568"
request_body_max_bytes = 1024
external_ip = "10.1.2.3"
internal_ip = "10.1.2.4"
[deployment.subnet]
net = "::/56"
[deployment.database]
Expand Down
108 changes: 65 additions & 43 deletions nexus/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ pub struct Server {
/// shared state used by API request handlers
pub apictx: Arc<ServerContext>,
/// dropshot server for external API (encrypted)
pub https_server_external: dropshot::HttpServer<Arc<ServerContext>>,
pub https_server_external: Option<dropshot::HttpServer<Arc<ServerContext>>>,
/// dropshot server for external API (unencrypted)
pub http_server_external: dropshot::HttpServer<Arc<ServerContext>>,
/// dropshot server for internal API
Expand All @@ -96,52 +96,83 @@ impl Server {
ServerContext::new(config.deployment.rack_id, ctxlog, &config)
.await?;

// We launch separate dropshot servers for the "encrypted" and
// "unencrypted" ports.
// Determine port choices

const HTTPS_PORT: u16 = 443;
const HTTP_PORT: u16 = 80;
let (external_http_port, external_https_port, internal_http_port) =
match config.deployment.port_picker {
omicron_common::nexus_config::PortPicker::NexusChoice => {
(80, 443, omicron_common::address::NEXUS_INTERNAL_PORT)
}
omicron_common::nexus_config::PortPicker::Zero => (0, 0, 0),
};

let dropshot_external_https_config = dropshot::ConfigDropshot {
bind_address: SocketAddr::new(
config.deployment.external_ip,
HTTPS_PORT,
),
request_body_max_bytes: 1048576,
tls: Some(dropshot::ConfigTls {
cert_file: PathBuf::from("/var/nexus/certs/cert.pem"),
key_file: PathBuf::from("/var/nexus/certs/key.pem"),
}),
};
// TODO: Consider removing this interface when all clients are using
// https?
let dropshot_external_http_config = dropshot::ConfigDropshot {
bind_address: SocketAddr::new(
config.deployment.external_ip,
HTTP_PORT,
),
request_body_max_bytes: 1048576,
tls: None,
};
// Launch the internal server.

let dropshot_internal_config = dropshot::ConfigDropshot {
bind_address: SocketAddr::new(
config.deployment.internal_ip,
omicron_common::address::NEXUS_INTERNAL_PORT,
internal_http_port,
),
request_body_max_bytes: 1048576,
..Default::default()
};

let https_server_starter_external = dropshot::HttpServerStarter::new(
&dropshot_external_https_config,
external_api(),
let http_server_starter_internal = dropshot::HttpServerStarter::new(
&dropshot_internal_config,
internal_api(),
Arc::clone(&apictx),
&log.new(o!("component" => "dropshot_external (encrypted)")),
&log.new(o!("component" => "dropshot_internal")),
)
.map_err(|error| format!("initializing external server: {}", error))?;
let https_server_external = https_server_starter_external.start();
.map_err(|error| format!("initializing internal server: {}", error))?;
let http_server_internal = http_server_starter_internal.start();

// Launch the external server(s).
//
// - The HTTP server is unconditionally started.
// - The HTTPS server is started if the necessary certificate files
// exist.
//
// TODO: Consider changing this disposition, making "HTTPS" the default,
// and returning an error if the certificates don't exist. Doing so
// would be the more secure long-term plan, but would make gradual
// deployment of this feature more difficult.

let cert_file = PathBuf::from("/var/nexus/certs/cert.pem");
let key_file = PathBuf::from("/var/nexus/certs/key.pem");

let https_server_external = if cert_file.exists() && key_file.exists() {
let dropshot_external_https_config = dropshot::ConfigDropshot {
bind_address: SocketAddr::new(
config.deployment.external_ip,
external_https_port,
),
request_body_max_bytes: 1048576,
tls: Some(dropshot::ConfigTls { cert_file, key_file }),
};
let https_server_starter_external =
dropshot::HttpServerStarter::new(
&dropshot_external_https_config,
external_api(),
Arc::clone(&apictx),
&log.new(
o!("component" => "dropshot_external (encrypted)"),
),
)
.map_err(|error| {
format!("initializing external server: {}", error)
})?;
Some(https_server_starter_external.start())
} else {
None
};

let dropshot_external_http_config = dropshot::ConfigDropshot {
bind_address: SocketAddr::new(
config.deployment.external_ip,
external_http_port,
),
request_body_max_bytes: 1048576,
tls: None,
};
let http_server_starter_external = dropshot::HttpServerStarter::new(
&dropshot_external_http_config,
external_api(),
Expand All @@ -151,15 +182,6 @@ impl Server {
.map_err(|error| format!("initializing external server: {}", error))?;
let http_server_external = http_server_starter_external.start();

let http_server_starter_internal = dropshot::HttpServerStarter::new(
&dropshot_internal_config,
internal_api(),
Arc::clone(&apictx),
&log.new(o!("component" => "dropshot_internal")),
)
.map_err(|error| format!("initializing internal server: {}", error))?;
let http_server_internal = http_server_starter_internal.start();

Ok(Server {
apictx,
https_server_external,
Expand Down
13 changes: 3 additions & 10 deletions nexus/tests/config.test.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,19 +41,12 @@ max_vpc_ipv4_subnet_prefix = 29
id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
rack_id = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"

#
external_ip = "127.0.0.1"
internal_ip = "127.0.0.1"
# NOTE: for the test suite, the port MUST be 0 (in order to bind to any
# available port) because the test suite will be running many servers
# concurrently.
#
[deployment.dropshot_external]
bind_address = "127.0.0.1:0"
request_body_max_bytes = 1048576

# port must be 0. see above
[deployment.dropshot_internal]
bind_address = "127.0.0.1:0"
request_body_max_bytes = 1048576
port_picker = "zero"

[deployment.subnet]
net = "fd00:1122:3344:0100::/56"
Expand Down
13 changes: 12 additions & 1 deletion nexus/tests/integration_tests/authn_http.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ use omicron_nexus::authn::external::HttpAuthnScheme;
use omicron_nexus::authn::external::SiloUserSilo;
use omicron_nexus::db::fixed_data::silo::SILO_ID;
use std::collections::HashMap;
use std::net::SocketAddr;
use std::sync::{Arc, Mutex};
use uuid::Uuid;

Expand Down Expand Up @@ -296,10 +297,20 @@ async fn start_whoami_server(
};

let log = logctx.log.new(o!());

assert!(matches!(
config.deployment.port_picker,
omicron_common::nexus_config::PortPicker::Zero
));
let config_dropshot = dropshot::ConfigDropshot {
bind_address: SocketAddr::new(config.deployment.external_ip, 0),
request_body_max_bytes: 1048576,
tls: None,
};
TestContext::new(
whoami_api,
server_state,
&config.deployment.dropshot_external,
&config_dropshot,
Some(logctx),
log,
)
Expand Down
Loading

0 comments on commit 494a6f1

Please sign in to comment.