[nexus] Populate IP pool, nexus service information, during rack setup (

#2358)

# Summary

My long-term goal is to have Nexus be in charge of provisioning all
services.

For that to be possible, Nexus must be able to internalize all input
during the handoff from RSS. This PR extends the RSS -> Nexus handoff to
include:

- What "Nexus Services" are being launched?
- What are the ranges of IP addresses that may be used for internal
services?
- What external IP addresses, from that pool, are currently in-use for
Nexus services?

# Nexus Changes

## Database Records
 
- Adds a `nexus_service` record, which just includes the information
about the in-use external IP address.

## IP Address Allocation

- Adds an `explicit_ip` option, which lets callers perform an allocation
with an explicit request for a single IP address. You might ask the
question: "Why not just directly create a record with the IP address in
question, if you want to create it?" We could! But we'd need to recreate
all the logic which validates that the IP address exists within the
known-to-the-DB IP ranges within the pool.
- The ability for an operator to "request Nexus execute with a specific
IP address" is a feature we want anyway, so this isn't wasted work.
- The implementation and tests for this behavior are mostly within
`nexus/src/db/queries/external_ip.rs`

## Rack Initialization

- Populates IP pools and Service records as a part of the RSS handoff.
- Implementation and tests exist within
`nexus/src/db/datastore/rack.rs`.

## Populate

- Move the body of some of the "populate" functions into their correct
spot in the datastore, which makes it easier to...
- ... call all the populate functions -- rather than just a chunk of
them -- from `omicron_nexus::db::datastore::datastore_test`.
- As a consequence, update some tests which assumed the rack would be
"half-populated" -- it's either fully populated, or not populated at
all.

# Sled Agent changes

- Explicitly pass the "IP pool ranges for internal services" up to
Nexus.
- In the future, it'll be possible to pass a larger range of addresses
than just those used by running Nexus services.

Fixes: #1958
Unblocks: #732

common/src/sql/dbinit.sql

@@ -136,6 +136,15 @@ CREATE INDEX ON omicron.public.service (
     sled_id
 );
 
+-- Extended information for services where "service.kind = nexus"
+-- The "id" columng of this table should match "id" column of the
+-- "omicron.public.service" table exactly.
+CREATE TABLE omicron.public.nexus_service (
+    id UUID PRIMARY KEY,
+    -- The external IP address used for Nexus' external interface.
+    external_ip_id UUID NOT NULL
+);
+
 CREATE TYPE omicron.public.physical_disk_kind AS ENUM (
   'm2',
   'u2'

nexus/db-model/src/external_ip.rs

@@ -18,6 +18,7 @@ use nexus_types::external_api::shared;
 use nexus_types::external_api::views;
 use omicron_common::api::external::Error;
 use std::convert::TryFrom;
+use std::net::IpAddr;
 use uuid::Uuid;
 
 impl_enum_type!(
@@ -90,6 +91,8 @@ pub struct IncompleteExternalIp {
     kind: IpKind,
     instance_id: Option<Uuid>,
     pool_id: Uuid,
+    // Optional address requesting that a specific IP address be allocated.
+    explicit_ip: Option<IpNetwork>,
 }
 
 impl IncompleteExternalIp {
@@ -106,6 +109,7 @@ impl IncompleteExternalIp {
             kind: IpKind::SNat,
             instance_id: Some(instance_id),
             pool_id,
+            explicit_ip: None,
         }
     }
 
@@ -118,6 +122,7 @@ impl IncompleteExternalIp {
             kind: IpKind::Ephemeral,
             instance_id: Some(instance_id),
             pool_id,
+            explicit_ip: None,
         }
     }
 
@@ -135,6 +140,24 @@ impl IncompleteExternalIp {
             kind: IpKind::Floating,
             instance_id: None,
             pool_id,
+            explicit_ip: None,
+        }
+    }
+
+    pub fn for_service_explicit(
+        id: Uuid,
+        pool_id: Uuid,
+        address: IpAddr,
+    ) -> Self {
+        Self {
+            id,
+            name: None,
+            description: None,
+            time_created: Utc::now(),
+            kind: IpKind::Service,
+            instance_id: None,
+            pool_id,
+            explicit_ip: Some(IpNetwork::from(address)),
         }
     }
 
@@ -147,6 +170,7 @@ impl IncompleteExternalIp {
             kind: IpKind::Service,
             instance_id: None,
             pool_id,
+            explicit_ip: None,
         }
     }
 
@@ -177,6 +201,10 @@ impl IncompleteExternalIp {
     pub fn pool_id(&self) -> &Uuid {
         &self.pool_id
     }
+
+    pub fn explicit_ip(&self) -> &Option<IpNetwork> {
+        &self.explicit_ip
+    }
 }
 
 impl TryFrom<IpKind> for shared::IpKind {

nexus/db-model/src/lib.rs

@@ -36,6 +36,7 @@ mod l4_port_range;
 mod macaddr;
 mod name;
 mod network_interface;
+mod nexus_service;
 mod organization;
 mod oximeter_info;
 mod physical_disk;
@@ -114,6 +115,7 @@ pub use ipv6net::*;
 pub use l4_port_range::*;
 pub use name::*;
 pub use network_interface::*;
+pub use nexus_service::*;
 pub use organization::*;
 pub use oximeter_info::*;
 pub use physical_disk::*;

nexus/db-model/src/nexus_service.rs

@@ -0,0 +1,20 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use crate::schema::nexus_service;
+use uuid::Uuid;
+
+/// Nexus-specific extended service information.
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = nexus_service)]
+pub struct NexusService {
+    pub id: Uuid,
+    pub external_ip_id: Uuid,
+}
+
+impl NexusService {
+    pub fn new(id: Uuid, external_ip_id: Uuid) -> Self {
+        Self { id, external_ip_id }
+    }
+}

nexus/db-model/src/schema.rs

@@ -408,6 +408,13 @@ table! {
     }
 }
 
+table! {
+    nexus_service (id) {
+        id -> Uuid,
+        external_ip_id -> Uuid,
+    }
+}
+
 table! {
     physical_disk (id) {
         id -> Uuid,

nexus/src/app/rack.rs

@@ -65,22 +65,6 @@ impl super::Nexus {
     ) -> Result<(), Error> {
         opctx.authorize(authz::Action::Modify, &authz::FLEET).await?;
 
-        // Convert from parameter -> DB type.
-        let services: Vec<_> = request
-            .services
-            .into_iter()
-            .map(|svc| {
-                db::model::Service::new(
-                    svc.service_id,
-                    svc.sled_id,
-                    svc.address,
-                    svc.kind.into(),
-                )
-            })
-            .collect();
-
-        // TODO(https://github.com/oxidecomputer/omicron/issues/1958): If nexus, add a pool?
-
         let datasets: Vec<_> = request
             .datasets
             .into_iter()
@@ -94,6 +78,7 @@ impl super::Nexus {
             })
             .collect();
 
+        let service_ip_pool_ranges = request.internal_services_ip_pool_ranges;
         let certificates: Vec<_> = request
             .certs
             .into_iter()
@@ -126,8 +111,9 @@ impl super::Nexus {
             .rack_set_initialized(
                 opctx,
                 rack_id,
-                services,
+                request.services,
                 datasets,
+                service_ip_pool_ranges,
                 certificates,
             )
             .await?;

nexus/src/db/datastore/external_ip.rs

@@ -14,17 +14,19 @@ use crate::db::model::ExternalIp;
 use crate::db::model::IncompleteExternalIp;
 use crate::db::model::IpKind;
 use crate::db::model::Name;
+use crate::db::pool::DbConnection;
 use crate::db::queries::external_ip::NextExternalIp;
 use crate::db::update_and_check::UpdateAndCheck;
 use crate::db::update_and_check::UpdateStatus;
-use async_bb8_diesel::AsyncRunQueryDsl;
+use async_bb8_diesel::{AsyncRunQueryDsl, PoolError};
 use chrono::Utc;
 use diesel::prelude::*;
 use nexus_types::identity::Resource;
 use omicron_common::api::external::CreateResult;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::Name as ExternalName;
+use std::net::IpAddr;
 use std::str::FromStr;
 use uuid::Uuid;
 
@@ -83,28 +85,65 @@ impl DataStore {
         opctx: &OpContext,
         data: IncompleteExternalIp,
     ) -> CreateResult<ExternalIp> {
-        NextExternalIp::new(data)
-            .get_result_async(self.pool_authorized(opctx).await?)
-            .await
-            .map_err(|e| {
-                use async_bb8_diesel::ConnectionError::Query;
-                use async_bb8_diesel::PoolError::Connection;
-                use diesel::result::Error::NotFound;
-                match e {
-                    Connection(Query(NotFound)) => Error::invalid_request(
-                        "No external IP addresses available",
-                    ),
-                    _ => public_error_from_diesel_pool(e, ErrorHandler::Server),
+        let conn = self.pool_authorized(opctx).await?;
+        Self::allocate_external_ip_on_connection(conn, data).await
+    }
+
+    /// Variant of [Self::allocate_external_ip] which may be called from a
+    /// transaction context.
+    pub(crate) async fn allocate_external_ip_on_connection<ConnErr>(
+        conn: &(impl async_bb8_diesel::AsyncConnection<DbConnection, ConnErr>
+              + Sync),
+        data: IncompleteExternalIp,
+    ) -> CreateResult<ExternalIp>
+    where
+        ConnErr: From<diesel::result::Error> + Send + 'static,
+        PoolError: From<ConnErr>,
+    {
+        let explicit_ip = data.explicit_ip().is_some();
+        NextExternalIp::new(data).get_result_async(conn).await.map_err(|e| {
+            use async_bb8_diesel::ConnectionError::Query;
+            use async_bb8_diesel::PoolError::Connection;
+            use diesel::result::Error::NotFound;
+            let e = PoolError::from(e);
+            match e {
+                Connection(Query(NotFound)) => {
+                    if explicit_ip {
+                        Error::invalid_request(
+                            "Requested external IP address not available",
+                        )
+                    } else {
+                        Error::invalid_request(
+                            "No external IP addresses available",
+                        )
+                    }
                 }
-            })
+                _ => crate::db::queries::external_ip::from_pool(e),
+            }
+        })
+    }
+
+    /// Allocates an explicit IP address for an internal service.
+    ///
+    /// Unlike the other IP allocation requests, this does not search for an
+    /// available IP address, it asks for one explicitly.
+    pub async fn allocate_explicit_service_ip(
+        &self,
+        opctx: &OpContext,
+        ip_id: Uuid,
+        ip: IpAddr,
+    ) -> CreateResult<ExternalIp> {
+        let (.., pool) = self.ip_pools_service_lookup(opctx).await?;
+        let data =
+            IncompleteExternalIp::for_service_explicit(ip_id, pool.id(), ip);
+        self.allocate_external_ip(opctx, data).await
     }
 
     /// Deallocate the external IP address with the provided ID.
     ///
     /// To support idempotency, such as in saga operations, this method returns
     /// an extra boolean, rather than the usual `DeleteResult`. The meaning of
     /// return values are:
-    ///
     /// - `Ok(true)`: The record was deleted during this call
     /// - `Ok(false)`: The record was already deleted, such as by a previous
     /// call

nexus/src/db/datastore/ip_pool.rs

@@ -21,10 +21,11 @@ use crate::db::model::IpPoolRange;
 use crate::db::model::IpPoolUpdate;
 use crate::db::model::Name;
 use crate::db::pagination::paginated;
+use crate::db::pool::DbConnection;
 use crate::db::queries::ip_pool::FilterOverlappingIpRanges;
 use crate::external_api::params;
 use crate::external_api::shared::IpRange;
-use async_bb8_diesel::AsyncRunQueryDsl;
+use async_bb8_diesel::{AsyncRunQueryDsl, PoolError};
 use chrono::Utc;
 use diesel::prelude::*;
 use ipnetwork::IpNetwork;
@@ -287,6 +288,24 @@ impl DataStore {
         authz_pool: &authz::IpPool,
         range: &IpRange,
     ) -> CreateResult<IpPoolRange> {
+        let conn = self.pool_authorized(opctx).await?;
+        Self::ip_pool_add_range_on_connection(conn, opctx, authz_pool, range)
+            .await
+    }
+
+    /// Variant of [Self::ip_pool_add_range] which may be called from a
+    /// transaction context.
+    pub(crate) async fn ip_pool_add_range_on_connection<ConnErr>(
+        conn: &(impl async_bb8_diesel::AsyncConnection<DbConnection, ConnErr>
+              + Sync),
+        opctx: &OpContext,
+        authz_pool: &authz::IpPool,
+        range: &IpRange,
+    ) -> CreateResult<IpPoolRange>
+    where
+        ConnErr: From<diesel::result::Error> + Send + 'static,
+        PoolError: From<ConnErr>,
+    {
         use db::schema::ip_pool_range::dsl;
         opctx.authorize(authz::Action::CreateChild, authz_pool).await?;
         let pool_id = authz_pool.id();
@@ -295,7 +314,7 @@ impl DataStore {
         let insert_query =
             diesel::insert_into(dsl::ip_pool_range).values(filter_subquery);
         IpPool::insert_resource(pool_id, insert_query)
-            .insert_and_get_result_async(self.pool_authorized(opctx).await?)
+            .insert_and_get_result_async(conn)
             .await
             .map_err(|e| {
                 use async_bb8_diesel::ConnectionError::Query;

nexus/src/db/datastore/mod.rs

@@ -81,6 +81,9 @@ pub use volume::CrucibleResources;
 // TODO: This should likely turn into a configuration option.
 pub(crate) const REGION_REDUNDANCY_THRESHOLD: usize = 3;
 
+/// The name of the built-in IP pool for Oxide services.
+pub const SERVICE_IP_POOL_NAME: &str = "oxide-service-pool";
+
 // Represents a query that is ready to be executed.
 //
 // This helper trait lets the statement either be executed or explained.
@@ -236,12 +239,20 @@ pub async fn datastore_test(
         authn::Context::internal_db_init(),
         Arc::clone(&datastore),
     );
+
+    // TODO: Can we just call "Populate" instead of doing this?
+    let rack_id = Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap();
     datastore.load_builtin_users(&opctx).await.unwrap();
     datastore.load_builtin_roles(&opctx).await.unwrap();
     datastore.load_builtin_role_asgns(&opctx).await.unwrap();
     datastore.load_builtin_silos(&opctx).await.unwrap();
     datastore.load_silo_users(&opctx).await.unwrap();
     datastore.load_silo_user_role_assignments(&opctx).await.unwrap();
+    datastore
+        .load_builtin_fleet_virtual_provisioning_collection(&opctx)
+        .await
+        .unwrap();
+    datastore.load_builtin_rack_data(&opctx, rack_id).await.unwrap();
 
     // Create an OpContext with the credentials of "test-privileged" for general
     // testing.
@@ -1067,14 +1078,28 @@ mod test {
 
         // Initialize the Rack.
         let result = datastore
-            .rack_set_initialized(&opctx, rack.id(), vec![], vec![], vec![])
+            .rack_set_initialized(
+                &opctx,
+                rack.id(),
+                vec![],
+                vec![],
+                vec![],
+                vec![],
+            )
             .await
             .unwrap();
         assert!(result.initialized);
 
         // Re-initialize the rack (check for idempotency)
         let result = datastore
-            .rack_set_initialized(&opctx, rack.id(), vec![], vec![], vec![])
+            .rack_set_initialized(
+                &opctx,
+                rack.id(),
+                vec![],
+                vec![],
+                vec![],
+                vec![],
+            )
             .await
             .unwrap();
         assert!(result.initialized);