Adds basic per-sled sequential IP address allocation (#891)

* Adds basic per-sled sequential IP address allocation

- Adds the `last_used_address` column to the `omicron.sled` table, which
  tracks the last IP address within the sled's prefix allocated to a
  service running on the sled
- Adds method for selecting the next IP address from the `sled` table,
  with a few basic tests for it
- Uses a static address when launching guest instances, providing it to
  the propolis server managing them.

* Review feedback

- Adds some comments and issue links
- Make allocation of IP addresses a separate saga action, to ensure
  idempotency. Also adds a generic helper, since this will likely be a
  common saga node.

common/src/sql/dbinit.sql

@@ -64,8 +64,12 @@ CREATE TABLE omicron.public.sled (
     time_deleted TIMESTAMPTZ,
     rcgen INT NOT NULL,
 
+    /* The IP address and bound port of the sled agent server. */
     ip INET NOT NULL,
-    port INT4 NOT NULL
+    port INT4 NOT NULL,
+
+    /* The last address allocated to an Oxide service on this sled. */
+    last_used_address INET NOT NULL
 );
 
 /*

nexus/src/db/datastore.rs

@@ -56,6 +56,7 @@ use omicron_common::api::external::{
 use omicron_common::api::internal::nexus::UpdateArtifact;
 use omicron_common::bail_unless;
 use std::convert::{TryFrom, TryInto};
+use std::net::Ipv6Addr;
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -2833,6 +2834,42 @@ impl DataStore {
 
         Ok(())
     }
+
+    /// Return the next available IPv6 address for an Oxide service running on
+    /// the provided sled.
+    pub async fn next_ipv6_address(
+        &self,
+        opctx: &OpContext,
+        sled_id: Uuid,
+    ) -> Result<Ipv6Addr, Error> {
+        use db::schema::sled::dsl;
+        let net = diesel::update(
+            dsl::sled.find(sled_id).filter(dsl::time_deleted.is_null()),
+        )
+        .set(dsl::last_used_address.eq(dsl::last_used_address + 1))
+        .returning(dsl::last_used_address)
+        .get_result_async(self.pool_authorized(opctx).await?)
+        .await
+        .map_err(|e| {
+            public_error_from_diesel_pool(
+                e,
+                ErrorHandler::NotFoundByLookup(
+                    ResourceType::Sled,
+                    LookupType::ById(sled_id),
+                ),
+            )
+        })?;
+
+        // TODO-correctness: We need to ensure that this address is actually
+        // within the sled's underlay prefix, once that's included in the
+        // database record.
+        match net {
+            ipnetwork::IpNetwork::V6(net) => Ok(net.ip()),
+            _ => Err(Error::InternalError {
+                internal_message: String::from("Sled IP address must be IPv6"),
+            }),
+        }
+    }
 }
 
 /// Constructs a DataStore for use in test suites that has preloaded the
@@ -3335,4 +3372,70 @@ mod test {
 
         let _ = db.cleanup().await;
     }
+
+    // Test sled-specific IPv6 address allocation
+    #[tokio::test]
+    async fn test_sled_ipv6_address_allocation() {
+        use crate::db::model::STATIC_IPV6_ADDRESS_OFFSET;
+        use std::net::Ipv6Addr;
+
+        let logctx = dev::test_setup_log("test_sled_ipv6_address_allocation");
+        let mut db = test_setup_database(&logctx.log).await;
+        let cfg = db::Config { url: db.pg_config().clone() };
+        let pool = Arc::new(db::Pool::new(&cfg));
+        let datastore = Arc::new(DataStore::new(Arc::clone(&pool)));
+        let opctx =
+            OpContext::for_tests(logctx.log.new(o!()), datastore.clone());
+
+        let addr1 = "[fd00:1de::1]:12345".parse().unwrap();
+        let sled1_id = "0de4b299-e0b4-46f0-d528-85de81a7095f".parse().unwrap();
+        let sled1 = db::model::Sled::new(sled1_id, addr1);
+        datastore.sled_upsert(sled1).await.unwrap();
+
+        let addr2 = "[fd00:1df::1]:12345".parse().unwrap();
+        let sled2_id = "66285c18-0c79-43e0-e54f-95271f271314".parse().unwrap();
+        let sled2 = db::model::Sled::new(sled2_id, addr2);
+        datastore.sled_upsert(sled2).await.unwrap();
+
+        let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap();
+        let expected_ip = Ipv6Addr::new(
+            0xfd00,
+            0x1de,
+            0,
+            0,
+            0,
+            0,
+            0,
+            2 + STATIC_IPV6_ADDRESS_OFFSET,
+        );
+        assert_eq!(ip, expected_ip);
+        let ip = datastore.next_ipv6_address(&opctx, sled1_id).await.unwrap();
+        let expected_ip = Ipv6Addr::new(
+            0xfd00,
+            0x1de,
+            0,
+            0,
+            0,
+            0,
+            0,
+            3 + STATIC_IPV6_ADDRESS_OFFSET,
+        );
+        assert_eq!(ip, expected_ip);
+
+        let ip = datastore.next_ipv6_address(&opctx, sled2_id).await.unwrap();
+        let expected_ip = Ipv6Addr::new(
+            0xfd00,
+            0x1df,
+            0,
+            0,
+            0,
+            0,
+            0,
+            2 + STATIC_IPV6_ADDRESS_OFFSET,
+        );
+        assert_eq!(ip, expected_ip);
+
+        let _ = db.cleanup().await;
+        logctx.cleanup_successful();
+    }
 }

nexus/src/db/model.rs

@@ -631,16 +631,50 @@ pub struct Sled {
     pub ip: ipnetwork::IpNetwork,
     // TODO: Make use of SqlU16
     pub port: i32,
+
+    /// The last IP address provided to an Oxide service on this sled
+    pub last_used_address: IpNetwork,
 }
 
+// TODO-correctness: We need a small offset here, while services and
+// their addresses are still hardcoded in the mock RSS config file at
+// `./smf/sled-agent/config-rss.toml`. This avoids conflicts with those
+// addresses, but should be removed when they are entirely under the
+// control of Nexus or RSS.
+//
+// See https://github.com/oxidecomputer/omicron/issues/732 for tracking issue.
+pub(crate) const STATIC_IPV6_ADDRESS_OFFSET: u16 = 20;
 impl Sled {
+    // TODO-cleanup: We should be using IPv6 only for Oxide services, including
+    // `std::net::Ipv6Addr` and `SocketAddrV6`. The v4/v6 enums should only be
+    // used for managing customer addressing information, or when needed to
+    // interact with the database.
     pub fn new(id: Uuid, addr: SocketAddr) -> Self {
+        let last_used_address = {
+            match addr.ip() {
+                IpAddr::V6(ip) => {
+                    let mut segments = ip.segments();
+                    segments[7] += STATIC_IPV6_ADDRESS_OFFSET;
+                    ipnetwork::IpNetwork::from(IpAddr::from(Ipv6Addr::from(
+                        segments,
+                    )))
+                }
+                IpAddr::V4(ip) => {
+                    // TODO-correctness: This match arm should disappear when we
+                    // support only IPv6 for underlay addressing.
+                    let x = u32::from_be_bytes(ip.octets())
+                        + u32::from(STATIC_IPV6_ADDRESS_OFFSET);
+                    ipnetwork::IpNetwork::from(IpAddr::from(Ipv4Addr::from(x)))
+                }
+            }
+        };
         Self {
             identity: SledIdentity::new(id),
             time_deleted: None,
             rcgen: Generation::new(),
             ip: addr.ip().into(),
             port: addr.port().into(),
+            last_used_address,
         }
     }
 
@@ -1143,10 +1177,10 @@ pub struct InstanceRuntimeState {
     pub sled_uuid: Uuid,
     #[column_name = "active_propolis_id"]
     pub propolis_uuid: Uuid,
-    #[column_name = "target_propolis_id"]
-    pub dst_propolis_uuid: Option<Uuid>,
     #[column_name = "active_propolis_ip"]
     pub propolis_ip: Option<ipnetwork::IpNetwork>,
+    #[column_name = "target_propolis_id"]
+    pub dst_propolis_uuid: Option<Uuid>,
     #[column_name = "migration_id"]
     pub migration_uuid: Option<Uuid>,
     #[column_name = "ncpus"]

nexus/src/db/schema.rs

@@ -71,8 +71,8 @@ table! {
         state_generation -> Int8,
         active_server_id -> Uuid,
         active_propolis_id -> Uuid,
-        target_propolis_id -> Nullable<Uuid>,
         active_propolis_ip -> Nullable<Inet>,
+        target_propolis_id -> Nullable<Uuid>,
         migration_id -> Nullable<Uuid>,
         ncpus -> Int8,
         memory -> Int8,
@@ -222,6 +222,7 @@ table! {
 
         ip -> Inet,
         port -> Int4,
+        last_used_address -> Inet,
     }
 }