Add type conversions for EarlyNetworkConfig upgrade

common/src/api/internal/shared.rs

@@ -68,9 +68,17 @@ pub struct SourceNatConfig {
     pub last_port: u16,
 }
 
+// We alias [`RackNetworkConfig`] to the current version of the protocol, so
+// that we can convert between versions as necessary.
+pub type RackNetworkConfig = RackNetworkConfigV1;
+
 /// Initial network configuration
+///
+/// TODO(AJS): It's unclear if this should be serde renamed to `RackNetworkConfig`
+/// I *think* it's useful to have the version be explicit in the name, but I'm open
+/// to discussion.
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
-pub struct RackNetworkConfig {
+pub struct RackNetworkConfigV1 {
     // TODO: #3591 Consider making infra-ip ranges implicit for uplinks
     /// First ip address to be used for configuring network infrastructure
     pub infra_ip_first: Ipv4Addr,
@@ -82,6 +90,41 @@ pub struct RackNetworkConfig {
     pub bgp: Vec<BgpConfig>,
 }
 
+/// Deprecated, use `RackNetworkConfig` instead. Cannot actually deprecate due to
+/// <https://github.com/serde-rs/serde/issues/2195>
+///
+/// Our first version of `RackNetworkConfig`. If this exists in the bootstore, we
+/// upgrade out of it into `RackNetworkConfigV1` or later versions if possible.
+/// Initial network configuration
+///
+// TODO(AJS): Should this actually exist in the OpenAPI spec or should we just
+// hide the old version inside sled-agent/src/bootstrap/early_networking.rs
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
+pub struct RackNetworkConfigV0 {
+    // TODO: #3591 Consider making infra-ip ranges implicit for uplinks
+    /// First ip address to be used for configuring network infrastructure
+    pub infra_ip_first: Ipv4Addr,
+    /// Last ip address to be used for configuring network infrastructure
+    pub infra_ip_last: Ipv4Addr,
+    /// Uplinks for connecting the rack to external networks
+    pub uplinks: Vec<UplinkConfig>,
+}
+
+impl From<RackNetworkConfigV0> for RackNetworkConfigV1 {
+    fn from(value: RackNetworkConfigV0) -> Self {
+        RackNetworkConfigV1 {
+            infra_ip_first: value.infra_ip_first,
+            infra_ip_last: value.infra_ip_last,
+            ports: value
+                .uplinks
+                .into_iter()
+                .map(|uplink| PortConfigV1::from(uplink))
+                .collect(),
+            bgp: vec![],
+        }
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]
 pub struct BgpConfig {
     /// The autonomous system number for the BGP configuration.
@@ -126,6 +169,23 @@ pub struct PortConfigV1 {
     pub bgp_peers: Vec<BgpPeerConfig>,
 }
 
+impl From<UplinkConfig> for PortConfigV1 {
+    fn from(value: UplinkConfig) -> Self {
+        PortConfigV1 {
+            routes: vec![RouteConfig {
+                destination: "0.0.0.0/0".parse().unwrap(),
+                nexthop: value.gateway_ip.into(),
+            }],
+            addresses: vec![value.uplink_cidr.into()],
+            switch: value.switch,
+            port: value.uplink_port,
+            uplink_port_speed: value.uplink_port_speed,
+            uplink_port_fec: value.uplink_port_fec,
+            bgp_peers: vec![],
+        }
+    }
+}
+
 /// Deprecated, use PortConfigV1 instead. Cannot actually deprecate due to
 /// <https://github.com/serde-rs/serde/issues/2195>
 #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)]

openapi/bootstrap-agent.json

@@ -626,7 +626,7 @@
             "description": "Initial rack network configuration",
             "allOf": [
               {
-                "$ref": "#/components/schemas/RackNetworkConfig"
+                "$ref": "#/components/schemas/RackNetworkConfigV1"
               }
             ]
           },
@@ -663,8 +663,8 @@
           "recovery_silo"
         ]
       },
-      "RackNetworkConfig": {
-        "description": "Initial network configuration",
+      "RackNetworkConfigV1": {
+        "description": "Initial network configuration\n\nTODO(AJS): It's unclear if this should be serde renamed to `RackNetworkConfig` I *think* it's useful to have the version be explicit in the name, but I'm open to discussion.",
         "type": "object",
         "properties": {
           "bgp": {

openapi/sled-agent.json

@@ -1692,14 +1692,33 @@
         ]
       },
       "EarlyNetworkConfig": {
-        "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates will come from Nexus in the future.",
+        "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.",
         "type": "object",
         "properties": {
+          "body": {
+            "$ref": "#/components/schemas/EarlyNetworkConfigBody"
+          },
           "generation": {
             "type": "integer",
             "format": "uint64",
             "minimum": 0
           },
+          "schema_version": {
+            "type": "integer",
+            "format": "uint32",
+            "minimum": 0
+          }
+        },
+        "required": [
+          "body",
+          "generation",
+          "schema_version"
+        ]
+      },
+      "EarlyNetworkConfigBody": {
+        "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.",
+        "type": "object",
+        "properties": {
           "ntp_servers": {
             "description": "The external NTP server addresses.",
             "type": "array",
@@ -1709,10 +1728,9 @@
           },
           "rack_network_config": {
             "nullable": true,
-            "description": "A copy of the initial rack network configuration when we are in generation `1`.",
             "allOf": [
               {
-                "$ref": "#/components/schemas/RackNetworkConfig"
+                "$ref": "#/components/schemas/RackNetworkConfigV1"
               }
             ]
           },
@@ -1722,7 +1740,6 @@
           }
         },
         "required": [
-          "generation",
           "ntp_servers",
           "rack_subnet"
         ]
@@ -2566,8 +2583,8 @@
         "minItems": 2,
         "maxItems": 2
       },
-      "RackNetworkConfig": {
-        "description": "Initial network configuration",
+      "RackNetworkConfigV1": {
+        "description": "Initial network configuration\n\nTODO(AJS): It's unclear if this should be serde renamed to `RackNetworkConfig` I *think* it's useful to have the version be explicit in the name, but I'm open to discussion.",
         "type": "object",
         "properties": {
           "bgp": {

schema/rss-sled-plan.json

@@ -472,7 +472,7 @@
           "description": "Initial rack network configuration",
           "anyOf": [
             {
-              "$ref": "#/definitions/RackNetworkConfig"
+              "$ref": "#/definitions/RackNetworkConfigV1"
             },
             {
               "type": "null"
@@ -503,8 +503,8 @@
         }
       }
     },
-    "RackNetworkConfig": {
-      "description": "Initial network configuration",
+    "RackNetworkConfigV1": {
+      "description": "Initial network configuration\n\nTODO(AJS): It's unclear if this should be serde renamed to `RackNetworkConfig` I *think* it's useful to have the version be explicit in the name, but I'm open to discussion.",
       "type": "object",
       "required": [
         "bgp",

sled-agent/src/bootstrap/early_networking.rs

@@ -20,7 +20,8 @@ use ipnetwork::IpNetwork;
 use omicron_common::address::{Ipv6Subnet, AZ_PREFIX, MGS_PORT};
 use omicron_common::address::{DDMD_PORT, DENDRITE_PORT};
 use omicron_common::api::internal::shared::{
-    PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, SwitchLocation,
+    PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV0,
+    RackNetworkConfigV1, SwitchLocation,
 };
 use omicron_common::backoff::{
     retry_notify, retry_policy_local, BackoffError, ExponentialBackoff,
@@ -568,25 +569,67 @@ fn retry_policy_switch_mapping() -> ExponentialBackoff {
 /// The fields in this structure are those from
 /// [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This
 /// is just for the initial rack configuration and cold boot purposes. Updates
-/// will come from Nexus in the future.
+/// come from Nexus.
 #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
 pub struct EarlyNetworkConfig {
-    // The version of data.
+    // The current generation number of data as stored in CRDB.
+    // The initial generation is set during RSS time and then only mutated
+    // by Nexus.
     pub generation: u64,
 
+    // Which version of the data structure do we have. This is to help with
+    // deserialization and conversion in future updates.
+    pub schema_version: u32,
+
+    // The actual configuration details
+    pub body: EarlyNetworkConfigBody,
+}
+
+/// This is the actual configuration of EarlyNetworking.
+///
+/// We nest it below the "header" of `generation` and `schema_version` so that
+/// we can perform partial deserialization of `EarlyNetworkConfig` to only read
+/// the header and defer deserialization of the body once we know the schema
+/// version. This is possible via the use of [`serde_json::value::RawValue`] in
+/// future (post-v1) deserialization paths.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+pub struct EarlyNetworkConfigBody {
     pub rack_subnet: Ipv6Addr,
 
     /// The external NTP server addresses.
     pub ntp_servers: Vec<String>,
 
-    /// A copy of the initial rack network configuration when we are in
-    /// generation `1`.
+    // Rack network configuration as delivered from RSS or Nexus
     pub rack_network_config: Option<RackNetworkConfig>,
 }
 
+// The first production version of the EarlyNetworkConfig.
+//
+// If this version is in the bootstore than we need to convert it to
+// `EarlyNetworkConfig`. We intend to convert this data on
+//
+// Once we do this for all customers that have initialized racks with the old version we
+// can go ahead and remove this type and its conversion code altogether.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+struct EarlyNetworkConfigV0 {
+    // The current generation number of data as stored in CRDB.
+    // The initial generation is set during RSS time and then only mutated
+    // by Nexus.
+    pub generation: u64,
+
+    pub rack_subnet: Ipv6Addr,
+
+    /// The external NTP server addresses.
+    pub ntp_servers: Vec<String>,
+
+    // Rack network configuration as delivered from RSS and only existing at
+    // generation 1
+    pub rack_network_config: Option<RackNetworkConfigV0>,
+}
+
 impl EarlyNetworkConfig {
     pub fn az_subnet(&self) -> Ipv6Subnet<AZ_PREFIX> {
-        Ipv6Subnet::<AZ_PREFIX>::new(self.rack_subnet)
+        Ipv6Subnet::<AZ_PREFIX>::new(self.body.rack_subnet)
     }
 }
 
@@ -603,13 +646,37 @@ impl From<EarlyNetworkConfig> for bootstore::NetworkConfig {
     }
 }
 
+// Note: This currently only converts between v0 and v1 or deserializes v1 of
+// `EarlyNetworkConfig`.
 impl TryFrom<bootstore::NetworkConfig> for EarlyNetworkConfig {
     type Error = serde_json::Error;
 
     fn try_from(
         value: bootstore::NetworkConfig,
     ) -> std::result::Result<Self, Self::Error> {
-        serde_json::from_slice(&value.blob)
+        // Try to deserialize the latest version of the data structure (v1). If
+        // that succeeds we are done.
+        if let Ok(val) =
+            serde_json::from_slice::<EarlyNetworkConfig>(&value.blob)
+        {
+            return Ok(val);
+        }
+
+        // We don't have the latest version. Try to deserialize v0 and then
+        // convert it to the latest version.
+        let v0 = serde_json::from_slice::<EarlyNetworkConfigV0>(&value.blob)?;
+
+        Ok(EarlyNetworkConfig {
+            generation: v0.generation,
+            schema_version: 1,
+            body: EarlyNetworkConfigBody {
+                rack_subnet: v0.rack_subnet,
+                ntp_servers: v0.ntp_servers,
+                rack_network_config: v0
+                    .rack_network_config
+                    .map(|v0_config| RackNetworkConfigV1::from(v0_config)),
+            },
+        })
     }
 }
 

sled-agent/src/rack_setup/service.rs

@@ -57,7 +57,8 @@
 use super::config::SetupServiceConfig as Config;
 use crate::bootstrap::config::BOOTSTRAP_AGENT_HTTP_PORT;
 use crate::bootstrap::early_networking::{
-    EarlyNetworkConfig, EarlyNetworkSetup, EarlyNetworkSetupError,
+    EarlyNetworkConfig, EarlyNetworkConfigBody, EarlyNetworkSetup,
+    EarlyNetworkSetupError,
 };
 use crate::bootstrap::params::BootstrapAddressDiscovery;
 use crate::bootstrap::params::StartSledAgentRequest;
@@ -895,9 +896,12 @@ impl ServiceInner {
         // from the bootstore".
         let early_network_config = EarlyNetworkConfig {
             generation: 1,
-            rack_subnet: config.rack_subnet,
-            ntp_servers: config.ntp_servers.clone(),
-            rack_network_config: config.rack_network_config.clone(),
+            schema_version: 1,
+            body: EarlyNetworkConfigBody {
+                rack_subnet: config.rack_subnet,
+                ntp_servers: config.ntp_servers.clone(),
+                rack_network_config: config.rack_network_config.clone(),
+            },
         };
         info!(self.log, "Writing Rack Network Configuration to bootstore");
         //NOTE(ry) this is where the early network config gets saved.

sled-agent/src/sim/http_entrypoints.rs

@@ -4,7 +4,9 @@
 
 //! HTTP entrypoint functions for the sled agent's exposed API
 
-use crate::bootstrap::early_networking::EarlyNetworkConfig;
+use crate::bootstrap::early_networking::{
+    EarlyNetworkConfig, EarlyNetworkConfigBody,
+};
 use crate::params::{
     DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody,
     InstancePutStateBody, InstancePutStateResponse, InstanceUnregisterResponse,
@@ -355,14 +357,17 @@ async fn read_network_bootstore_config(
 ) -> Result<HttpResponseOk<EarlyNetworkConfig>, HttpError> {
     let config = EarlyNetworkConfig {
         generation: 0,
-        rack_subnet: Ipv6Addr::UNSPECIFIED,
-        ntp_servers: Vec::new(),
-        rack_network_config: Some(RackNetworkConfig {
-            infra_ip_first: Ipv4Addr::UNSPECIFIED,
-            infra_ip_last: Ipv4Addr::UNSPECIFIED,
-            ports: Vec::new(),
-            bgp: Vec::new(),
-        }),
+        schema_version: 1,
+        body: EarlyNetworkConfigBody {
+            rack_subnet: Ipv6Addr::UNSPECIFIED,
+            ntp_servers: Vec::new(),
+            rack_network_config: Some(RackNetworkConfig {
+                infra_ip_first: Ipv4Addr::UNSPECIFIED,
+                infra_ip_last: Ipv4Addr::UNSPECIFIED,
+                ports: Vec::new(),
+                bgp: Vec::new(),
+            }),
+        },
     };
     Ok(HttpResponseOk(config))
 }

sled-agent/src/sled_agent.rs

@@ -412,7 +412,7 @@ impl SledAgent {
                 EarlyNetworkConfig::try_from(serialized_config)
                     .map_err(|err| BackoffError::transient(err.to_string()))?;
 
-            Ok(early_network_config.rack_network_config)
+            Ok(early_network_config.body.rack_network_config)
         };
         let rack_network_config: Option<RackNetworkConfig> =
             retry_notify::<_, String, _, _, _, _>(