From 9191af67b9aff710dac93ada6012b1e9b7d0c79c Mon Sep 17 00:00:00 2001 From: Greg Colombo Date: Fri, 20 Oct 2023 09:57:34 -0700 Subject: [PATCH 1/3] Update virtual provisioning counters on instance stop/start (#4277) Only charge virtual provisioning collections for instances when those instances are running. Charges are taken in the instance start saga and dropped when a sled agent tries to transition an instance to a stopped state. Unlike sled resource charges, provisioning charges are tied to instance states, not to VMM lifetimes. This ensures that a user is not charged twice for an instance (e.g. for quota management purposes) while the instance is migrating. See RFD 427 for more discussion. Also fix a small idempotency issue in the cleanup path for VMM resources. Tests: updated integration tests; manually checked virtual provisioning table values in a dev cluster & checked the values on the utilization graphs. Fixes #4257. --- nexus/db-model/src/schema.rs | 5 + .../virtual_provisioning_collection.rs | 13 +- .../virtual_provisioning_collection_update.rs | 29 ++++ nexus/src/app/instance.rs | 50 +++++- nexus/src/app/sagas/instance_create.rs | 56 ------- nexus/src/app/sagas/instance_delete.rs | 29 ---- nexus/src/app/sagas/instance_start.rs | 66 ++++++++ nexus/tests/integration_tests/instances.rs | 143 ++++++++++++++++-- 8 files changed, 285 insertions(+), 106 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 61a05754c6..9189b6db7b 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -812,6 +812,11 @@ table! { } } +allow_tables_to_appear_in_same_query! { + virtual_provisioning_resource, + instance +} + table! { zpool (id) { id -> Uuid, diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index 18ff58735e..83856e10c7 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -272,7 +272,11 @@ impl DataStore { Ok(provisions) } - /// Transitively updates all CPU/RAM provisions from project -> fleet. + /// Transitively removes the CPU and memory charges for an instance from the + /// instance's project, silo, and fleet, provided that the instance's state + /// generation is less than `max_instance_gen`. This allows a caller who is + /// about to apply generation G to an instance to avoid deleting resources + /// if its update was superseded. pub async fn virtual_provisioning_collection_delete_instance( &self, opctx: &OpContext, @@ -280,10 +284,15 @@ impl DataStore { project_id: Uuid, cpus_diff: i64, ram_diff: ByteCount, + max_instance_gen: i64, ) -> Result, Error> { let provisions = VirtualProvisioningCollectionUpdate::new_delete_instance( - id, cpus_diff, ram_diff, project_id, + id, + max_instance_gen, + cpus_diff, + ram_diff, + project_id, ) .get_results_async(&*self.pool_connection_authorized(opctx).await?) .await diff --git a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs index b7271f3f49..0a383eb6f1 100644 --- a/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs +++ b/nexus/db-queries/src/db/queries/virtual_provisioning_collection_update.rs @@ -368,10 +368,12 @@ impl VirtualProvisioningCollectionUpdate { pub fn new_delete_instance( id: uuid::Uuid, + max_instance_gen: i64, cpus_diff: i64, ram_diff: ByteCount, project_id: uuid::Uuid, ) -> Self { + use crate::db::schema::instance::dsl as instance_dsl; use virtual_provisioning_collection::dsl as collection_dsl; use virtual_provisioning_resource::dsl as resource_dsl; @@ -379,9 +381,36 @@ impl VirtualProvisioningCollectionUpdate { // We should delete the record if it exists. DoUpdate::new_for_delete(id), // The query to actually delete the record. + // + // The filter condition here ensures that the provisioning record is + // only deleted if the corresponding instance has a generation + // number less than the supplied `max_instance_gen`. This allows a + // caller that is about to apply an instance update that will stop + // the instance and that bears generation G to avoid deleting + // resources if the instance generation was already advanced to or + // past G. + // + // If the relevant instance ID is not in the database, then some + // other operation must have ensured the instance was previously + // stopped (because that's the only way it could have been deleted), + // and that operation should have cleaned up the resources already, + // in which case there's nothing to do here. + // + // There is an additional "direct" filter on the target resource ID + // to avoid a full scan of the resource table. UnreferenceableSubquery( diesel::delete(resource_dsl::virtual_provisioning_resource) .filter(resource_dsl::id.eq(id)) + .filter( + resource_dsl::id.nullable().eq(instance_dsl::instance + .filter(instance_dsl::id.eq(id)) + .filter( + instance_dsl::state_generation + .lt(max_instance_gen), + ) + .select(instance_dsl::id) + .single_value()), + ) .returning(virtual_provisioning_resource::all_columns), ), // Within this project, silo, fleet... diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 1adcd8f9c0..17d033c5a0 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1290,6 +1290,14 @@ impl super::Nexus { "propolis_id" => %propolis_id, "vmm_state" => ?new_runtime_state.vmm_state); + // Grab the current state of the instance in the DB to reason about + // whether this update is stale or not. + let (.., authz_instance, db_instance) = + LookupPath::new(&opctx, &self.db_datastore) + .instance_id(*instance_id) + .fetch() + .await?; + // Update OPTE and Dendrite if the instance's active sled assignment // changed or a migration was retired. If these actions fail, sled agent // is expected to retry this update. @@ -1303,12 +1311,6 @@ impl super::Nexus { // // In the future, this should be replaced by a call to trigger a // networking state update RPW. - let (.., authz_instance, db_instance) = - LookupPath::new(&opctx, &self.db_datastore) - .instance_id(*instance_id) - .fetch() - .await?; - self.ensure_updated_instance_network_config( opctx, &authz_instance, @@ -1317,6 +1319,27 @@ impl super::Nexus { ) .await?; + // If the supplied instance state indicates that the instance no longer + // has an active VMM, attempt to delete the virtual provisioning record + // + // As with updating networking state, this must be done before + // committing the new runtime state to the database: once the DB is + // written, a new start saga can arrive and start the instance, which + // will try to create its own virtual provisioning charges, which will + // race with this operation. + if new_runtime_state.instance_state.propolis_id.is_none() { + self.db_datastore + .virtual_provisioning_collection_delete_instance( + opctx, + *instance_id, + db_instance.project_id, + i64::from(db_instance.ncpus.0 .0), + db_instance.memory, + (&new_runtime_state.instance_state.gen).into(), + ) + .await?; + } + // Write the new instance and VMM states back to CRDB. This needs to be // done before trying to clean up the VMM, since the datastore will only // allow a VMM to be marked as deleted if it is already in a terminal @@ -1337,7 +1360,20 @@ impl super::Nexus { // If the VMM is now in a terminal state, make sure its resources get // cleaned up. - if let Ok((_, true)) = result { + // + // For idempotency, only check to see if the update was successfully + // processed and ignore whether the VMM record was actually updated. + // This is required to handle the case where this routine is called + // once, writes the terminal VMM state, fails before all per-VMM + // resources are released, returns a retriable error, and is retried: + // the per-VMM resources still need to be cleaned up, but the DB update + // will return Ok(_, false) because the database was already updated. + // + // Unlike the pre-update cases, it is legal to do this cleanup *after* + // committing state to the database, because a terminated VMM cannot be + // reused (restarting or migrating its former instance will use new VMM + // IDs). + if result.is_ok() { let propolis_terminated = matches!( new_runtime_state.vmm_state.state, InstanceState::Destroyed | InstanceState::Failed diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index 5d55aaf0fe..153e0323e7 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -13,7 +13,6 @@ use crate::external_api::params; use nexus_db_model::NetworkInterfaceKind; use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup::LookupPath; -use nexus_db_queries::db::model::ByteCount as DbByteCount; use nexus_db_queries::db::queries::network_interface::InsertError as InsertNicError; use nexus_db_queries::{authn, authz, db}; use nexus_defaults::DEFAULT_PRIMARY_NIC_NAME; @@ -75,10 +74,6 @@ struct DiskAttachParams { declare_saga_actions! { instance_create; - VIRTUAL_RESOURCES_ACCOUNT -> "no_result" { - + sic_account_virtual_resources - - sic_account_virtual_resources_undo - } CREATE_INSTANCE_RECORD -> "instance_record" { + sic_create_instance_record - sic_delete_instance_record @@ -131,7 +126,6 @@ impl NexusSaga for SagaInstanceCreate { })?, )); - builder.append(virtual_resources_account_action()); builder.append(create_instance_record_action()); // Helper function for appending subsagas to our parent saga. @@ -728,56 +722,6 @@ async fn ensure_instance_disk_attach_state( Ok(()) } -async fn sic_account_virtual_resources( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let instance_id = sagactx.lookup::("instance_id")?; - - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - osagactx - .datastore() - .virtual_provisioning_collection_insert_instance( - &opctx, - instance_id, - params.project_id, - i64::from(params.create_params.ncpus.0), - DbByteCount(params.create_params.memory), - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - -async fn sic_account_virtual_resources_undo( - sagactx: NexusActionContext, -) -> Result<(), anyhow::Error> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let instance_id = sagactx.lookup::("instance_id")?; - - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - osagactx - .datastore() - .virtual_provisioning_collection_delete_instance( - &opctx, - instance_id, - params.project_id, - i64::from(params.create_params.ncpus.0), - DbByteCount(params.create_params.memory), - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - async fn sic_create_instance_record( sagactx: NexusActionContext, ) -> Result { diff --git a/nexus/src/app/sagas/instance_delete.rs b/nexus/src/app/sagas/instance_delete.rs index 7da497136e..1605465c74 100644 --- a/nexus/src/app/sagas/instance_delete.rs +++ b/nexus/src/app/sagas/instance_delete.rs @@ -9,7 +9,6 @@ use super::NexusActionContext; use super::NexusSaga; use crate::app::sagas::declare_saga_actions; use nexus_db_queries::{authn, authz, db}; -use nexus_types::identity::Resource; use omicron_common::api::external::{Error, ResourceType}; use omicron_common::api::internal::shared::SwitchLocation; use serde::Deserialize; @@ -40,9 +39,6 @@ declare_saga_actions! { DEALLOCATE_EXTERNAL_IP -> "no_result3" { + sid_deallocate_external_ip } - VIRTUAL_RESOURCES_ACCOUNT -> "no_result4" { - + sid_account_virtual_resources - } } // instance delete saga: definition @@ -64,7 +60,6 @@ impl NexusSaga for SagaInstanceDelete { builder.append(instance_delete_record_action()); builder.append(delete_network_interfaces_action()); builder.append(deallocate_external_ip_action()); - builder.append(virtual_resources_account_action()); Ok(builder.build()?) } } @@ -135,30 +130,6 @@ async fn sid_deallocate_external_ip( Ok(()) } -async fn sid_account_virtual_resources( - sagactx: NexusActionContext, -) -> Result<(), ActionError> { - let osagactx = sagactx.user_data(); - let params = sagactx.saga_params::()?; - let opctx = crate::context::op_context_for_saga_action( - &sagactx, - ¶ms.serialized_authn, - ); - - osagactx - .datastore() - .virtual_provisioning_collection_delete_instance( - &opctx, - params.instance.id(), - params.instance.project_id, - i64::from(params.instance.ncpus.0 .0), - params.instance.memory, - ) - .await - .map_err(ActionError::action_failed)?; - Ok(()) -} - #[cfg(test)] mod test { use crate::{ diff --git a/nexus/src/app/sagas/instance_start.rs b/nexus/src/app/sagas/instance_start.rs index 068d2e7005..76773d6369 100644 --- a/nexus/src/app/sagas/instance_start.rs +++ b/nexus/src/app/sagas/instance_start.rs @@ -52,6 +52,11 @@ declare_saga_actions! { - sis_move_to_starting_undo } + ADD_VIRTUAL_RESOURCES -> "virtual_resources" { + + sis_account_virtual_resources + - sis_account_virtual_resources_undo + } + // TODO(#3879) This can be replaced with an action that triggers the NAT RPW // once such an RPW is available. DPD_ENSURE -> "dpd_ensure" { @@ -98,6 +103,7 @@ impl NexusSaga for SagaInstanceStart { builder.append(alloc_propolis_ip_action()); builder.append(create_vmm_record_action()); builder.append(mark_as_starting_action()); + builder.append(add_virtual_resources_action()); builder.append(dpd_ensure_action()); builder.append(v2p_ensure_action()); builder.append(ensure_registered_action()); @@ -305,6 +311,66 @@ async fn sis_move_to_starting_undo( Ok(()) } +async fn sis_account_virtual_resources( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let instance_id = params.db_instance.id(); + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + osagactx + .datastore() + .virtual_provisioning_collection_insert_instance( + &opctx, + instance_id, + params.db_instance.project_id, + i64::from(params.db_instance.ncpus.0 .0), + nexus_db_model::ByteCount(*params.db_instance.memory), + ) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + +async fn sis_account_virtual_resources_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let instance_id = params.db_instance.id(); + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let started_record = + sagactx.lookup::("started_record")?; + + osagactx + .datastore() + .virtual_provisioning_collection_delete_instance( + &opctx, + instance_id, + params.db_instance.project_id, + i64::from(params.db_instance.ncpus.0 .0), + nexus_db_model::ByteCount(*params.db_instance.memory), + // Use the next instance generation number as the generation limit + // to ensure the provisioning counters are released. (The "mark as + // starting" undo step will "publish" this new state generation when + // it moves the instance back to Stopped.) + (&started_record.runtime().gen.next()).into(), + ) + .await + .map_err(ActionError::action_failed)?; + Ok(()) +} + async fn sis_dpd_ensure( sagactx: NexusActionContext, ) -> Result<(), ActionError> { diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 9208e21652..ea633be9dc 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -942,7 +942,7 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { assert_metrics(cptestctx, project_id, 0, 0, 0).await; - // Create an instance. + // Create and start an instance. let instance_name = "just-rainsticks"; create_instance(client, PROJECT_NAME, instance_name).await; let virtual_provisioning_collection = datastore @@ -955,27 +955,22 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { ByteCount::from_gibibytes_u32(1), ); - // Stop the instance + // Stop the instance. This should cause the relevant resources to be + // deprovisioned. let instance = instance_post(&client, instance_name, InstanceOp::Stop).await; instance_simulate(nexus, &instance.identity.id).await; let instance = instance_get(&client, &get_instance_url(&instance_name)).await; assert_eq!(instance.runtime.run_state, InstanceState::Stopped); - // NOTE: I think it's arguably "more correct" to identify that the - // number of CPUs being used by guests at this point is actually "0", - // not "4", because the instance is stopped (same re: RAM usage). - // - // However, for implementation reasons, this is complicated (we have a - // tendency to update the runtime without checking the prior state, which - // makes edge-triggered behavior trickier to notice). + let virtual_provisioning_collection = datastore .virtual_provisioning_collection_get(&opctx, project_id) .await .unwrap(); - let expected_cpus = 4; + let expected_cpus = 0; let expected_ram = - i64::try_from(ByteCount::from_gibibytes_u32(1).to_bytes()).unwrap(); + i64::try_from(ByteCount::from_gibibytes_u32(0).to_bytes()).unwrap(); assert_eq!(virtual_provisioning_collection.cpus_provisioned, expected_cpus); assert_eq!( i64::from(virtual_provisioning_collection.ram_provisioned.0), @@ -983,7 +978,7 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { ); assert_metrics(cptestctx, project_id, 0, expected_cpus, expected_ram).await; - // Stop the instance + // Delete the instance. NexusRequest::object_delete(client, &get_instance_url(&instance_name)) .authn_as(AuthnMode::PrivilegedUser) .execute() @@ -999,6 +994,130 @@ async fn test_instance_metrics(cptestctx: &ControlPlaneTestContext) { assert_metrics(cptestctx, project_id, 0, 0, 0).await; } +#[nexus_test] +async fn test_instance_metrics_with_migration( + cptestctx: &ControlPlaneTestContext, +) { + let client = &cptestctx.external_client; + let apictx = &cptestctx.server.apictx(); + let nexus = &apictx.nexus; + let instance_name = "bird-ecology"; + + // Create a second sled to migrate to/from. + let default_sled_id: Uuid = + nexus_test_utils::SLED_AGENT_UUID.parse().unwrap(); + let update_dir = Utf8Path::new("/should/be/unused"); + let other_sled_id = Uuid::new_v4(); + let _other_sa = nexus_test_utils::start_sled_agent( + cptestctx.logctx.log.new(o!("sled_id" => other_sled_id.to_string())), + cptestctx.server.get_http_server_internal_address().await, + other_sled_id, + &update_dir, + sim::SimMode::Explicit, + ) + .await + .unwrap(); + + let project_id = create_org_and_project(&client).await; + let instance_url = get_instance_url(instance_name); + + // Explicitly create an instance with no disks. Simulated sled agent assumes + // that disks are co-located with their instances. + let instance = nexus_test_utils::resource_helpers::create_instance_with( + client, + PROJECT_NAME, + instance_name, + ¶ms::InstanceNetworkInterfaceAttachment::Default, + Vec::::new(), + Vec::::new(), + ) + .await; + let instance_id = instance.identity.id; + + // Poke the instance into an active state. + instance_simulate(nexus, &instance_id).await; + let instance_next = instance_get(&client, &instance_url).await; + assert_eq!(instance_next.runtime.run_state, InstanceState::Running); + + // The instance should be provisioned while it's in the running state. + let nexus = &apictx.nexus; + let datastore = nexus.datastore(); + let check_provisioning_state = |cpus: i64, mem_gib: u32| async move { + let opctx = OpContext::for_tests( + cptestctx.logctx.log.new(o!()), + datastore.clone(), + ); + let virtual_provisioning_collection = datastore + .virtual_provisioning_collection_get(&opctx, project_id) + .await + .unwrap(); + assert_eq!( + virtual_provisioning_collection.cpus_provisioned, + cpus.clone() + ); + assert_eq!( + virtual_provisioning_collection.ram_provisioned.0, + ByteCount::from_gibibytes_u32(mem_gib) + ); + }; + + check_provisioning_state(4, 1).await; + + // Request migration to the other sled. This reserves resources on the + // target sled, but shouldn't change the virtual provisioning counters. + let original_sled = nexus + .instance_sled_id(&instance_id) + .await + .unwrap() + .expect("running instance should have a sled"); + + let dst_sled_id = if original_sled == default_sled_id { + other_sled_id + } else { + default_sled_id + }; + + let migrate_url = + format!("/v1/instances/{}/migrate", &instance_id.to_string()); + let _ = NexusRequest::new( + RequestBuilder::new(client, Method::POST, &migrate_url) + .body(Some(¶ms::InstanceMigrate { dst_sled_id })) + .expect_status(Some(StatusCode::OK)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body::() + .unwrap(); + + check_provisioning_state(4, 1).await; + + // Complete migration on the target. Simulated migrations always succeed. + // After this the instance should be running and should continue to appear + // to be provisioned. + instance_simulate_on_sled(cptestctx, nexus, dst_sled_id, instance_id).await; + let instance = instance_get(&client, &instance_url).await; + assert_eq!(instance.runtime.run_state, InstanceState::Running); + + check_provisioning_state(4, 1).await; + + // Now stop the instance. This should retire the instance's active Propolis + // and cause the virtual provisioning charges to be released. Note that + // the source sled still has an active resource charge for the source + // instance (whose demise wasn't simulated here), but this is intentionally + // not reflected in the virtual provisioning counters (which reflect the + // logical states of instances ignoring migration). + let instance = + instance_post(&client, instance_name, InstanceOp::Stop).await; + instance_simulate(nexus, &instance.identity.id).await; + let instance = + instance_get(&client, &get_instance_url(&instance_name)).await; + assert_eq!(instance.runtime.run_state, InstanceState::Stopped); + + check_provisioning_state(0, 0).await; +} + #[nexus_test] async fn test_instances_create_stopped_start( cptestctx: &ControlPlaneTestContext, From 0cfc8706242ac6f8b14657d3af065ab3a57507ba Mon Sep 17 00:00:00 2001 From: Ryan Goodfellow Date: Fri, 20 Oct 2023 20:48:10 -0700 Subject: [PATCH 2/3] BGP support (#3986) --- .../buildomat/jobs/build-and-test-linux.sh | 4 +- .github/buildomat/jobs/clippy.sh | 1 + .github/buildomat/jobs/deploy.sh | 10 +- .github/buildomat/jobs/package.sh | 6 +- .gitignore | 2 + Cargo.lock | 26 + Cargo.toml | 3 + bootstore/src/schemes/v0/storage.rs | 4 +- clients/bootstrap-agent-client/src/lib.rs | 2 + clients/ddm-admin-client/build.rs | 17 +- clients/mg-admin-client/Cargo.toml | 26 + clients/mg-admin-client/build.rs | 102 ++ clients/mg-admin-client/src/lib.rs | 83 ++ clients/nexus-client/src/lib.rs | 2 + clients/sled-agent-client/src/lib.rs | 32 +- clients/wicketd-client/src/lib.rs | 8 +- common/src/address.rs | 9 + common/src/api/external/mod.rs | 85 +- common/src/api/internal/shared.rs | 110 +- common/src/nexus_config.rs | 20 +- dev-tools/omdb/tests/env.out | 18 +- dev-tools/omdb/tests/successes.out | 20 +- env.sh | 1 + illumos-utils/src/destructor.rs | 2 +- installinator/src/bootstrap.rs | 2 +- installinator/src/dispatch.rs | 2 +- internal-dns/src/config.rs | 34 +- internal-dns/src/names.rs | 13 +- nexus/Cargo.toml | 1 + nexus/db-macros/src/lookup.rs | 4 +- nexus/db-model/src/bgp.rs | 37 + nexus/db-model/src/bootstore.rs | 13 + nexus/db-model/src/lib.rs | 2 + nexus/db-model/src/rack.rs | 3 + nexus/db-model/src/schema.rs | 21 +- nexus/db-model/src/service_kind.rs | 2 + nexus/db-model/src/switch_interface.rs | 9 +- nexus/db-model/src/switch_port.rs | 188 +++- nexus/db-queries/src/db/datastore/bgp.rs | 351 +++++++ .../db-queries/src/db/datastore/bootstore.rs | 37 + nexus/db-queries/src/db/datastore/mod.rs | 2 + nexus/db-queries/src/db/datastore/rack.rs | 25 + .../src/db/datastore/switch_port.rs | 294 +++--- nexus/src/app/bgp.rs | 162 +++ nexus/src/app/mod.rs | 67 +- nexus/src/app/rack.rs | 226 ++++- nexus/src/app/sagas/mod.rs | 1 - .../app/sagas/switch_port_settings_apply.rs | 760 +++++++++++++- .../app/sagas/switch_port_settings_clear.rs | 209 +++- .../app/sagas/switch_port_settings_update.rs | 5 - nexus/src/app/switch_port.rs | 110 +- nexus/src/external_api/http_entrypoints.rs | 203 +++- nexus/src/lib.rs | 12 +- nexus/test-utils/src/lib.rs | 62 ++ nexus/tests/integration_tests/address_lots.rs | 10 +- nexus/tests/integration_tests/endpoints.rs | 77 ++ .../tests/integration_tests/initialization.rs | 9 + nexus/tests/integration_tests/schema.rs | 2 + nexus/tests/integration_tests/switch_port.rs | 92 +- nexus/tests/output/nexus_tags.txt | 8 + nexus/types/src/external_api/params.rs | 111 ++- nexus/types/src/internal_api/params.rs | 2 + openapi/bootstrap-agent.json | 239 +++-- openapi/nexus-internal.json | 253 +++-- openapi/nexus.json | 938 +++++++++++++++++- openapi/sled-agent.json | 401 ++++++++ openapi/wicketd.json | 241 +++-- package-manifest.toml | 40 +- schema/crdb/8.0.0/up01.sql | 1 + schema/crdb/8.0.0/up02.sql | 1 + schema/crdb/8.0.0/up03.sql | 1 + schema/crdb/8.0.0/up04.sql | 5 + schema/crdb/8.0.0/up05.sql | 11 + schema/crdb/8.0.0/up06.sql | 1 + schema/crdb/8.0.0/up07.sql | 1 + schema/crdb/8.0.0/up08.sql | 1 + schema/crdb/8.0.0/up09.sql | 1 + schema/crdb/8.0.0/up10.sql | 1 + schema/crdb/8.0.0/up11.sql | 1 + schema/crdb/8.0.0/up12.sql | 1 + schema/crdb/8.0.0/up13.sql | 1 + schema/crdb/8.0.0/up14.sql | 4 + schema/crdb/dbinit.sql | 45 +- schema/rss-sled-plan.json | 243 +++-- sled-agent/Cargo.toml | 2 +- sled-agent/src/bootstrap/early_networking.rs | 290 +++++- sled-agent/src/bootstrap/maghemite.rs | 2 +- sled-agent/src/bootstrap/secret_retriever.rs | 6 +- sled-agent/src/bootstrap/server.rs | 2 +- sled-agent/src/http_entrypoints.rs | 82 +- sled-agent/src/params.rs | 21 +- sled-agent/src/rack_setup/plan/service.rs | 17 +- sled-agent/src/rack_setup/service.rs | 51 +- sled-agent/src/services.rs | 67 +- sled-agent/src/sim/http_entrypoints.rs | 57 ++ sled-agent/src/sim/server.rs | 2 +- sled-agent/src/sled_agent.rs | 27 +- .../gimlet-standalone/config-rss.toml | 21 +- smf/sled-agent/non-gimlet/config-rss.toml | 21 +- test-utils/src/dev/dendrite.rs | 2 +- test-utils/src/dev/maghemite.rs | 155 +++ test-utils/src/dev/mod.rs | 1 + tools/build-global-zone-packages.sh | 4 +- .../build-trampoline-global-zone-packages.sh | 4 +- tools/ci_download_maghemite_mgd | 168 ++++ tools/ci_download_maghemite_openapi | 13 +- tools/ci_download_softnpu_machinery | 2 +- tools/create_virtual_hardware.sh | 6 +- tools/delete-reservoir.sh | 6 + tools/dendrite_openapi_version | 2 +- tools/dendrite_stub_checksums | 6 +- tools/install_builder_prerequisites.sh | 4 + tools/install_runner_prerequisites.sh | 5 +- ..._version => maghemite_ddm_openapi_version} | 2 +- tools/maghemite_mg_openapi_version | 2 + tools/maghemite_mgd_checksums | 2 + tools/update_maghemite.sh | 33 +- update-engine/src/context.rs | 2 +- wicket/src/rack_setup/config_template.toml | 30 +- wicket/src/rack_setup/config_toml.rs | 186 +++- wicket/src/ui/main.rs | 2 +- wicket/src/ui/panes/rack_setup.rs | 63 +- wicket/src/ui/wrap.rs | 2 +- wicketd/Cargo.toml | 1 + wicketd/src/installinator_progress.rs | 2 +- wicketd/src/preflight_check/uplink.rs | 275 ++--- wicketd/src/rss_config.rs | 69 +- workspace-hack/Cargo.toml | 2 + 128 files changed, 6883 insertions(+), 1028 deletions(-) create mode 100644 clients/mg-admin-client/Cargo.toml create mode 100644 clients/mg-admin-client/build.rs create mode 100644 clients/mg-admin-client/src/lib.rs create mode 100644 nexus/db-model/src/bootstore.rs create mode 100644 nexus/db-queries/src/db/datastore/bgp.rs create mode 100644 nexus/db-queries/src/db/datastore/bootstore.rs create mode 100644 nexus/src/app/bgp.rs delete mode 100644 nexus/src/app/sagas/switch_port_settings_update.rs create mode 100644 schema/crdb/8.0.0/up01.sql create mode 100644 schema/crdb/8.0.0/up02.sql create mode 100644 schema/crdb/8.0.0/up03.sql create mode 100644 schema/crdb/8.0.0/up04.sql create mode 100644 schema/crdb/8.0.0/up05.sql create mode 100644 schema/crdb/8.0.0/up06.sql create mode 100644 schema/crdb/8.0.0/up07.sql create mode 100644 schema/crdb/8.0.0/up08.sql create mode 100644 schema/crdb/8.0.0/up09.sql create mode 100644 schema/crdb/8.0.0/up10.sql create mode 100644 schema/crdb/8.0.0/up11.sql create mode 100644 schema/crdb/8.0.0/up12.sql create mode 100644 schema/crdb/8.0.0/up13.sql create mode 100644 schema/crdb/8.0.0/up14.sql create mode 100644 test-utils/src/dev/maghemite.rs create mode 100755 tools/ci_download_maghemite_mgd create mode 100755 tools/delete-reservoir.sh rename tools/{maghemite_openapi_version => maghemite_ddm_openapi_version} (59%) create mode 100644 tools/maghemite_mg_openapi_version create mode 100644 tools/maghemite_mgd_checksums diff --git a/.github/buildomat/jobs/build-and-test-linux.sh b/.github/buildomat/jobs/build-and-test-linux.sh index f33d1a8cfa..715effd080 100755 --- a/.github/buildomat/jobs/build-and-test-linux.sh +++ b/.github/buildomat/jobs/build-and-test-linux.sh @@ -1,8 +1,8 @@ #!/bin/bash #: -#: name = "build-and-test (ubuntu-20.04)" +#: name = "build-and-test (ubuntu-22.04)" #: variety = "basic" -#: target = "ubuntu-20.04" +#: target = "ubuntu-22.04" #: rust_toolchain = "1.72.1" #: output_rules = [ #: "/var/tmp/omicron_tmp/*", diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh index dba1021919..5fd31adb76 100755 --- a/.github/buildomat/jobs/clippy.sh +++ b/.github/buildomat/jobs/clippy.sh @@ -29,3 +29,4 @@ ptime -m bash ./tools/install_builder_prerequisites.sh -y banner clippy ptime -m cargo xtask clippy +ptime -m cargo doc diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index bdc1a9cce8..ff9b44fc40 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -232,11 +232,11 @@ infra_ip_first = \"$UPLINK_IP\" /^infra_ip_last/c\\ infra_ip_last = \"$UPLINK_IP\" } - /^\\[\\[rack_network_config.uplinks/,/^\$/ { - /^gateway_ip/c\\ -gateway_ip = \"$GATEWAY_IP\" - /^uplink_cidr/c\\ -uplink_cidr = \"$UPLINK_IP/32\" + /^\\[\\[rack_network_config.ports/,/^\$/ { + /^routes/c\\ +routes = \\[{nexthop = \"$GATEWAY_IP\", destination = \"0.0.0.0/0\"}\\] + /^addresses/c\\ +addresses = \\[\"$UPLINK_IP/32\"\\] } " pkg/config-rss.toml diff -u pkg/config-rss.toml{~,} || true diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 64c087524e..c1cb04124d 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -71,7 +71,7 @@ tarball_src_dir="$(pwd)/out/versioned" stamp_packages() { for package in "$@"; do # TODO: remove once https://github.com/oxidecomputer/omicron-package/pull/54 lands - if [[ $package == maghemite ]]; then + if [[ $package == mg-ddm-gz ]]; then echo "0.0.0" > VERSION tar rvf "out/$package.tar" VERSION rm VERSION @@ -90,7 +90,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- \ -t host target create -i standard -m gimlet -s asic -r multi-sled ptime -m cargo run --locked --release --bin omicron-package -- \ -t host package -stamp_packages omicron-sled-agent maghemite propolis-server overlay +stamp_packages omicron-sled-agent mg-ddm-gz propolis-server overlay # Create global zone package @ /work/global-zone-packages.tar.gz ptime -m ./tools/build-global-zone-packages.sh "$tarball_src_dir" /work @@ -135,7 +135,7 @@ ptime -m cargo run --locked --release --bin omicron-package -- \ -t recovery target create -i trampoline ptime -m cargo run --locked --release --bin omicron-package -- \ -t recovery package -stamp_packages installinator maghemite +stamp_packages installinator mg-ddm-gz # Create trampoline global zone package @ /work/trampoline-global-zone-packages.tar.gz ptime -m ./tools/build-trampoline-global-zone-packages.sh "$tarball_src_dir" /work diff --git a/.gitignore b/.gitignore index 574e867c02..1d7177320f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ core *.vdev debug.out rusty-tags.vi +*.sw* +tags diff --git a/Cargo.lock b/Cargo.lock index b38cda6a45..06d5f2fb70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4161,6 +4161,29 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mg-admin-client" +version = "0.1.0" +dependencies = [ + "anyhow", + "either", + "omicron-common 0.1.0", + "omicron-workspace-hack", + "omicron-zone-package", + "progenitor", + "progenitor-client", + "quote", + "reqwest", + "rustfmt-wrapper", + "serde", + "serde_json", + "sled-hardware", + "slog", + "thiserror", + "tokio", + "toml 0.7.8", +] + [[package]] name = "mime" version = "0.3.17" @@ -5078,6 +5101,7 @@ dependencies = [ "itertools 0.11.0", "lazy_static", "macaddr", + "mg-admin-client", "mime_guess", "newtype_derive", "nexus-db-model", @@ -5460,6 +5484,7 @@ dependencies = [ "schemars", "semver 1.0.18", "serde", + "serde_json", "sha2", "signature 2.1.0", "similar", @@ -10116,6 +10141,7 @@ dependencies = [ "installinator-artifactd", "installinator-common", "internal-dns 0.1.0", + "ipnetwork", "itertools 0.11.0", "omicron-certificates", "omicron-common 0.1.0", diff --git a/Cargo.toml b/Cargo.toml index b213f3adff..e9eea3c4ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "clients/dpd-client", "clients/gateway-client", "clients/installinator-artifact-client", + "clients/mg-admin-client", "clients/nexus-client", "clients/oxide-client", "clients/oximeter-client", @@ -82,6 +83,7 @@ default-members = [ "clients/oximeter-client", "clients/sled-agent-client", "clients/wicketd-client", + "clients/mg-admin-client", "common", "dev-tools/crdb-seed", "dev-tools/omdb", @@ -227,6 +229,7 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } mime_guess = "2.0.4" mockall = "0.11" newtype_derive = "0.1.6" +mg-admin-client = { path = "clients/mg-admin-client" } nexus-client = { path = "clients/nexus-client" } nexus-db-model = { path = "nexus/db-model" } nexus-db-queries = { path = "nexus/db-queries" } diff --git a/bootstore/src/schemes/v0/storage.rs b/bootstore/src/schemes/v0/storage.rs index ee31d24f05..327acc6058 100644 --- a/bootstore/src/schemes/v0/storage.rs +++ b/bootstore/src/schemes/v0/storage.rs @@ -5,9 +5,9 @@ //! Storage for the v0 bootstore scheme //! //! We write two pieces of data to M.2 devices in production via -//! [`omicron_common::Ledger`]: +//! [`omicron_common::ledger::Ledger`]: //! -//! 1. [`super::Fsm::State`] for bootstore state itself +//! 1. [`super::State`] for bootstore state itself //! 2. A network config blob required for pre-rack-unlock configuration //! diff --git a/clients/bootstrap-agent-client/src/lib.rs b/clients/bootstrap-agent-client/src/lib.rs index 3f8b20e1f5..19ecb599f3 100644 --- a/clients/bootstrap-agent-client/src/lib.rs +++ b/clients/bootstrap-agent-client/src/lib.rs @@ -20,6 +20,8 @@ progenitor::generate_api!( derives = [schemars::JsonSchema], replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, } ); diff --git a/clients/ddm-admin-client/build.rs b/clients/ddm-admin-client/build.rs index e3c1345eda..da74ee9962 100644 --- a/clients/ddm-admin-client/build.rs +++ b/clients/ddm-admin-client/build.rs @@ -21,20 +21,21 @@ fn main() -> Result<()> { println!("cargo:rerun-if-changed=../../package-manifest.toml"); let config: Config = toml::from_str(&manifest) - .context("failed to parse ../../package-manifest.toml")?; - let maghemite = config + .context("failed to parse ../package-manifest.toml")?; + + let ddm = config .packages - .get("maghemite") - .context("missing maghemite package in ../../package-manifest.toml")?; + .get("mg-ddm-gz") + .context("missing mg-ddm-gz package in ../package-manifest.toml")?; - let local_path = match &maghemite.source { + let local_path = match &ddm.source { PackageSource::Prebuilt { commit, .. } => { // Report a relatively verbose error if we haven't downloaded the requisite // openapi spec. let local_path = format!("../../out/downloads/ddm-admin-{commit}.json"); if !Path::new(&local_path).exists() { - bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); + bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_ddm_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); } println!("cargo:rerun-if-changed={local_path}"); local_path @@ -51,7 +52,9 @@ fn main() -> Result<()> { } _ => { - bail!("maghemite external package must have type `prebuilt` or `manual`") + bail!( + "mg-ddm external package must have type `prebuilt` or `manual`" + ) } }; diff --git a/clients/mg-admin-client/Cargo.toml b/clients/mg-admin-client/Cargo.toml new file mode 100644 index 0000000000..c444fee32f --- /dev/null +++ b/clients/mg-admin-client/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "mg-admin-client" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +either.workspace = true +progenitor-client.workspace = true +reqwest = { workspace = true, features = ["json", "stream", "rustls-tls"] } +serde.workspace = true +slog.workspace = true +thiserror.workspace = true +tokio.workspace = true +omicron-common.workspace = true +sled-hardware.workspace = true +omicron-workspace-hack.workspace = true + +[build-dependencies] +anyhow.workspace = true +omicron-zone-package.workspace = true +progenitor.workspace = true +quote.workspace = true +rustfmt-wrapper.workspace = true +serde_json.workspace = true +toml.workspace = true diff --git a/clients/mg-admin-client/build.rs b/clients/mg-admin-client/build.rs new file mode 100644 index 0000000000..dcc7ae61cb --- /dev/null +++ b/clients/mg-admin-client/build.rs @@ -0,0 +1,102 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2022 Oxide Computer Company + +use anyhow::bail; +use anyhow::Context; +use anyhow::Result; +use omicron_zone_package::config::Config; +use omicron_zone_package::package::PackageSource; +use quote::quote; +use std::env; +use std::fs; +use std::path::Path; + +fn main() -> Result<()> { + // Find the current maghemite repo commit from our package manifest. + let manifest = fs::read_to_string("../../package-manifest.toml") + .context("failed to read ../../package-manifest.toml")?; + println!("cargo:rerun-if-changed=../../package-manifest.toml"); + + let config: Config = toml::from_str(&manifest) + .context("failed to parse ../../package-manifest.toml")?; + let mg = config + .packages + .get("mgd") + .context("missing mgd package in ../../package-manifest.toml")?; + + let local_path = match &mg.source { + PackageSource::Prebuilt { commit, .. } => { + // Report a relatively verbose error if we haven't downloaded the requisite + // openapi spec. + let local_path = + format!("../../out/downloads/mg-admin-{commit}.json"); + if !Path::new(&local_path).exists() { + bail!("{local_path} doesn't exist; rerun `tools/ci_download_maghemite_openapi` (after updating `tools/maghemite_mg_openapi_version` if the maghemite commit in package-manifest.toml has changed)"); + } + println!("cargo:rerun-if-changed={local_path}"); + local_path + } + + PackageSource::Manual => { + let local_path = + "../../out/downloads/mg-admin-manual.json".to_string(); + if !Path::new(&local_path).exists() { + bail!("{local_path} doesn't exist, please copy manually built mg-admin.json there!"); + } + println!("cargo:rerun-if-changed={local_path}"); + local_path + } + + _ => { + bail!("mgd external package must have type `prebuilt` or `manual`") + } + }; + + let spec = { + let bytes = fs::read(&local_path) + .with_context(|| format!("failed to read {local_path}"))?; + serde_json::from_slice(&bytes).with_context(|| { + format!("failed to parse {local_path} as openapi spec") + })? + }; + + let code = progenitor::Generator::new( + progenitor::GenerationSettings::new() + .with_inner_type(quote!(slog::Logger)) + .with_pre_hook(quote! { + |log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + } + }) + .with_post_hook(quote! { + |log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + } + }), + ) + .generate_tokens(&spec) + .with_context(|| { + format!("failed to generate progenitor client from {local_path}") + })?; + + let content = rustfmt_wrapper::rustfmt(code).with_context(|| { + format!("rustfmt failed on progenitor code from {local_path}") + })?; + + let out_file = + Path::new(&env::var("OUT_DIR").expect("OUT_DIR env var not set")) + .join("mg-admin-client.rs"); + + fs::write(&out_file, content).with_context(|| { + format!("failed to write client to {}", out_file.display()) + })?; + + Ok(()) +} diff --git a/clients/mg-admin-client/src/lib.rs b/clients/mg-admin-client/src/lib.rs new file mode 100644 index 0000000000..bb1d925c73 --- /dev/null +++ b/clients/mg-admin-client/src/lib.rs @@ -0,0 +1,83 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +#![allow(clippy::redundant_closure_call)] +#![allow(clippy::needless_lifetimes)] +#![allow(clippy::match_single_binding)] +#![allow(clippy::clone_on_copy)] +#![allow(rustdoc::broken_intra_doc_links)] +#![allow(rustdoc::invalid_html_tags)] + +#[allow(dead_code)] +mod inner { + include!(concat!(env!("OUT_DIR"), "/mg-admin-client.rs")); +} + +pub use inner::types; +pub use inner::Error; + +use inner::Client as InnerClient; +use omicron_common::api::external::BgpPeerState; +use slog::Logger; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use thiserror::Error; + +// TODO-cleanup Is it okay to hardcode this port number here? +const MGD_PORT: u16 = 4676; + +#[derive(Debug, Error)] +pub enum MgError { + #[error("Failed to construct an HTTP client: {0}")] + HttpClient(#[from] reqwest::Error), + + #[error("Failed making HTTP request to mgd: {0}")] + MgApi(#[from] Error), +} + +impl From for BgpPeerState { + fn from(s: inner::types::FsmStateKind) -> BgpPeerState { + use inner::types::FsmStateKind; + match s { + FsmStateKind::Idle => BgpPeerState::Idle, + FsmStateKind::Connect => BgpPeerState::Connect, + FsmStateKind::Active => BgpPeerState::Active, + FsmStateKind::OpenSent => BgpPeerState::OpenSent, + FsmStateKind::OpenConfirm => BgpPeerState::OpenConfirm, + FsmStateKind::SessionSetup => BgpPeerState::SessionSetup, + FsmStateKind::Established => BgpPeerState::Established, + } + } +} + +#[derive(Debug, Clone)] +pub struct Client { + pub inner: InnerClient, + pub log: Logger, +} + +impl Client { + /// Creates a new [`Client`] that points to localhost + pub fn localhost(log: &Logger) -> Result { + Self::new(log, SocketAddr::new(Ipv6Addr::LOCALHOST.into(), MGD_PORT)) + } + + pub fn new(log: &Logger, mgd_addr: SocketAddr) -> Result { + let dur = std::time::Duration::from_secs(60); + let log = log.new(slog::o!("MgAdminClient" => mgd_addr)); + + let inner = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build()?; + let inner = InnerClient::new_with_client( + &format!("http://{mgd_addr}"), + inner, + log.clone(), + ); + Ok(Self { inner, log }) + } +} diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index 33a68cb3ce..23ceb114fc 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -23,6 +23,8 @@ progenitor::generate_api!( }), replace = { Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, MacAddr = omicron_common::api::external::MacAddr, Name = omicron_common::api::external::Name, NewPasswordHash = omicron_passwords::NewPasswordHash, diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 3daac7dd60..0df21d894e 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -5,11 +5,33 @@ //! Interface for making API requests to a Sled Agent use async_trait::async_trait; -use omicron_common::generate_logging_api; use std::convert::TryFrom; use uuid::Uuid; -generate_logging_api!("../../openapi/sled-agent.json"); +progenitor::generate_api!( + spec = "../../openapi/sled-agent.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + //TODO trade the manual transformations later in this file for the + // replace directives below? + replace = { + //Ipv4Network = ipnetwork::Ipv4Network, + SwitchLocation = omicron_common::api::external::SwitchLocation, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, + PortFec = omicron_common::api::internal::shared::PortFec, + PortSpeed = omicron_common::api::internal::shared::PortSpeed, + } +); impl omicron_common::api::external::ClientError for types::Error { fn message(&self) -> String { @@ -269,6 +291,12 @@ impl From for types::Ipv4Net { } } +impl From for types::Ipv4Network { + fn from(n: ipnetwork::Ipv4Network) -> Self { + Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) + } +} + impl From for types::Ipv6Net { fn from(n: ipnetwork::Ipv6Network) -> Self { Self::try_from(n.to_string()).unwrap_or_else(|e| panic!("{}: {}", n, e)) diff --git a/clients/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs index ff45232520..982ec13780 100644 --- a/clients/wicketd-client/src/lib.rs +++ b/clients/wicketd-client/src/lib.rs @@ -42,8 +42,12 @@ progenitor::generate_api!( RackInitId = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, RackResetId = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, RackOperationStatus = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, - RackNetworkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + RackNetworkConfigV1 = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, UplinkConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + PortConfigV1 = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpPeerConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + BgpConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, + RouteConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigInsensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfigSensitive = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, CurrentRssUserConfig = { derives = [ PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize ] }, @@ -52,6 +56,8 @@ progenitor::generate_api!( replace = { Duration = std::time::Duration, Ipv4Network = ipnetwork::Ipv4Network, + Ipv6Network = ipnetwork::Ipv6Network, + IpNetwork = ipnetwork::IpNetwork, PutRssUserConfigInsensitive = wicket_common::rack_setup::PutRssUserConfigInsensitive, EventReportForWicketdEngineSpec = wicket_common::update_events::EventReport, StepEventForWicketdEngineSpec = wicket_common::update_events::StepEvent, diff --git a/common/src/address.rs b/common/src/address.rs index baa344ef22..992e8f0406 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -39,6 +39,7 @@ pub const CLICKHOUSE_PORT: u16 = 8123; pub const CLICKHOUSE_KEEPER_PORT: u16 = 9181; pub const OXIMETER_PORT: u16 = 12223; pub const DENDRITE_PORT: u16 = 12224; +pub const MGD_PORT: u16 = 4676; pub const DDMD_PORT: u16 = 8000; pub const MGS_PORT: u16 = 12225; pub const WICKETD_PORT: u16 = 12226; @@ -172,6 +173,14 @@ impl Ipv6Subnet { } } +impl From for Ipv6Subnet { + fn from(net: Ipv6Network) -> Self { + // Ensure the address is set to within-prefix only components. + let net = Ipv6Network::new(net.network(), N).unwrap(); + Self { net: Ipv6Net(net) } + } +} + // We need a custom Deserialize to ensure that the subnet is what we expect. impl<'de, const N: u8> Deserialize<'de> for Ipv6Subnet { fn deserialize(deserializer: D) -> Result diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 53512408af..fcea57220d 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -12,6 +12,7 @@ pub mod http_pagination; use dropshot::HttpError; pub use error::*; +pub use crate::api::internal::shared::SwitchLocation; use anyhow::anyhow; use anyhow::Context; use api_identity::ObjectIdentity; @@ -98,6 +99,13 @@ pub struct DataPageParams<'a, NameType> { } impl<'a, NameType> DataPageParams<'a, NameType> { + pub fn max_page() -> Self { + Self { + marker: None, + direction: dropshot::PaginationOrder::Ascending, + limit: NonZeroU32::new(u32::MAX).unwrap(), + } + } /// Maps the marker type to a new type. /// /// Equivalent to [std::option::Option::map], because that's what it calls. @@ -400,7 +408,7 @@ impl SemverVersion { /// This is the official ECMAScript-compatible validation regex for /// semver: - /// https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + /// const VALIDATION_REGEX: &str = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"; } @@ -690,6 +698,8 @@ pub enum ResourceType { AddressLot, AddressLotBlock, BackgroundTask, + BgpConfig, + BgpAnnounceSet, Fleet, Silo, SiloUser, @@ -2459,9 +2469,6 @@ pub struct SwitchPortBgpPeerConfig { /// The port settings object this BGP configuration belongs to. pub port_settings_id: Uuid, - /// The id for the set of prefixes announced in this peer configuration. - pub bgp_announce_set_id: Uuid, - /// The id of the global BGP configuration referenced by this peer /// configuration. pub bgp_config_id: Uuid, @@ -2476,7 +2483,9 @@ pub struct SwitchPortBgpPeerConfig { } /// A base BGP configuration. -#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +#[derive( + ObjectIdentity, Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, +)] pub struct BgpConfig { #[serde(flatten)] pub identity: IdentityMetadata, @@ -2528,6 +2537,72 @@ pub struct SwitchPortAddressConfig { pub interface_name: String, } +/// The current state of a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum BgpPeerState { + /// Initial state. Refuse all incomming BGP connections. No resources + /// allocated to peer. + Idle, + + /// Waiting for the TCP connection to be completed. + Connect, + + /// Trying to acquire peer by listening for and accepting a TCP connection. + Active, + + /// Waiting for open message from peer. + OpenSent, + + /// Waiting for keepaliave or notification from peer. + OpenConfirm, + + /// Synchronizing with peer. + SessionSetup, + + /// Session established. Able to exchange update, notification and keepliave + /// messages with peers. + Established, +} + +/// The current status of a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct BgpPeerStatus { + /// IP address of the peer. + pub addr: IpAddr, + + /// Local autonomous system number. + pub local_asn: u32, + + /// Remote autonomous system number. + pub remote_asn: u32, + + /// State of the peer. + pub state: BgpPeerState, + + /// Time of last state change. + pub state_duration_millis: u64, + + /// Switch with the peer session. + pub switch: SwitchLocation, +} + +/// A route imported from a BGP peer. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct BgpImportedRouteIpv4 { + /// The destination network prefix. + pub prefix: Ipv4Net, + + /// The nexthop the prefix is reachable through. + pub nexthop: Ipv4Addr, + + /// BGP identifier of the originating router. + pub id: u32, + + /// Switch the route is imported into. + pub switch: SwitchLocation, +} + #[cfg(test)] mod test { use serde::Deserialize; diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 9e3f3ec1f6..784da8fcc6 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -5,7 +5,7 @@ //! Types shared between Nexus and Sled Agent. use crate::api::external::{self, Name}; -use ipnetwork::Ipv4Network; +use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{ @@ -68,18 +68,88 @@ pub struct SourceNatConfig { pub last_port: u16, } +// We alias [`RackNetworkConfig`] to the current version of the protocol, so +// that we can convert between versions as necessary. +pub type RackNetworkConfig = RackNetworkConfigV1; + /// Initial network configuration #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] -pub struct RackNetworkConfig { +pub struct RackNetworkConfigV1 { + pub rack_subnet: Ipv6Network, // TODO: #3591 Consider making infra-ip ranges implicit for uplinks /// First ip address to be used for configuring network infrastructure pub infra_ip_first: Ipv4Addr, /// Last ip address to be used for configuring network infrastructure pub infra_ip_last: Ipv4Addr, /// Uplinks for connecting the rack to external networks - pub uplinks: Vec, + pub ports: Vec, + /// BGP configurations for connecting the rack to external networks + pub bgp: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpConfig { + /// The autonomous system number for the BGP configuration. + pub asn: u32, + /// The set of prefixes for the BGP router to originate. + pub originate: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct BgpPeerConfig { + /// The autonomous sysetm number of the router the peer belongs to. + pub asn: u32, + /// Switch port the peer is reachable on. + pub port: String, + /// Address of the peer. + pub addr: Ipv4Addr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct RouteConfig { + /// The destination of the route. + pub destination: IpNetwork, + /// The nexthop/gateway address. + pub nexthop: IpAddr, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct PortConfigV1 { + /// The set of routes associated with this port. + pub routes: Vec, + /// This port's addresses. + pub addresses: Vec, + /// Switch the port belongs to. + pub switch: SwitchLocation, + /// Nmae of the port this config applies to. + pub port: String, + /// Port speed. + pub uplink_port_speed: PortSpeed, + /// Port forward error correction type. + pub uplink_port_fec: PortFec, + /// BGP peers on this port + pub bgp_peers: Vec, } +impl From for PortConfigV1 { + fn from(value: UplinkConfig) -> Self { + PortConfigV1 { + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: value.gateway_ip.into(), + }], + addresses: vec![value.uplink_cidr.into()], + switch: value.switch, + port: value.uplink_port, + uplink_port_speed: value.uplink_port_speed, + uplink_port_fec: value.uplink_port_fec, + bgp_peers: vec![], + } + } +} + +/// Deprecated, use PortConfigV1 instead. Cannot actually deprecate due to +/// #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] pub struct UplinkConfig { /// Gateway address @@ -99,9 +169,41 @@ pub struct UplinkConfig { pub uplink_vid: Option, } +/// A set of switch uplinks. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct SwitchPorts { + pub uplinks: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct HostPortConfig { + /// Switchport to use for external connectivity + pub port: String, + + /// IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport + /// (must be in infra_ip pool) + pub addrs: Vec, +} + +impl From for HostPortConfig { + fn from(x: PortConfigV1) -> Self { + Self { port: x.port, addrs: x.addresses } + } +} + /// Identifies switch physical location #[derive( - Clone, Copy, Debug, Deserialize, Serialize, PartialEq, JsonSchema, Hash, Eq, + Clone, + Copy, + Debug, + Deserialize, + Serialize, + PartialEq, + JsonSchema, + Hash, + Eq, + PartialOrd, + Ord, )] #[serde(rename_all = "snake_case")] pub enum SwitchLocation { diff --git a/common/src/nexus_config.rs b/common/src/nexus_config.rs index 6b0960643e..da50356d2e 100644 --- a/common/src/nexus_config.rs +++ b/common/src/nexus_config.rs @@ -241,6 +241,12 @@ pub struct DpdConfig { pub address: SocketAddr, } +/// Configuration for the `Dendrite` dataplane daemon. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct MgdConfig { + pub address: SocketAddr, +} + // A deserializable type that does no validation on the tunable parameters. #[derive(Clone, Debug, Deserialize, PartialEq)] struct UnvalidatedTunables { @@ -388,6 +394,9 @@ pub struct PackageConfig { /// `Dendrite` dataplane daemon configuration #[serde(default)] pub dendrite: HashMap, + /// Maghemite mgd daemon configuration + #[serde(default)] + pub mgd: HashMap, /// Background task configuration pub background_tasks: BackgroundTaskConfig, /// Default Crucible region allocation strategy @@ -469,7 +478,7 @@ mod test { use crate::nexus_config::{ BackgroundTaskConfig, ConfigDropshotWithTls, Database, DeploymentConfig, DnsTasksConfig, DpdConfig, ExternalEndpointsConfig, - InternalDns, LoadErrorKind, + InternalDns, LoadErrorKind, MgdConfig, }; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; @@ -605,6 +614,8 @@ mod test { type = "from_dns" [dendrite.switch0] address = "[::1]:12224" + [mgd.switch0] + address = "[::1]:4676" [background_tasks] dns_internal.period_secs_config = 1 dns_internal.period_secs_servers = 2 @@ -686,6 +697,13 @@ mod test { .unwrap(), } )]), + mgd: HashMap::from([( + SwitchLocation::Switch0, + MgdConfig { + address: SocketAddr::from_str("[::1]:4676") + .unwrap(), + } + )]), background_tasks: BackgroundTaskConfig { dns_internal: DnsTasksConfig { period_secs_config: Duration::from_secs(1), diff --git a/dev-tools/omdb/tests/env.out b/dev-tools/omdb/tests/env.out index 8e345b78d1..7cbac1565d 100644 --- a/dev-tools/omdb/tests/env.out +++ b/dev-tools/omdb/tests/env.out @@ -2,12 +2,12 @@ EXECUTING COMMAND: omdb ["db", "--db-url", "postgresql://root@[::1]:REDACTED_POR termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "--db-url", "junk", "sleds"] termination: Exited(2) @@ -165,25 +165,25 @@ EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["--dns-server", "[::1]:REDACTED_PORT", "db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: database URL not specified. Will search DNS. note: (override with --db-url or OMDB_DB_URL) note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 6fd84c5eb3..3ebf7046d4 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -8,7 +8,7 @@ external oxide-dev.test 2 create silo: "tes --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "diff", "external", "2"] termination: Exited(0) @@ -24,7 +24,7 @@ changes: names added: 1, names removed: 0 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "dns", "names", "external", "2"] termination: Exited(0) @@ -36,7 +36,7 @@ External zone: oxide-dev.test --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-instances"] termination: Exited(0) @@ -49,10 +49,12 @@ Dendrite REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT ExternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 InternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT sim-b6d65341 +Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 +Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT sim-b6d65341 --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "services", "list-by-sled"] termination: Exited(0) @@ -67,22 +69,24 @@ sled: sim-b6d65341 (id REDACTED_UUID_REDACTED_UUID_REDACTED) ExternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT InternalDns REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT Nexus REDACTED_UUID_REDACTED_UUID_REDACTED [::ffff:127.0.0.1]:REDACTED_PORT + Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT + Mgd REDACTED_UUID_REDACTED_UUID_REDACTED [::1]:REDACTED_PORT --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["db", "sleds"] termination: Exited(0) --------------------------------------------- stdout: -SERIAL IP ROLE ID -sim-b6d65341 [::1]:REDACTED_PORT - REDACTED_UUID_REDACTED_UUID_REDACTED +SERIAL IP ROLE ID +sim-b6d65341 [::1]:REDACTED_PORT scrimlet REDACTED_UUID_REDACTED_UUID_REDACTED --------------------------------------------- stderr: note: using database URL postgresql://root@[::1]:REDACTED_PORT/omicron?sslmode=disable -note: database schema version matches expected (7.0.0) +note: database schema version matches expected (8.0.0) ============================================= EXECUTING COMMAND: omdb ["mgs", "inventory"] termination: Exited(0) diff --git a/env.sh b/env.sh index 5b1e2b34ac..483a89f597 100644 --- a/env.sh +++ b/env.sh @@ -9,5 +9,6 @@ OMICRON_WS="$(cd $(dirname "${BASH_SOURCE[0]}") && echo $PWD)" export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" +export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" unset OMICRON_WS set +o xtrace diff --git a/illumos-utils/src/destructor.rs b/illumos-utils/src/destructor.rs index e019f2562f..ccc5b15486 100644 --- a/illumos-utils/src/destructor.rs +++ b/illumos-utils/src/destructor.rs @@ -21,7 +21,7 @@ use tokio::sync::mpsc; type SharedBoxFuture = Shared + Send>>>; -/// Future stored within [Destructor]. +/// Future stored within [`Destructor`]. struct ShutdownWaitFuture(SharedBoxFuture>); impl Future for ShutdownWaitFuture { diff --git a/installinator/src/bootstrap.rs b/installinator/src/bootstrap.rs index 2854293d8a..71c76809db 100644 --- a/installinator/src/bootstrap.rs +++ b/installinator/src/bootstrap.rs @@ -20,7 +20,7 @@ use sled_hardware::underlay::BootstrapInterface; use slog::info; use slog::Logger; -const MG_DDM_SERVICE_FMRI: &str = "svc:/system/illumos/mg-ddm"; +const MG_DDM_SERVICE_FMRI: &str = "svc:/oxide/mg-ddm"; const MG_DDM_MANIFEST_PATH: &str = "/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"; // TODO-cleanup The implementation of this function is heavily derived from diff --git a/installinator/src/dispatch.rs b/installinator/src/dispatch.rs index 9c06aeac77..9bec14664c 100644 --- a/installinator/src/dispatch.rs +++ b/installinator/src/dispatch.rs @@ -104,7 +104,7 @@ impl DebugDiscoverOpts { /// Options shared by both [`DebugDiscoverOpts`] and [`InstallOpts`]. #[derive(Debug, Args)] struct DiscoverOpts { - /// The mechanism by which to discover peers: bootstrap or list:[::1]:8000 + /// The mechanism by which to discover peers: bootstrap or `list:[::1]:8000` #[clap(long, default_value_t = DiscoveryMechanism::Bootstrap)] mechanism: DiscoveryMechanism, } diff --git a/internal-dns/src/config.rs b/internal-dns/src/config.rs index e5272cd23a..86dd6e802e 100644 --- a/internal-dns/src/config.rs +++ b/internal-dns/src/config.rs @@ -63,8 +63,9 @@ use crate::names::{ServiceName, DNS_ZONE}; use anyhow::{anyhow, ensure}; use dns_service_client::types::{DnsConfigParams, DnsConfigZone, DnsRecord}; +use omicron_common::api::internal::shared::SwitchLocation; use std::collections::BTreeMap; -use std::net::Ipv6Addr; +use std::net::{Ipv6Addr, SocketAddrV6}; use uuid::Uuid; /// Zones that can be referenced within the internal DNS system. @@ -136,6 +137,8 @@ pub struct DnsConfigBuilder { /// network sleds: BTreeMap, + scrimlets: BTreeMap, + /// set of hosts of type "zone" that have been configured so far, mapping /// each zone's unique uuid to its sole IPv6 address on the control plane /// network @@ -175,6 +178,7 @@ impl DnsConfigBuilder { DnsConfigBuilder { sleds: BTreeMap::new(), zones: BTreeMap::new(), + scrimlets: BTreeMap::new(), service_instances_zones: BTreeMap::new(), service_instances_sleds: BTreeMap::new(), } @@ -205,6 +209,15 @@ impl DnsConfigBuilder { } } + pub fn host_scrimlet( + &mut self, + switch_location: SwitchLocation, + addr: SocketAddrV6, + ) -> anyhow::Result<()> { + self.scrimlets.insert(switch_location, addr); + Ok(()) + } + /// Add a new dendrite host of type "zone" to the configuration /// /// Returns a [`Zone`] that can be used with [`Self::service_backend_zone()`] to @@ -351,6 +364,23 @@ impl DnsConfigBuilder { (zone.dns_name(), vec![DnsRecord::Aaaa(zone_ip)]) }); + let scrimlet_srv_records = + self.scrimlets.clone().into_iter().map(|(location, addr)| { + let srv = DnsRecord::Srv(dns_service_client::types::Srv { + prio: 0, + weight: 0, + port: addr.port(), + target: format!("{location}.scrimlet.{}", DNS_ZONE), + }); + (ServiceName::Scrimlet(location).dns_name(), vec![srv]) + }); + + let scrimlet_aaaa_records = + self.scrimlets.into_iter().map(|(location, addr)| { + let aaaa = DnsRecord::Aaaa(*addr.ip()); + (format!("{location}.scrimlet"), vec![aaaa]) + }); + // Assemble the set of SRV records, which implicitly point back at // zones' AAAA records. let srv_records_zones = self.service_instances_zones.into_iter().map( @@ -399,6 +429,8 @@ impl DnsConfigBuilder { .chain(zone_records) .chain(srv_records_sleds) .chain(srv_records_zones) + .chain(scrimlet_aaaa_records) + .chain(scrimlet_srv_records) .collect(); DnsConfigParams { diff --git a/internal-dns/src/names.rs b/internal-dns/src/names.rs index 44ed9228e2..e0c9b79555 100644 --- a/internal-dns/src/names.rs +++ b/internal-dns/src/names.rs @@ -4,6 +4,7 @@ //! Well-known DNS names and related types for internal DNS (see RFD 248) +use omicron_common::api::internal::shared::SwitchLocation; use uuid::Uuid; /// Name for the control plane DNS zone @@ -32,7 +33,9 @@ pub enum ServiceName { Crucible(Uuid), BoundaryNtp, InternalNtp, - Maghemite, + Maghemite, //TODO change to Dpd - maghemite has several services. + Mgd, + Scrimlet(SwitchLocation), } impl ServiceName { @@ -55,6 +58,8 @@ impl ServiceName { ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", ServiceName::Maghemite => "maghemite", + ServiceName::Mgd => "mgd", + ServiceName::Scrimlet(_) => "scrimlet", } } @@ -76,7 +81,8 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp - | ServiceName::Maghemite => { + | ServiceName::Maghemite + | ServiceName::Mgd => { format!("_{}._tcp", self.service_kind()) } ServiceName::SledAgent(id) => { @@ -85,6 +91,9 @@ impl ServiceName { ServiceName::Crucible(id) => { format!("_{}._tcp.{}", self.service_kind(), id) } + ServiceName::Scrimlet(location) => { + format!("_{location}._scrimlet._tcp") + } } } diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 3de6dac7c0..323386ba25 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -22,6 +22,7 @@ crucible-agent-client.workspace = true crucible-pantry-client.workspace = true dns-service-client.workspace = true dpd-client.workspace = true +mg-admin-client.workspace = true dropshot.workspace = true fatfs.workspace = true futures.workspace = true diff --git a/nexus/db-macros/src/lookup.rs b/nexus/db-macros/src/lookup.rs index 38cab15e30..f2362f5bc5 100644 --- a/nexus/db-macros/src/lookup.rs +++ b/nexus/db-macros/src/lookup.rs @@ -15,7 +15,7 @@ use std::ops::Deref; // INPUT (arguments to the macro) // -/// Arguments for [`lookup_resource!`] +/// Arguments for [`super::lookup_resource!`] // NOTE: this is only "pub" for the `cargo doc` link on [`lookup_resource!`]. #[derive(serde::Deserialize)] pub struct Input { @@ -167,7 +167,7 @@ impl Resource { // MACRO IMPLEMENTATION // -/// Implementation of [`lookup_resource!`] +/// Implementation of [`super::lookup_resource!`] pub fn lookup_resource( raw_input: TokenStream, ) -> Result { diff --git a/nexus/db-model/src/bgp.rs b/nexus/db-model/src/bgp.rs index 532b9cce36..cc9ebfb4f5 100644 --- a/nexus/db-model/src/bgp.rs +++ b/nexus/db-model/src/bgp.rs @@ -6,8 +6,10 @@ use crate::schema::{bgp_announce_set, bgp_announcement, bgp_config}; use crate::SqlU32; use db_macros::Resource; use ipnetwork::IpNetwork; +use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_common::api::external::IdentityMetadataCreateParams; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -26,6 +28,7 @@ pub struct BgpConfig { #[diesel(embed)] pub identity: BgpConfigIdentity, pub asn: SqlU32, + pub bgp_announce_set_id: Uuid, pub vrf: Option, } @@ -39,6 +42,26 @@ impl Into for BgpConfig { } } +impl BgpConfig { + pub fn from_config_create( + c: ¶ms::BgpConfigCreate, + bgp_announce_set_id: Uuid, + ) -> BgpConfig { + BgpConfig { + identity: BgpConfigIdentity::new( + Uuid::new_v4(), + IdentityMetadataCreateParams { + name: c.identity.name.clone(), + description: c.identity.description.clone(), + }, + ), + asn: c.asn.into(), + bgp_announce_set_id, + vrf: c.vrf.as_ref().map(|x| x.to_string()), + } + } +} + #[derive( Queryable, Insertable, @@ -55,6 +78,20 @@ pub struct BgpAnnounceSet { pub identity: BgpAnnounceSetIdentity, } +impl From for BgpAnnounceSet { + fn from(x: params::BgpAnnounceSetCreate) -> BgpAnnounceSet { + BgpAnnounceSet { + identity: BgpAnnounceSetIdentity::new( + Uuid::new_v4(), + IdentityMetadataCreateParams { + name: x.identity.name.clone(), + description: x.identity.description.clone(), + }, + ), + } + } +} + impl Into for BgpAnnounceSet { fn into(self) -> external::BgpAnnounceSet { external::BgpAnnounceSet { identity: self.identity() } diff --git a/nexus/db-model/src/bootstore.rs b/nexus/db-model/src/bootstore.rs new file mode 100644 index 0000000000..38afd37f54 --- /dev/null +++ b/nexus/db-model/src/bootstore.rs @@ -0,0 +1,13 @@ +use crate::schema::bootstore_keys; +use serde::{Deserialize, Serialize}; + +pub const NETWORK_KEY: &str = "network_key"; + +#[derive( + Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, +)] +#[diesel(table_name = bootstore_keys)] +pub struct BootstoreKeys { + pub key: String, + pub generation: i64, +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index f1447fc503..f399605f55 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -12,6 +12,7 @@ extern crate newtype_derive; mod address_lot; mod bgp; mod block_size; +mod bootstore; mod bytecount; mod certificate; mod collection; @@ -100,6 +101,7 @@ pub use self::unsigned::*; pub use address_lot::*; pub use bgp::*; pub use block_size::*; +pub use bootstore::*; pub use bytecount::*; pub use certificate::*; pub use collection::*; diff --git a/nexus/db-model/src/rack.rs b/nexus/db-model/src/rack.rs index 0f1ef2a853..580ec155b4 100644 --- a/nexus/db-model/src/rack.rs +++ b/nexus/db-model/src/rack.rs @@ -4,6 +4,7 @@ use crate::schema::rack; use db_macros::Asset; +use ipnetwork::IpNetwork; use nexus_types::{external_api::views, identity::Asset}; use uuid::Uuid; @@ -15,6 +16,7 @@ pub struct Rack { pub identity: RackIdentity, pub initialized: bool, pub tuf_base_url: Option, + pub rack_subnet: Option, } impl Rack { @@ -23,6 +25,7 @@ impl Rack { identity: RackIdentity::new(id), initialized: false, tuf_base_url: None, + rack_subnet: None, } } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 9189b6db7b..e079432e5a 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -144,6 +144,8 @@ table! { lldp_service_config_id -> Uuid, link_name -> Text, mtu -> Int4, + fec -> crate::SwitchLinkFecEnum, + speed -> crate::SwitchLinkSpeedEnum, } } @@ -188,7 +190,7 @@ table! { } table! { - switch_port_settings_route_config (port_settings_id, interface_name, dst, gw, vid) { + switch_port_settings_route_config (port_settings_id, interface_name, dst, gw) { port_settings_id -> Uuid, interface_name -> Text, dst -> Inet, @@ -200,10 +202,14 @@ table! { table! { switch_port_settings_bgp_peer_config (port_settings_id, interface_name, addr) { port_settings_id -> Uuid, - bgp_announce_set_id -> Uuid, bgp_config_id -> Uuid, interface_name -> Text, addr -> Inet, + hold_time -> Int8, + idle_hold_time -> Int8, + delay_open -> Int8, + connect_retry -> Int8, + keepalive -> Int8, } } @@ -216,6 +222,7 @@ table! { time_modified -> Timestamptz, time_deleted -> Nullable, asn -> Int8, + bgp_announce_set_id -> Uuid, vrf -> Nullable, } } @@ -673,6 +680,7 @@ table! { time_modified -> Timestamptz, initialized -> Bool, tuf_base_url -> Nullable, + rack_subnet -> Nullable, } } @@ -1132,6 +1140,13 @@ table! { } } +table! { + bootstore_keys (key, generation) { + key -> Text, + generation -> Int8, + } +} + table! { db_metadata (singleton) { singleton -> Bool, @@ -1147,7 +1162,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(7, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(8, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-model/src/service_kind.rs b/nexus/db-model/src/service_kind.rs index c2598434d5..4210c3ee20 100644 --- a/nexus/db-model/src/service_kind.rs +++ b/nexus/db-model/src/service_kind.rs @@ -30,6 +30,7 @@ impl_enum_type!( Oximeter => b"oximeter" Tfport => b"tfport" Ntp => b"ntp" + Mgd => b"mgd" ); impl TryFrom for ServiceUsingCertificate { @@ -88,6 +89,7 @@ impl From for ServiceKind { | internal_api::params::ServiceKind::InternalNtp => { ServiceKind::Ntp } + internal_api::params::ServiceKind::Mgd => ServiceKind::Mgd, } } } diff --git a/nexus/db-model/src/switch_interface.rs b/nexus/db-model/src/switch_interface.rs index 9ac7e4323a..f0c4b91de6 100644 --- a/nexus/db-model/src/switch_interface.rs +++ b/nexus/db-model/src/switch_interface.rs @@ -64,7 +64,14 @@ impl Into for DbSwitchInterfaceKind { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_vlan_interface_config)] pub struct SwitchVlanInterfaceConfig { diff --git a/nexus/db-model/src/switch_port.rs b/nexus/db-model/src/switch_port.rs index e9c0697450..f6df50ef97 100644 --- a/nexus/db-model/src/switch_port.rs +++ b/nexus/db-model/src/switch_port.rs @@ -2,7 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::impl_enum_type; use crate::schema::{ lldp_config, lldp_service_config, switch_port, switch_port_settings, switch_port_settings_address_config, switch_port_settings_bgp_peer_config, @@ -11,11 +10,14 @@ use crate::schema::{ switch_port_settings_port_config, switch_port_settings_route_config, }; use crate::SqlU16; +use crate::{impl_enum_type, SqlU32}; use db_macros::Resource; +use diesel::AsChangeset; use ipnetwork::IpNetwork; use nexus_types::external_api::params; use nexus_types::identity::Resource; use omicron_common::api::external; +use omicron_common::api::internal::shared::{PortFec, PortSpeed}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -42,6 +44,110 @@ impl_enum_type!( Sfp28x4 => b"Sfp28x4" ); +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_fec"))] + pub struct SwitchLinkFecEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkFecEnum)] + pub enum SwitchLinkFec; + + Firecode => b"Firecode" + None => b"None" + Rs => b"Rs" +); + +impl_enum_type!( + #[derive(SqlType, Debug, Clone, Copy)] + #[diesel(postgres_type(name = "switch_link_speed"))] + pub struct SwitchLinkSpeedEnum; + + #[derive( + Clone, + Copy, + Debug, + AsExpression, + FromSqlRow, + PartialEq, + Serialize, + Deserialize + )] + #[diesel(sql_type = SwitchLinkSpeedEnum)] + pub enum SwitchLinkSpeed; + + Speed0G => b"0G" + Speed1G => b"1G" + Speed10G => b"10G" + Speed25G => b"25G" + Speed40G => b"40G" + Speed50G => b"50G" + Speed100G => b"100G" + Speed200G => b"200G" + Speed400G => b"400G" +); + +impl From for PortFec { + fn from(value: SwitchLinkFec) -> Self { + match value { + SwitchLinkFec::Firecode => PortFec::Firecode, + SwitchLinkFec::None => PortFec::None, + SwitchLinkFec::Rs => PortFec::Rs, + } + } +} + +impl From for SwitchLinkFec { + fn from(value: params::LinkFec) -> Self { + match value { + params::LinkFec::Firecode => SwitchLinkFec::Firecode, + params::LinkFec::None => SwitchLinkFec::None, + params::LinkFec::Rs => SwitchLinkFec::Rs, + } + } +} + +impl From for PortSpeed { + fn from(value: SwitchLinkSpeed) -> Self { + match value { + SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G, + SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G, + SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G, + SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G, + SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G, + SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G, + SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G, + SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G, + SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G, + } + } +} + +impl From for SwitchLinkSpeed { + fn from(value: params::LinkSpeed) -> Self { + match value { + params::LinkSpeed::Speed0G => SwitchLinkSpeed::Speed0G, + params::LinkSpeed::Speed1G => SwitchLinkSpeed::Speed1G, + params::LinkSpeed::Speed10G => SwitchLinkSpeed::Speed10G, + params::LinkSpeed::Speed25G => SwitchLinkSpeed::Speed25G, + params::LinkSpeed::Speed40G => SwitchLinkSpeed::Speed40G, + params::LinkSpeed::Speed50G => SwitchLinkSpeed::Speed50G, + params::LinkSpeed::Speed100G => SwitchLinkSpeed::Speed100G, + params::LinkSpeed::Speed200G => SwitchLinkSpeed::Speed200G, + params::LinkSpeed::Speed400G => SwitchLinkSpeed::Speed400G, + } + } +} + impl From for SwitchPortGeometry { fn from(g: params::SwitchPortGeometry) -> Self { match g { @@ -225,7 +331,14 @@ impl Into for SwitchPortConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_link_config)] pub struct SwitchPortLinkConfig { @@ -233,6 +346,8 @@ pub struct SwitchPortLinkConfig { pub lldp_service_config_id: Uuid, pub link_name: String, pub mtu: SqlU16, + pub fec: SwitchLinkFec, + pub speed: SwitchLinkSpeed, } impl SwitchPortLinkConfig { @@ -241,11 +356,15 @@ impl SwitchPortLinkConfig { lldp_service_config_id: Uuid, link_name: String, mtu: u16, + fec: SwitchLinkFec, + speed: SwitchLinkSpeed, ) -> Self { Self { port_settings_id, lldp_service_config_id, link_name, + fec, + speed, mtu: mtu.into(), } } @@ -263,7 +382,14 @@ impl Into for SwitchPortLinkConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = lldp_service_config)] pub struct LldpServiceConfig { @@ -321,7 +447,14 @@ impl Into for LldpConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_interface_config)] pub struct SwitchInterfaceConfig { @@ -362,7 +495,14 @@ impl Into for SwitchInterfaceConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_route_config)] pub struct SwitchPortRouteConfig { @@ -398,31 +538,51 @@ impl Into for SwitchPortRouteConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_bgp_peer_config)] pub struct SwitchPortBgpPeerConfig { pub port_settings_id: Uuid, - pub bgp_announce_set_id: Uuid, pub bgp_config_id: Uuid, pub interface_name: String, pub addr: IpNetwork, + pub hold_time: SqlU32, + pub idle_hold_time: SqlU32, + pub delay_open: SqlU32, + pub connect_retry: SqlU32, + pub keepalive: SqlU32, } impl SwitchPortBgpPeerConfig { + #[allow(clippy::too_many_arguments)] pub fn new( port_settings_id: Uuid, - bgp_announce_set_id: Uuid, bgp_config_id: Uuid, interface_name: String, addr: IpNetwork, + hold_time: SqlU32, + idle_hold_time: SqlU32, + delay_open: SqlU32, + connect_retry: SqlU32, + keepalive: SqlU32, ) -> Self { Self { port_settings_id, - bgp_announce_set_id, bgp_config_id, interface_name, addr, + hold_time, + idle_hold_time, + delay_open, + connect_retry, + keepalive, } } } @@ -431,7 +591,6 @@ impl Into for SwitchPortBgpPeerConfig { fn into(self) -> external::SwitchPortBgpPeerConfig { external::SwitchPortBgpPeerConfig { port_settings_id: self.port_settings_id, - bgp_announce_set_id: self.bgp_announce_set_id, bgp_config_id: self.bgp_config_id, interface_name: self.interface_name.clone(), addr: self.addr.ip(), @@ -440,7 +599,14 @@ impl Into for SwitchPortBgpPeerConfig { } #[derive( - Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, + Queryable, + Insertable, + Selectable, + Clone, + Debug, + Serialize, + Deserialize, + AsChangeset, )] #[diesel(table_name = switch_port_settings_address_config)] pub struct SwitchPortAddressConfig { diff --git a/nexus/db-queries/src/db/datastore/bgp.rs b/nexus/db-queries/src/db/datastore/bgp.rs new file mode 100644 index 0000000000..ff314a2564 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/bgp.rs @@ -0,0 +1,351 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::error::TransactionError; +use crate::db::model::Name; +use crate::db::model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; +use crate::db::pagination::paginated; +use async_bb8_diesel::{AsyncConnection, AsyncRunQueryDsl}; +use chrono::Utc; +use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; +use nexus_types::external_api::params; +use nexus_types::identity::Resource; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::{ + CreateResult, DeleteResult, Error, ListResultVec, LookupResult, NameOrId, + ResourceType, +}; +use ref_cast::RefCast; +use uuid::Uuid; + +impl DataStore { + pub async fn bgp_config_set( + &self, + opctx: &OpContext, + config: ¶ms::BgpConfigCreate, + ) -> CreateResult { + use db::schema::bgp_config::dsl; + use db::schema::{ + bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, + }; + let pool = self.pool_connection_authorized(opctx).await?; + + pool.transaction_async(|conn| async move { + let id: Uuid = match &config.bgp_announce_set_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => *id, + }; + + let config = BgpConfig::from_config_create(config, id); + + let result = diesel::insert_into(dsl::bgp_config) + .values(config.clone()) + .returning(BgpConfig::as_returning()) + .get_result_async(&conn) + .await?; + Ok(result) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_config_delete( + &self, + opctx: &OpContext, + sel: ¶ms::BgpConfigSelector, + ) -> DeleteResult { + use db::schema::bgp_config; + use db::schema::bgp_config::dsl as bgp_config_dsl; + + use db::schema::switch_port_settings_bgp_peer_config as sps_bgp_peer_config; + use db::schema::switch_port_settings_bgp_peer_config::dsl as sps_bgp_peer_config_dsl; + + #[derive(Debug)] + enum BgpConfigDeleteError { + ConfigInUse, + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let name_or_id = sel.name_or_id.clone(); + + let id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => { + bgp_config_dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(bgp_config::id) + .limit(1) + .first_async::(&conn) + .await? + } + }; + + let count = + sps_bgp_peer_config_dsl::switch_port_settings_bgp_peer_config + .filter(sps_bgp_peer_config::bgp_config_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(TxnError::CustomError( + BgpConfigDeleteError::ConfigInUse, + )); + } + + diesel::update(bgp_config_dsl::bgp_config) + .filter(bgp_config_dsl::id.eq(id)) + .set(bgp_config_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| match e { + TxnError::CustomError(BgpConfigDeleteError::ConfigInUse) => { + Error::invalid_request("BGP config in use") + } + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn bgp_config_get( + &self, + opctx: &OpContext, + name_or_id: &NameOrId, + ) -> LookupResult { + use db::schema::bgp_config; + use db::schema::bgp_config::dsl; + let pool = self.pool_connection_authorized(opctx).await?; + + let name_or_id = name_or_id.clone(); + + let config = match name_or_id { + NameOrId::Name(name) => dsl::bgp_config + .filter(bgp_config::name.eq(name.to_string())) + .select(BgpConfig::as_select()) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), + NameOrId::Id(id) => dsl::bgp_config + .filter(bgp_config::id.eq(id)) + .select(BgpConfig::as_select()) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)), + }?; + + Ok(config) + } + + pub async fn bgp_config_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + use db::schema::bgp_config::dsl; + + let pool = self.pool_connection_authorized(opctx).await?; + + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::bgp_config, dsl::id, &pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::bgp_config, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::time_deleted.is_null()) + .select(BgpConfig::as_select()) + .load_async(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_announce_list( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> ListResultVec { + use db::schema::{ + bgp_announce_set, bgp_announce_set::dsl as announce_set_dsl, + bgp_announcement::dsl as announce_dsl, + }; + + #[derive(Debug)] + enum BgpAnnounceListError { + AnnounceSetNotFound(Name), + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let name_or_id = sel.name_or_id.clone(); + + let announce_id: Uuid = match name_or_id { + NameOrId::Id(id) => id, + NameOrId::Name(name) => announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::time_deleted.is_null()) + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await + .map_err(|_| { + TxnError::CustomError( + BgpAnnounceListError::AnnounceSetNotFound( + Name::from(name.clone()), + ), + ) + })?, + }; + + let result = announce_dsl::bgp_announcement + .filter(announce_dsl::announce_set_id.eq(announce_id)) + .select(BgpAnnouncement::as_select()) + .load_async(&conn) + .await?; + + Ok(result) + }) + .await + .map_err(|e| match e { + TxnError::CustomError( + BgpAnnounceListError::AnnounceSetNotFound(name), + ) => Error::not_found_by_name(ResourceType::BgpAnnounceSet, &name), + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn bgp_create_announce_set( + &self, + opctx: &OpContext, + announce: ¶ms::BgpAnnounceSetCreate, + ) -> CreateResult<(BgpAnnounceSet, Vec)> { + use db::schema::bgp_announce_set::dsl as announce_set_dsl; + use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; + + let pool = self.pool_connection_authorized(opctx).await?; + pool.transaction_async(|conn| async move { + let bas: BgpAnnounceSet = announce.clone().into(); + + let db_as: BgpAnnounceSet = + diesel::insert_into(announce_set_dsl::bgp_announce_set) + .values(bas.clone()) + .returning(BgpAnnounceSet::as_returning()) + .get_result_async::(&conn) + .await?; + + let mut db_annoucements = Vec::new(); + for a in &announce.announcement { + let an = BgpAnnouncement { + announce_set_id: db_as.id(), + address_lot_block_id: bas.identity.id, + network: a.network.into(), + }; + let an = + diesel::insert_into(bgp_announcement_dsl::bgp_announcement) + .values(an.clone()) + .returning(BgpAnnouncement::as_returning()) + .get_result_async::(&conn) + .await?; + db_annoucements.push(an); + } + + Ok((db_as, db_annoucements)) + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn bgp_delete_announce_set( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> DeleteResult { + use db::schema::bgp_announce_set; + use db::schema::bgp_announce_set::dsl as announce_set_dsl; + use db::schema::bgp_announcement::dsl as bgp_announcement_dsl; + + use db::schema::bgp_config; + use db::schema::bgp_config::dsl as bgp_config_dsl; + + #[derive(Debug)] + enum BgpAnnounceSetDeleteError { + AnnounceSetInUse, + } + type TxnError = TransactionError; + + let pool = self.pool_connection_authorized(opctx).await?; + let name_or_id = sel.name_or_id.clone(); + + pool.transaction_async(|conn| async move { + let id: Uuid = match name_or_id { + NameOrId::Name(name) => { + announce_set_dsl::bgp_announce_set + .filter(bgp_announce_set::name.eq(name.to_string())) + .select(bgp_announce_set::id) + .limit(1) + .first_async::(&conn) + .await? + } + NameOrId::Id(id) => id, + }; + + let count = bgp_config_dsl::bgp_config + .filter(bgp_config::bgp_announce_set_id.eq(id)) + .count() + .execute_async(&conn) + .await?; + + if count > 0 { + return Err(TxnError::CustomError( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + )); + } + + diesel::update(announce_set_dsl::bgp_announce_set) + .filter(announce_set_dsl::id.eq(id)) + .set(announce_set_dsl::time_deleted.eq(Utc::now())) + .execute_async(&conn) + .await?; + + diesel::delete(bgp_announcement_dsl::bgp_announcement) + .filter(bgp_announcement_dsl::announce_set_id.eq(id)) + .execute_async(&conn) + .await?; + + Ok(()) + }) + .await + .map_err(|e| match e { + TxnError::CustomError( + BgpAnnounceSetDeleteError::AnnounceSetInUse, + ) => Error::invalid_request("BGP announce set in use"), + TxnError::Database(e) => { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } +} diff --git a/nexus/db-queries/src/db/datastore/bootstore.rs b/nexus/db-queries/src/db/datastore/bootstore.rs new file mode 100644 index 0000000000..44f7a2036e --- /dev/null +++ b/nexus/db-queries/src/db/datastore/bootstore.rs @@ -0,0 +1,37 @@ +use super::DataStore; +use crate::context::OpContext; +use crate::db; +use crate::db::error::{public_error_from_diesel, ErrorHandler}; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::ExpressionMethods; +use diesel::SelectableHelper; +use nexus_db_model::BootstoreKeys; +use omicron_common::api::external::LookupResult; + +impl DataStore { + pub async fn bump_bootstore_generation( + &self, + opctx: &OpContext, + key: String, + ) -> LookupResult { + use db::schema::bootstore_keys; + use db::schema::bootstore_keys::dsl; + + let conn = self.pool_connection_authorized(opctx).await?; + + let bks = diesel::insert_into(dsl::bootstore_keys) + .values(BootstoreKeys { + key: key.clone(), + generation: 2, // RSS starts with a generation of 1 + }) + .on_conflict(bootstore_keys::key) + .do_update() + .set(bootstore_keys::generation.eq(dsl::generation + 1)) + .returning(BootstoreKeys::as_returning()) + .get_result_async(&*conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(bks.generation) + } +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index a77e20647a..f5283e263e 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -48,6 +48,8 @@ use std::sync::Arc; use uuid::Uuid; mod address_lot; +mod bgp; +mod bootstore; mod certificate; mod console_session; mod dataset; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index f5f7524aab..ae982d86f8 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -32,6 +32,7 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::Error as DieselError; use diesel::upsert::excluded; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::DnsZone; use nexus_db_model::ExternalIp; @@ -61,6 +62,7 @@ use uuid::Uuid; #[derive(Clone)] pub struct RackInit { pub rack_id: Uuid, + pub rack_subnet: IpNetwork, pub services: Vec, pub datasets: Vec, pub service_ip_pool_ranges: Vec, @@ -190,6 +192,28 @@ impl DataStore { }) } + pub async fn update_rack_subnet( + &self, + opctx: &OpContext, + rack: &Rack, + ) -> Result<(), Error> { + debug!( + opctx.log, + "updating rack subnet for rack {} to {:#?}", + rack.id(), + rack.rack_subnet + ); + use db::schema::rack::dsl; + diesel::update(dsl::rack) + .filter(dsl::id.eq(rack.id())) + .set(dsl::rack_subnet.eq(rack.rack_subnet)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + // The following methods which return a `TxnError` take a `conn` parameter // which comes from the transaction created in `rack_set_initialized`. @@ -681,6 +705,7 @@ mod test { fn default() -> Self { RackInit { rack_id: Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap(), + rack_subnet: nexus_test_utils::RACK_SUBNET.parse().unwrap(), services: vec![], datasets: vec![], service_ip_pool_ranges: vec![], diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 45be594be6..f2126bd968 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -97,41 +97,79 @@ pub struct SwitchPortSettingsGroupCreateResult { } impl DataStore { - // port settings + pub async fn switch_port_settings_exist( + &self, + opctx: &OpContext, + name: Name, + ) -> LookupResult { + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; + + let pool = self.pool_connection_authorized(opctx).await?; + + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::name.eq(name)) + .select(switch_port_settings::id) + .limit(1) + .first_async::(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn switch_ports_using_settings( + &self, + opctx: &OpContext, + switch_port_settings_id: Uuid, + ) -> LookupResult> { + use db::schema::switch_port::{self, dsl}; + + let pool = self.pool_connection_authorized(opctx).await?; + + dsl::switch_port + .filter(switch_port::port_settings_id.eq(switch_port_settings_id)) + .select((switch_port::id, switch_port::port_name)) + .load_async(&*pool) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } pub async fn switch_port_settings_create( &self, opctx: &OpContext, params: ¶ms::SwitchPortSettingsCreate, ) -> CreateResult { - use db::schema::address_lot::dsl as address_lot_dsl; - use db::schema::bgp_announce_set::dsl as bgp_announce_set_dsl; - use db::schema::bgp_config::dsl as bgp_config_dsl; - use db::schema::lldp_service_config::dsl as lldp_config_dsl; - use db::schema::switch_port_settings::dsl as port_settings_dsl; - use db::schema::switch_port_settings_address_config::dsl as address_config_dsl; - use db::schema::switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl; - use db::schema::switch_port_settings_interface_config::dsl as interface_config_dsl; - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; - use db::schema::switch_vlan_interface_config::dsl as vlan_config_dsl; + use db::schema::{ + address_lot::dsl as address_lot_dsl, + //XXX ANNOUNCE bgp_announce_set::dsl as bgp_announce_set_dsl, + bgp_config::dsl as bgp_config_dsl, + lldp_service_config::dsl as lldp_config_dsl, + switch_port_settings::dsl as port_settings_dsl, + switch_port_settings_address_config::dsl as address_config_dsl, + switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl, + switch_port_settings_interface_config::dsl as interface_config_dsl, + switch_port_settings_link_config::dsl as link_config_dsl, + switch_port_settings_port_config::dsl as port_config_dsl, + switch_port_settings_route_config::dsl as route_config_dsl, + switch_vlan_interface_config::dsl as vlan_config_dsl, + }; #[derive(Debug)] enum SwitchPortSettingsCreateError { AddressLotNotFound, - BgpAnnounceSetNotFound, + //XXX ANNOUNCE BgpAnnounceSetNotFound, BgpConfigNotFound, ReserveBlock(ReserveBlockError), } type TxnError = TransactionError; + type SpsCreateError = SwitchPortSettingsCreateError; let conn = self.pool_connection_authorized(opctx).await?; // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage conn.transaction_async(|conn| async move { - // create the top level port settings object let port_settings = SwitchPortSettings::new(¶ms.identity); let db_port_settings: SwitchPortSettings = @@ -189,6 +227,8 @@ impl DataStore { lldp_svc_config.id, link_name.clone(), c.mtu, + c.fec.into(), + c.speed.into(), )); } result.link_lldp = @@ -260,33 +300,6 @@ impl DataStore { let mut bgp_peer_config = Vec::new(); for (interface_name, p) in ¶ms.bgp_peers { - - // add the bgp peer - // TODO this requires pluming in the API to create - // - bgp configs - // - announce sets - // - announcements - - use db::schema::bgp_announce_set; - let announce_set_id = match &p.bgp_announce_set { - NameOrId::Id(id) => *id, - NameOrId::Name(name) => { - let name = name.to_string(); - bgp_announce_set_dsl::bgp_announce_set - .filter(bgp_announce_set::time_deleted.is_null()) - .filter(bgp_announce_set::name.eq(name)) - .select(bgp_announce_set::id) - .limit(1) - .first_async::(&conn) - .await - .map_err(|_| { - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpAnnounceSetNotFound, - ) - })? - } - }; - use db::schema::bgp_config; let bgp_config_id = match &p.bgp_config { NameOrId::Id(id) => *id, @@ -309,10 +322,14 @@ impl DataStore { bgp_peer_config.push(SwitchPortBgpPeerConfig::new( psid, - announce_set_id, bgp_config_id, interface_name.clone(), p.addr.into(), + p.hold_time.into(), + p.idle_hold_time.into(), + p.delay_open.into(), + p.connect_retry.into(), + p.keepalive.into(), )); } @@ -389,16 +406,10 @@ impl DataStore { }) .await .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpAnnounceSetNotFound) => { - Error::invalid_request("BGP announce set not found") - } - TxnError::CustomError( - SwitchPortSettingsCreateError::AddressLotNotFound) => { + TxnError::CustomError(SpsCreateError::AddressLotNotFound) => { Error::invalid_request("AddressLot not found") } - TxnError::CustomError( - SwitchPortSettingsCreateError::BgpConfigNotFound) => { + TxnError::CustomError(SpsCreateError::BgpConfigNotFound) => { Error::invalid_request("BGP config not found") } TxnError::CustomError( @@ -475,30 +486,31 @@ impl DataStore { .await?; // delete the port config object - use db::schema::switch_port_settings_port_config; - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; + use db::schema::switch_port_settings_port_config::{ + self as sps_port_config, dsl as port_config_dsl, + }; diesel::delete(port_config_dsl::switch_port_settings_port_config) - .filter(switch_port_settings_port_config::port_settings_id.eq(id)) + .filter(sps_port_config::port_settings_id.eq(id)) .execute_async(&conn) .await?; // delete the link configs - use db::schema::switch_port_settings_link_config; - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; + use db::schema::switch_port_settings_link_config::{ + self as sps_link_config, dsl as link_config_dsl, + }; let links: Vec = diesel::delete( link_config_dsl::switch_port_settings_link_config ) .filter( - switch_port_settings_link_config::port_settings_id.eq(id) + sps_link_config::port_settings_id.eq(id) ) .returning(SwitchPortLinkConfig::as_returning()) .get_results_async(&conn) .await?; // delete lldp configs - use db::schema::lldp_service_config; - use db::schema::lldp_service_config::dsl as lldp_config_dsl; + use db::schema::lldp_service_config::{self, dsl as lldp_config_dsl}; let lldp_svc_ids: Vec = links .iter() .map(|link| link.lldp_service_config_id) @@ -509,26 +521,25 @@ impl DataStore { .await?; // delete interface configs - use db::schema::switch_port_settings_interface_config; - use db::schema::switch_port_settings_interface_config::dsl - as interface_config_dsl; + use db::schema::switch_port_settings_interface_config::{ + self as sps_interface_config, dsl as interface_config_dsl, + }; let interfaces: Vec = diesel::delete( interface_config_dsl::switch_port_settings_interface_config ) .filter( - switch_port_settings_interface_config::port_settings_id.eq( - id - ) + sps_interface_config::port_settings_id.eq(id) ) .returning(SwitchInterfaceConfig::as_returning()) .get_results_async(&conn) .await?; // delete any vlan interfaces - use db::schema::switch_vlan_interface_config; - use db::schema::switch_vlan_interface_config::dsl as vlan_config_dsl; + use db::schema::switch_vlan_interface_config::{ + self, dsl as vlan_config_dsl, + }; let interface_ids: Vec = interfaces .iter() .map(|interface| interface.id) @@ -566,22 +577,26 @@ impl DataStore { .await?; // delete address configs - use db::schema::switch_port_settings_address_config as address_config; - use db::schema::switch_port_settings_address_config::dsl - as address_config_dsl; + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; - let ps = diesel::delete(address_config_dsl::switch_port_settings_address_config) - .filter(address_config::port_settings_id.eq(id)) - .returning(SwitchPortAddressConfig::as_returning()) - .get_result_async(&conn) - .await?; + let port_settings_addrs = diesel::delete( + address_config_dsl::switch_port_settings_address_config, + ) + .filter(address_config::port_settings_id.eq(id)) + .returning(SwitchPortAddressConfig::as_returning()) + .get_results_async(&conn) + .await?; use db::schema::address_lot_rsvd_block::dsl as rsvd_block_dsl; - diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) - .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) - .execute_async(&conn) - .await?; + for ps in &port_settings_addrs { + diesel::delete(rsvd_block_dsl::address_lot_rsvd_block) + .filter(rsvd_block_dsl::id.eq(ps.rsvd_address_lot_block_id)) + .execute_async(&conn) + .await?; + } Ok(()) }) @@ -650,10 +665,10 @@ impl DataStore { // TODO https://github.com/oxidecomputer/omicron/issues/2811 // Audit external networking database transaction usage conn.transaction_async(|conn| async move { - // get the top level port settings object - use db::schema::switch_port_settings::dsl as port_settings_dsl; - use db::schema::switch_port_settings; + use db::schema::switch_port_settings::{ + self, dsl as port_settings_dsl, + }; let id = match name_or_id { NameOrId::Id(id) => *id, @@ -668,23 +683,27 @@ impl DataStore { .await .map_err(|_| { TxnError::CustomError( - SwitchPortSettingsGetError::NotFound(name.clone()) + SwitchPortSettingsGetError::NotFound( + name.clone(), + ), ) })? } }; - let settings: SwitchPortSettings = port_settings_dsl::switch_port_settings - .filter(switch_port_settings::time_deleted.is_null()) - .filter(switch_port_settings::id.eq(id)) - .select(SwitchPortSettings::as_select()) - .limit(1) - .first_async::(&conn) - .await?; + let settings: SwitchPortSettings = + port_settings_dsl::switch_port_settings + .filter(switch_port_settings::time_deleted.is_null()) + .filter(switch_port_settings::id.eq(id)) + .select(SwitchPortSettings::as_select()) + .limit(1) + .first_async::(&conn) + .await?; // get the port config - use db::schema::switch_port_settings_port_config::dsl as port_config_dsl; - use db::schema::switch_port_settings_port_config as port_config; + use db::schema::switch_port_settings_port_config::{ + self as port_config, dsl as port_config_dsl, + }; let port: SwitchPortConfig = port_config_dsl::switch_port_settings_port_config .filter(port_config::port_settings_id.eq(id)) @@ -694,11 +713,13 @@ impl DataStore { .await?; // initialize result - let mut result = SwitchPortSettingsCombinedResult::new(settings, port); + let mut result = + SwitchPortSettingsCombinedResult::new(settings, port); // get the link configs - use db::schema::switch_port_settings_link_config::dsl as link_config_dsl; - use db::schema::switch_port_settings_link_config as link_config; + use db::schema::switch_port_settings_link_config::{ + self as link_config, dsl as link_config_dsl, + }; result.links = link_config_dsl::switch_port_settings_link_config .filter(link_config::port_settings_id.eq(id)) @@ -706,25 +727,25 @@ impl DataStore { .load_async::(&conn) .await?; - let lldp_svc_ids: Vec = result.links + let lldp_svc_ids: Vec = result + .links .iter() .map(|link| link.lldp_service_config_id) .collect(); - use db::schema::lldp_service_config::dsl as lldp_dsl; use db::schema::lldp_service_config as lldp_config; - result.link_lldp = - lldp_dsl::lldp_service_config - .filter(lldp_config::id.eq_any(lldp_svc_ids)) - .select(LldpServiceConfig::as_select()) - .limit(1) - .load_async::(&conn) - .await?; + use db::schema::lldp_service_config::dsl as lldp_dsl; + result.link_lldp = lldp_dsl::lldp_service_config + .filter(lldp_config::id.eq_any(lldp_svc_ids)) + .select(LldpServiceConfig::as_select()) + .limit(1) + .load_async::(&conn) + .await?; // get the interface configs - use db::schema::switch_port_settings_interface_config::dsl - as interface_config_dsl; - use db::schema::switch_port_settings_interface_config as interface_config; + use db::schema::switch_port_settings_interface_config::{ + self as interface_config, dsl as interface_config_dsl, + }; result.interfaces = interface_config_dsl::switch_port_settings_interface_config @@ -733,37 +754,35 @@ impl DataStore { .load_async::(&conn) .await?; - use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; use db::schema::switch_vlan_interface_config as vlan_config; - let interface_ids: Vec = result.interfaces + use db::schema::switch_vlan_interface_config::dsl as vlan_dsl; + let interface_ids: Vec = result + .interfaces .iter() .map(|interface| interface.id) .collect(); - result.vlan_interfaces = - vlan_dsl::switch_vlan_interface_config - .filter( - vlan_config::interface_config_id.eq_any(interface_ids) - ) + result.vlan_interfaces = vlan_dsl::switch_vlan_interface_config + .filter(vlan_config::interface_config_id.eq_any(interface_ids)) .select(SwitchVlanInterfaceConfig::as_select()) .load_async::(&conn) .await?; - // get the route configs - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; - use db::schema::switch_port_settings_route_config as route_config; + use db::schema::switch_port_settings_route_config::{ + self as route_config, dsl as route_config_dsl, + }; - result.routes = - route_config_dsl::switch_port_settings_route_config - .filter(route_config::port_settings_id.eq(id)) - .select(SwitchPortRouteConfig::as_select()) - .load_async::(&conn) - .await?; + result.routes = route_config_dsl::switch_port_settings_route_config + .filter(route_config::port_settings_id.eq(id)) + .select(SwitchPortRouteConfig::as_select()) + .load_async::(&conn) + .await?; // get the bgp peer configs - use db::schema::switch_port_settings_bgp_peer_config::dsl as bgp_peer_dsl; - use db::schema::switch_port_settings_bgp_peer_config as bgp_peer; + use db::schema::switch_port_settings_bgp_peer_config::{ + self as bgp_peer, dsl as bgp_peer_dsl, + }; result.bgp_peers = bgp_peer_dsl::switch_port_settings_bgp_peer_config @@ -773,9 +792,9 @@ impl DataStore { .await?; // get the address configs - use db::schema::switch_port_settings_address_config::dsl - as address_config_dsl; - use db::schema::switch_port_settings_address_config as address_config; + use db::schema::switch_port_settings_address_config::{ + self as address_config, dsl as address_config_dsl, + }; result.addresses = address_config_dsl::switch_port_settings_address_config @@ -785,14 +804,15 @@ impl DataStore { .await?; Ok(result) - }) .await .map_err(|e| match e { - TxnError::CustomError( - SwitchPortSettingsGetError::NotFound(name)) => { - Error::not_found_by_name(ResourceType::SwitchPortSettings, &name) - } + TxnError::CustomError(SwitchPortSettingsGetError::NotFound( + name, + )) => Error::not_found_by_name( + ResourceType::SwitchPortSettings, + &name, + ), TxnError::Database(e) => match e { DieselError::DatabaseError(_, _) => { let name = name_or_id.to_string(); @@ -803,7 +823,7 @@ impl DataStore { &name, ), ) - }, + } _ => public_error_from_diesel(e, ErrorHandler::Server), }, }) @@ -1083,8 +1103,10 @@ impl DataStore { &self, opctx: &OpContext, ) -> ListResultVec { - use db::schema::switch_port::dsl as switch_port_dsl; - use db::schema::switch_port_settings_route_config::dsl as route_config_dsl; + use db::schema::{ + switch_port::dsl as switch_port_dsl, + switch_port_settings_route_config::dsl as route_config_dsl, + }; switch_port_dsl::switch_port .filter(switch_port_dsl::port_settings_id.is_not_null()) diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs new file mode 100644 index 0000000000..e800d72bdd --- /dev/null +++ b/nexus/src/app/bgp.rs @@ -0,0 +1,162 @@ +use crate::app::authz; +use crate::external_api::params; +use nexus_db_model::{BgpAnnounceSet, BgpAnnouncement, BgpConfig}; +use nexus_db_queries::context::OpContext; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::{ + BgpImportedRouteIpv4, BgpPeerStatus, CreateResult, DeleteResult, Ipv4Net, + ListResultVec, LookupResult, NameOrId, +}; + +impl super::Nexus { + pub async fn bgp_config_set( + &self, + opctx: &OpContext, + config: ¶ms::BgpConfigCreate, + ) -> CreateResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = self.db_datastore.bgp_config_set(opctx, config).await?; + Ok(result) + } + + pub async fn bgp_config_get( + &self, + opctx: &OpContext, + name_or_id: NameOrId, + ) -> LookupResult { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_config_get(opctx, &name_or_id).await + } + + pub async fn bgp_config_list( + &self, + opctx: &OpContext, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_config_list(opctx, pagparams).await + } + + pub async fn bgp_config_delete( + &self, + opctx: &OpContext, + sel: ¶ms::BgpConfigSelector, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = self.db_datastore.bgp_config_delete(opctx, sel).await?; + Ok(result) + } + + pub async fn bgp_create_announce_set( + &self, + opctx: &OpContext, + announce: ¶ms::BgpAnnounceSetCreate, + ) -> CreateResult<(BgpAnnounceSet, Vec)> { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = + self.db_datastore.bgp_create_announce_set(opctx, announce).await?; + Ok(result) + } + + pub async fn bgp_announce_list( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + self.db_datastore.bgp_announce_list(opctx, sel).await + } + + pub async fn bgp_delete_announce_set( + &self, + opctx: &OpContext, + sel: ¶ms::BgpAnnounceSetSelector, + ) -> DeleteResult { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let result = + self.db_datastore.bgp_delete_announce_set(opctx, sel).await?; + Ok(result) + } + + pub async fn bgp_peer_status( + &self, + opctx: &OpContext, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let mut result = Vec::new(); + for (switch, client) in &self.mg_clients { + let router_info = match client.inner.get_routers().await { + Ok(result) => result.into_inner(), + Err(e) => { + error!( + self.log, + "failed to get routers from {switch}: {e}" + ); + continue; + } + }; + + for r in &router_info { + for (addr, info) in &r.peers { + let Ok(addr) = addr.parse() else { + continue; + }; + result.push(BgpPeerStatus { + switch: *switch, + addr, + local_asn: r.asn, + remote_asn: info.asn.unwrap_or(0), + state: info.state.into(), + state_duration_millis: info.duration_millis, + }); + } + } + } + Ok(result) + } + + pub async fn bgp_imported_routes_ipv4( + &self, + opctx: &OpContext, + sel: ¶ms::BgpRouteSelector, + ) -> ListResultVec { + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + let mut result = Vec::new(); + for (switch, client) in &self.mg_clients { + let imported: Vec = match client + .inner + .get_imported4(&mg_admin_client::types::GetImported4Request { + asn: sel.asn, + }) + .await + { + Ok(result) => result + .into_inner() + .into_iter() + .map(|x| BgpImportedRouteIpv4 { + switch: *switch, + prefix: Ipv4Net( + ipnetwork::Ipv4Network::new( + x.prefix.value, + x.prefix.length, + ) + .unwrap(), + ), + nexthop: x.nexthop, + id: x.id, + }) + .collect(), + Err(e) => { + error!( + self.log, + "failed to get BGP imported from {switch}: {e}" + ); + continue; + } + }; + + result.extend_from_slice(&imported); + } + Ok(result) + } +} diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 23ded83150..7db93a158a 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -20,6 +20,7 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use omicron_common::address::DENDRITE_PORT; +use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::api::external::Error; use omicron_common::api::internal::shared::SwitchLocation; @@ -34,6 +35,7 @@ use uuid::Uuid; // by resource. mod address_lot; pub(crate) mod background; +mod bgp; mod certificate; mod device_auth; mod disk; @@ -162,6 +164,9 @@ pub struct Nexus { /// Mapping of SwitchLocations to their respective Dendrite Clients dpd_clients: HashMap>, + /// Map switch location to maghemite admin clients. + mg_clients: HashMap>, + /// Background tasks background_tasks: background::BackgroundTasks, @@ -216,7 +221,13 @@ impl Nexus { let mut dpd_clients: HashMap> = HashMap::new(); - // Currently static dpd configuration mappings are still required for testing + let mut mg_clients: HashMap< + SwitchLocation, + Arc, + > = HashMap::new(); + + // Currently static dpd configuration mappings are still required for + // testing for (location, config) in &config.pkg.dendrite { let address = config.address.ip().to_string(); let port = config.address.port(); @@ -226,6 +237,11 @@ impl Nexus { ); dpd_clients.insert(*location, Arc::new(dpd_client)); } + for (location, config) in &config.pkg.mgd { + let mg_client = mg_admin_client::Client::new(&log, config.address) + .map_err(|e| format!("mg admin client: {e}"))?; + mg_clients.insert(*location, Arc::new(mg_client)); + } if config.pkg.dendrite.is_empty() { loop { let result = resolver @@ -259,6 +275,42 @@ impl Nexus { } } } + if config.pkg.mgd.is_empty() { + loop { + let result = resolver + // TODO this should be ServiceName::Mgd, but in the upgrade + // path, that does not exist because RSS has not + // created it. So we just piggyback on Dendrite's SRV + // record. + .lookup_all_ipv6(ServiceName::Dendrite) + .await + .map_err(|e| format!("Cannot lookup mgd addresses: {e}")); + match result { + Ok(addrs) => { + let mappings = map_switch_zone_addrs( + &log.new(o!("component" => "Nexus")), + addrs, + ) + .await; + for (location, addr) in &mappings { + let port = MGD_PORT; + let mgd_client = mg_admin_client::Client::new( + &log, + std::net::SocketAddr::new((*addr).into(), port), + ) + .map_err(|e| format!("mg admin client: {e}"))?; + mg_clients.insert(*location, Arc::new(mgd_client)); + } + break; + } + Err(e) => { + warn!(log, "Failed to lookup mgd address: {e}"); + tokio::time::sleep(std::time::Duration::from_secs(1)) + .await; + } + } + } + } // Connect to clickhouse - but do so lazily. // Clickhouse may not be executing when Nexus starts. @@ -343,6 +395,7 @@ impl Nexus { .external_dns_servers .clone(), dpd_clients, + mg_clients, background_tasks, default_region_allocation_strategy: config .pkg @@ -352,6 +405,12 @@ impl Nexus { // TODO-cleanup all the extra Arcs here seems wrong let nexus = Arc::new(nexus); + let bootstore_opctx = OpContext::for_background( + log.new(o!("component" => "Bootstore")), + Arc::clone(&authz), + authn::Context::internal_api(), + Arc::clone(&db_datastore), + ); let opctx = OpContext::for_background( log.new(o!("component" => "SagaRecoverer")), Arc::clone(&authz), @@ -391,6 +450,12 @@ impl Nexus { for task in task_nexus.background_tasks.driver.tasks() { task_nexus.background_tasks.driver.activate(task); } + if let Err(e) = task_nexus + .initial_bootstore_sync(&bootstore_opctx) + .await + { + error!(task_log, "failed to run bootstore sync: {e}"); + } } Err(_) => { error!(task_log, "populate failed"); diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 3ac4b9063d..907c3ffa78 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -5,11 +5,14 @@ //! Rack management use super::silo::silo_dns_name; +use crate::external_api::params; use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; +use ipnetwork::IpNetwork; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -33,17 +36,21 @@ use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; -use omicron_common::api::external::IpNet; -use omicron_common::api::external::Ipv4Net; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use omicron_common::api::internal::shared::ExternalPortDiscovery; +use sled_agent_client::types::EarlyNetworkConfigBody; +use sled_agent_client::types::{ + BgpConfig, BgpPeerConfig, EarlyNetworkConfig, PortConfigV1, + RackNetworkConfigV1, RouteConfig as SledRouteConfig, +}; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; +use std::net::Ipv4Addr; use std::str::FromStr; use uuid::Uuid; @@ -188,10 +195,18 @@ impl super::Nexus { mapped_fleet_roles, }; + let rack_network_config = request.rack_network_config.as_ref().ok_or( + Error::InvalidRequest { + message: "cannot initialize a rack without a network config" + .into(), + }, + )?; + self.db_datastore .rack_set_initialized( opctx, RackInit { + rack_subnet: rack_network_config.rack_subnet.into(), rack_id, services: request.services, datasets, @@ -380,7 +395,7 @@ impl super::Nexus { })?; for (idx, uplink_config) in - rack_network_config.uplinks.iter().enumerate() + rack_network_config.ports.iter().enumerate() { let switch = uplink_config.switch.to_string(); let switch_location = Name::from_str(&switch).map_err(|e| { @@ -449,31 +464,32 @@ impl super::Nexus { addresses: HashMap::new(), }; - let uplink_address = - IpNet::V4(Ipv4Net(uplink_config.uplink_cidr)); - let address = Address { - address_lot: NameOrId::Name(address_lot_name.clone()), - address: uplink_address, - }; - port_settings_params.addresses.insert( - "phy0".to_string(), - AddressConfig { addresses: vec![address] }, - ); - - let dst = IpNet::from_str("0.0.0.0/0").map_err(|e| { - Error::internal_error(&format!( - "failed to parse provided default route CIDR: {e}" - )) - })?; - - let gw = IpAddr::V4(uplink_config.gateway_ip); - let vid = uplink_config.uplink_vid; - let route = Route { dst, gw, vid }; - - port_settings_params.routes.insert( - "phy0".to_string(), - RouteConfig { routes: vec![route] }, - ); + let addresses: Vec
= uplink_config + .addresses + .iter() + .map(|a| Address { + address_lot: NameOrId::Name(address_lot_name.clone()), + address: (*a).into(), + }) + .collect(); + + port_settings_params + .addresses + .insert("phy0".to_string(), AddressConfig { addresses }); + + let routes: Vec = uplink_config + .routes + .iter() + .map(|r| Route { + dst: r.destination.into(), + gw: r.nexthop, + vid: None, + }) + .collect(); + + port_settings_params + .routes + .insert("phy0".to_string(), RouteConfig { routes }); match self .db_datastore @@ -498,9 +514,7 @@ impl super::Nexus { opctx, rack_id, switch_location.into(), - Name::from_str(&uplink_config.uplink_port) - .unwrap() - .into(), + Name::from_str(&uplink_config.port).unwrap().into(), ) .await?; @@ -515,6 +529,7 @@ impl super::Nexus { } // TODO - https://github.com/oxidecomputer/omicron/issues/3277 // record port speed }; + self.initial_bootstore_sync(&opctx).await?; Ok(()) } @@ -548,4 +563,153 @@ impl super::Nexus { tokio::time::sleep(std::time::Duration::from_secs(2)).await; } } + + pub(crate) async fn initial_bootstore_sync( + &self, + opctx: &OpContext, + ) -> Result<(), Error> { + let mut rack = self.rack_lookup(opctx, &self.rack_id).await?; + if rack.rack_subnet.is_some() { + return Ok(()); + } + let addr = self + .sled_list(opctx, &DataPageParams::max_page()) + .await? + .get(0) + .ok_or(Error::InternalError { + internal_message: "no sleds at time of bootstore sync".into(), + })? + .address(); + + let sa = sled_agent_client::Client::new( + &format!("http://{}", addr), + self.log.clone(), + ); + + let result = sa + .read_network_bootstore_config_cache() + .await + .map_err(|e| Error::InternalError { + internal_message: format!("read bootstore network config: {e}"), + })? + .into_inner(); + + rack.rack_subnet = + result.body.rack_network_config.map(|x| x.rack_subnet.into()); + + self.datastore().update_rack_subnet(opctx, &rack).await?; + + Ok(()) + } + + pub(crate) async fn bootstore_network_config( + &self, + opctx: &OpContext, + ) -> Result { + let rack = self.rack_lookup(opctx, &self.rack_id).await?; + + let subnet = match rack.rack_subnet { + Some(IpNetwork::V6(subnet)) => subnet, + Some(IpNetwork::V4(_)) => { + return Err(Error::InternalError { + internal_message: "rack subnet not IPv6".into(), + }) + } + None => { + return Err(Error::InternalError { + internal_message: "rack subnet not set".into(), + }) + } + }; + + let db_ports = self.active_port_settings(opctx).await?; + let mut ports = Vec::new(); + let mut bgp = Vec::new(); + for (port, info) in &db_ports { + let mut peer_info = Vec::new(); + for p in &info.bgp_peers { + let bgp_config = + self.bgp_config_get(&opctx, p.bgp_config_id.into()).await?; + let announcements = self + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: bgp_config.bgp_announce_set_id.into(), + }, + ) + .await?; + let addr = match p.addr { + ipnetwork::IpNetwork::V4(addr) => addr, + ipnetwork::IpNetwork::V6(_) => continue, //TODO v6 + }; + peer_info.push((p, bgp_config.asn.0, addr.ip())); + bgp.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let p = PortConfigV1 { + routes: info + .routes + .iter() + .map(|r| SledRouteConfig { + destination: r.dst, + nexthop: r.gw.ip(), + }) + .collect(), + addresses: info.addresses.iter().map(|a| a.address).collect(), + bgp_peers: peer_info + .iter() + .map(|(_p, asn, addr)| BgpPeerConfig { + addr: *addr, + asn: *asn, + port: port.port_name.clone(), + }) + .collect(), + switch: port.switch_location.parse().unwrap(), + port: port.port_name.clone(), + uplink_port_fec: info + .links + .get(0) //TODO breakout support + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: info + .links + .get(0) //TODO breakout support + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), + }; + + ports.push(p); + } + + let result = EarlyNetworkConfig { + generation: 0, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: Vec::new(), //TODO + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: subnet, + //TODO(ry) you are here. We need to remove these too. They are + // inconsistent with a generic set of addresses on ports. + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports, + bgp, + }), + }, + }; + + Ok(result) + } } diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs index 0ae17c7237..83e0e9b8b4 100644 --- a/nexus/src/app/sagas/mod.rs +++ b/nexus/src/app/sagas/mod.rs @@ -36,7 +36,6 @@ pub mod snapshot_create; pub mod snapshot_delete; pub mod switch_port_settings_apply; pub mod switch_port_settings_clear; -pub mod switch_port_settings_update; pub mod test_saga; pub mod volume_delete; pub mod volume_remove_rop; diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index 687613f0cc..93dc45751a 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -7,6 +7,7 @@ use crate::app::sagas::retry_until_known_result; use crate::app::sagas::{ declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError, }; +use crate::Nexus; use anyhow::Error; use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::{ @@ -15,18 +16,37 @@ use dpd_client::types::{ }; use dpd_client::{Ipv4Cidr, Ipv6Cidr}; use ipnetwork::IpNetwork; +use mg_admin_client::types::Prefix4; +use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, BgpRoute}; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed, NETWORK_KEY}; +use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::{authn, db}; -use omicron_common::api::external::{self, NameOrId}; -use omicron_common::api::internal::shared::SwitchLocation; +use nexus_types::external_api::params; +use omicron_common::api::external::{self, DataPageParams, NameOrId}; +use omicron_common::api::internal::shared::{ + ParseSwitchLocationError, SwitchLocation, +}; use serde::{Deserialize, Serialize}; +use sled_agent_client::types::PortConfigV1; +use sled_agent_client::types::RouteConfig; +use sled_agent_client::types::{BgpConfig, EarlyNetworkConfig}; +use sled_agent_client::types::{ + BgpPeerConfig as OmicronBgpPeerConfig, HostPortConfig, +}; use std::collections::HashMap; use std::net::IpAddr; +use std::net::SocketAddrV6; use std::str::FromStr; use std::sync::Arc; use steno::ActionError; use uuid::Uuid; +// This is more of an implementation detail of the BGP implementation. It +// defines the maximum time the peering engine will wait for external messages +// before breaking to check for shutdown conditions. +const BGP_SESSION_RESOLUTION: u64 = 100; + // switch port settings apply saga: input parameters #[derive(Debug, Deserialize, Serialize)] @@ -52,6 +72,18 @@ declare_saga_actions! { + spa_ensure_switch_port_settings - spa_undo_ensure_switch_port_settings } + ENSURE_SWITCH_PORT_UPLINK -> "ensure_switch_port_uplink" { + + spa_ensure_switch_port_uplink + - spa_undo_ensure_switch_port_uplink + } + ENSURE_SWITCH_PORT_BGP_SETTINGS -> "ensure_switch_port_bgp_settings" { + + spa_ensure_switch_port_bgp_settings + - spa_undo_ensure_switch_port_bgp_settings + } + ENSURE_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "ensure_switch_port_bootstore_network_settings" { + + spa_ensure_switch_port_bootstore_network_settings + - spa_undo_ensure_switch_port_bootstore_network_settings + } } // switch port settings apply saga: definition @@ -74,6 +106,9 @@ impl NexusSaga for SagaSwitchPortSettingsApply { builder.append(associate_switch_port_action()); builder.append(get_switch_port_settings_action()); builder.append(ensure_switch_port_settings_action()); + builder.append(ensure_switch_port_uplink_action()); + builder.append(ensure_switch_port_bgp_settings_action()); + builder.append(ensure_switch_port_bootstore_network_settings_action()); Ok(builder.build()?) } } @@ -91,10 +126,10 @@ async fn spa_associate_switch_port( ); // first get the current association so we fall back to this on failure - let port = nexus - .get_switch_port(&opctx, params.switch_port_id) - .await - .map_err(ActionError::action_failed)?; + let port = + nexus.get_switch_port(&opctx, params.switch_port_id).await.map_err( + |e| ActionError::action_failed(format!("get switch port: {e}")), + )?; // update the switch port settings association nexus @@ -105,7 +140,11 @@ async fn spa_associate_switch_port( UpdatePrecondition::DontCare, ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "set switch port settings id {e}" + )) + })?; Ok(port.port_settings_id) } @@ -127,7 +166,9 @@ async fn spa_get_switch_port_settings( &NameOrId::Id(params.switch_port_settings_id), ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!("get switch port settings: {e}")) + })?; Ok(port_settings) } @@ -142,22 +183,42 @@ pub(crate) fn api_to_dpd_port_settings( v6_routes: HashMap::new(), }; - // TODO handle breakouts - // https://github.com/oxidecomputer/omicron/issues/3062 + //TODO breakouts let link_id = LinkId(0); - let link_settings = LinkSettings { - // TODO Allow user to configure link properties - // https://github.com/oxidecomputer/omicron/issues/3061 - params: LinkCreate { - autoneg: false, - kr: false, - fec: PortFec::None, - speed: PortSpeed::Speed100G, - }, - addrs: settings.addresses.iter().map(|a| a.address.ip()).collect(), - }; - dpd_port_settings.links.insert(link_id.to_string(), link_settings); + for l in settings.links.iter() { + dpd_port_settings.links.insert( + link_id.to_string(), + LinkSettings { + params: LinkCreate { + autoneg: false, + kr: false, + fec: match l.fec { + SwitchLinkFec::Firecode => PortFec::Firecode, + SwitchLinkFec::Rs => PortFec::Rs, + SwitchLinkFec::None => PortFec::None, + }, + speed: match l.speed { + SwitchLinkSpeed::Speed0G => PortSpeed::Speed0G, + SwitchLinkSpeed::Speed1G => PortSpeed::Speed1G, + SwitchLinkSpeed::Speed10G => PortSpeed::Speed10G, + SwitchLinkSpeed::Speed25G => PortSpeed::Speed25G, + SwitchLinkSpeed::Speed40G => PortSpeed::Speed40G, + SwitchLinkSpeed::Speed50G => PortSpeed::Speed50G, + SwitchLinkSpeed::Speed100G => PortSpeed::Speed100G, + SwitchLinkSpeed::Speed200G => PortSpeed::Speed200G, + SwitchLinkSpeed::Speed400G => PortSpeed::Speed400G, + }, + }, + //TODO won't work for breakouts + addrs: settings + .addresses + .iter() + .map(|a| a.address.ip()) + .collect(), + }, + ); + } for r in &settings.routes { match &r.dst { @@ -214,20 +275,28 @@ async fn spa_ensure_switch_port_settings( let settings = sagactx .lookup::("switch_port_settings")?; - let port_id: PortId = PortId::from_str(¶ms.switch_port_name) - .map_err(|e| ActionError::action_failed(e.to_string()))?; + let port_id: PortId = + PortId::from_str(¶ms.switch_port_name).map_err(|e| { + ActionError::action_failed(format!("parse port id: {e}")) + })?; let dpd_client: Arc = select_dendrite_client(&sagactx).await?; - let dpd_port_settings = api_to_dpd_port_settings(&settings) - .map_err(ActionError::action_failed)?; + let dpd_port_settings = + api_to_dpd_port_settings(&settings).map_err(|e| { + ActionError::action_failed(format!( + "translate api port settings to dpd port settings: {e}", + )) + })?; retry_until_known_result(log, || async { dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await }) .await - .map_err(|e| ActionError::action_failed(e.to_string()))?; + .map_err(|e| { + ActionError::action_failed(format!("dpd port settings apply {e}")) + })?; Ok(()) } @@ -270,10 +339,16 @@ async fn spa_undo_ensure_switch_port_settings( let settings = nexus .switch_port_settings_get(&opctx, &NameOrId::Id(id)) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!("switch port settings get: {e}")) + })?; - let dpd_port_settings = api_to_dpd_port_settings(&settings) - .map_err(ActionError::action_failed)?; + let dpd_port_settings = + api_to_dpd_port_settings(&settings).map_err(|e| { + ActionError::action_failed(format!( + "translate api to dpd port settings {e}" + )) + })?; retry_until_known_result(log, || async { dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await @@ -284,6 +359,326 @@ async fn spa_undo_ensure_switch_port_settings( Ok(()) } +async fn spa_ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let settings = sagactx + .lookup::("switch_port_settings") + .map_err(|e| { + ActionError::action_failed(format!( + "lookup switch port settings: {e}" + )) + })?; + + ensure_switch_port_bgp_settings(sagactx, settings).await +} + +pub(crate) async fn ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, + settings: SwitchPortSettingsCombinedResult, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client: {e}")) + })?; + + let mut bgp_peer_configs = Vec::new(); + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("get bgp config: {e}")) + })?; + + let announcements = nexus + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: NameOrId::Id(config.bgp_announce_set_id), + }, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get bgp announcements: {e}" + )) + })?; + + // TODO picking the first configured address by default, but this needs + // to be something that can be specified in the API. + let nexthop = match settings.addresses.get(0) { + Some(switch_port_addr) => Ok(switch_port_addr.address.ip()), + None => Err(ActionError::action_failed( + "at least one address required for bgp peering".to_string(), + )), + }?; + + let nexthop = match nexthop { + IpAddr::V4(nexthop) => Ok(nexthop), + IpAddr::V6(_) => Err(ActionError::action_failed( + "IPv6 nexthop not yet supported".to_string(), + )), + }?; + + let mut prefixes = Vec::new(); + for a in &announcements { + let value = match a.network.ip() { + IpAddr::V4(value) => Ok(value), + IpAddr::V6(_) => Err(ActionError::action_failed( + "IPv6 announcement not yet supported".to_string(), + )), + }?; + prefixes.push(Prefix4 { value, length: a.network.prefix() }); + } + + let bpc = BgpPeerConfig { + asn: *config.asn, + name: format!("{}", peer.addr.ip()), //TODO user defined name? + host: format!("{}:179", peer.addr.ip()), + hold_time: peer.hold_time.0.into(), + idle_hold_time: peer.idle_hold_time.0.into(), + delay_open: peer.delay_open.0.into(), + connect_retry: peer.connect_retry.0.into(), + keepalive: peer.keepalive.0.into(), + resolution: BGP_SESSION_RESOLUTION, + routes: vec![BgpRoute { nexthop, prefixes }], + }; + + bgp_peer_configs.push(bpc); + } + + mg_client + .inner + .bgp_apply(&ApplyRequest { + peer_group: params.switch_port_name.clone(), + peers: bgp_peer_configs, + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("apply bgp settings: {e}")) + })?; + + Ok(()) +} +async fn spa_undo_ensure_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + use mg_admin_client::types::DeleteNeighborRequest; + + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let settings = sagactx + .lookup::("switch_port_settings") + .map_err(|e| { + ActionError::action_failed(format!( + "lookup switch port settings (bgp undo): {e}" + )) + })?; + + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client (undo): {e}")) + })?; + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete bgp config: {e}")) + })?; + + mg_client + .inner + .delete_neighbor(&DeleteNeighborRequest { + asn: *config.asn, + addr: peer.addr.ip(), + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete neighbor: {e}")) + })?; + } + + Ok(()) +} + +async fn spa_ensure_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let mut config = + nexus.bootstore_network_config(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "read nexus bootstore network config: {e}" + )) + })?; + + let generation = nexus + .datastore() + .bump_bootstore_generation(&opctx, NETWORK_KEY.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "bump bootstore network generation number: {e}" + )) + })?; + + config.generation = generation as u64; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_undo_ensure_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + // The overall saga update failed but the bootstore udpate succeeded. + // Between now and then other updates may have happened which prevent us + // from simply undoing the changes we did before, as we may inadvertently + // roll back changes at the intersection of this failed update and other + // succesful updates. The only thing we can really do here is attempt a + // complete update of the bootstore network settings based on the current + // state in the Nexus databse which, we assume to be consistent at any point + // in time. + + let nexus = sagactx.user_data().nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let config = nexus.bootstore_network_config(&opctx).await?; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_ensure_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + ensure_switch_port_uplink(sagactx, false, None).await +} + +async fn spa_undo_ensure_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), Error> { + Ok(ensure_switch_port_uplink(sagactx, true, None).await?) +} + +pub(crate) async fn ensure_switch_port_uplink( + sagactx: NexusActionContext, + skip_self: bool, + inject: Option, +) -> Result<(), ActionError> { + let params = sagactx.saga_params::()?; + + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let osagactx = sagactx.user_data(); + let nexus = osagactx.nexus(); + + let switch_port = nexus + .get_switch_port(&opctx, params.switch_port_id) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for uplink: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err(|e| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + })?; + + let mut uplinks: Vec = Vec::new(); + + // The sled agent uplinks interface is an all or nothing interface, so we + // need to get all the uplink configs for all the ports. + let active_ports = + nexus.active_port_settings(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "get active switch port settings: {e}" + )) + })?; + + for (port, info) in &active_ports { + // Since we are undoing establishing uplinks for the settings + // associated with this port we skip adding this ports uplinks + // to the list - effectively removing them. + if skip_self && port.id == switch_port.id { + continue; + } + uplinks.push(HostPortConfig { + port: port.port_name.clone(), + addrs: info.addresses.iter().map(|a| a.address).collect(), + }) + } + + if let Some(id) = inject { + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let settings = nexus + .switch_port_settings_get(&opctx, &id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port settings for injection: {e}" + )) + })?; + uplinks.push(HostPortConfig { + port: params.switch_port_name.clone(), + addrs: settings.addresses.iter().map(|a| a.address).collect(), + }) + } + + let sc = switch_sled_agent(switch_location, &sagactx).await?; + sc.uplink_ensure(&sled_agent_client::types::SwitchPorts { uplinks }) + .await + .map_err(|e| { + ActionError::action_failed(format!("ensure uplink: {e}")) + })?; + + Ok(()) +} + // a common route representation for dendrite and port settings #[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] pub(crate) struct Route { @@ -316,7 +711,11 @@ async fn spa_disassociate_switch_port( UpdatePrecondition::Value(params.switch_port_settings_id), ) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "set switch port settings id for disassociate: {e}" + )) + })?; Ok(()) } @@ -335,12 +734,21 @@ pub(crate) async fn select_dendrite_client( let switch_port = nexus .get_switch_port(&opctx, params.switch_port_id) .await - .map_err(ActionError::action_failed)?; + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for dendrite client selection {e}" + )) + })?; + let switch_location: SwitchLocation = - switch_port - .switch_location - .parse() - .map_err(ActionError::action_failed)?; + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + let dpd_client: Arc = osagactx .nexus() .dpd_clients @@ -353,3 +761,283 @@ pub(crate) async fn select_dendrite_client( .clone(); Ok(dpd_client) } + +pub(crate) async fn select_mg_client( + sagactx: &NexusActionContext, +) -> Result, ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let switch_port = nexus + .get_switch_port(&opctx, params.switch_port_id) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get switch port for mg client selection: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + + let mg_client: Arc = osagactx + .nexus() + .mg_clients + .get(&switch_location) + .ok_or_else(|| { + ActionError::action_failed(format!( + "requested switch not available: {switch_location}" + )) + })? + .clone(); + Ok(mg_client) +} + +pub(crate) async fn get_scrimlet_address( + _location: SwitchLocation, + nexus: &Arc, +) -> Result { + /* TODO this depends on DNS entries only coming from RSS, it's broken + on the upgrade path + nexus + .resolver() + .await + .lookup_socket_v6(ServiceName::Scrimlet(location)) + .await + .map_err(|e| e.to_string()) + .map_err(|e| { + ActionError::action_failed(format!( + "scrimlet dns lookup failed {e}", + )) + }) + */ + let opctx = &nexus.opctx_for_internal_api(); + Ok(nexus + .sled_list(opctx, &DataPageParams::max_page()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get_scrimlet_address: failed to list sleds: {e}" + )) + })? + .into_iter() + .find(|x| x.is_scrimlet()) + .ok_or(ActionError::action_failed( + "get_scrimlet_address: no scrimlets found".to_string(), + ))? + .address()) +} + +#[derive(Clone, Debug)] +pub struct EarlyNetworkPortUpdate { + port: PortConfigV1, + bgp_configs: Vec, +} + +pub(crate) async fn bootstore_update( + nexus: &Arc, + opctx: &OpContext, + switch_port_id: Uuid, + switch_port_name: &str, + settings: &SwitchPortSettingsCombinedResult, +) -> Result { + let switch_port = + nexus.get_switch_port(&opctx, switch_port_id).await.map_err(|e| { + ActionError::action_failed(format!( + "get switch port for uplink: {e}" + )) + })?; + + let switch_location: SwitchLocation = + switch_port.switch_location.parse().map_err( + |e: ParseSwitchLocationError| { + ActionError::action_failed(format!( + "get switch location for uplink: {e:?}", + )) + }, + )?; + + let mut peer_info = Vec::new(); + let mut bgp_configs = Vec::new(); + for p in &settings.bgp_peers { + let bgp_config = nexus + .bgp_config_get(&opctx, p.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("get bgp config: {e}")) + })?; + + let announcements = nexus + .bgp_announce_list( + &opctx, + ¶ms::BgpAnnounceSetSelector { + name_or_id: NameOrId::Id(bgp_config.bgp_announce_set_id), + }, + ) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "get bgp announcements: {e}" + )) + })?; + + peer_info.push((p, bgp_config.asn.0)); + bgp_configs.push(BgpConfig { + asn: bgp_config.asn.0, + originate: announcements + .iter() + .filter_map(|a| match a.network { + IpNetwork::V4(net) => Some(net.into()), + //TODO v6 + _ => None, + }) + .collect(), + }); + } + + let update = EarlyNetworkPortUpdate { + port: PortConfigV1 { + routes: settings + .routes + .iter() + .map(|r| RouteConfig { destination: r.dst, nexthop: r.gw.ip() }) + .collect(), + addresses: settings.addresses.iter().map(|a| a.address).collect(), + switch: switch_location, + port: switch_port_name.into(), + uplink_port_fec: settings + .links + .get(0) + .map(|l| l.fec) + .unwrap_or(SwitchLinkFec::None) + .into(), + uplink_port_speed: settings + .links + .get(0) + .map(|l| l.speed) + .unwrap_or(SwitchLinkSpeed::Speed100G) + .into(), + bgp_peers: peer_info + .iter() + .filter_map(|(p, asn)| { + //TODO v6 + match p.addr.ip() { + IpAddr::V4(addr) => Some(OmicronBgpPeerConfig { + asn: *asn, + port: switch_port_name.into(), + addr, + }), + IpAddr::V6(_) => { + warn!(opctx.log, "IPv6 peers not yet supported"); + None + } + } + }) + .collect(), + }, + bgp_configs, + }; + + Ok(update) +} + +pub(crate) async fn read_bootstore_config( + sa: &sled_agent_client::Client, +) -> Result { + Ok(sa + .read_network_bootstore_config_cache() + .await + .map_err(|e| { + ActionError::action_failed(format!( + "read bootstore network config: {e}" + )) + })? + .into_inner()) +} + +pub(crate) async fn write_bootstore_config( + sa: &sled_agent_client::Client, + config: &EarlyNetworkConfig, +) -> Result<(), ActionError> { + sa.write_network_bootstore_config(config).await.map_err(|e| { + ActionError::action_failed(format!( + "write bootstore network config: {e}" + )) + })?; + Ok(()) +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct BootstoreNetworkPortChange { + previous_port_config: Option, + changed_bgp_configs: Vec, + added_bgp_configs: Vec, +} + +pub(crate) fn apply_bootstore_update( + config: &mut EarlyNetworkConfig, + update: &EarlyNetworkPortUpdate, +) -> Result { + let mut change = BootstoreNetworkPortChange::default(); + + let rack_net_config = match &mut config.body.rack_network_config { + Some(cfg) => cfg, + None => { + return Err(ActionError::action_failed( + "rack network config not yet initialized".to_string(), + )) + } + }; + + for port in &mut rack_net_config.ports { + if port.port == update.port.port { + change.previous_port_config = Some(port.clone()); + *port = update.port.clone(); + break; + } + } + if change.previous_port_config.is_none() { + rack_net_config.ports.push(update.port.clone()); + } + + for updated_bgp in &update.bgp_configs { + let mut exists = false; + for resident_bgp in &mut rack_net_config.bgp { + if resident_bgp.asn == updated_bgp.asn { + change.changed_bgp_configs.push(resident_bgp.clone()); + *resident_bgp = updated_bgp.clone(); + exists = true; + break; + } + } + if !exists { + change.added_bgp_configs.push(updated_bgp.clone()); + } + } + rack_net_config.bgp.extend_from_slice(&change.added_bgp_configs); + + Ok(change) +} + +pub(crate) async fn switch_sled_agent( + location: SwitchLocation, + sagactx: &NexusActionContext, +) -> Result { + let nexus = sagactx.user_data().nexus(); + let sled_agent_addr = get_scrimlet_address(location, nexus).await?; + Ok(sled_agent_client::Client::new( + &format!("http://{}", sled_agent_addr), + sagactx.user_data().log().clone(), + )) +} diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs index 0c0f4ec01b..14544b0f55 100644 --- a/nexus/src/app/sagas/switch_port_settings_clear.rs +++ b/nexus/src/app/sagas/switch_port_settings_clear.rs @@ -5,17 +5,25 @@ use super::switch_port_settings_apply::select_dendrite_client; use super::NexusActionContext; use crate::app::sagas::retry_until_known_result; -use crate::app::sagas::switch_port_settings_apply::api_to_dpd_port_settings; +use crate::app::sagas::switch_port_settings_apply::{ + api_to_dpd_port_settings, apply_bootstore_update, bootstore_update, + ensure_switch_port_bgp_settings, ensure_switch_port_uplink, + read_bootstore_config, select_mg_client, switch_sled_agent, + write_bootstore_config, +}; use crate::app::sagas::{ declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError, }; use anyhow::Error; use dpd_client::types::PortId; +use mg_admin_client::types::DeleteNeighborRequest; +use nexus_db_model::NETWORK_KEY; use nexus_db_queries::authn; use nexus_db_queries::db::datastore::UpdatePrecondition; -use omicron_common::api::external::{self, NameOrId}; +use omicron_common::api::external::{self, NameOrId, SwitchLocation}; use serde::{Deserialize, Serialize}; use std::str::FromStr; +use std::sync::Arc; use steno::ActionError; use uuid::Uuid; @@ -36,6 +44,18 @@ declare_saga_actions! { + spa_clear_switch_port_settings - spa_undo_clear_switch_port_settings } + CLEAR_SWITCH_PORT_UPLINK -> "clear_switch_port_uplink" { + + spa_clear_switch_port_uplink + - spa_undo_clear_switch_port_uplink + } + CLEAR_SWITCH_PORT_BGP_SETTINGS -> "clear_switch_port_bgp_settings" { + + spa_clear_switch_port_bgp_settings + - spa_undo_clear_switch_port_bgp_settings + } + CLEAR_SWITCH_PORT_BOOTSTORE_NETWORK_SETTINGS -> "clear_switch_port_bootstore_network_settings" { + + spa_clear_switch_port_bootstore_network_settings + - spa_undo_clear_switch_port_bootstore_network_settings + } } #[derive(Debug)] @@ -54,6 +74,9 @@ impl NexusSaga for SagaSwitchPortSettingsClear { ) -> Result { builder.append(disassociate_switch_port_action()); builder.append(clear_switch_port_settings_action()); + builder.append(clear_switch_port_uplink_action()); + builder.append(clear_switch_port_bgp_settings_action()); + builder.append(clear_switch_port_bootstore_network_settings_action()); Ok(builder.build()?) } } @@ -181,3 +204,185 @@ async fn spa_undo_clear_switch_port_settings( Ok(()) } + +async fn spa_clear_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + ensure_switch_port_uplink(sagactx, true, None).await +} + +async fn spa_undo_clear_switch_port_uplink( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| external::Error::internal_error(&e.to_string()))?; + + Ok(ensure_switch_port_uplink(sagactx, false, id).await?) +} + +async fn spa_clear_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| { + ActionError::action_failed(format!( + "original port settings id lookup: {e}" + )) + })?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + let mg_client: Arc = + select_mg_client(&sagactx).await.map_err(|e| { + ActionError::action_failed(format!("select mg client (undo): {e}")) + })?; + + for peer in settings.bgp_peers { + let config = nexus + .bgp_config_get(&opctx, peer.bgp_config_id.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete bgp config: {e}")) + })?; + + mg_client + .inner + .delete_neighbor(&DeleteNeighborRequest { + asn: *config.asn, + addr: peer.addr.ip(), + }) + .await + .map_err(|e| { + ActionError::action_failed(format!("delete neighbor: {e}")) + })?; + } + + Ok(()) +} + +async fn spa_undo_clear_switch_port_bgp_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = + sagactx.lookup::>("original_switch_port_settings_id")?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = + nexus.switch_port_settings_get(&opctx, &NameOrId::Id(id)).await?; + + Ok(ensure_switch_port_bgp_settings(sagactx, settings).await?) +} + +async fn spa_clear_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), ActionError> { + let nexus = sagactx.user_data().nexus(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + let mut config = + nexus.bootstore_network_config(&opctx).await.map_err(|e| { + ActionError::action_failed(format!( + "read nexus bootstore network config: {e}" + )) + })?; + + let generation = nexus + .datastore() + .bump_bootstore_generation(&opctx, NETWORK_KEY.into()) + .await + .map_err(|e| { + ActionError::action_failed(format!( + "bump bootstore network generation number: {e}" + )) + })?; + + config.generation = generation as u64; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} + +async fn spa_undo_clear_switch_port_bootstore_network_settings( + sagactx: NexusActionContext, +) -> Result<(), Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let nexus = osagactx.nexus(); + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + + let orig_port_settings_id = sagactx + .lookup::>("original_switch_port_settings_id") + .map_err(|e| { + ActionError::action_failed(format!( + "original port settings id lookup: {e}" + )) + })?; + + let id = match orig_port_settings_id { + Some(id) => id, + None => return Ok(()), + }; + + let settings = nexus + .switch_port_settings_get(&opctx, &NameOrId::Id(id)) + .await + .map_err(ActionError::action_failed)?; + + // Just choosing the sled agent associated with switch0 for no reason. + let sa = switch_sled_agent(SwitchLocation::Switch0, &sagactx).await?; + + // Read the current bootstore config, perform the update and write it back. + let mut config = read_bootstore_config(&sa).await?; + let update = bootstore_update( + &nexus, + &opctx, + params.switch_port_id, + ¶ms.port_name, + &settings, + ) + .await?; + apply_bootstore_update(&mut config, &update)?; + write_bootstore_config(&sa, &config).await?; + + Ok(()) +} diff --git a/nexus/src/app/sagas/switch_port_settings_update.rs b/nexus/src/app/sagas/switch_port_settings_update.rs deleted file mode 100644 index 23120bdbf4..0000000000 --- a/nexus/src/app/sagas/switch_port_settings_update.rs +++ /dev/null @@ -1,5 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -// TODO https://github.com/oxidecomputer/omicron/issues/3002 diff --git a/nexus/src/app/switch_port.rs b/nexus/src/app/switch_port.rs index 996290b684..03b874727b 100644 --- a/nexus/src/app/switch_port.rs +++ b/nexus/src/app/switch_port.rs @@ -2,33 +2,114 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +//XXX +#![allow(unused_imports)] + use crate::app::sagas; use crate::external_api::params; use db::datastore::SwitchPortSettingsCombinedResult; +use ipnetwork::IpNetwork; +use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::authn; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::db::model::{SwitchPort, SwitchPortSettings}; +use nexus_types::identity::Resource; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::{ self, CreateResult, DataPageParams, DeleteResult, ListResultVec, LookupResult, Name, NameOrId, UpdateResult, }; +use sled_agent_client::types::BgpConfig; +use sled_agent_client::types::BgpPeerConfig; +use sled_agent_client::types::{ + EarlyNetworkConfig, PortConfigV1, RackNetworkConfigV1, RouteConfig, +}; use std::sync::Arc; use uuid::Uuid; impl super::Nexus { - pub(crate) async fn switch_port_settings_create( - &self, + pub(crate) async fn switch_port_settings_post( + self: &Arc, opctx: &OpContext, params: params::SwitchPortSettingsCreate, ) -> CreateResult { opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + + //TODO(ry) race conditions on exists check versus update/create. + // Normally I would use a DB lock here, but not sure what + // the Omicron way of doing things here is. + + match self + .db_datastore + .switch_port_settings_exist( + opctx, + params.identity.name.clone().into(), + ) + .await + { + Ok(id) => self.switch_port_settings_update(opctx, id, params).await, + Err(_) => self.switch_port_settings_create(opctx, params).await, + } + } + + pub async fn switch_port_settings_create( + self: &Arc, + opctx: &OpContext, + params: params::SwitchPortSettingsCreate, + ) -> CreateResult { self.db_datastore.switch_port_settings_create(opctx, ¶ms).await } + pub(crate) async fn switch_port_settings_update( + self: &Arc, + opctx: &OpContext, + switch_port_settings_id: Uuid, + new_settings: params::SwitchPortSettingsCreate, + ) -> CreateResult { + // delete old settings + self.switch_port_settings_delete( + opctx, + ¶ms::SwitchPortSettingsSelector { + port_settings: Some(NameOrId::Id(switch_port_settings_id)), + }, + ) + .await?; + + // create new settings + let result = self + .switch_port_settings_create(opctx, new_settings.clone()) + .await?; + + // run the port settings apply saga for each port referencing the + // updated settings + + let ports = self + .db_datastore + .switch_ports_using_settings(opctx, switch_port_settings_id) + .await?; + + for (switch_port_id, switch_port_name) in ports.into_iter() { + let saga_params = sagas::switch_port_settings_apply::Params { + serialized_authn: authn::saga::Serialized::for_opctx(opctx), + switch_port_id, + switch_port_settings_id: result.settings.id(), + switch_port_name: switch_port_name.to_string(), + }; + + self.execute_saga::< + sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply + >( + saga_params, + ) + .await?; + } + + Ok(result) + } + pub(crate) async fn switch_port_settings_delete( &self, opctx: &OpContext, @@ -151,7 +232,9 @@ impl super::Nexus { switch_port_name: port.to_string(), }; - self.execute_saga::( + self.execute_saga::< + sagas::switch_port_settings_apply::SagaSwitchPortSettingsApply + >( saga_params, ) .await?; @@ -215,4 +298,25 @@ impl super::Nexus { Ok(()) } + + // TODO it would likely be better to do this as a one shot db query. + pub(crate) async fn active_port_settings( + &self, + opctx: &OpContext, + ) -> LookupResult> { + let mut ports = Vec::new(); + let port_list = + self.switch_port_list(opctx, &DataPageParams::max_page()).await?; + + for p in port_list { + if let Some(id) = p.port_settings_id { + ports.push(( + p.clone(), + self.switch_port_settings_get(opctx, &id.into()).await?, + )); + } + } + + LookupResult::Ok(ports) + } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 1fddfba85b..990704904a 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -63,6 +63,11 @@ use omicron_common::api::external::http_pagination::ScanParams; use omicron_common::api::external::AddressLot; use omicron_common::api::external::AddressLotBlock; use omicron_common::api::external::AddressLotCreateResponse; +use omicron_common::api::external::BgpAnnounceSet; +use omicron_common::api::external::BgpAnnouncement; +use omicron_common::api::external::BgpConfig; +use omicron_common::api::external::BgpImportedRouteIpv4; +use omicron_common::api::external::BgpPeerStatus; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Disk; use omicron_common::api::external::Error; @@ -250,6 +255,15 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(networking_switch_port_apply_settings)?; api.register(networking_switch_port_clear_settings)?; + api.register(networking_bgp_config_create)?; + api.register(networking_bgp_config_list)?; + api.register(networking_bgp_status)?; + api.register(networking_bgp_imported_routes_ipv4)?; + api.register(networking_bgp_config_delete)?; + api.register(networking_bgp_announce_set_create)?; + api.register(networking_bgp_announce_set_list)?; + api.register(networking_bgp_announce_set_delete)?; + // Fleet-wide API operations api.register(silo_list)?; api.register(silo_create)?; @@ -2642,7 +2656,7 @@ async fn networking_switch_port_settings_create( let nexus = &apictx.nexus; let params = new_settings.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let result = nexus.switch_port_settings_create(&opctx, params).await?; + let result = nexus.switch_port_settings_post(&opctx, params).await?; let settings: SwitchPortSettingsView = result.into(); Ok(HttpResponseCreated(settings)) @@ -2810,6 +2824,193 @@ async fn networking_switch_port_clear_settings( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// Create a new BGP configuration. +#[endpoint { + method = POST, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_create( + rqctx: RequestContext>, + config: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let config = config.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_config_set(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Get BGP configurations. +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_list( + rqctx: RequestContext>, + query_params: Query>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let configs = nexus + .bgp_config_list(&opctx, &paginated_by) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + configs, + &marker_for_name_or_id, + )?)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get BGP peer status +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-status", + tags = ["system/networking"], +}] +async fn networking_bgp_status( + rqctx: RequestContext>, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.nexus; + let result = nexus.bgp_peer_status(&opctx).await?; + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get imported IPv4 BGP routes. +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-routes-ipv4", + tags = ["system/networking"], +}] +async fn networking_bgp_imported_routes_ipv4( + rqctx: RequestContext>, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let handler = async { + let nexus = &apictx.nexus; + let sel = query_params.into_inner(); + let result = nexus.bgp_imported_routes_ipv4(&opctx, &sel).await?; + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Delete a BGP configuration. +#[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp", + tags = ["system/networking"], +}] +async fn networking_bgp_config_delete( + rqctx: RequestContext>, + sel: Query, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = sel.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_config_delete(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Create a new BGP announce set. +#[endpoint { + method = POST, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_create( + rqctx: RequestContext>, + config: TypedBody, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let config = config.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus.bgp_create_announce_set(&opctx, &config).await?; + Ok(HttpResponseCreated::(result.0.into())) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +//TODO pagination? the normal by-name/by-id stuff does not work here +/// Get originated routes for a given BGP configuration. +#[endpoint { + method = GET, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_list( + rqctx: RequestContext>, + query_params: Query, +) -> Result>, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = query_params.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let result = nexus + .bgp_announce_list(&opctx, &sel) + .await? + .into_iter() + .map(|p| p.into()) + .collect(); + Ok(HttpResponseOk(result)) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Delete a BGP announce set. +#[endpoint { + method = DELETE, + path = "/v1/system/networking/bgp-announce", + tags = ["system/networking"], +}] +async fn networking_bgp_announce_set_delete( + rqctx: RequestContext>, + selector: Query, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let sel = selector.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus.bgp_delete_announce_set(&opctx, &sel).await?; + Ok(HttpResponseUpdatedNoContent {}) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Images /// List images diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index 0ada48e203..01aca36e1d 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -32,12 +32,12 @@ use internal_api::http_entrypoints::internal_api; use nexus_types::internal_api::params::ServiceKind; use omicron_common::address::IpRange; use omicron_common::api::internal::shared::{ - ExternalPortDiscovery, SwitchLocation, + ExternalPortDiscovery, RackNetworkConfig, SwitchLocation, }; use omicron_common::FileKv; use slog::Logger; use std::collections::HashMap; -use std::net::{SocketAddr, SocketAddrV6}; +use std::net::{Ipv4Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; use uuid::Uuid; @@ -289,7 +289,13 @@ impl nexus_test_interface::NexusServer for Server { vec!["qsfp0".parse().unwrap()], )]), ), - rack_network_config: None, + rack_network_config: Some(RackNetworkConfig { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports: Vec::new(), + bgp: Vec::new(), + }), }, ) .await diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 34c218b3e2..701a6e8ba9 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -58,6 +58,7 @@ pub const RACK_UUID: &str = "c19a698f-c6f9-4a17-ae30-20d711b8f7dc"; pub const SWITCH_UUID: &str = "dae4e1f1-410e-4314-bff1-fec0504be07e"; pub const OXIMETER_UUID: &str = "39e6175b-4df2-4730-b11d-cbc1e60a2e78"; pub const PRODUCER_UUID: &str = "a6458b7d-87c3-4483-be96-854d814c20de"; +pub const RACK_SUBNET: &str = "fd00:1122:3344:01::/56"; /// The reported amount of hardware threads for an emulated sled agent. pub const TEST_HARDWARE_THREADS: u32 = 16; @@ -86,6 +87,7 @@ pub struct ControlPlaneTestContext { pub oximeter: Oximeter, pub producer: ProducerServer, pub dendrite: HashMap, + pub mgd: HashMap, pub external_dns_zone_name: String, pub external_dns: dns_server::TransientServer, pub internal_dns: dns_server::TransientServer, @@ -108,6 +110,9 @@ impl ControlPlaneTestContext { for (_, mut dendrite) in self.dendrite { dendrite.cleanup().await.unwrap(); } + for (_, mut mgd) in self.mgd { + mgd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } @@ -237,6 +242,7 @@ pub struct ControlPlaneTestContextBuilder<'a, N: NexusServer> { pub oximeter: Option, pub producer: Option, pub dendrite: HashMap, + pub mgd: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -274,6 +280,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { oximeter: None, producer: None, dendrite: HashMap::new(), + mgd: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, external_dns_zone_name: None, @@ -398,6 +405,32 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { ); } + pub async fn start_mgd(&mut self, switch_location: SwitchLocation) { + let log = &self.logctx.log; + debug!(log, "Starting mgd for {switch_location}"); + + // Set up an instance of mgd + let mgd = dev::maghemite::MgdInstance::start(0).await.unwrap(); + let port = mgd.port; + self.mgd.insert(switch_location, mgd); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); + + debug!(log, "mgd port is {port}"); + + let config = omicron_common::nexus_config::MgdConfig { + address: std::net::SocketAddr::V6(address), + }; + self.config.pkg.mgd.insert(switch_location, config); + + let sled_id = Uuid::parse_str(SLED_AGENT_UUID).unwrap(); + self.rack_init_builder.add_service( + address, + ServiceKind::Mgd, + internal_dns::ServiceName::Mgd, + sled_id, + ); + } + pub async fn start_oximeter(&mut self) { let log = &self.logctx.log; debug!(log, "Starting Oximeter"); @@ -528,8 +561,11 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { &format!("http://{}", internal_dns_address), log.clone(), ); + let dns_config = self.rack_init_builder.internal_dns_config.clone().build(); + + slog::info!(log, "DNS population: {:#?}", dns_config); dns_config_client.dns_config_put(&dns_config).await.expect( "Failed to send initial DNS records to internal DNS server", ); @@ -669,6 +705,25 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { ); } + pub async fn scrimlet_dns_setup(&mut self) { + let sled_agent = self + .sled_agent + .as_ref() + .expect("Cannot set up scrimlet DNS without sled agent"); + + let sa = match sled_agent.http_server.local_addr() { + SocketAddr::V6(sa) => sa, + SocketAddr::V4(_) => panic!("expected SocketAddrV6 for sled agent"), + }; + + for loc in [SwitchLocation::Switch0, SwitchLocation::Switch1] { + self.rack_init_builder + .internal_dns_config + .host_scrimlet(loc, sa) + .expect("add switch0 scrimlet dns entry"); + } + } + // Set up an external DNS server. pub async fn start_external_dns(&mut self) { let log = self.logctx.log.new(o!("component" => "external_dns_server")); @@ -742,6 +797,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { producer: self.producer.unwrap(), logctx: self.logctx, dendrite: self.dendrite, + mgd: self.mgd, external_dns_zone_name: self.external_dns_zone_name.unwrap(), external_dns: self.external_dns.unwrap(), internal_dns: self.internal_dns.unwrap(), @@ -772,6 +828,9 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { for (_, mut dendrite) in self.dendrite { dendrite.cleanup().await.unwrap(); } + for (_, mut mgd) in self.mgd { + mgd.cleanup().await.unwrap(); + } self.logctx.cleanup_successful(); } } @@ -862,11 +921,14 @@ async fn setup_with_config_impl( builder.start_clickhouse().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.start_internal_dns().await; builder.start_external_dns().await; builder.start_nexus_internal().await; builder.start_sled(sim_mode).await; builder.start_crucible_pantry().await; + builder.scrimlet_dns_setup().await; // Give Nexus necessary information to find the Crucible Pantry let dns_config = builder.populate_internal_dns().await; diff --git a/nexus/tests/integration_tests/address_lots.rs b/nexus/tests/integration_tests/address_lots.rs index b4659daa62..40c8865929 100644 --- a/nexus/tests/integration_tests/address_lots.rs +++ b/nexus/tests/integration_tests/address_lots.rs @@ -27,8 +27,8 @@ type ControlPlaneTestContext = async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { let client = &ctx.external_client; - // Verify there are no lots - let lots = NexusRequest::iter_collection_authn::( + // Verify there is only one system lot + let lots = NexusRequest::iter_collection_authn::( client, "/v1/system/networking/address-lot", "", @@ -37,7 +37,7 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .await .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 0, "Expected no lots"); + assert_eq!(lots.len(), 1, "Expected one lot"); // Create a lot let params = AddressLotCreate { @@ -111,8 +111,8 @@ async fn test_address_lot_basic_crud(ctx: &ControlPlaneTestContext) { .expect("Failed to list address lots") .all_items; - assert_eq!(lots.len(), 1, "Expected 1 lot"); - assert_eq!(lots[0], address_lot); + assert_eq!(lots.len(), 2, "Expected 2 lots"); + assert_eq!(lots[1], address_lot); // Verify there are lot blocks let blist = NexusRequest::iter_collection_authn::( diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index e9ae11c21f..8fba22fb2f 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -420,6 +420,40 @@ lazy_static! { }; } +lazy_static! { + pub static ref DEMO_BGP_CONFIG_CREATE_URL: String = + format!("/v1/system/networking/bgp?name_or_id=as47"); + pub static ref DEMO_BGP_CONFIG: params::BgpConfigCreate = + params::BgpConfigCreate { + identity: IdentityMetadataCreateParams { + name: "as47".parse().unwrap(), + description: "BGP config for AS47".into(), + }, + bgp_announce_set_id: NameOrId::Name("instances".parse().unwrap()), + asn: 47, + vrf: None, + }; + pub static ref DEMO_BGP_ANNOUNCE_SET_URL: String = + format!("/v1/system/networking/bgp-announce?name_or_id=a-bag-of-addrs"); + pub static ref DEMO_BGP_ANNOUNCE: params::BgpAnnounceSetCreate = + params::BgpAnnounceSetCreate { + identity: IdentityMetadataCreateParams { + name: "a-bag-of-addrs".parse().unwrap(), + description: "a bag of addrs".into(), + }, + announcement: vec![params::BgpAnnouncementCreate { + address_lot_block: NameOrId::Name( + "some-block".parse().unwrap(), + ), + network: "10.0.0.0/16".parse().unwrap(), + }], + }; + pub static ref DEMO_BGP_STATUS_URL: String = + format!("/v1/system/networking/bgp-status"); + pub static ref DEMO_BGP_ROUTES_IPV4_URL: String = + format!("/v1/system/networking/bgp-routes-ipv4?asn=47"); +} + lazy_static! { // Project Images pub static ref DEMO_IMAGE_NAME: Name = "demo-image".parse().unwrap(); @@ -1876,5 +1910,48 @@ lazy_static! { AllowedMethod::GetNonexistent ], }, + VerifyEndpoint { + url: &DEMO_BGP_CONFIG_CREATE_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_BGP_CONFIG).unwrap(), + ), + AllowedMethod::Get, + AllowedMethod::Delete + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_ANNOUNCE_SET_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Post( + serde_json::to_value(&*DEMO_BGP_ANNOUNCE).unwrap(), + ), + AllowedMethod::GetNonexistent, + AllowedMethod::Delete + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_STATUS_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + ], + }, + + VerifyEndpoint { + url: &DEMO_BGP_ROUTES_IPV4_URL, + visibility: Visibility::Public, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + ], + } ]; } diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 2d4c76dc99..43a4ac8f2e 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -29,6 +29,8 @@ async fn test_nexus_boots_before_cockroach() { builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.start_internal_dns().await; builder.start_external_dns().await; @@ -144,6 +146,11 @@ async fn test_nexus_boots_before_dendrite() { builder.start_dendrite(SwitchLocation::Switch1).await; info!(log, "Started Dendrite"); + info!(log, "Starting mgd"); + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; + info!(log, "Started mgd"); + info!(log, "Populating internal DNS records"); builder.populate_internal_dns().await; info!(log, "Populated internal DNS records"); @@ -166,6 +173,8 @@ async fn nexus_schema_test_setup( builder.start_external_dns().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.populate_internal_dns().await; } diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index f7d6c1da6a..e75211b834 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -58,6 +58,8 @@ async fn test_setup<'a>( builder.start_external_dns().await; builder.start_dendrite(SwitchLocation::Switch0).await; builder.start_dendrite(SwitchLocation::Switch1).await; + builder.start_mgd(SwitchLocation::Switch0).await; + builder.start_mgd(SwitchLocation::Switch1).await; builder.populate_internal_dns().await; builder } diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index 3d3d6c9f5f..fada45694d 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -10,8 +10,10 @@ use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params::{ Address, AddressConfig, AddressLotBlockCreate, AddressLotCreate, - LinkConfig, LldpServiceConfig, Route, RouteConfig, SwitchInterfaceConfig, - SwitchInterfaceKind, SwitchPortApplySettings, SwitchPortSettingsCreate, + BgpAnnounceSetCreate, BgpAnnouncementCreate, BgpConfigCreate, + BgpPeerConfig, LinkConfig, LinkFec, LinkSpeed, LldpServiceConfig, Route, + RouteConfig, SwitchInterfaceConfig, SwitchInterfaceKind, + SwitchPortApplySettings, SwitchPortSettingsCreate, }; use nexus_types::external_api::views::Rack; use omicron_common::api::external::{ @@ -33,10 +35,16 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { description: "an address parking lot".into(), }, kind: AddressLotKind::Infra, - blocks: vec![AddressLotBlockCreate { - first_address: "203.0.113.10".parse().unwrap(), - last_address: "203.0.113.20".parse().unwrap(), - }], + blocks: vec![ + AddressLotBlockCreate { + first_address: "203.0.113.10".parse().unwrap(), + last_address: "203.0.113.20".parse().unwrap(), + }, + AddressLotBlockCreate { + first_address: "1.2.3.0".parse().unwrap(), + last_address: "1.2.3.255".parse().unwrap(), + }, + ], }; NexusRequest::objects_post( @@ -49,6 +57,49 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .await .unwrap(); + // Create BGP announce set + let announce_set = BgpAnnounceSetCreate { + identity: IdentityMetadataCreateParams { + name: "instances".parse().unwrap(), + description: "autonomous system 47 announcements".into(), + }, + announcement: vec![BgpAnnouncementCreate { + address_lot_block: NameOrId::Name("parkinglot".parse().unwrap()), + network: "1.2.3.0/24".parse().unwrap(), + }], + }; + + NexusRequest::objects_post( + client, + "/v1/system/networking/bgp-announce", + &announce_set, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Create BGP config + let bgp_config = BgpConfigCreate { + identity: IdentityMetadataCreateParams { + name: "as47".parse().unwrap(), + description: "autonomous system 47".into(), + }, + bgp_announce_set_id: NameOrId::Name("instances".parse().unwrap()), + asn: 47, + vrf: None, + }; + + NexusRequest::objects_post( + client, + "/v1/system/networking/bgp", + &bgp_config, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + // Create port settings let mut settings = SwitchPortSettingsCreate::new(IdentityMetadataCreateParams { @@ -61,6 +112,8 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { LinkConfig { mtu: 4700, lldp: LldpServiceConfig { enabled: false, lldp_config: None }, + fec: LinkFec::None, + speed: LinkSpeed::Speed100G, }, ); // interfaces @@ -191,6 +244,33 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { .parsed_body() .unwrap(); + // Update port settings. Should not see conflict. + settings.bgp_peers.insert( + "phy0".into(), + BgpPeerConfig { + bgp_config: NameOrId::Name("as47".parse().unwrap()), //TODO + bgp_announce_set: NameOrId::Name("instances".parse().unwrap()), //TODO + interface_name: "phy0".to_string(), + addr: "1.2.3.4".parse().unwrap(), + hold_time: 6, + idle_hold_time: 6, + delay_open: 0, + connect_retry: 3, + keepalive: 2, + }, + ); + let _created: SwitchPortSettingsView = NexusRequest::objects_post( + client, + "/v1/system/networking/switch-port-settings", + &settings, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + // There should be one switch port to begin with, see // Server::start_and_populate in nexus/src/lib.rs diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 1d7f5556c2..e55eaa4df6 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -145,6 +145,14 @@ networking_address_lot_block_list GET /v1/system/networking/address- networking_address_lot_create POST /v1/system/networking/address-lot networking_address_lot_delete DELETE /v1/system/networking/address-lot/{address_lot} networking_address_lot_list GET /v1/system/networking/address-lot +networking_bgp_announce_set_create POST /v1/system/networking/bgp-announce +networking_bgp_announce_set_delete DELETE /v1/system/networking/bgp-announce +networking_bgp_announce_set_list GET /v1/system/networking/bgp-announce +networking_bgp_config_create POST /v1/system/networking/bgp +networking_bgp_config_delete DELETE /v1/system/networking/bgp +networking_bgp_config_list GET /v1/system/networking/bgp +networking_bgp_imported_routes_ipv4 GET /v1/system/networking/bgp-routes-ipv4 +networking_bgp_status GET /v1/system/networking/bgp-status networking_loopback_address_create POST /v1/system/networking/loopback-address networking_loopback_address_delete DELETE /v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask} networking_loopback_address_list GET /v1/system/networking/loopback-address diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index b4e0e705d8..a0169ae777 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -1325,6 +1325,42 @@ pub enum SwitchPortGeometry { Sfp28x4, } +/// The forward error correction mode of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkFec { + /// Firecode foward error correction. + Firecode, + /// No forward error correction. + None, + /// Reed-Solomon forward error correction. + Rs, +} + +/// The speed of a link. +#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum LinkSpeed { + /// Zero gigabits per second. + Speed0G, + /// 1 gigabit per second. + Speed1G, + /// 10 gigabits per second. + Speed10G, + /// 25 gigabits per second. + Speed25G, + /// 40 gigabits per second. + Speed40G, + /// 50 gigabits per second. + Speed50G, + /// 100 gigabits per second. + Speed100G, + /// 200 gigabits per second. + Speed200G, + /// 400 gigabits per second. + Speed400G, +} + /// Switch link configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct LinkConfig { @@ -1333,6 +1369,12 @@ pub struct LinkConfig { /// The link-layer discovery protocol (LLDP) configuration for the link. pub lldp: LldpServiceConfig, + + /// The forward error correction mode of the link. + pub fec: LinkFec, + + /// The speed of the link. + pub speed: LinkSpeed, } /// The LLDP configuration associated with a port. LLDP may be either enabled or @@ -1406,6 +1448,20 @@ pub struct Route { pub vid: Option, } +/// Select a BGP config by a name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpConfigSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: NameOrId, +} + +/// List BGP configs with an optional name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpConfigListSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: Option, +} + /// A BGP peer configuration for an interface. Includes the set of announcements /// that will be advertised to the peer identified by `addr`. The `bgp_config` /// parameter is a reference to global BGP parameters. The `interface_name` @@ -1427,21 +1483,59 @@ pub struct BgpPeerConfig { /// The address of the host to peer with. pub addr: IpAddr, + + /// How long to hold peer connections between keppalives (seconds). + pub hold_time: u32, + + /// How long to hold a peer in idle before attempting a new session + /// (seconds). + pub idle_hold_time: u32, + + /// How long to delay sending an open request after establishing a TCP + /// session (seconds). + pub delay_open: u32, + + /// How long to to wait between TCP connection retries (seconds). + pub connect_retry: u32, + + /// How often to send keepalive requests (seconds). + pub keepalive: u32, } /// Parameters for creating a named set of BGP announcements. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct CreateBgpAnnounceSet { +pub struct BgpAnnounceSetCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, /// The announcements in this set. - pub announcement: Vec, + pub announcement: Vec, +} + +/// Select a BGP announce set by a name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpAnnounceSetSelector { + /// A name or id to use when selecting BGP port settings + pub name_or_id: NameOrId, +} + +/// List BGP announce set with an optional name or id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpAnnounceListSelector { + /// A name or id to use when selecting BGP config. + pub name_or_id: Option, +} + +/// Selector used for querying imported BGP routes. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpRouteSelector { + /// The ASN to filter on. Required. + pub asn: u32, } /// A BGP announcement tied to a particular address lot block. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct BgpAnnouncement { +pub struct BgpAnnouncementCreate { /// Address lot this announcement is drawn from. pub address_lot_block: NameOrId, @@ -1452,18 +1546,27 @@ pub struct BgpAnnouncement { /// Parameters for creating a BGP configuration. This includes and autonomous /// system number (ASN) and a virtual routing and forwarding (VRF) identifier. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub struct CreateBgpConfig { +pub struct BgpConfigCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, /// The autonomous system number of this BGP configuration. pub asn: u32, + pub bgp_announce_set_id: NameOrId, + /// Optional virtual routing and forwarding identifier for this BGP /// configuration. pub vrf: Option, } +/// Select a BGP status information by BGP config id. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct BgpStatusSelector { + /// A name or id of the BGP configuration to get status for + pub name_or_id: NameOrId, +} + /// A set of addresses associated with a port configuration. #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct AddressConfig { diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index e2a5e3d094..c0991ebb17 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -182,6 +182,7 @@ pub enum ServiceKind { Tfport, BoundaryNtp { snat: SourceNatConfig, nic: ServiceNic }, InternalNtp, + Mgd, } impl fmt::Display for ServiceKind { @@ -200,6 +201,7 @@ impl fmt::Display for ServiceKind { Tfport => "tfport", CruciblePantry => "crucible_pantry", BoundaryNtp { .. } | InternalNtp => "ntp", + Mgd => "mgd", }; write!(f, "{}", s) } diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 682512cc24..6dcf756737 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -241,6 +241,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapAddressDiscovery": { "oneOf": [ { @@ -333,6 +380,26 @@ "request_id" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -375,6 +442,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -406,6 +477,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -492,7 +626,7 @@ "description": "Initial rack network configuration", "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] }, @@ -529,10 +663,17 @@ "recovery_silo" ] }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -543,18 +684,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { @@ -747,6 +893,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -770,67 +938,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 67db222155..411c52ddff 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -767,6 +767,53 @@ "serial_number" ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BinRangedouble": { "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", "oneOf": [ @@ -3653,6 +3700,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -3695,6 +3762,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -4038,6 +4109,69 @@ "PhysicalDiskPutResponse": { "type": "object" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -4268,7 +4402,7 @@ "description": "Initial rack network configuration", "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] }, @@ -4299,10 +4433,17 @@ "services" ] }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -4313,18 +4454,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RecoverySiloConfig": { @@ -4346,6 +4492,28 @@ "user_password_hash" ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "Saga": { "description": "Sagas\n\nThese are currently only intended for observability by developers. We will eventually want to flesh this out into something more observable for end users.", "type": "object", @@ -4822,6 +4990,20 @@ "required": [ "type" ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "mgd" + ] + } + }, + "required": [ + "type" + ] } ] }, @@ -5090,67 +5272,6 @@ "SwitchPutResponse": { "type": "object" }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "UserId": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", diff --git a/openapi/nexus.json b/openapi/nexus.json index 9dda94f283..456f2aebd6 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -5165,6 +5165,318 @@ } } }, + "/v1/system/networking/bgp": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get BGP configurations.", + "operationId": "networking_bgp_config_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP config.", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfigResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, + "post": { + "tags": [ + "system/networking" + ], + "summary": "Create a new BGP configuration.", + "operationId": "networking_bgp_config_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfigCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/networking" + ], + "summary": "Delete a BGP configuration.", + "operationId": "networking_bgp_config_delete", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP config.", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-announce": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get originated routes for a given BGP configuration.", + "operationId": "networking_bgp_announce_set_list", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP port settings", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpAnnouncement", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnouncement" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "post": { + "tags": [ + "system/networking" + ], + "summary": "Create a new BGP announce set.", + "operationId": "networking_bgp_announce_set_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpAnnounceSetCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BgpAnnounceSet" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/networking" + ], + "summary": "Delete a BGP announce set.", + "operationId": "networking_bgp_announce_set_delete", + "parameters": [ + { + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP port settings", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-routes-ipv4": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get imported IPv4 BGP routes.", + "operationId": "networking_bgp_imported_routes_ipv4", + "parameters": [ + { + "in": "query", + "name": "asn", + "description": "The ASN to filter on. Required.", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpImportedRouteIpv4", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpImportedRouteIpv4" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/networking/bgp-status": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get BGP peer status", + "operationId": "networking_bgp_status", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_BgpPeerStatus", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerStatus" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/system/networking/loopback-address": { "get": { "tags": [ @@ -7741,40 +8053,289 @@ "$ref": "#/components/schemas/AddressLotBlock" } }, - "lot": { - "description": "The address lot that was created.", + "lot": { + "description": "The address lot that was created.", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLot" + } + ] + } + }, + "required": [ + "blocks", + "lot" + ] + }, + "AddressLotKind": { + "description": "The kind associated with an address lot.", + "oneOf": [ + { + "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", + "type": "string", + "enum": [ + "infra" + ] + }, + { + "description": "Pool address lots are used by IP pools.", + "type": "string", + "enum": [ + "pool" + ] + } + ] + }, + "AddressLotResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/AddressLot" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" + ] + }, + "Baseboard": { + "description": "Properties that uniquely identify an Oxide hardware component", + "type": "object", + "properties": { + "part": { + "type": "string" + }, + "revision": { + "type": "integer", + "format": "int64" + }, + "serial": { + "type": "string" + } + }, + "required": [ + "part", + "revision", + "serial" + ] + }, + "BgpAnnounceSet": { + "description": "Represents a BGP announce set by id. The id can be used with other API calls to view and manage the announce set.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "BgpAnnounceSetCreate": { + "description": "Parameters for creating a named set of BGP announcements.", + "type": "object", + "properties": { + "announcement": { + "description": "The announcements in this set.", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnouncementCreate" + } + }, + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + } + }, + "required": [ + "announcement", + "description", + "name" + ] + }, + "BgpAnnouncement": { + "description": "A BGP announcement tied to an address lot block.", + "type": "object", + "properties": { + "address_lot_block_id": { + "description": "The address block the IP network being announced is drawn from.", + "type": "string", + "format": "uuid" + }, + "announce_set_id": { + "description": "The id of the set this announcement is a part of.", + "type": "string", + "format": "uuid" + }, + "network": { + "description": "The IP network being announced.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + } + }, + "required": [ + "address_lot_block_id", + "announce_set_id", + "network" + ] + }, + "BgpAnnouncementCreate": { + "description": "A BGP announcement tied to a particular address lot block.", + "type": "object", + "properties": { + "address_lot_block": { + "description": "Address lot this announcement is drawn from.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + }, + "network": { + "description": "The network being announced.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + } + }, + "required": [ + "address_lot_block", + "network" + ] + }, + "BgpConfig": { + "description": "A base BGP configuration.", + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number of this BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] + }, + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" + }, + "vrf": { + "nullable": true, + "description": "Optional virtual routing and forwarding identifier for this BGP configuration.", + "type": "string" + } + }, + "required": [ + "asn", + "description", + "id", + "name", + "time_created", + "time_modified" + ] + }, + "BgpConfigCreate": { + "description": "Parameters for creating a BGP configuration. This includes and autonomous system number (ASN) and a virtual routing and forwarding (VRF) identifier.", + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number of this BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "bgp_announce_set_id": { + "$ref": "#/components/schemas/NameOrId" + }, + "description": { + "type": "string" + }, + "name": { + "$ref": "#/components/schemas/Name" + }, + "vrf": { + "nullable": true, + "description": "Optional virtual routing and forwarding identifier for this BGP configuration.", "allOf": [ { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/Name" } ] } }, "required": [ - "blocks", - "lot" - ] - }, - "AddressLotKind": { - "description": "The kind associated with an address lot.", - "oneOf": [ - { - "description": "Infrastructure address lots are used for network infrastructure like addresses assigned to rack switches.", - "type": "string", - "enum": [ - "infra" - ] - }, - { - "description": "Pool address lots are used by IP pools.", - "type": "string", - "enum": [ - "pool" - ] - } + "asn", + "bgp_announce_set_id", + "description", + "name" ] }, - "AddressLotResultsPage": { + "BgpConfigResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -7782,7 +8343,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/AddressLot" + "$ref": "#/components/schemas/BgpConfig" } }, "next_page": { @@ -7795,25 +8356,43 @@ "items" ] }, - "Baseboard": { - "description": "Properties that uniquely identify an Oxide hardware component", + "BgpImportedRouteIpv4": { + "description": "A route imported from a BGP peer.", "type": "object", "properties": { - "part": { - "type": "string" - }, - "revision": { + "id": { + "description": "BGP identifier of the originating router.", "type": "integer", - "format": "int64" + "format": "uint32", + "minimum": 0 }, - "serial": { - "type": "string" + "nexthop": { + "description": "The nexthop the prefix is reachable through.", + "type": "string", + "format": "ipv4" + }, + "prefix": { + "description": "The destination network prefix.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Net" + } + ] + }, + "switch": { + "description": "Switch the route is imported into.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] } }, "required": [ - "part", - "revision", - "serial" + "id", + "nexthop", + "prefix", + "switch" ] }, "BgpPeerConfig": { @@ -7841,16 +8420,158 @@ } ] }, + "connect_retry": { + "description": "How long to to wait between TCP connection retries (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "delay_open": { + "description": "How long to delay sending an open request after establishing a TCP session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "hold_time": { + "description": "How long to hold peer connections between keppalives (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "idle_hold_time": { + "description": "How long to hold a peer in idle before attempting a new session (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, "interface_name": { "description": "The name of interface to peer on. This is relative to the port configuration this BGP peer configuration is a part of. For example this value could be phy0 to refer to a primary physical interface. Or it could be vlan47 to refer to a VLAN interface.", "type": "string" + }, + "keepalive": { + "description": "How often to send keepalive requests (seconds).", + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ "addr", "bgp_announce_set", "bgp_config", - "interface_name" + "connect_retry", + "delay_open", + "hold_time", + "idle_hold_time", + "interface_name", + "keepalive" + ] + }, + "BgpPeerState": { + "description": "The current state of a BGP peer.", + "oneOf": [ + { + "description": "Initial state. Refuse all incomming BGP connections. No resources allocated to peer.", + "type": "string", + "enum": [ + "idle" + ] + }, + { + "description": "Waiting for the TCP connection to be completed.", + "type": "string", + "enum": [ + "connect" + ] + }, + { + "description": "Trying to acquire peer by listening for and accepting a TCP connection.", + "type": "string", + "enum": [ + "active" + ] + }, + { + "description": "Waiting for open message from peer.", + "type": "string", + "enum": [ + "open_sent" + ] + }, + { + "description": "Waiting for keepaliave or notification from peer.", + "type": "string", + "enum": [ + "open_confirm" + ] + }, + { + "description": "Synchronizing with peer.", + "type": "string", + "enum": [ + "session_setup" + ] + }, + { + "description": "Session established. Able to exchange update, notification and keepliave messages with peers.", + "type": "string", + "enum": [ + "established" + ] + } + ] + }, + "BgpPeerStatus": { + "description": "The current status of a BGP peer.", + "type": "object", + "properties": { + "addr": { + "description": "IP address of the peer.", + "type": "string", + "format": "ip" + }, + "local_asn": { + "description": "Local autonomous system number.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "remote_asn": { + "description": "Remote autonomous system number.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "state": { + "description": "State of the peer.", + "allOf": [ + { + "$ref": "#/components/schemas/BgpPeerState" + } + ] + }, + "state_duration_millis": { + "description": "Time of last state change.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "switch": { + "description": "Switch with the peer session.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + } + }, + "required": [ + "addr", + "local_asn", + "remote_asn", + "state", + "state_duration_millis", + "switch" ] }, "BinRangedouble": { @@ -11747,6 +12468,14 @@ "description": "Switch link configuration.", "type": "object", "properties": { + "fec": { + "description": "The forward error correction mode of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkFec" + } + ] + }, "lldp": { "description": "The link-layer discovery protocol (LLDP) configuration for the link.", "allOf": [ @@ -11760,11 +12489,115 @@ "type": "integer", "format": "uint16", "minimum": 0 + }, + "speed": { + "description": "The speed of the link.", + "allOf": [ + { + "$ref": "#/components/schemas/LinkSpeed" + } + ] } }, "required": [ + "fec", "lldp", - "mtu" + "mtu", + "speed" + ] + }, + "LinkFec": { + "description": "The forward error correction mode of a link.", + "oneOf": [ + { + "description": "Firecode foward error correction.", + "type": "string", + "enum": [ + "firecode" + ] + }, + { + "description": "No forward error correction.", + "type": "string", + "enum": [ + "none" + ] + }, + { + "description": "Reed-Solomon forward error correction.", + "type": "string", + "enum": [ + "rs" + ] + } + ] + }, + "LinkSpeed": { + "description": "The speed of a link.", + "oneOf": [ + { + "description": "Zero gigabits per second.", + "type": "string", + "enum": [ + "speed0_g" + ] + }, + { + "description": "1 gigabit per second.", + "type": "string", + "enum": [ + "speed1_g" + ] + }, + { + "description": "10 gigabits per second.", + "type": "string", + "enum": [ + "speed10_g" + ] + }, + { + "description": "25 gigabits per second.", + "type": "string", + "enum": [ + "speed25_g" + ] + }, + { + "description": "40 gigabits per second.", + "type": "string", + "enum": [ + "speed40_g" + ] + }, + { + "description": "50 gigabits per second.", + "type": "string", + "enum": [ + "speed50_g" + ] + }, + { + "description": "100 gigabits per second.", + "type": "string", + "enum": [ + "speed100_g" + ] + }, + { + "description": "200 gigabits per second.", + "type": "string", + "enum": [ + "speed200_g" + ] + }, + { + "description": "400 gigabits per second.", + "type": "string", + "enum": [ + "speed400_g" + ] + } ] }, "LldpServiceConfig": { @@ -13539,6 +14372,25 @@ } ] }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, "SwitchPort": { "description": "A switch port represents a physical external port on a rack switch.", "type": "object", @@ -13635,11 +14487,6 @@ "type": "string", "format": "ip" }, - "bgp_announce_set_id": { - "description": "The id for the set of prefixes announced in this peer configuration.", - "type": "string", - "format": "uuid" - }, "bgp_config_id": { "description": "The id of the global BGP configuration referenced by this peer configuration.", "type": "string", @@ -13657,7 +14504,6 @@ }, "required": [ "addr", - "bgp_announce_set_id", "bgp_config_id", "interface_name", "port_settings_id" diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 7831193fc2..486662853c 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -289,6 +289,55 @@ } } }, + "/network-bootstore-config": { + "get": { + "summary": "This API endpoint is only reading the local sled agent's view of the", + "description": "bootstore. The boostore is a distributed data store that is eventually consistent. Reads from individual nodes may not represent the latest state.", + "operationId": "read_network_bootstore_config_cache", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "operationId": "write_network_bootstore_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EarlyNetworkConfig" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/services": { "put": { "operationId": "services_put", @@ -338,6 +387,32 @@ } } }, + "/switch-ports": { + "post": { + "operationId": "uplink_ensure", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchPorts" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/timesync": { "get": { "operationId": "timesync_get", @@ -863,6 +938,53 @@ } }, "schemas": { + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BundleUtilization": { "description": "The portion of a debug dataset used for zone bundles.", "type": "object", @@ -1601,6 +1723,54 @@ "secs" ] }, + "EarlyNetworkConfig": { + "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/EarlyNetworkConfigBody" + }, + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "EarlyNetworkConfigBody": { + "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "ntp_servers": { + "description": "The external NTP server addresses.", + "type": "array", + "items": { + "type": "string" + } + }, + "rack_network_config": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/RackNetworkConfigV1" + } + ] + } + }, + "required": [ + "ntp_servers" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", @@ -1667,6 +1837,26 @@ } ] }, + "HostPortConfig": { + "type": "object", + "properties": { + "addrs": { + "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "port": { + "description": "Switchport to use for external connectivity", + "type": "string" + } + }, + "required": [ + "addrs", + "port" + ] + }, "InstanceCpuCount": { "description": "The number of CPUs in an Instance", "type": "integer", @@ -2133,6 +2323,26 @@ } ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "Ipv4Net": { "example": "192.168.1.0/24", "title": "An IPv4 subnet", @@ -2140,6 +2350,10 @@ "type": "string", "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$" }, + "Ipv4Network": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" + }, "Ipv6Net": { "example": "fd12:3456::/64", "title": "An IPv6 subnet", @@ -2147,6 +2361,10 @@ "type": "string", "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "KnownArtifactKind": { "description": "Kinds of update artifacts, as used by Nexus to determine what updates are available and by sled-agent to determine how to apply an update when asked.", "type": "string", @@ -2281,6 +2499,93 @@ } ] }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, + "PortFec": { + "description": "Switchport FEC options", + "type": "string", + "enum": [ + "firecode", + "none", + "rs" + ] + }, + "PortSpeed": { + "description": "Switchport Speed options", + "type": "string", + "enum": [ + "speed0_g", + "speed1_g", + "speed10_g", + "speed25_g", + "speed40_g", + "speed50_g", + "speed100_g", + "speed200_g", + "speed400_g" + ] + }, "PriorityDimension": { "description": "A dimension along with bundles can be sorted, to determine priority.", "oneOf": [ @@ -2309,6 +2614,68 @@ "minItems": 2, "maxItems": 2 }, + "RackNetworkConfigV1": { + "description": "Initial network configuration", + "type": "object", + "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, + "infra_ip_first": { + "description": "First ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "infra_ip_last": { + "description": "Last ip address to be used for configuring network infrastructure", + "type": "string", + "format": "ipv4" + }, + "ports": { + "description": "Uplinks for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/PortConfigV1" + } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" + } + }, + "required": [ + "bgp", + "infra_ip_first", + "infra_ip_last", + "ports", + "rack_subnet" + ] + }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -2823,6 +3190,40 @@ "format": "uint8", "minimum": 0 }, + "SwitchLocation": { + "description": "Identifies switch physical location", + "oneOf": [ + { + "description": "Switch in upper slot", + "type": "string", + "enum": [ + "switch0" + ] + }, + { + "description": "Switch in lower slot", + "type": "string", + "enum": [ + "switch1" + ] + } + ] + }, + "SwitchPorts": { + "description": "A set of switch uplinks.", + "type": "object", + "properties": { + "uplinks": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HostPortConfig" + } + } + }, + "required": [ + "uplinks" + ] + }, "TimeSync": { "type": "object", "properties": { diff --git a/openapi/wicketd.json b/openapi/wicketd.json index 8b4da8970f..75db82e8e1 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -838,6 +838,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Ipv4Network" + } + } + }, + "required": [ + "asn", + "originate" + ] + }, + "BgpPeerConfig": { + "type": "object", + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + }, + "required": [ + "addr", + "asn", + "port" + ] + }, "BootstrapSledDescription": { "type": "object", "properties": { @@ -1025,7 +1072,7 @@ "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] } @@ -1339,6 +1386,26 @@ "installable" ] }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Network" + } + ] + } + ] + }, "IpRange": { "oneOf": [ { @@ -1381,6 +1448,10 @@ "last" ] }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -1404,6 +1475,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/components/schemas/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/components/schemas/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/components/schemas/PortSpeed" + } + ] + } + }, + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ] + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -1973,7 +2107,7 @@ } }, "rack_network_config": { - "$ref": "#/components/schemas/RackNetworkConfig" + "$ref": "#/components/schemas/RackNetworkConfigV1" } }, "required": [ @@ -1990,10 +2124,17 @@ "type": "string", "format": "uuid" }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -2004,18 +2145,23 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/components/schemas/UplinkConfig" + "$ref": "#/components/schemas/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/components/schemas/Ipv6Network" } }, "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ] }, "RackOperationStatus": { @@ -2332,6 +2478,28 @@ } ] }, + "RouteConfig": { + "type": "object", + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination", + "nexthop" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -4457,67 +4625,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/components/schemas/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/components/schemas/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/components/schemas/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/components/schemas/PortSpeed" - } - ] - }, - "uplink_vid": { - "nullable": true, - "description": "VLAN id to use for uplink", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ] - }, "IgnitionCommand": { "description": "Ignition command.", "type": "string", diff --git a/package-manifest.toml b/package-manifest.toml index a88f8170d0..3404f5f44f 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -412,7 +412,7 @@ source.commit = "334df299a56cd0d33e1227ed4ce4d2fe7478d541" source.sha256 = "531e0654de94b6e805836c35aa88b8a1ac691184000a03976e2b7825061e904e" output.type = "zone" -[package.maghemite] +[package.mg-ddm-gz] service_name = "mg-ddm" # Note: unlike every other package, `maghemite` is not restricted to either the # "standard" or "trampoline" image; it is included in both. @@ -422,10 +422,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d" +source.commit = "2f25a2005521f643317879b46692141b4127608a" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "e57fe791ee898d59890c5779fbd4dce598250fb6ed53832024212bcdeec0cc5b" +source.sha256 = "e808388cd080a3325fb5429314a5674809bcde24ad0456b58b57c87cbaa1300d" output.type = "tarball" [package.mg-ddm] @@ -438,10 +438,25 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "12703675393459e74139f8140e0b3c4c4f129d5d" +source.commit = "2f25a2005521f643317879b46692141b4127608a" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "3aa0d32b1d2b6be7091b9c665657296e924a86a00ca38756e9f45a1e629fd92b" +source.sha256 = "27e4845fd11b9768559eb9835309387e83c95628a6a292977e734e8bc7f9fa0f" +output.type = "zone" +output.intermediate_only = true + +[package.mgd] +service_name = "mgd" +source.type = "prebuilt" +source.repo = "maghemite" +# Updating the commit hash here currently requires also updating +# `tools/maghemite_openapi_version`. Failing to do so will cause a failure when +# building `ddm-admin-client` (which will instruct you to update +# `tools/maghemite_openapi_version`). +source.commit = "2f25a2005521f643317879b46692141b4127608a" +# The SHA256 digest is automatically posted to: +# https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt +source.sha256 = "16878501f5440590674acd82bee6ce5dcf3d1326531c25064dd9c060ab6440a4" output.type = "zone" output.intermediate_only = true @@ -458,8 +473,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" +source.commit = "c0cbc39b55fac54b95468304c497e00f3d3cf686" +source.sha256 = "3706e0e8230b7f76407ec0acea9020b9efc7d6c78b74c304102fd8e62cac6760" output.type = "zone" output.intermediate_only = true @@ -483,8 +498,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "76ff76d3526323c3fcbe2351cf9fbda4840e0dc11cd0eb6b71a3e0bd36c5e5e8" +source.commit = "c0cbc39b55fac54b95468304c497e00f3d3cf686" +source.sha256 = "f0847927f7d7197d9a5c4267a0bd0af609d18fd8d6d9b80755c370872c5297fa" output.type = "zone" output.intermediate_only = true @@ -501,8 +516,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "7712104585266a2898da38c1345210ad26f9e71d" -source.sha256 = "b8e5c176070f9bc9ea0028de1999c77d66ea3438913664163975964effe4481b" +source.commit = "c0cbc39b55fac54b95468304c497e00f3d3cf686" +source.sha256 = "33b5897db1fe7b57d282531724ecd7bf74f5156f9aa23f10c6f0d9b54c38a987" output.type = "zone" output.intermediate_only = true @@ -534,6 +549,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "xcvradm.tar.gz" ] @@ -555,6 +571,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "sp-sim-stub.tar.gz" ] @@ -576,6 +593,7 @@ source.packages = [ "wicketd.tar.gz", "wicket.tar.gz", "mg-ddm.tar.gz", + "mgd.tar.gz", "switch_zone_setup.tar.gz", "sp-sim-softnpu.tar.gz" ] diff --git a/schema/crdb/8.0.0/up01.sql b/schema/crdb/8.0.0/up01.sql new file mode 100644 index 0000000000..c617a0b634 --- /dev/null +++ b/schema/crdb/8.0.0/up01.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.service_kind ADD VALUE IF NOT EXISTS 'mgd'; diff --git a/schema/crdb/8.0.0/up02.sql b/schema/crdb/8.0.0/up02.sql new file mode 100644 index 0000000000..119e7b9a86 --- /dev/null +++ b/schema/crdb/8.0.0/up02.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.bgp_config ADD COLUMN IF NOT EXISTS bgp_announce_set_id UUID NOT NULL; diff --git a/schema/crdb/8.0.0/up03.sql b/schema/crdb/8.0.0/up03.sql new file mode 100644 index 0000000000..3705d4091e --- /dev/null +++ b/schema/crdb/8.0.0/up03.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config DROP COLUMN IF EXISTS bgp_announce_set_id; diff --git a/schema/crdb/8.0.0/up04.sql b/schema/crdb/8.0.0/up04.sql new file mode 100644 index 0000000000..c5a91796dd --- /dev/null +++ b/schema/crdb/8.0.0/up04.sql @@ -0,0 +1,5 @@ +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); diff --git a/schema/crdb/8.0.0/up05.sql b/schema/crdb/8.0.0/up05.sql new file mode 100644 index 0000000000..4d94bafb9f --- /dev/null +++ b/schema/crdb/8.0.0/up05.sql @@ -0,0 +1,11 @@ +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); diff --git a/schema/crdb/8.0.0/up06.sql b/schema/crdb/8.0.0/up06.sql new file mode 100644 index 0000000000..e27800969c --- /dev/null +++ b/schema/crdb/8.0.0/up06.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; diff --git a/schema/crdb/8.0.0/up07.sql b/schema/crdb/8.0.0/up07.sql new file mode 100644 index 0000000000..c84ae8e5d2 --- /dev/null +++ b/schema/crdb/8.0.0/up07.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; diff --git a/schema/crdb/8.0.0/up08.sql b/schema/crdb/8.0.0/up08.sql new file mode 100644 index 0000000000..c84480feba --- /dev/null +++ b/schema/crdb/8.0.0/up08.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS hold_time INT8; diff --git a/schema/crdb/8.0.0/up09.sql b/schema/crdb/8.0.0/up09.sql new file mode 100644 index 0000000000..82f645c753 --- /dev/null +++ b/schema/crdb/8.0.0/up09.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS idle_hold_time INT8; diff --git a/schema/crdb/8.0.0/up10.sql b/schema/crdb/8.0.0/up10.sql new file mode 100644 index 0000000000..a672953991 --- /dev/null +++ b/schema/crdb/8.0.0/up10.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS delay_open INT8; diff --git a/schema/crdb/8.0.0/up11.sql b/schema/crdb/8.0.0/up11.sql new file mode 100644 index 0000000000..63f16a011f --- /dev/null +++ b/schema/crdb/8.0.0/up11.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS connect_retry INT8; diff --git a/schema/crdb/8.0.0/up12.sql b/schema/crdb/8.0.0/up12.sql new file mode 100644 index 0000000000..431d10cd3c --- /dev/null +++ b/schema/crdb/8.0.0/up12.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.switch_port_settings_bgp_peer_config ADD COLUMN IF NOT EXISTS keepalive INT8; diff --git a/schema/crdb/8.0.0/up13.sql b/schema/crdb/8.0.0/up13.sql new file mode 100644 index 0000000000..44bfd90b8c --- /dev/null +++ b/schema/crdb/8.0.0/up13.sql @@ -0,0 +1 @@ +ALTER TABLE omicron.public.rack ADD COLUMN IF NOT EXISTS rack_subnet INET; diff --git a/schema/crdb/8.0.0/up14.sql b/schema/crdb/8.0.0/up14.sql new file mode 100644 index 0000000000..18ce39e61c --- /dev/null +++ b/schema/crdb/8.0.0/up14.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS omicron.public.bootstore_keys ( + key TEXT NOT NULL PRIMARY KEY, + generation INT8 NOT NULL +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 4d0589b3a0..0fdaf5083c 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -63,7 +63,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.rack ( initialized BOOL NOT NULL, /* Used to configure the updates service URL */ - tuf_base_url STRING(512) + tuf_base_url STRING(512), + + /* The IPv6 underlay /56 prefix for the rack */ + rack_subnet INET ); /* @@ -198,7 +201,8 @@ CREATE TYPE IF NOT EXISTS omicron.public.service_kind AS ENUM ( 'nexus', 'ntp', 'oximeter', - 'tfport' + 'tfport', + 'mgd' ); CREATE TABLE IF NOT EXISTS omicron.public.service ( @@ -2441,10 +2445,14 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_route_config ( CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_bgp_peer_config ( port_settings_id UUID, - bgp_announce_set_id UUID NOT NULL, bgp_config_id UUID NOT NULL, interface_name TEXT, addr INET, + hold_time INT8, + idle_hold_time INT8, + delay_open INT8, + connect_retry INT8, + keepalive INT8, /* TODO https://github.com/oxidecomputer/omicron/issues/3013 */ PRIMARY KEY (port_settings_id, interface_name, addr) @@ -2458,7 +2466,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.bgp_config ( time_modified TIMESTAMPTZ NOT NULL, time_deleted TIMESTAMPTZ, asn INT8 NOT NULL, - vrf TEXT + vrf TEXT, + bgp_announce_set_id UUID NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS lookup_bgp_config_by_name ON omicron.public.bgp_config ( @@ -2500,6 +2509,11 @@ CREATE TABLE IF NOT EXISTS omicron.public.switch_port_settings_address_config ( PRIMARY KEY (port_settings_id, address, interface_name) ); +CREATE TABLE IF NOT EXISTS omicron.public.bootstore_keys ( + key TEXT NOT NULL PRIMARY KEY, + generation INT8 NOT NULL +); + /* * The `sled_instance` view's definition needs to be modified in a separate * transaction from the transaction that created it. @@ -2559,6 +2573,27 @@ FROM WHERE instance.time_deleted IS NULL AND vmm.time_deleted IS NULL; +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_fec AS ENUM ( + 'Firecode', + 'None', + 'Rs' +); + +CREATE TYPE IF NOT EXISTS omicron.public.switch_link_speed AS ENUM ( + '0G', + '1G', + '10G', + '25G', + '40G', + '50G', + '100G', + '200G', + '400G' +); + +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS fec omicron.public.switch_link_fec; +ALTER TABLE omicron.public.switch_port_settings_link_config ADD COLUMN IF NOT EXISTS speed omicron.public.switch_link_speed; + CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( -- There should only be one row of this table for the whole DB. -- It's a little goofy, but filter on "singleton = true" before querying @@ -2585,7 +2620,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '7.0.0', NULL) + ( TRUE, NOW(), NOW(), '8.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 4a8b02d23d..39a9a68acc 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -91,6 +91,53 @@ } ] }, + "BgpConfig": { + "type": "object", + "required": [ + "asn", + "originate" + ], + "properties": { + "asn": { + "description": "The autonomous system number for the BGP configuration.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "originate": { + "description": "The set of prefixes for the BGP router to originate.", + "type": "array", + "items": { + "$ref": "#/definitions/Ipv4Network" + } + } + } + }, + "BgpPeerConfig": { + "type": "object", + "required": [ + "addr", + "asn", + "port" + ], + "properties": { + "addr": { + "description": "Address of the peer.", + "type": "string", + "format": "ipv4" + }, + "asn": { + "description": "The autonomous sysetm number of the router the peer belongs to.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" + } + } + }, "BootstrapAddressDiscovery": { "oneOf": [ { @@ -149,6 +196,27 @@ } } }, + "IpNetwork": { + "oneOf": [ + { + "title": "v4", + "allOf": [ + { + "$ref": "#/definitions/Ipv4Network" + } + ] + }, + { + "title": "v6", + "allOf": [ + { + "$ref": "#/definitions/Ipv6Network" + } + ] + } + ], + "x-rust-type": "ipnetwork::IpNetwork" + }, "IpRange": { "oneOf": [ { @@ -201,6 +269,11 @@ "type": "string", "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$", + "x-rust-type": "ipnetwork::Ipv6Network" + }, "Ipv6Range": { "description": "A non-decreasing IPv6 address range, inclusive of both ends.\n\nThe first address must be less than or equal to the last address.", "type": "object", @@ -244,6 +317,69 @@ "description": "Password hashes must be in PHC (Password Hashing Competition) string format. Passwords must be hashed with Argon2id. Password hashes may be rejected if the parameters appear not to be secure enough.", "type": "string" }, + "PortConfigV1": { + "type": "object", + "required": [ + "addresses", + "bgp_peers", + "port", + "routes", + "switch", + "uplink_port_fec", + "uplink_port_speed" + ], + "properties": { + "addresses": { + "description": "This port's addresses.", + "type": "array", + "items": { + "$ref": "#/definitions/IpNetwork" + } + }, + "bgp_peers": { + "description": "BGP peers on this port", + "type": "array", + "items": { + "$ref": "#/definitions/BgpPeerConfig" + } + }, + "port": { + "description": "Nmae of the port this config applies to.", + "type": "string" + }, + "routes": { + "description": "The set of routes associated with this port.", + "type": "array", + "items": { + "$ref": "#/definitions/RouteConfig" + } + }, + "switch": { + "description": "Switch the port belongs to.", + "allOf": [ + { + "$ref": "#/definitions/SwitchLocation" + } + ] + }, + "uplink_port_fec": { + "description": "Port forward error correction type.", + "allOf": [ + { + "$ref": "#/definitions/PortFec" + } + ] + }, + "uplink_port_speed": { + "description": "Port speed.", + "allOf": [ + { + "$ref": "#/definitions/PortSpeed" + } + ] + } + } + }, "PortFec": { "description": "Switchport FEC options", "type": "string", @@ -336,7 +472,7 @@ "description": "Initial rack network configuration", "anyOf": [ { - "$ref": "#/definitions/RackNetworkConfig" + "$ref": "#/definitions/RackNetworkConfigV1" }, { "type": "null" @@ -367,15 +503,24 @@ } } }, - "RackNetworkConfig": { + "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", "required": [ + "bgp", "infra_ip_first", "infra_ip_last", - "uplinks" + "ports", + "rack_subnet" ], "properties": { + "bgp": { + "description": "BGP configurations for connecting the rack to external networks", + "type": "array", + "items": { + "$ref": "#/definitions/BgpConfig" + } + }, "infra_ip_first": { "description": "First ip address to be used for configuring network infrastructure", "type": "string", @@ -386,12 +531,15 @@ "type": "string", "format": "ipv4" }, - "uplinks": { + "ports": { "description": "Uplinks for connecting the rack to external networks", "type": "array", "items": { - "$ref": "#/definitions/UplinkConfig" + "$ref": "#/definitions/PortConfigV1" } + }, + "rack_subnet": { + "$ref": "#/definitions/Ipv6Network" } } }, @@ -414,6 +562,28 @@ } } }, + "RouteConfig": { + "type": "object", + "required": [ + "destination", + "nexthop" + ], + "properties": { + "destination": { + "description": "The destination of the route.", + "allOf": [ + { + "$ref": "#/definitions/IpNetwork" + } + ] + }, + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + } + } + }, "StartSledAgentRequest": { "description": "Configuration information for launching a Sled Agent.", "type": "object", @@ -484,69 +654,6 @@ } ] }, - "UplinkConfig": { - "type": "object", - "required": [ - "gateway_ip", - "switch", - "uplink_cidr", - "uplink_port", - "uplink_port_fec", - "uplink_port_speed" - ], - "properties": { - "gateway_ip": { - "description": "Gateway address", - "type": "string", - "format": "ipv4" - }, - "switch": { - "description": "Switch to use for uplink", - "allOf": [ - { - "$ref": "#/definitions/SwitchLocation" - } - ] - }, - "uplink_cidr": { - "description": "IP Address and prefix (e.g., `192.168.0.1/16`) to apply to switchport (must be in infra_ip pool)", - "allOf": [ - { - "$ref": "#/definitions/Ipv4Network" - } - ] - }, - "uplink_port": { - "description": "Switchport to use for external connectivity", - "type": "string" - }, - "uplink_port_fec": { - "description": "Forward Error Correction setting for the uplink port", - "allOf": [ - { - "$ref": "#/definitions/PortFec" - } - ] - }, - "uplink_port_speed": { - "description": "Speed for the Switchport", - "allOf": [ - { - "$ref": "#/definitions/PortSpeed" - } - ] - }, - "uplink_vid": { - "description": "VLAN id to use for uplink", - "type": [ - "integer", - "null" - ], - "format": "uint16", - "minimum": 0.0 - } - } - }, "UserId": { "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 636c9665ef..ff9644773a 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -55,7 +55,7 @@ reqwest = { workspace = true, features = ["rustls-tls", "stream"] } schemars = { workspace = true, features = [ "chrono", "uuid1" ] } semver.workspace = true serde.workspace = true -serde_json.workspace = true +serde_json = {workspace = true, features = ["raw_value"]} sha3.workspace = true sled-agent-client.workspace = true sled-hardware.workspace = true diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 61d4c84af3..6c19080e9c 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -7,29 +7,31 @@ use anyhow::{anyhow, Context}; use bootstore::schemes::v0 as bootstore; use ddm_admin_client::{Client as DdmAdminClient, DdmError}; -use dpd_client::types::Ipv6Entry; +use dpd_client::types::{Ipv6Entry, RouteSettingsV6}; use dpd_client::types::{ LinkCreate, LinkId, LinkSettings, PortId, PortSettings, RouteSettingsV4, }; use dpd_client::Client as DpdClient; -use dpd_client::Ipv4Cidr; use futures::future; use gateway_client::Client as MgsClient; use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; use internal_dns::ServiceName; -use omicron_common::address::{Ipv6Subnet, AZ_PREFIX, MGS_PORT}; +use ipnetwork::{IpNetwork, Ipv6Network}; +use omicron_common::address::{Ipv6Subnet, MGS_PORT}; use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; use omicron_common::api::internal::shared::{ - PortFec, PortSpeed, RackNetworkConfig, SwitchLocation, UplinkConfig, + PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV1, + SwitchLocation, UplinkConfig, }; use omicron_common::backoff::{ retry_notify, retry_policy_local, BackoffError, ExponentialBackoff, ExponentialBackoffBuilder, }; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use slog::Logger; use std::collections::{HashMap, HashSet}; -use std::net::{IpAddr, Ipv6Addr, SocketAddrV6}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddrV6}; use std::time::{Duration, Instant}; use thiserror::Error; @@ -107,11 +109,11 @@ impl<'a> EarlyNetworkSetup<'a> { resolver: &DnsResolver, config: &RackNetworkConfig, ) -> HashSet { - // Which switches have uplinks? + // Which switches have configured ports? let uplinked_switches = config - .uplinks + .ports .iter() - .map(|uplink_config| uplink_config.switch) + .map(|port_config| port_config.switch) .collect::>(); // If we have no uplinks, we have nothing to look up. @@ -342,7 +344,7 @@ impl<'a> EarlyNetworkSetup<'a> { &mut self, rack_network_config: &RackNetworkConfig, switch_zone_underlay_ip: Ipv6Addr, - ) -> Result, EarlyNetworkSetupError> { + ) -> Result, EarlyNetworkSetupError> { // First, we have to know which switch we are: ask MGS. info!( self.log, @@ -385,10 +387,10 @@ impl<'a> EarlyNetworkSetup<'a> { }; // We now know which switch we are: filter the uplinks to just ours. - let our_uplinks = rack_network_config - .uplinks + let our_ports = rack_network_config + .ports .iter() - .filter(|uplink| uplink.switch == switch_location) + .filter(|port| port.switch == switch_location) .cloned() .collect::>(); @@ -396,7 +398,7 @@ impl<'a> EarlyNetworkSetup<'a> { self.log, "Initializing {} Uplinks on {switch_location:?} at \ {switch_zone_underlay_ip}", - our_uplinks.len(), + our_ports.len(), ); let dpd = DpdClient::new( &format!("http://[{}]:{}", switch_zone_underlay_ip, DENDRITE_PORT), @@ -408,9 +410,9 @@ impl<'a> EarlyNetworkSetup<'a> { // configure uplink for each requested uplink in configuration that // matches our switch_location - for uplink_config in &our_uplinks { + for port_config in &our_ports { let (ipv6_entry, dpd_port_settings, port_id) = - self.build_uplink_config(uplink_config)?; + self.build_port_config(port_config)?; self.wait_for_dendrite(&dpd).await; @@ -446,14 +448,14 @@ impl<'a> EarlyNetworkSetup<'a> { ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr)); } - Ok(our_uplinks) + Ok(our_ports) } - fn build_uplink_config( + fn build_port_config( &self, - uplink_config: &UplinkConfig, + port_config: &PortConfigV1, ) -> Result<(Ipv6Entry, PortSettings, PortId), EarlyNetworkSetupError> { - info!(self.log, "Building Uplink Configuration"); + info!(self.log, "Building Port Configuration"); let ipv6_entry = Ipv6Entry { addr: BOUNDARY_SERVICES_ADDR.parse().map_err(|e| { EarlyNetworkSetupError::BadConfig(format!( @@ -469,41 +471,57 @@ impl<'a> EarlyNetworkSetup<'a> { v6_routes: HashMap::new(), }; let link_id = LinkId(0); + + let mut addrs = Vec::new(); + for a in &port_config.addresses { + addrs.push(a.ip()); + } + // TODO We're discarding the `uplink_cidr.prefix()` here and only using // the IP address; at some point we probably need to give the full CIDR // to dendrite? - let addr = IpAddr::V4(uplink_config.uplink_cidr.ip()); let link_settings = LinkSettings { // TODO Allow user to configure link properties // https://github.com/oxidecomputer/omicron/issues/3061 params: LinkCreate { autoneg: false, kr: false, - fec: convert_fec(&uplink_config.uplink_port_fec), - speed: convert_speed(&uplink_config.uplink_port_speed), + fec: convert_fec(&port_config.uplink_port_fec), + speed: convert_speed(&port_config.uplink_port_speed), }, - addrs: vec![addr], + //addrs: vec![addr], + addrs, }; dpd_port_settings.links.insert(link_id.to_string(), link_settings); - let port_id: PortId = - uplink_config.uplink_port.parse().map_err(|e| { - EarlyNetworkSetupError::BadConfig(format!( - concat!( - "could not use value provided to", - "rack_network_config.uplink_port as PortID: {}" - ), - e - )) - })?; - dpd_port_settings.v4_routes.insert( - Ipv4Cidr { prefix: "0.0.0.0".parse().unwrap(), prefix_len: 0 } - .to_string(), - RouteSettingsV4 { - link_id: link_id.0, - vid: uplink_config.uplink_vid, - nexthop: uplink_config.gateway_ip, - }, - ); + let port_id: PortId = port_config.port.parse().map_err(|e| { + EarlyNetworkSetupError::BadConfig(format!( + concat!( + "could not use value provided to", + "rack_network_config.uplink_port as PortID: {}" + ), + e + )) + })?; + + for r in &port_config.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v4_routes.insert( + dst.to_string(), + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, + ); + } + if let (IpNetwork::V6(dst), IpAddr::V6(nexthop)) = + (r.destination, r.nexthop) + { + dpd_port_settings.v6_routes.insert( + dst.to_string(), + RouteSettingsV6 { link_id: link_id.0, nexthop, vid: None }, + ); + } + } + Ok((ipv6_entry, dpd_port_settings, port_id)) } @@ -546,33 +564,68 @@ fn retry_policy_switch_mapping() -> ExponentialBackoff { .build() } +// The first production version of the `EarlyNetworkConfig`. +// +// If this version is in the bootstore than we need to convert it to +// `EarlyNetworkConfigV1`. +// +// Once we do this for all customers that have initialized racks with the +// old version we can go ahead and remove this type and its conversion code +// altogether. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +struct EarlyNetworkConfigV0 { + // The current generation number of data as stored in CRDB. + // The initial generation is set during RSS time and then only mutated + // by Nexus. + pub generation: u64, + + pub rack_subnet: Ipv6Addr, + + /// The external NTP server addresses. + pub ntp_servers: Vec, + + // Rack network configuration as delivered from RSS and only existing at + // generation 1 + pub rack_network_config: Option, +} + /// Network configuration required to bring up the control plane /// /// The fields in this structure are those from /// [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This /// is just for the initial rack configuration and cold boot purposes. Updates -/// will come from Nexus in the future. -#[derive(Clone, Debug, Deserialize, Serialize)] +/// come from Nexus. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] pub struct EarlyNetworkConfig { - // The version of data. Always `1` when created from RSS. + // The current generation number of data as stored in CRDB. + // The initial generation is set during RSS time and then only mutated + // by Nexus. pub generation: u64, - pub rack_subnet: Ipv6Addr, + // Which version of the data structure do we have. This is to help with + // deserialization and conversion in future updates. + pub schema_version: u32, + + // The actual configuration details + pub body: EarlyNetworkConfigBody, +} +/// This is the actual configuration of EarlyNetworking. +/// +/// We nest it below the "header" of `generation` and `schema_version` so that +/// we can perform partial deserialization of `EarlyNetworkConfig` to only read +/// the header and defer deserialization of the body once we know the schema +/// version. This is possible via the use of [`serde_json::value::RawValue`] in +/// future (post-v1) deserialization paths. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq)] +pub struct EarlyNetworkConfigBody { /// The external NTP server addresses. pub ntp_servers: Vec, - /// A copy of the initial rack network configuration when we are in - /// generation `1`. + // Rack network configuration as delivered from RSS or Nexus pub rack_network_config: Option, } -impl EarlyNetworkConfig { - pub fn az_subnet(&self) -> Ipv6Subnet { - Ipv6Subnet::::new(self.rack_subnet) - } -} - impl From for bootstore::NetworkConfig { fn from(value: EarlyNetworkConfig) -> Self { // Can this ever actually fail? @@ -586,13 +639,77 @@ impl From for bootstore::NetworkConfig { } } +// Note: This currently only converts between v0 and v1 or deserializes v1 of +// `EarlyNetworkConfig`. impl TryFrom for EarlyNetworkConfig { type Error = serde_json::Error; fn try_from( value: bootstore::NetworkConfig, ) -> std::result::Result { - serde_json::from_slice(&value.blob) + // Try to deserialize the latest version of the data structure (v1). If + // that succeeds we are done. + if let Ok(val) = + serde_json::from_slice::(&value.blob) + { + return Ok(val); + } + + // We don't have the latest version. Try to deserialize v0 and then + // convert it to the latest version. + let v0 = serde_json::from_slice::(&value.blob)?; + + Ok(EarlyNetworkConfig { + generation: v0.generation, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: v0.ntp_servers, + rack_network_config: v0.rack_network_config.map(|v0_config| { + RackNetworkConfigV0::to_v1(v0.rack_subnet, v0_config) + }), + }, + }) + } +} + +/// Deprecated, use `RackNetworkConfig` instead. Cannot actually deprecate due to +/// +/// +/// Our first version of `RackNetworkConfig`. If this exists in the bootstore, we +/// upgrade out of it into `RackNetworkConfigV1` or later versions if possible. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] +pub struct RackNetworkConfigV0 { + // TODO: #3591 Consider making infra-ip ranges implicit for uplinks + /// First ip address to be used for configuring network infrastructure + pub infra_ip_first: Ipv4Addr, + /// Last ip address to be used for configuring network infrastructure + pub infra_ip_last: Ipv4Addr, + /// Uplinks for connecting the rack to external networks + pub uplinks: Vec, +} + +impl RackNetworkConfigV0 { + /// Convert from `RackNetworkConfigV0` to `RackNetworkConfigV1` + /// + /// We cannot use `From for `RackNetworkConfigV1` + /// because the `rack_subnet` field does not exist in `RackNetworkConfigV0` + /// and must be passed in from the `EarlyNetworkConfigV0` struct which + /// contains the `RackNetworkConfivV0` struct. + pub fn to_v1( + rack_subnet: Ipv6Addr, + v0: RackNetworkConfigV0, + ) -> RackNetworkConfigV1 { + RackNetworkConfigV1 { + rack_subnet: Ipv6Network::new(rack_subnet, 56).unwrap(), + infra_ip_first: v0.infra_ip_first, + infra_ip_last: v0.infra_ip_last, + ports: v0 + .uplinks + .into_iter() + .map(|uplink| PortConfigV1::from(uplink)) + .collect(), + bgp: vec![], + } } } @@ -621,3 +738,66 @@ fn convert_fec(fec: &PortFec) -> dpd_client::types::PortFec { PortFec::Rs => dpd_client::types::PortFec::Rs, } } + +#[cfg(test)] +mod tests { + use super::*; + use omicron_common::api::internal::shared::RouteConfig; + + #[test] + fn serialized_early_network_config_v0_to_v1_conversion() { + let v0 = EarlyNetworkConfigV0 { + generation: 1, + rack_subnet: Ipv6Addr::UNSPECIFIED, + ntp_servers: Vec::new(), + rack_network_config: Some(RackNetworkConfigV0 { + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + uplinks: vec![UplinkConfig { + gateway_ip: Ipv4Addr::UNSPECIFIED, + switch: SwitchLocation::Switch0, + uplink_port: "Port0".to_string(), + uplink_port_speed: PortSpeed::Speed100G, + uplink_port_fec: PortFec::None, + uplink_cidr: "192.168.0.1/16".parse().unwrap(), + uplink_vid: None, + }], + }), + }; + + let v0_serialized = serde_json::to_vec(&v0).unwrap(); + let bootstore_conf = + bootstore::NetworkConfig { generation: 1, blob: v0_serialized }; + + let v1 = EarlyNetworkConfig::try_from(bootstore_conf).unwrap(); + let v0_rack_network_config = v0.rack_network_config.unwrap(); + let uplink = v0_rack_network_config.uplinks[0].clone(); + let expected = EarlyNetworkConfig { + generation: 1, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: v0.ntp_servers.clone(), + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: Ipv6Network::new(v0.rack_subnet, 56).unwrap(), + infra_ip_first: v0_rack_network_config.infra_ip_first, + infra_ip_last: v0_rack_network_config.infra_ip_last, + ports: vec![PortConfigV1 { + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: uplink.gateway_ip.into(), + }], + addresses: vec![uplink.uplink_cidr.into()], + switch: uplink.switch, + port: uplink.uplink_port, + uplink_port_speed: uplink.uplink_port_speed, + uplink_port_fec: uplink.uplink_port_fec, + bgp_peers: vec![], + }], + bgp: vec![], + }), + }, + }; + + assert_eq!(expected, v1); + } +} diff --git a/sled-agent/src/bootstrap/maghemite.rs b/sled-agent/src/bootstrap/maghemite.rs index 1adc677b23..2cf0eaf190 100644 --- a/sled-agent/src/bootstrap/maghemite.rs +++ b/sled-agent/src/bootstrap/maghemite.rs @@ -8,7 +8,7 @@ use illumos_utils::addrobj::AddrObject; use slog::Logger; use thiserror::Error; -const SERVICE_FMRI: &str = "svc:/system/illumos/mg-ddm"; +const SERVICE_FMRI: &str = "svc:/oxide/mg-ddm"; const MANIFEST_PATH: &str = "/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"; #[derive(Debug, Error)] diff --git a/sled-agent/src/bootstrap/secret_retriever.rs b/sled-agent/src/bootstrap/secret_retriever.rs index 5cae06310c..1d5ac10ac5 100644 --- a/sled-agent/src/bootstrap/secret_retriever.rs +++ b/sled-agent/src/bootstrap/secret_retriever.rs @@ -14,9 +14,9 @@ use std::sync::OnceLock; static MAYBE_LRTQ_RETRIEVER: OnceLock = OnceLock::new(); -/// A [`key-manager::SecretRetriever`] that either uses a -/// [`LocalSecretRetriever`] or [`LrtqSecretRetriever`] under the hood depending -/// upon how many sleds are in the cluster at rack init time. +/// A [`key_manager::SecretRetriever`] that either uses a +/// [`HardcodedSecretRetriever`] or [`LrtqSecretRetriever`] under the +/// hood depending upon how many sleds are in the cluster at rack init time. pub struct LrtqOrHardcodedSecretRetriever {} impl LrtqOrHardcodedSecretRetriever { diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 0cbbf0678b..9ed3ad582d 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -528,7 +528,7 @@ fn start_dropshot_server( /// /// TODO-correctness Subsequent steps may assume all M.2s that will ever be /// present are present once we return from this function; see -/// https://github.com/oxidecomputer/omicron/issues/3815. +/// . async fn wait_for_boot_m2(storage_resources: &StorageResources, log: &Logger) { // Wait for at least the M.2 we booted from to show up. loop { diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 2ab8273e39..68330d0c0e 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -5,6 +5,7 @@ //! HTTP entrypoint functions for the sled agent's exposed API use super::sled_agent::SledAgent; +use crate::bootstrap::early_networking::EarlyNetworkConfig; use crate::params::{ CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, @@ -14,6 +15,7 @@ use crate::params::{ }; use crate::sled_agent::Error as SledAgentError; use crate::zone_bundle; +use bootstore::schemes::v0::NetworkConfig; use camino::Utf8PathBuf; use dropshot::{ endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseCreated, @@ -24,9 +26,10 @@ use illumos_utils::opte::params::{ DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, }; use omicron_common::api::external::Error; -use omicron_common::api::internal::nexus::DiskRuntimeState; -use omicron_common::api::internal::nexus::SledInstanceState; -use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::nexus::{ + DiskRuntimeState, SledInstanceState, UpdateArtifactId, +}; +use omicron_common::api::internal::shared::SwitchPorts; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -62,6 +65,9 @@ pub fn api() -> SledApiDescription { api.register(update_artifact)?; api.register(vpc_firewall_rules_put)?; api.register(zpools_get)?; + api.register(uplink_ensure)?; + api.register(read_network_bootstore_config_cache)?; + api.register(write_network_bootstore_config)?; Ok(()) } @@ -630,3 +636,73 @@ async fn timesync_get( let sa = rqctx.context(); Ok(HttpResponseOk(sa.timesync_get().await.map_err(|e| Error::from(e))?)) } + +#[endpoint { + method = POST, + path = "/switch-ports", +}] +async fn uplink_ensure( + rqctx: RequestContext, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + sa.ensure_scrimlet_host_ports(body.into_inner().uplinks).await?; + Ok(HttpResponseUpdatedNoContent()) +} + +/// This API endpoint is only reading the local sled agent's view of the +/// bootstore. The boostore is a distributed data store that is eventually +/// consistent. Reads from individual nodes may not represent the latest state. +#[endpoint { + method = GET, + path = "/network-bootstore-config", +}] +async fn read_network_bootstore_config_cache( + rqctx: RequestContext, +) -> Result, HttpError> { + let sa = rqctx.context(); + let bs = sa.bootstore(); + + let config = bs.get_network_config().await.map_err(|e| { + HttpError::for_internal_error(format!("failed to get bootstore: {e}")) + })?; + + let config = match config { + Some(config) => EarlyNetworkConfig::try_from(config).map_err(|e| { + HttpError::for_internal_error(format!( + "deserialize early network config: {e}" + )) + })?, + None => { + return Err(HttpError::for_unavail( + None, + "early network config does not exist yet".into(), + )); + } + }; + + Ok(HttpResponseOk(config)) +} + +#[endpoint { + method = PUT, + path = "/network-bootstore-config", +}] +async fn write_network_bootstore_config( + rqctx: RequestContext, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + let bs = sa.bootstore(); + let config = body.into_inner(); + + bs.update_network_config(NetworkConfig::from(config)).await.map_err( + |e| { + HttpError::for_internal_error(format!( + "failed to write updated config to boot store: {e}" + )) + }, + )?; + + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index e1c8b05cde..5fda3c1ae6 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -351,10 +351,12 @@ pub enum ServiceType { #[serde(skip)] Uplink, #[serde(skip)] - Maghemite { + MgDdm { mode: String, }, #[serde(skip)] + Mgd, + #[serde(skip)] SpSim, CruciblePantry { address: SocketAddrV6, @@ -404,7 +406,8 @@ impl std::fmt::Display for ServiceType { ServiceType::CruciblePantry { .. } => write!(f, "crucible/pantry"), ServiceType::BoundaryNtp { .. } | ServiceType::InternalNtp { .. } => write!(f, "ntp"), - ServiceType::Maghemite { .. } => write!(f, "mg-ddm"), + ServiceType::MgDdm { .. } => write!(f, "mg-ddm"), + ServiceType::Mgd => write!(f, "mgd"), ServiceType::SpSim => write!(f, "sp-sim"), ServiceType::Clickhouse { .. } => write!(f, "clickhouse"), ServiceType::ClickhouseKeeper { .. } => { @@ -421,13 +424,7 @@ impl crate::smf_helper::Service for ServiceType { self.to_string() } fn smf_name(&self) -> String { - match self { - // NOTE: This style of service-naming is deprecated - ServiceType::Maghemite { .. } => { - format!("svc:/system/illumos/{}", self.service_name()) - } - _ => format!("svc:/oxide/{}", self.service_name()), - } + format!("svc:/oxide/{}", self.service_name()) } fn should_import(&self) -> bool { true @@ -527,7 +524,8 @@ impl TryFrom for sled_agent_client::types::ServiceType { | St::Dendrite { .. } | St::Tfport { .. } | St::Uplink - | St::Maghemite { .. } => Err(AutonomousServiceOnlyError), + | St::Mgd + | St::MgDdm { .. } => Err(AutonomousServiceOnlyError), } } } @@ -826,7 +824,8 @@ impl ServiceZoneRequest { | ServiceType::SpSim | ServiceType::Wicketd { .. } | ServiceType::Dendrite { .. } - | ServiceType::Maghemite { .. } + | ServiceType::MgDdm { .. } + | ServiceType::Mgd | ServiceType::Tfport { .. } | ServiceType::Uplink => { return Err(AutonomousServiceOnlyError); diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 2183aa7b63..3dac5d7d1e 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -19,10 +19,11 @@ use internal_dns::{ServiceName, DNS_ZONE}; use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, ReservedRackSubnet, DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, DNS_REDUNDANCY, MAX_DNS_REDUNDANCY, - MGS_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, RSS_RESERVED_ADDRESSES, + MGD_PORT, MGS_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, RSS_RESERVED_ADDRESSES, SLED_PREFIX, }; use omicron_common::api::external::{MacAddr, Vni}; +use omicron_common::api::internal::shared::SwitchLocation; use omicron_common::api::internal::shared::{ NetworkInterface, NetworkInterfaceKind, SourceNatConfig, }; @@ -276,7 +277,7 @@ impl Plan { "No scrimlets observed".to_string(), )); } - for sled in scrimlets { + for (i, sled) in scrimlets.iter().enumerate() { let address = get_switch_zone_address(sled.subnet); let zone = dns_builder.host_dendrite(sled.sled_id, address).unwrap(); @@ -294,6 +295,18 @@ impl Plan { MGS_PORT, ) .unwrap(); + dns_builder + .service_backend_zone(ServiceName::Mgd, &zone, MGD_PORT) + .unwrap(); + + // TODO only works for single rack + let sled_address = get_sled_address(sled.subnet); + let switch_location = if i == 0 { + SwitchLocation::Switch0 + } else { + SwitchLocation::Switch1 + }; + dns_builder.host_scrimlet(switch_location, sled_address).unwrap(); } // We'll stripe most services across all available Sleds, round-robin diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 805c889295..7f6469d2c0 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -57,7 +57,8 @@ use super::config::SetupServiceConfig as Config; use crate::bootstrap::config::BOOTSTRAP_AGENT_HTTP_PORT; use crate::bootstrap::early_networking::{ - EarlyNetworkConfig, EarlyNetworkSetup, EarlyNetworkSetupError, + EarlyNetworkConfig, EarlyNetworkConfigBody, EarlyNetworkSetup, + EarlyNetworkSetupError, }; use crate::bootstrap::params::BootstrapAddressDiscovery; use crate::bootstrap::params::StartSledAgentRequest; @@ -575,17 +576,25 @@ impl ServiceInner { let rack_network_config = match &config.rack_network_config { Some(config) => { - let value = NexusTypes::RackNetworkConfig { + let value = NexusTypes::RackNetworkConfigV1 { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| NexusTypes::UplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| NexusTypes::PortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| NexusTypes::RouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), switch: config.switch.into(), - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: config .uplink_port_speed .clone() @@ -594,7 +603,23 @@ impl ServiceInner { .uplink_port_fec .clone() .into(), - uplink_vid: config.uplink_vid, + bgp_peers: config + .bgp_peers + .iter() + .map(|b| NexusTypes::BgpPeerConfig { + addr: b.addr, + asn: b.asn, + port: b.port.clone(), + }) + .collect(), + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| NexusTypes::BgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), }; @@ -872,9 +897,11 @@ impl ServiceInner { // from the bootstore". let early_network_config = EarlyNetworkConfig { generation: 1, - rack_subnet: config.rack_subnet, - ntp_servers: config.ntp_servers.clone(), - rack_network_config: config.rack_network_config.clone(), + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: config.ntp_servers.clone(), + rack_network_config: config.rack_network_config.clone(), + }, }; info!(self.log, "Writing Rack Network Configuration to bootstore"); bootstore.update_network_config(early_network_config.into()).await?; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 06d3ae1977..a9be0e7c4a 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -77,7 +77,9 @@ use omicron_common::address::WICKETD_NEXUS_PROXY_PORT; use omicron_common::address::WICKETD_PORT; use omicron_common::address::{Ipv6Subnet, NEXUS_TECHPORT_EXTERNAL_PORT}; use omicron_common::api::external::Generation; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + HostPortConfig, RackNetworkConfig, +}; use omicron_common::backoff::{ retry_notify, retry_policy_internal_service_aggressive, retry_policy_local, BackoffError, @@ -96,8 +98,8 @@ use sled_hardware::underlay::BOOTSTRAP_PREFIX; use sled_hardware::Baseboard; use sled_hardware::SledMode; use slog::Logger; +use std::collections::BTreeMap; use std::collections::HashSet; -use std::collections::{BTreeMap, HashMap}; use std::iter; use std::iter::FromIterator; use std::net::{IpAddr, Ipv6Addr, SocketAddr}; @@ -758,7 +760,7 @@ impl ServiceManager { } } } - ServiceType::Maghemite { .. } => { + ServiceType::MgDdm { .. } => { // If on a non-gimlet, sled-agent can be configured to map // links into the switch zone. Validate those links here. for link in &self.inner.switch_zone_maghemite_links { @@ -1953,8 +1955,13 @@ impl ServiceManager { // Nothing to do here - this service is special and // configured in `ensure_switch_zone_uplinks_configured` } - ServiceType::Maghemite { mode } => { - info!(self.inner.log, "Setting up Maghemite service"); + ServiceType::Mgd => { + info!(self.inner.log, "Setting up mgd service"); + smfh.setprop("config/admin_host", "::")?; + smfh.refresh()?; + } + ServiceType::MgDdm { mode } => { + info!(self.inner.log, "Setting up mg-ddm service"); smfh.setprop("config/mode", &mode)?; smfh.setprop("config/admin_host", "::")?; @@ -2015,8 +2022,8 @@ impl ServiceManager { )?; if is_gimlet { - // Maghemite for a scrimlet needs to be configured to - // talk to dendrite + // Ddm for a scrimlet needs to be configured to talk to + // dendrite smfh.setprop("config/dpd_host", "[::1]")?; smfh.setprop("config/dpd_port", DENDRITE_PORT)?; } @@ -2505,7 +2512,8 @@ impl ServiceManager { ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, - ServiceType::Maghemite { mode: "transit".to_string() }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, ] } @@ -2528,7 +2536,8 @@ impl ServiceManager { ServiceType::ManagementGatewayService, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, - ServiceType::Maghemite { mode: "transit".to_string() }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, ServiceType::SpSim, ] @@ -2583,10 +2592,20 @@ impl ServiceManager { let log = &self.inner.log; // Configure uplinks via DPD in our switch zone. - let our_uplinks = EarlyNetworkSetup::new(log) + let our_ports = EarlyNetworkSetup::new(log) .init_switch_config(rack_network_config, switch_zone_ip) - .await?; + .await? + .into_iter() + .map(From::from) + .collect(); + self.ensure_scrimlet_host_ports(our_ports).await + } + + pub async fn ensure_scrimlet_host_ports( + &self, + our_ports: Vec, + ) -> Result<(), Error> { // We expect the switch zone to be running, as we're called immediately // after `ensure_zone()` above and we just successfully configured // uplinks via DPD running in our switch zone. If somehow we're in any @@ -2617,22 +2636,14 @@ impl ServiceManager { smfh.delpropgroup("uplinks")?; smfh.addpropgroup("uplinks", "application")?; - // When naming the uplink ports, we need to append `_0`, `_1`, etc., for - // each use of any given port. We use a hashmap of counters of port name - // -> number of uplinks to correctly supply that suffix. - let mut port_count = HashMap::new(); - for uplink_config in &our_uplinks { - let this_port_count: &mut usize = - port_count.entry(&uplink_config.uplink_port).or_insert(0); - smfh.addpropvalue_type( - &format!( - "uplinks/{}_{}", - uplink_config.uplink_port, *this_port_count - ), - &uplink_config.uplink_cidr.to_string(), - "astring", - )?; - *this_port_count += 1; + for port_config in &our_ports { + for addr in &port_config.addrs { + smfh.addpropvalue_type( + &format!("uplinks/{}_0", port_config.port,), + &addr.to_string(), + "astring", + )?; + } } smfh.refresh()?; @@ -2868,7 +2879,7 @@ impl ServiceManager { // Only configured in // `ensure_switch_zone_uplinks_configured` } - ServiceType::Maghemite { mode } => { + ServiceType::MgDdm { mode } => { smfh.delpropvalue("config/mode", "*")?; smfh.addpropvalue("config/mode", &mode)?; smfh.refresh()?; diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 08f6c7d10b..f77da11b0e 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -4,6 +4,9 @@ //! HTTP entrypoint functions for the sled agent's exposed API +use crate::bootstrap::early_networking::{ + EarlyNetworkConfig, EarlyNetworkConfigBody, +}; use crate::params::{ DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, InstancePutStateResponse, InstanceUnregisterResponse, @@ -19,11 +22,15 @@ use dropshot::RequestContext; use dropshot::TypedBody; use illumos_utils::opte::params::DeleteVirtualNetworkInterfaceHost; use illumos_utils::opte::params::SetVirtualNetworkInterfaceHost; +use ipnetwork::Ipv6Network; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::SwitchPorts; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use uuid::Uuid; @@ -46,6 +53,9 @@ pub fn api() -> SledApiDescription { api.register(vpc_firewall_rules_put)?; api.register(set_v2p)?; api.register(del_v2p)?; + api.register(uplink_ensure)?; + api.register(read_network_bootstore_config)?; + api.register(write_network_bootstore_config)?; Ok(()) } @@ -327,3 +337,50 @@ async fn del_v2p( Ok(HttpResponseUpdatedNoContent()) } + +#[endpoint { + method = POST, + path = "/switch-ports", +}] +async fn uplink_ensure( + _rqctx: RequestContext>, + _body: TypedBody, +) -> Result { + Ok(HttpResponseUpdatedNoContent()) +} + +#[endpoint { + method = GET, + path = "/network-bootstore-config", +}] +async fn read_network_bootstore_config( + _rqctx: RequestContext>, +) -> Result, HttpError> { + let config = EarlyNetworkConfig { + generation: 0, + schema_version: 1, + body: EarlyNetworkConfigBody { + ntp_servers: Vec::new(), + rack_network_config: Some(RackNetworkConfig { + rack_subnet: Ipv6Network::new(Ipv6Addr::UNSPECIFIED, 56) + .unwrap(), + infra_ip_first: Ipv4Addr::UNSPECIFIED, + infra_ip_last: Ipv4Addr::UNSPECIFIED, + ports: Vec::new(), + bgp: Vec::new(), + }), + }, + }; + Ok(HttpResponseOk(config)) +} + +#[endpoint { + method = PUT, + path = "/network-bootstore-config", +}] +async fn write_network_bootstore_config( + _rqctx: RequestContext>, + _body: TypedBody, +) -> Result { + Ok(HttpResponseUpdatedNoContent()) +} diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 595d83a7ee..1f2fe8e1d8 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -94,7 +94,7 @@ impl Server { &config.id, &NexusTypes::SledAgentStartupInfo { sa_address: sa_address.to_string(), - role: NexusTypes::SledRole::Gimlet, + role: NexusTypes::SledRole::Scrimlet, baseboard: NexusTypes::Baseboard { serial_number: format!( "sim-{}", diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index b6f910220e..52513f081d 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -38,7 +38,9 @@ use omicron_common::api::external::Vni; use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; -use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::api::internal::shared::{ + HostPortConfig, RackNetworkConfig, +}; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::InstanceRuntimeState, internal::nexus::UpdateArtifactId, @@ -237,6 +239,9 @@ struct SledAgentInner { // Object managing zone bundles. zone_bundler: zone_bundle::ZoneBundler, + + // A handle to the bootstore. + bootstore: bootstore::NodeHandle, } impl SledAgentInner { @@ -407,7 +412,7 @@ impl SledAgent { EarlyNetworkConfig::try_from(serialized_config) .map_err(|err| BackoffError::transient(err.to_string()))?; - Ok(early_network_config.rack_network_config) + Ok(early_network_config.body.rack_network_config) }; let rack_network_config: Option = retry_notify::<_, String, _, _, _, _>( @@ -458,6 +463,7 @@ impl SledAgent { nexus_request_queue: NexusRequestQueue::new(), rack_network_config, zone_bundler, + bootstore: bootstore.clone(), }), log: log.clone(), }; @@ -769,7 +775,7 @@ impl SledAgent { /// Idempotently ensures that a given instance is registered with this sled, /// i.e., that it can be addressed by future calls to - /// [`instance_ensure_state`]. + /// [`Self::instance_ensure_state`]. pub async fn instance_ensure_registered( &self, instance_id: Uuid, @@ -918,4 +924,19 @@ impl SledAgent { pub async fn timesync_get(&self) -> Result { self.inner.services.timesync_get().await.map_err(Error::from) } + + pub async fn ensure_scrimlet_host_ports( + &self, + uplinks: Vec, + ) -> Result<(), Error> { + self.inner + .services + .ensure_scrimlet_host_ports(uplinks) + .await + .map_err(Error::from) + } + + pub fn bootstore(&self) -> bootstore::NodeHandle { + self.inner.bootstore.clone() + } } diff --git a/smf/sled-agent/gimlet-standalone/config-rss.toml b/smf/sled-agent/gimlet-standalone/config-rss.toml index c6fbab49de..29a7a79eba 100644 --- a/smf/sled-agent/gimlet-standalone/config-rss.toml +++ b/smf/sled-agent/gimlet-standalone/config-rss.toml @@ -88,27 +88,34 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.30" +# Configurations for BGP routers to run on the scrimlets. +bgp = [] + # You can configure multiple uplinks by repeating the following stanza -[[rack_network_config.uplinks]] -# The gateway IP for the rack's external network -gateway_ip = "192.168.1.199" +[[rack_network_config.ports]] +# Routes associated with this port. +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# Addresses associated with this port. +addresses = ["192.168.1.30/32"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. -uplink_port = "qsfp0" +port = "qsfp0" +# The speed of this port. uplink_port_speed = "40G" +# The forward error correction mode for this port. uplink_port_fec="none" -# For softnpu, an address within the "infra" block above that will be used for -# the softnpu uplink port. You can just pick the first address in that pool. -uplink_cidr = "192.168.1.30/32" # Switch to use for the uplink. For single-rack deployments this can be # "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu # and dendrite stub environments, use "switch0" switch = "switch0" +# Neighbors we expect to peer with over BGP on this port. +bgp_peers = [] # Configuration for the initial Silo, user, and password. # diff --git a/smf/sled-agent/non-gimlet/config-rss.toml b/smf/sled-agent/non-gimlet/config-rss.toml index 8a009dd687..fea3cfa5d8 100644 --- a/smf/sled-agent/non-gimlet/config-rss.toml +++ b/smf/sled-agent/non-gimlet/config-rss.toml @@ -88,27 +88,34 @@ last = "192.168.1.29" # Configuration to bring up Boundary Services and make Nexus reachable from the # outside. See docs/how-to-run.adoc for more on what to put here. [rack_network_config] +rack_subnet = "fd00:1122:3344:01::/56" # A range of IP addresses used by Boundary Services on the external network. In # a real system, these would be addresses of the uplink ports on the Sidecar. # With softnpu, only one address is used. infra_ip_first = "192.168.1.30" infra_ip_last = "192.168.1.30" +# Configurations for BGP routers to run on the scrimlets. +bgp = [] + # You can configure multiple uplinks by repeating the following stanza -[[rack_network_config.uplinks]] -# The gateway IP for the rack's external network -gateway_ip = "192.168.1.199" +[[rack_network_config.ports]] +# Routes associated with this port. +routes = [{nexthop = "192.168.1.199", destination = "0.0.0.0/0"}] +# Addresses associated with this port. +addresses = ["192.168.1.30/32"] # Name of the uplink port. This should always be "qsfp0" when using softnpu. -uplink_port = "qsfp0" +port = "qsfp0" +# The speed of this port. uplink_port_speed = "40G" +# The forward error correction mode for this port. uplink_port_fec="none" -# For softnpu, an address within the "infra" block above that will be used for -# the softnpu uplink port. You can just pick the first address in that pool. -uplink_cidr = "192.168.1.30/32" # Switch to use for the uplink. For single-rack deployments this can be # "switch0" (upper slot) or "switch1" (lower slot). For single-node softnpu # and dendrite stub environments, use "switch0" switch = "switch0" +# Neighbors we expect to peer with over BGP on this port. +bgp_peers = [] # Configuration for the initial Silo, user, and password. # diff --git a/test-utils/src/dev/dendrite.rs b/test-utils/src/dev/dendrite.rs index 520bf12401..8938595aa2 100644 --- a/test-utils/src/dev/dendrite.rs +++ b/test-utils/src/dev/dendrite.rs @@ -19,7 +19,7 @@ use tokio::{ /// Specifies the amount of time we will wait for `dpd` to launch, /// which is currently confirmed by watching `dpd`'s log output /// for a message specifying the address and port `dpd` is listening on. -pub const DENDRITE_TIMEOUT: Duration = Duration::new(5, 0); +pub const DENDRITE_TIMEOUT: Duration = Duration::new(30, 0); /// Represents a running instance of the Dendrite dataplane daemon (dpd). pub struct DendriteInstance { diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs new file mode 100644 index 0000000000..fa1f353896 --- /dev/null +++ b/test-utils/src/dev/maghemite.rs @@ -0,0 +1,155 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tools for managing Maghemite during development + +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use tempfile::TempDir; +use tokio::{ + fs::File, + io::{AsyncBufReadExt, BufReader}, + time::{sleep, Instant}, +}; + +/// Specifies the amount of time we will wait for `mgd` to launch, +/// which is currently confirmed by watching `mgd`'s log output +/// for a message specifying the address and port `mgd` is listening on. +pub const MGD_TIMEOUT: Duration = Duration::new(5, 0); + +pub struct MgdInstance { + /// Port number the mgd instance is listening on. This can be provided + /// manually, or dynamically determined if a value of 0 is provided. + pub port: u16, + /// Arguments provided to the `mgd` cli command. + pub args: Vec, + /// Child process spawned by running `mgd` + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `mgd` are stored. + pub data_dir: Option, +} + +impl MgdInstance { + pub async fn start(mut port: u16) -> Result { + let temp_dir = TempDir::new()?; + + let args = vec![ + "run".to_string(), + "--admin-addr".into(), + "::1".into(), + "--admin-port".into(), + port.to_string(), + "--no-bgp-dispatcher".into(), + "--data-dir".into(), + temp_dir.path().display().to_string(), + ]; + + let child = tokio::process::Command::new("mgd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file(temp_dir.path(), "mgd_stdout")?)) + .stderr(Stdio::from(redirect_file(temp_dir.path(), "mgd_stderr")?)) + .spawn() + .with_context(|| { + format!("failed to spawn `mgd` (with args: {:?})", &args) + })?; + + let child = Some(child); + + let temp_dir = temp_dir.into_path(); + if port == 0 { + port = discover_port( + temp_dir.join("mgd_stdout").display().to_string(), + ) + .await + .with_context(|| { + format!( + "failed to discover mgd port from files in {}", + temp_dir.display() + ) + })?; + } + + Ok(Self { port, args, child, data_dir: Some(temp_dir) }) + } + + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; + } + Ok(()) + } +} + +impl Drop for MgdInstance { + fn drop(&mut self) { + if self.child.is_some() || self.data_dir.is_some() { + eprintln!( + "WARN: dropped MgdInstance without cleaning it up first \ + (there may still be a child process running and a \ + temporary directory leaked)" + ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: mgd temporary directory leaked: {}", + path.display() + ); + } + } + } +} + +fn redirect_file( + temp_dir_path: &Path, + label: &str, +) -> Result { + let out_path = temp_dir_path.join(label); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&out_path) + .with_context(|| format!("open \"{}\"", out_path.display())) +} + +async fn discover_port(logfile: String) -> Result { + let timeout = Instant::now() + MGD_TIMEOUT; + tokio::time::timeout_at(timeout, find_mgd_port_in_log(logfile)) + .await + .context("time out while discovering mgd port number")? +} + +async fn find_mgd_port_in_log(logfile: String) -> Result { + let re = regex::Regex::new(r#""local_addr":"\[::1\]:?([0-9]+)""#).unwrap(); + let reader = BufReader::new(File::open(logfile).await?); + let mut lines = reader.lines(); + loop { + match lines.next_line().await? { + Some(line) => { + if let Some(cap) = re.captures(&line) { + // unwrap on get(1) should be ok, since captures() returns + // `None` if there are no matches found + let port = cap.get(1).unwrap(); + let result = port.as_str().parse::()?; + return Ok(result); + } + } + None => { + sleep(Duration::from_millis(10)).await; + } + } + } +} diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index dbd66fe1f8..e29da9c51e 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -8,6 +8,7 @@ pub mod clickhouse; pub mod db; pub mod dendrite; +pub mod maghemite; pub mod poll; #[cfg(feature = "seed-gen")] pub mod seed; diff --git a/tools/build-global-zone-packages.sh b/tools/build-global-zone-packages.sh index 54af9d6327..fc1ab42ade 100755 --- a/tools/build-global-zone-packages.sh +++ b/tools/build-global-zone-packages.sh @@ -12,7 +12,7 @@ out_dir="$(readlink -f "${2:-"$tarball_src_dir"}")" # Make sure needed packages exist deps=( "$tarball_src_dir/omicron-sled-agent.tar" - "$tarball_src_dir/maghemite.tar" + "$tarball_src_dir/mg-ddm-gz.tar" "$tarball_src_dir/propolis-server.tar.gz" "$tarball_src_dir/overlay.tar.gz" ) @@ -46,7 +46,7 @@ cd - pkg_dir="$tmp_gz/root/opt/oxide/mg-ddm" mkdir -p "$pkg_dir" cd "$pkg_dir" -tar -xvfz "$tarball_src_dir/maghemite.tar" +tar -xvfz "$tarball_src_dir/mg-ddm-gz.tar" cd - # propolis should be bundled with this OS: Put the propolis-server zone image diff --git a/tools/build-trampoline-global-zone-packages.sh b/tools/build-trampoline-global-zone-packages.sh index 87013fb563..d8df0f8921 100755 --- a/tools/build-trampoline-global-zone-packages.sh +++ b/tools/build-trampoline-global-zone-packages.sh @@ -12,7 +12,7 @@ out_dir="$(readlink -f "${2:-$tarball_src_dir}")" # Make sure needed packages exist deps=( "$tarball_src_dir"/installinator.tar - "$tarball_src_dir"/maghemite.tar + "$tarball_src_dir"/mg-ddm-gz.tar ) for dep in "${deps[@]}"; do if [[ ! -e $dep ]]; then @@ -44,7 +44,7 @@ cd - pkg_dir="$tmp_trampoline/root/opt/oxide/mg-ddm" mkdir -p "$pkg_dir" cd "$pkg_dir" -tar -xvfz "$tarball_src_dir/maghemite.tar" +tar -xvfz "$tarball_src_dir/mg-ddm-gz.tar" cd - # Create the final output and we're done diff --git a/tools/ci_download_maghemite_mgd b/tools/ci_download_maghemite_mgd new file mode 100755 index 0000000000..eff680d7fd --- /dev/null +++ b/tools/ci_download_maghemite_mgd @@ -0,0 +1,168 @@ +#!/bin/bash + +# +# ci_download_maghemite_mgd: fetches the maghemite mgd binary tarball, unpacks +# it, and creates a copy called mgd, all in the current directory +# + +set -o pipefail +set -o xtrace +set -o errexit + +SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +ARG0="$(basename "${BASH_SOURCE[0]}")" + +source "$SOURCE_DIR/maghemite_mgd_checksums" +source "$SOURCE_DIR/maghemite_mg_openapi_version" + +TARGET_DIR="out" +# Location where intermediate artifacts are downloaded / unpacked. +DOWNLOAD_DIR="$TARGET_DIR/downloads" +# Location where the final mgd directory should end up. +DEST_DIR="./$TARGET_DIR/mgd" +BIN_DIR="$DEST_DIR/root/opt/oxide/mgd/bin" + +ARTIFACT_URL="https://buildomat.eng.oxide.computer/public/file" + +REPO='oxidecomputer/maghemite' +PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/image/$COMMIT" + +function main +{ + # + # Process command-line arguments. We generally don't expect any, but + # we allow callers to specify a value to override OSTYPE, just for + # testing. + # + if [[ $# != 0 ]]; then + CIDL_OS="$1" + shift + else + CIDL_OS="$OSTYPE" + fi + + if [[ $# != 0 ]]; then + echo "unexpected arguments" >&2 + exit 2 + fi + + # Configure this program + configure_os "$CIDL_OS" + + CIDL_SHA256FUNC="do_sha256sum" + TARBALL_FILENAME="mgd.tar.gz" + PACKAGE_URL="$PACKAGE_BASE_URL/$TARBALL_FILENAME" + TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME" + + # Download the file. + echo "URL: $PACKAGE_URL" + echo "Local file: $TARBALL_FILE" + + mkdir -p "$DOWNLOAD_DIR" + mkdir -p "$DEST_DIR" + + fetch_and_verify + + do_untar "$TARBALL_FILE" + + do_assemble + + $SET_BINARIES +} + +function fail +{ + echo "$ARG0: $@" >&2 + exit 1 +} + +function configure_os +{ + echo "current directory: $PWD" + echo "configuring based on OS: \"$1\"" + case "$1" in + linux-gnu*) + SET_BINARIES="linux_binaries" + ;; + solaris*) + SET_BINARIES="" + ;; + *) + echo "WARNING: binaries for $1 are not published by maghemite" + echo "Dynamic routing apis will be unavailable" + SET_BINARIES="unsupported_os" + ;; + esac +} + +function do_download_curl +{ + curl --silent --show-error --fail --location --output "$2" "$1" +} + +function do_sha256sum +{ + sha256sum < "$1" | awk '{print $1}' +} + +function do_untar +{ + tar xzf "$1" -C "$DOWNLOAD_DIR" +} + +function do_assemble +{ + rm -r "$DEST_DIR" || true + mkdir "$DEST_DIR" + cp -r "$DOWNLOAD_DIR/root" "$DEST_DIR/root" +} + +function fetch_and_verify +{ + local DO_DOWNLOAD="true" + if [[ -f "$TARBALL_FILE" ]]; then + # If the file exists with a valid checksum, we can skip downloading. + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" == "$CIDL_SHA256" ]]; then + DO_DOWNLOAD="false" + fi + fi + + if [ "$DO_DOWNLOAD" == "true" ]; then + echo "Downloading..." + do_download_curl "$PACKAGE_URL" "$TARBALL_FILE" || \ + fail "failed to download file" + + # Verify the sha256sum. + calculated_sha256="$($CIDL_SHA256FUNC "$TARBALL_FILE")" || \ + fail "failed to calculate sha256sum" + if [[ "$calculated_sha256" != "$CIDL_SHA256" ]]; then + fail "sha256sum mismatch \ + (expected $CIDL_SHA256, found $calculated_sha256)" + fi + fi + +} + +function linux_binaries +{ + PACKAGE_BASE_URL="$ARTIFACT_URL/$REPO/linux/$COMMIT" + CIDL_SHA256="$MGD_LINUX_SHA256" + CIDL_SHA256FUNC="do_sha256sum" + TARBALL_FILENAME="mgd" + PACKAGE_URL="$PACKAGE_BASE_URL/$TARBALL_FILENAME" + TARBALL_FILE="$DOWNLOAD_DIR/$TARBALL_FILENAME" + fetch_and_verify + chmod +x "$DOWNLOAD_DIR/mgd" + cp "$DOWNLOAD_DIR/mgd" "$BIN_DIR" +} + +function unsupported_os +{ + mkdir -p "$BIN_DIR" + echo "echo 'unsupported os' && exit 1" >> "$BIN_DIR/dpd" + chmod +x "$BIN_DIR/dpd" +} + +main "$@" diff --git a/tools/ci_download_maghemite_openapi b/tools/ci_download_maghemite_openapi index 37ff4f5547..db53f68d2c 100755 --- a/tools/ci_download_maghemite_openapi +++ b/tools/ci_download_maghemite_openapi @@ -15,10 +15,7 @@ TARGET_DIR="out" # Location where intermediate artifacts are downloaded / unpacked. DOWNLOAD_DIR="$TARGET_DIR/downloads" -source "$SOURCE_DIR/maghemite_openapi_version" -URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/ddm-admin.json" -LOCAL_FILE="$DOWNLOAD_DIR/ddm-admin-$COMMIT.json" function main { @@ -83,4 +80,14 @@ function do_sha256sum $SHA < "$1" | awk '{print $1}' } +source "$SOURCE_DIR/maghemite_ddm_openapi_version" +URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/ddm-admin.json" +LOCAL_FILE="$DOWNLOAD_DIR/ddm-admin-$COMMIT.json" + +main "$@" + +source "$SOURCE_DIR/maghemite_mg_openapi_version" +URL="https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/openapi/$COMMIT/mg-admin.json" +LOCAL_FILE="$DOWNLOAD_DIR/mg-admin-$COMMIT.json" + main "$@" diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index 7975a310f0..cb5ea40210 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="eb27e6a00f1082c9faac7cf997e57d0609f7a309" +SOFTNPU_COMMIT="c1c42398c82b0220c8b5fa3bfba9c7a3bcaa0943" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index 95c2aa63df..248cbcde73 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -44,6 +44,9 @@ function ensure_simulated_links { if [[ -z "$(get_vnic_name_if_exists "sc0_1")" ]]; then dladm create-vnic -t "sc0_1" -l "$PHYSICAL_LINK" -m a8:e1:de:01:70:1d + if [[ -v PROMISC_FILT_OFF ]]; then + dladm set-linkprop -p promisc-filtered=off sc0_1 + fi fi success "Vnic sc0_1 exists" } @@ -58,7 +61,8 @@ function ensure_softnpu_zone { out/npuzone/npuzone create sidecar \ --omicron-zone \ --ports sc0_0,tfportrear0_0 \ - --ports sc0_1,tfportqsfp0_0 + --ports sc0_1,tfportqsfp0_0 \ + --sidecar-lite-branch omicron-tracking } "$SOURCE_DIR"/scrimlet/softnpu-init.sh success "softnpu zone exists" diff --git a/tools/delete-reservoir.sh b/tools/delete-reservoir.sh new file mode 100755 index 0000000000..77e814f0c7 --- /dev/null +++ b/tools/delete-reservoir.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +size=`pfexec /usr/lib/rsrvrctl -q | grep Free | awk '{print $3}'` +let x=$size/1024 + +pfexec /usr/lib/rsrvrctl -r $x diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index b1f210a647..9a2ea85ac0 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="7712104585266a2898da38c1345210ad26f9e71d" +COMMIT="c0cbc39b55fac54b95468304c497e00f3d3cf686" SHA2="cb3f0cfbe6216d2441d34e0470252e0fb142332e47b33b65c24ef7368a694b6d" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 9538bc0d00..fe52c59381 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="486b0b016c0df06947810b90f3a3dd40423f0ee6f255ed079dc8e5618c9a7281" -CIDL_SHA256_LINUX_DPD="af97aaf7e1046a5c651d316c384171df6387b4c54c8ae4a3ef498e532eaa5a4c" -CIDL_SHA256_LINUX_SWADM="909e400dcc9880720222c6dc3919404d83687f773f668160f66f38b51a81c188" +CIDL_SHA256_ILLUMOS="3706e0e8230b7f76407ec0acea9020b9efc7d6c78b74c304102fd8e62cac6760" +CIDL_SHA256_LINUX_DPD="b275a1c688eae1024b9ce1cbb766a66e37072e84b4a6cbc18746c903739ccf51" +CIDL_SHA256_LINUX_SWADM="7e604cc4b67c1a711a63ece2a8d0e2e7c8ef2b9ac6bb433b3c2e02f5f66018ba" diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 62603ecac7..d3ecd8eaa8 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -197,6 +197,10 @@ retry ./tools/ci_download_dendrite_openapi # asic and running dendrite instance retry ./tools/ci_download_dendrite_stub +# Download mgd. This is required to run tests that invovle dynamic external +# routing +retry ./tools/ci_download_maghemite_mgd + # Download transceiver-control. This is used as the source for the # xcvradm binary which is bundled with the switch zone. retry ./tools/ci_download_transceiver_control diff --git a/tools/install_runner_prerequisites.sh b/tools/install_runner_prerequisites.sh index 7ece993bc9..42347f518d 100755 --- a/tools/install_runner_prerequisites.sh +++ b/tools/install_runner_prerequisites.sh @@ -105,6 +105,7 @@ function install_packages { 'pkg-config' 'brand/omicron1/tools' 'library/libxmlsec1' + 'chrony' ) # Install/update the set of packages. @@ -119,13 +120,15 @@ function install_packages { exit "$rc" fi + pfexec svcadm enable chrony + pkg list -v "${packages[@]}" elif [[ "${HOST_OS}" == "Linux" ]]; then packages=( 'ca-certificates' 'libpq5' 'libsqlite3-0' - 'libssl1.1' + 'libssl3' 'libxmlsec1-openssl' ) sudo apt-get update diff --git a/tools/maghemite_openapi_version b/tools/maghemite_ddm_openapi_version similarity index 59% rename from tools/maghemite_openapi_version rename to tools/maghemite_ddm_openapi_version index 8f84b30cb1..a315c31d4b 100644 --- a/tools/maghemite_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="12703675393459e74139f8140e0b3c4c4f129d5d" +COMMIT="2f25a2005521f643317879b46692141b4127608a" SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version new file mode 100644 index 0000000000..acd5a5f546 --- /dev/null +++ b/tools/maghemite_mg_openapi_version @@ -0,0 +1,2 @@ +COMMIT="2f25a2005521f643317879b46692141b4127608a" +SHA2="d0f7611e5ecd049b0f83bcfa843942401f155a0be36d9a2dfd73b8341d5f816e" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums new file mode 100644 index 0000000000..45fbec5274 --- /dev/null +++ b/tools/maghemite_mgd_checksums @@ -0,0 +1,2 @@ +CIDL_SHA256="16878501f5440590674acd82bee6ce5dcf3d1326531c25064dd9c060ab6440a4" +MGD_LINUX_SHA256="45e5ddc9d81cfcb94917f9c58942c3a7211fb34a3c563fbfc2434b0a97306b3d" diff --git a/tools/update_maghemite.sh b/tools/update_maghemite.sh index a4a9b1291e..eebece1aa5 100755 --- a/tools/update_maghemite.sh +++ b/tools/update_maghemite.sh @@ -15,8 +15,9 @@ function usage { } PACKAGES=( - "maghemite" + "mg-ddm-gz" "mg-ddm" + "mgd" ) REPO="oxidecomputer/maghemite" @@ -26,13 +27,14 @@ REPO="oxidecomputer/maghemite" function update_openapi { TARGET_COMMIT="$1" DRY_RUN="$2" - SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "ddm-admin.json" "openapi") + DAEMON="$3" + SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "${DAEMON}-admin.json" "openapi") OUTPUT=$(printf "COMMIT=\"%s\"\nSHA2=\"%s\"\n" "$TARGET_COMMIT" "$SHA") if [ -n "$DRY_RUN" ]; then OPENAPI_PATH="/dev/null" else - OPENAPI_PATH="$SOURCE_DIR/maghemite_openapi_version" + OPENAPI_PATH="$SOURCE_DIR/maghemite_${DAEMON}_openapi_version" fi echo "Updating Maghemite OpenAPI from: $TARGET_COMMIT" set -x @@ -40,6 +42,27 @@ function update_openapi { set +x } +function update_mgd { + TARGET_COMMIT="$1" + DRY_RUN="$2" + DAEMON="$3" + SHA=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "image") + OUTPUT=$(printf "CIDL_SHA256=\"%s\"\n" "$SHA") + + SHA_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "linux") + OUTPUT_LINUX=$(printf "MGD_LINUX_SHA256=\"%s\"\n" "$SHA_LINUX") + + if [ -n "$DRY_RUN" ]; then + MGD_PATH="/dev/null" + else + MGD_PATH="$SOURCE_DIR/maghemite_mgd_checksums" + fi + echo "Updating Maghemite mgd from: $TARGET_COMMIT" + set -x + echo "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH + set +x +} + function main { TARGET_COMMIT="" DRY_RUN="" @@ -60,7 +83,9 @@ function main { TARGET_COMMIT=$(get_latest_commit_from_gh "$REPO" "$TARGET_COMMIT") install_toml2json do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}" - update_openapi "$TARGET_COMMIT" "$DRY_RUN" + update_openapi "$TARGET_COMMIT" "$DRY_RUN" ddm + update_openapi "$TARGET_COMMIT" "$DRY_RUN" mg + update_mgd "$TARGET_COMMIT" "$DRY_RUN" do_update_packages "$TARGET_COMMIT" "$DRY_RUN" "$REPO" "${PACKAGES[@]}" } diff --git a/update-engine/src/context.rs b/update-engine/src/context.rs index d232d931a2..cd85687cf9 100644 --- a/update-engine/src/context.rs +++ b/update-engine/src/context.rs @@ -223,7 +223,7 @@ impl StepContext { } } -/// Tracker for [`StepContext::add_nested_report`]. +/// Tracker for [`StepContext::send_nested_report`]. /// /// Nested event reports might contain events already seen in prior runs: /// `NestedEventBuffer` deduplicates those events such that only deltas are sent diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/rack_setup/config_template.toml index 4b193a0c29..617b61fadc 100644 --- a/wicket/src/rack_setup/config_template.toml +++ b/wicket/src/rack_setup/config_template.toml @@ -40,18 +40,24 @@ bootstrap_sleds = [] # TODO: docs on network config [rack_network_config] +rack_subnet = "" infra_ip_first = "" infra_ip_last = "" -[[rack_network_config.uplinks]] +[[rack_network_config.ports]] +# Routes associated with this port. +# { nexthop = "1.2.3.4", destination = "0.0.0.0/0" } +routes = [] + +# Addresses associated with this port. +# "1.2.3.4/24" +addresses = [] + # Either `switch0` or `switch1`, matching the hardware. switch = "" -# IP address this uplink should use as its gateway. -gateway_ip = "" - # qsfp0, qsfp1, ... -uplink_port = "" +port = "" # `speed40_g`, `speed100_g`, ... uplink_port_speed = "" @@ -59,8 +65,14 @@ uplink_port_speed = "" # `none`, `firecode`, or `rs` uplink_port_fec = "" -# IP address and prefix for this uplink; e.g., `192.168.100.100/16` -uplink_cidr = "" +# A list of bgp peers +# { addr = "1.7.0.1", asn = 47, port = "qsfp0" } +bgp_peers = [] + +# Optional BGP configuration. Remove this section if not needed. +[[rack_network_config.bgp]] +# The autonomous system numer +asn = 0 -# VLAN ID for this uplink; omit if no VLAN ID is needed -uplink_vid = 1234 +# Prefixes to originate e.g., ["10.0.0.0/16"] +originate = [] diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/rack_setup/config_toml.rs index 5f0bb9e876..e087c9aa7c 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/rack_setup/config_toml.rs @@ -18,7 +18,7 @@ use toml_edit::Value; use wicketd_client::types::BootstrapSledDescription; use wicketd_client::types::CurrentRssUserConfigInsensitive; use wicketd_client::types::IpRange; -use wicketd_client::types::RackNetworkConfig; +use wicketd_client::types::RackNetworkConfigV1; use wicketd_client::types::SpType; static TEMPLATE: &str = include_str!("config_template.toml"); @@ -176,7 +176,7 @@ fn build_sleds_array(sleds: &[BootstrapSledDescription]) -> Array { fn populate_network_table( table: &mut Table, - config: Option<&RackNetworkConfig>, + config: Option<&RackNetworkConfigV1>, ) { // Helper function to serialize enums into their appropriate string // representations. @@ -195,6 +195,7 @@ fn populate_network_table( }; for (property, value) in [ + ("rack_subnet", config.rack_subnet.to_string()), ("infra_ip_first", config.infra_ip_first.to_string()), ("infra_ip_last", config.infra_ip_last.to_string()), ] { @@ -202,20 +203,17 @@ fn populate_network_table( Value::String(Formatted::new(value)); } - // If `config.uplinks` is empty, we'll leave the template uplinks in place; - // otherwise, replace it with the user's uplinks. - if !config.uplinks.is_empty() { - *table.get_mut("uplinks").unwrap().as_array_of_tables_mut().unwrap() = + if !config.ports.is_empty() { + *table.get_mut("ports").unwrap().as_array_of_tables_mut().unwrap() = config - .uplinks + .ports .iter() .map(|cfg| { let mut uplink = Table::new(); - let mut last_key = None; + let mut _last_key = None; for (property, value) in [ ("switch", cfg.switch.to_string()), - ("gateway_ip", cfg.gateway_ip.to_string()), - ("uplink_port", cfg.uplink_port.to_string()), + ("port", cfg.port.to_string()), ( "uplink_port_speed", enum_to_toml_string(&cfg.uplink_port_speed), @@ -224,63 +222,121 @@ fn populate_network_table( "uplink_port_fec", enum_to_toml_string(&cfg.uplink_port_fec), ), - ("uplink_cidr", cfg.uplink_cidr.to_string()), ] { uplink.insert( property, Item::Value(Value::String(Formatted::new(value))), ); - last_key = Some(property); + _last_key = Some(property); } - if let Some(uplink_vid) = cfg.uplink_vid { - uplink.insert( - "uplink_vid", - Item::Value(Value::Integer(Formatted::new( - i64::from(uplink_vid), - ))), + let mut routes = Array::new(); + for r in &cfg.routes { + let mut route = InlineTable::new(); + route.insert( + "nexthop", + Value::String(Formatted::new( + r.nexthop.to_string(), + )), + ); + route.insert( + "destination", + Value::String(Formatted::new( + r.destination.to_string(), + )), ); - } else { - // Unwraps: We know `last_key` is `Some(_)`, because we - // set it in every iteration of the loop above, and we - // know it's present in `uplink` because we set it to - // the `property` we just inserted. - let last = uplink.get_mut(last_key.unwrap()).unwrap(); - - // Every item we insert is an `Item::Value`, so we can - // unwrap this conversion. - last.as_value_mut() - .unwrap() - .decor_mut() - .set_suffix("\n# uplink_vid ="); + routes.push(Value::InlineTable(route)); } + uplink.insert("routes", Item::Value(Value::Array(routes))); + let mut addresses = Array::new(); + for a in &cfg.addresses { + addresses + .push(Value::String(Formatted::new(a.to_string()))) + } + uplink.insert( + "addresses", + Item::Value(Value::Array(addresses)), + ); + + let mut peers = Array::new(); + for p in &cfg.bgp_peers { + let mut peer = InlineTable::new(); + peer.insert( + "addr", + Value::String(Formatted::new(p.addr.to_string())), + ); + peer.insert( + "asn", + Value::Integer(Formatted::new(p.asn as i64)), + ); + peer.insert( + "port", + Value::String(Formatted::new(p.port.to_string())), + ); + peers.push(Value::InlineTable(peer)); + } + uplink + .insert("bgp_peers", Item::Value(Value::Array(peers))); uplink }) .collect(); } + if !config.bgp.is_empty() { + *table.get_mut("bgp").unwrap().as_array_of_tables_mut().unwrap() = + config + .bgp + .iter() + .map(|cfg| { + let mut bgp = Table::new(); + bgp.insert( + "asn", + Item::Value(Value::Integer(Formatted::new( + cfg.asn as i64, + ))), + ); + + let mut originate = Array::new(); + for o in &cfg.originate { + originate + .push(Value::String(Formatted::new(o.to_string()))); + } + bgp.insert( + "originate", + Item::Value(Value::Array(originate)), + ); + bgp + }) + .collect(); + } } #[cfg(test)] mod tests { use super::*; - use omicron_common::api::internal::shared::RackNetworkConfig as InternalRackNetworkConfig; + use omicron_common::api::internal::shared::RackNetworkConfigV1 as InternalRackNetworkConfig; use std::net::Ipv6Addr; use wicket_common::rack_setup::PutRssUserConfigInsensitive; use wicketd_client::types::Baseboard; + use wicketd_client::types::BgpConfig; + use wicketd_client::types::BgpPeerConfig; + use wicketd_client::types::PortConfigV1; use wicketd_client::types::PortFec; use wicketd_client::types::PortSpeed; + use wicketd_client::types::RouteConfig; use wicketd_client::types::SpIdentifier; use wicketd_client::types::SwitchLocation; - use wicketd_client::types::UplinkConfig; fn put_config_from_current_config( value: CurrentRssUserConfigInsensitive, ) -> PutRssUserConfigInsensitive { + use omicron_common::api::internal::shared::BgpConfig as InternalBgpConfig; + use omicron_common::api::internal::shared::BgpPeerConfig as InternalBgpPeerConfig; + use omicron_common::api::internal::shared::PortConfigV1 as InternalPortConfig; use omicron_common::api::internal::shared::PortFec as InternalPortFec; use omicron_common::api::internal::shared::PortSpeed as InternalPortSpeed; + use omicron_common::api::internal::shared::RouteConfig as InternalRouteConfig; use omicron_common::api::internal::shared::SwitchLocation as InternalSwitchLocation; - use omicron_common::api::internal::shared::UplinkConfig as InternalUplinkConfig; let rnc = value.rack_network_config.unwrap(); @@ -310,14 +366,32 @@ mod tests { external_dns_ips: value.external_dns_ips, ntp_servers: value.ntp_servers, rack_network_config: InternalRackNetworkConfig { + rack_subnet: rnc.rack_subnet, infra_ip_first: rnc.infra_ip_first, infra_ip_last: rnc.infra_ip_last, - uplinks: rnc - .uplinks + ports: rnc + .ports .iter() - .map(|config| InternalUplinkConfig { - gateway_ip: config.gateway_ip, - uplink_port: config.uplink_port.clone(), + .map(|config| InternalPortConfig { + routes: config + .routes + .iter() + .map(|r| InternalRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| InternalBgpPeerConfig { + asn: p.asn, + port: p.port.clone(), + addr: p.addr, + }) + .collect(), + port: config.port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => InternalPortSpeed::Speed0G, PortSpeed::Speed1G => InternalPortSpeed::Speed1G, @@ -340,8 +414,6 @@ mod tests { PortFec::None => InternalPortFec::None, PortFec::Rs => InternalPortFec::Rs, }, - uplink_cidr: config.uplink_cidr, - uplink_vid: config.uplink_vid, switch: match config.switch { SwitchLocation::Switch0 => { InternalSwitchLocation::Switch0 @@ -352,6 +424,14 @@ mod tests { }, }) .collect(), + bgp: rnc + .bgp + .iter() + .map(|config| InternalBgpConfig { + asn: config.asn, + originate: config.originate.clone(), + }) + .collect(), }, } } @@ -392,18 +472,30 @@ mod tests { )], external_dns_ips: vec!["10.0.0.1".parse().unwrap()], ntp_servers: vec!["ntp1.com".into(), "ntp2.com".into()], - rack_network_config: Some(RackNetworkConfig { + rack_network_config: Some(RackNetworkConfigV1 { + rack_subnet: "fd00:1122:3344:01::/56".parse().unwrap(), infra_ip_first: "172.30.0.1".parse().unwrap(), infra_ip_last: "172.30.0.10".parse().unwrap(), - uplinks: vec![UplinkConfig { - gateway_ip: "172.30.0.10".parse().unwrap(), - uplink_cidr: "172.30.0.1/24".parse().unwrap(), + ports: vec![PortConfigV1 { + addresses: vec!["172.30.0.1/24".parse().unwrap()], + routes: vec![RouteConfig { + destination: "0.0.0.0/0".parse().unwrap(), + nexthop: "172.30.0.10".parse().unwrap(), + }], + bgp_peers: vec![BgpPeerConfig { + asn: 47, + addr: "10.2.3.4".parse().unwrap(), + port: "port0".into(), + }], uplink_port_speed: PortSpeed::Speed400G, uplink_port_fec: PortFec::Firecode, - uplink_port: "port0".into(), - uplink_vid: None, + port: "port0".into(), switch: SwitchLocation::Switch0, }], + bgp: vec![BgpConfig { + asn: 47, + originate: vec!["10.0.0.0/16".parse().unwrap()], + }], }), }; let template = TomlTemplate::populate(&config).to_string(); diff --git a/wicket/src/ui/main.rs b/wicket/src/ui/main.rs index 42cc6bf587..58ea6c1771 100644 --- a/wicket/src/ui/main.rs +++ b/wicket/src/ui/main.rs @@ -23,7 +23,7 @@ use wicketd_client::types::GetLocationResponse; /// This structure allows us to maintain similar styling and navigation /// throughout wicket with a minimum of code. /// -/// Specific functionality is put inside [`Pane`]s, which can be customized +/// Specific functionality is put inside Panes, which can be customized /// as needed. pub struct MainScreen { #[allow(unused)] diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index 212ddff4da..086d01ce9d 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -695,56 +695,61 @@ fn rss_config_text<'a>( }; if let Some(cfg) = insensitive.rack_network_config.as_ref() { - for (i, uplink) in cfg.uplinks.iter().enumerate() { + for (i, uplink) in cfg.ports.iter().enumerate() { let mut items = vec![ vec![ - Span::styled(" • Switch : ", label_style), + Span::styled(" • Switch : ", label_style), Span::styled(uplink.switch.to_string(), ok_style), ], vec![ - Span::styled(" • Gateway IP : ", label_style), - Span::styled(uplink.gateway_ip.to_string(), ok_style), - ], - vec![ - Span::styled(" • Uplink CIDR : ", label_style), - Span::styled(uplink.uplink_cidr.to_string(), ok_style), - ], - vec![ - Span::styled(" • Uplink port : ", label_style), - Span::styled(uplink.uplink_port.clone(), ok_style), - ], - vec![ - Span::styled(" • Uplink port speed: ", label_style), + Span::styled(" • Speed : ", label_style), Span::styled( uplink.uplink_port_speed.to_string(), ok_style, ), ], vec![ - Span::styled(" • Uplink port FEC : ", label_style), + Span::styled(" • FEC : ", label_style), Span::styled(uplink.uplink_port_fec.to_string(), ok_style), ], ]; - if let Some(uplink_vid) = uplink.uplink_vid { - items.push(vec![ - Span::styled(" • Uplink VLAN id : ", label_style), - Span::styled(uplink_vid.to_string(), ok_style), - ]); - } else { - items.push(vec![ - Span::styled(" • Uplink VLAN id : ", label_style), - Span::styled("none", ok_style), - ]); - } + + let routes = uplink.routes.iter().map(|r| { + vec![ + Span::styled(" • Route : ", label_style), + Span::styled( + format!("{} -> {}", r.destination, r.nexthop), + ok_style, + ), + ] + }); + + let addresses = uplink.addresses.iter().map(|a| { + vec![ + Span::styled(" • Address : ", label_style), + Span::styled(a.to_string(), ok_style), + ] + }); + + let peers = uplink.bgp_peers.iter().map(|p| { + vec![ + Span::styled(" • BGP peer : ", label_style), + Span::styled(format!("{} ASN={}", p.addr, p.asn), ok_style), + ] + }); + + items.extend(routes); + items.extend(addresses); + items.extend(peers); append_list( &mut spans, - Cow::from(format!("Uplink {}: ", i + 1)), + Cow::from(format!("Port {}: ", i + 1)), items, ); } } else { - append_list(&mut spans, "Uplinks: ".into(), vec![]); + append_list(&mut spans, "Ports: ".into(), vec![]); } append_list( diff --git a/wicket/src/ui/wrap.rs b/wicket/src/ui/wrap.rs index 6cd5f7010a..9cd57d45d5 100644 --- a/wicket/src/ui/wrap.rs +++ b/wicket/src/ui/wrap.rs @@ -324,7 +324,7 @@ impl<'a> Fragment for StyledWord<'a> { /// Forcibly break spans wider than `line_width` into smaller spans. /// -/// This simply calls [`Span::break_apart`] on spans that are too wide. +/// This simply calls [`StyledWord::break_apart`] on spans that are too wide. fn break_words<'a, I>(spans: I, line_width: usize) -> Vec> where I: IntoIterator>, diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index f11fda9750..655f3bb803 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -25,6 +25,7 @@ hubtools.workspace = true http.workspace = true hyper.workspace = true illumos-utils.workspace = true +ipnetwork.workspace = true internal-dns.workspace = true itertools.workspace = true reqwest.workspace = true diff --git a/wicketd/src/installinator_progress.rs b/wicketd/src/installinator_progress.rs index ba3f743171..77baec2c94 100644 --- a/wicketd/src/installinator_progress.rs +++ b/wicketd/src/installinator_progress.rs @@ -165,7 +165,7 @@ enum RunningUpdate { /// Reports from the installinator have been received. /// /// This is an `UnboundedSender` to avoid cancel-safety issues (see - /// https://github.com/oxidecomputer/omicron/pull/3579). + /// ). ReportsReceived(watch::Sender), /// All messages have been received. diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs index 58955d04d6..ebcba90645 100644 --- a/wicketd/src/preflight_check/uplink.rs +++ b/wicketd/src/preflight_check/uplink.rs @@ -17,12 +17,13 @@ use dpd_client::ClientState as DpdClientState; use either::Either; use illumos_utils::zone::SVCCFG; use illumos_utils::PFEXEC; +use ipnetwork::IpNetwork; use omicron_common::address::DENDRITE_PORT; +use omicron_common::api::internal::shared::PortConfigV1; use omicron_common::api::internal::shared::PortFec as OmicronPortFec; use omicron_common::api::internal::shared::PortSpeed as OmicronPortSpeed; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::api::internal::shared::SwitchLocation; -use omicron_common::api::internal::shared::UplinkConfig; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -32,7 +33,6 @@ use slog::Logger; use std::collections::BTreeSet; use std::collections::HashMap; use std::net::IpAddr; -use std::net::Ipv4Addr; use std::str::FromStr; use std::sync::Arc; use std::sync::Mutex; @@ -66,8 +66,6 @@ const CHRONYD: &str = "/usr/sbin/chronyd"; const IPADM: &str = "/usr/sbin/ipadm"; const ROUTE: &str = "/usr/sbin/route"; -const DPD_DEFAULT_IPV4_CIDR: &str = "0.0.0.0/0"; - pub(super) async fn run_local_uplink_preflight_check( network_config: RackNetworkConfig, dns_servers: Vec, @@ -90,7 +88,7 @@ pub(super) async fn run_local_uplink_preflight_check( let mut engine = UpdateEngine::new(log, sender); for uplink in network_config - .uplinks + .ports .iter() .filter(|uplink| uplink.switch == our_switch_location) { @@ -131,7 +129,7 @@ pub(super) async fn run_local_uplink_preflight_check( fn add_steps_for_single_local_uplink_preflight_check<'a>( engine: &mut UpdateEngine<'a>, dpd_client: &'a DpdClient, - uplink: &'a UplinkConfig, + uplink: &'a PortConfigV1, dns_servers: &'a [IpAddr], ntp_servers: &'a [String], dns_name_to_query: Option<&'a str>, @@ -153,7 +151,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Timeout we give to chronyd during the NTP check, in seconds. const CHRONYD_CHECK_TIMEOUT_SECS: &str = "30"; - let registrar = engine.for_component(uplink.uplink_port.clone()); + let registrar = engine.for_component(uplink.port.clone()); let prev_step = registrar .new_step( @@ -162,7 +160,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( |_cx| async { // Check that the port name is valid and that it has no links // configured already. - let port_id = PortId::from_str(&uplink.uplink_port) + let port_id = PortId::from_str(&uplink.port) .map_err(UplinkPreflightTerminalError::InvalidPortName)?; let links = dpd_client .link_list(&port_id) @@ -192,11 +190,11 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( { Ok(_response) => { let metadata = vec![format!( - "configured {}/{}: ip {}, gateway {}", + "configured {}/{}: ips {:#?}, routes {:#?}", *port_id, link_id.0, - uplink.uplink_cidr, - uplink.gateway_ip + uplink.addresses, + uplink.routes )]; StepSuccess::new((port_id, link_id)) .with_metadata(metadata) @@ -298,93 +296,99 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Tell the `uplink` service about the IP address we created on // the switch when configuring the uplink. let uplink_property = - UplinkProperty(format!("uplinks/{}_0", uplink.uplink_port)); - let uplink_cidr = uplink.uplink_cidr.to_string(); - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_SMF_NAME, - "addpropvalue", - &uplink_property.0, - "astring:", - &uplink_cidr, - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkAddProperty(level1)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - if let Err(err) = execute_command(&[ - SVCCFG, - "-s", - UPLINK_DEFAULT_SMF_NAME, - "refresh", - ]) - .await - { - return StepWarning::new( - Err(L2Failure::UplinkRefresh(level1, uplink_property)), - format!("could not add uplink property: {err}"), - ) - .into(); - }; - - // Wait for the `uplink` service to create the IP address. - let start_waiting_addr = Instant::now(); - 'waiting_for_addr: loop { - let ipadm_out = match execute_command(&[ - IPADM, - "show-addr", - "-p", - "-o", - "addr", + UplinkProperty(format!("uplinks/{}_0", uplink.port)); + + for addr in &uplink.addresses { + let uplink_cidr = addr.to_string(); + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_SMF_NAME, + "addpropvalue", + &uplink_property.0, + "astring:", + &uplink_cidr, ]) .await { - Ok(stdout) => stdout, - Err(err) => { - return StepWarning::new( - Err(L2Failure::RunIpadm( - level1, - uplink_property, - )), - format!("failed running ipadm: {err}"), - ) - .into(); - } + return StepWarning::new( + Err(L2Failure::UplinkAddProperty(level1)), + format!("could not add uplink property: {err}"), + ) + .into(); }; - for line in ipadm_out.split('\n') { - if line == uplink_cidr { - break 'waiting_for_addr; - } - } - - // We did not find `uplink_cidr` in the output of ipadm; - // sleep a bit and try again, unless we've been waiting too - // long already. - if start_waiting_addr.elapsed() < UPLINK_SVC_WAIT_TIMEOUT { - tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; - } else { + if let Err(err) = execute_command(&[ + SVCCFG, + "-s", + UPLINK_DEFAULT_SMF_NAME, + "refresh", + ]) + .await + { return StepWarning::new( - Err(L2Failure::WaitingForHostAddr( + Err(L2Failure::UplinkRefresh( level1, uplink_property, )), - format!( - "timed out waiting for `uplink` to \ - create {uplink_cidr}" - ), + format!("could not add uplink property: {err}"), ) .into(); + }; + + // Wait for the `uplink` service to create the IP address. + let start_waiting_addr = Instant::now(); + 'waiting_for_addr: loop { + let ipadm_out = match execute_command(&[ + IPADM, + "show-addr", + "-p", + "-o", + "addr", + ]) + .await + { + Ok(stdout) => stdout, + Err(err) => { + return StepWarning::new( + Err(L2Failure::RunIpadm( + level1, + uplink_property, + )), + format!("failed running ipadm: {err}"), + ) + .into(); + } + }; + + for line in ipadm_out.split('\n') { + if line == uplink_cidr { + break 'waiting_for_addr; + } + } + + // We did not find `uplink_cidr` in the output of ipadm; + // sleep a bit and try again, unless we've been waiting too + // long already. + if start_waiting_addr.elapsed() + < UPLINK_SVC_WAIT_TIMEOUT + { + tokio::time::sleep(UPLINK_SVC_RETRY_DELAY).await; + } else { + return StepWarning::new( + Err(L2Failure::WaitingForHostAddr( + level1, + uplink_property, + )), + format!( + "timed out waiting for `uplink` to \ + create {uplink_cidr}" + ), + ) + .into(); + } } } - let metadata = vec![format!("configured {}", uplink_property.0)]; StepSuccess::new(Ok(L2Success { level1, uplink_property })) @@ -410,27 +414,29 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - // Add the gateway as the default route in illumos. - if let Err(err) = execute_command(&[ - ROUTE, - "add", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - { - return StepWarning::new( - Err(RoutingFailure::HostDefaultRoute(level2)), - format!("could not add default route: {err}"), - ) - .into(); - }; + for r in &uplink.routes { + // Add the gateway as the default route in illumos. + if let Err(err) = execute_command(&[ + ROUTE, + "add", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + { + return StepWarning::new( + Err(RoutingFailure::HostDefaultRoute(level2)), + format!("could not add default route: {err}"), + ) + .into(); + }; + } StepSuccess::new(Ok(RoutingSuccess { level2 })) .with_metadata(vec![format!( - "added default route to {}", - uplink.gateway_ip + "added routes {:#?}", + uplink.routes, )]) .into() }, @@ -595,21 +601,24 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } }; - if remove_host_route { - execute_command(&[ - ROUTE, - "delete", - "-inet", - "default", - &uplink.gateway_ip.to_string(), - ]) - .await - .map_err(|err| { - UplinkPreflightTerminalError::RemoveHostRoute { - err, - gateway_ip: uplink.gateway_ip, - } - })?; + for r in &uplink.routes { + if remove_host_route { + execute_command(&[ + ROUTE, + "delete", + "-inet", + &r.destination.to_string(), + &r.nexthop.to_string(), + ]) + .await + .map_err(|err| { + UplinkPreflightTerminalError::RemoveHostRoute { + err, + destination: r.destination, + nexthop: r.nexthop, + } + })?; + } } StepSuccess::new(Ok(level2)).into() @@ -730,7 +739,7 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( } fn build_port_settings( - uplink: &UplinkConfig, + uplink: &PortConfigV1, link_id: &LinkId, ) -> PortSettings { // Map from omicron_common types to dpd_client types @@ -758,10 +767,12 @@ fn build_port_settings( v6_routes: HashMap::new(), }; + let addrs = uplink.addresses.iter().map(|a| a.ip()).collect(); + port_settings.links.insert( link_id.to_string(), LinkSettings { - addrs: vec![IpAddr::V4(uplink.uplink_cidr.ip())], + addrs, params: LinkCreate { // TODO we should take these parameters too // https://github.com/oxidecomputer/omicron/issues/3061 @@ -773,14 +784,16 @@ fn build_port_settings( }, ); - port_settings.v4_routes.insert( - DPD_DEFAULT_IPV4_CIDR.parse().unwrap(), - RouteSettingsV4 { - link_id: link_id.0, - nexthop: uplink.gateway_ip, - vid: uplink.uplink_vid, - }, - ); + for r in &uplink.routes { + if let (IpNetwork::V4(dst), IpAddr::V4(nexthop)) = + (r.destination, r.nexthop) + { + port_settings.v4_routes.insert( + dst.to_string(), + RouteSettingsV4 { link_id: link_id.0, nexthop, vid: None }, + ); + } + } port_settings } @@ -890,8 +903,10 @@ pub(crate) enum UplinkPreflightTerminalError { err: DpdError, port_id: PortId, }, - #[error("failed to remove host OS route to gateway {gateway_ip}: {err}")] - RemoveHostRoute { err: String, gateway_ip: Ipv4Addr }, + #[error( + "failed to remove host OS route {destination} -> {nexthop}: {err}" + )] + RemoveHostRoute { err: String, destination: IpNetwork, nexthop: IpAddr }, #[error("failed to remove uplink SMF property {property:?}: {err}")] RemoveSmfProperty { property: String, err: String }, #[error("failed to refresh uplink service config: {0}")] diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 1dc9f84985..a96acc56a0 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -453,18 +453,21 @@ impl From<&'_ CurrentRssConfig> for CurrentRssUserConfig { fn validate_rack_network_config( config: &RackNetworkConfig, -) -> Result { +) -> Result { + use bootstrap_agent_client::types::BgpConfig as BaBgpConfig; + use bootstrap_agent_client::types::BgpPeerConfig as BaBgpPeerConfig; + use bootstrap_agent_client::types::PortConfigV1 as BaPortConfigV1; use bootstrap_agent_client::types::PortFec as BaPortFec; use bootstrap_agent_client::types::PortSpeed as BaPortSpeed; + use bootstrap_agent_client::types::RouteConfig as BaRouteConfig; use bootstrap_agent_client::types::SwitchLocation as BaSwitchLocation; - use bootstrap_agent_client::types::UplinkConfig as BaUplinkConfig; use omicron_common::api::internal::shared::PortFec; use omicron_common::api::internal::shared::PortSpeed; use omicron_common::api::internal::shared::SwitchLocation; // Ensure that there is at least one uplink - if config.uplinks.is_empty() { - return Err(anyhow!("Must have at least one uplink configured")); + if config.ports.is_empty() { + return Err(anyhow!("Must have at least one port configured")); } // Make sure `infra_ip_first`..`infra_ip_last` is a well-defined range... @@ -475,34 +478,55 @@ fn validate_rack_network_config( }, )?; - // iterate through each UplinkConfig - for uplink_config in &config.uplinks { - // ... and check that it contains `uplink_ip`. - if uplink_config.uplink_cidr.ip() < infra_ip_range.first - || uplink_config.uplink_cidr.ip() > infra_ip_range.last - { - bail!( + // TODO this implies a single contiguous range for port IPs which is over + // constraining + // iterate through each port config + for port_config in &config.ports { + for addr in &port_config.addresses { + // ... and check that it contains `uplink_ip`. + if addr.ip() < infra_ip_range.first + || addr.ip() > infra_ip_range.last + { + bail!( "`uplink_cidr`'s IP address must be in the range defined by \ `infra_ip_first` and `infra_ip_last`" ); + } } } // TODO Add more client side checks on `rack_network_config` contents? - Ok(bootstrap_agent_client::types::RackNetworkConfig { + Ok(bootstrap_agent_client::types::RackNetworkConfigV1 { + rack_subnet: config.rack_subnet, infra_ip_first: config.infra_ip_first, infra_ip_last: config.infra_ip_last, - uplinks: config - .uplinks + ports: config + .ports .iter() - .map(|config| BaUplinkConfig { - gateway_ip: config.gateway_ip, + .map(|config| BaPortConfigV1 { + port: config.port.clone(), + routes: config + .routes + .iter() + .map(|r| BaRouteConfig { + destination: r.destination, + nexthop: r.nexthop, + }) + .collect(), + addresses: config.addresses.clone(), + bgp_peers: config + .bgp_peers + .iter() + .map(|p| BaBgpPeerConfig { + addr: p.addr, + asn: p.asn, + port: p.port.clone(), + }) + .collect(), switch: match config.switch { SwitchLocation::Switch0 => BaSwitchLocation::Switch0, SwitchLocation::Switch1 => BaSwitchLocation::Switch1, }, - uplink_cidr: config.uplink_cidr, - uplink_port: config.uplink_port.clone(), uplink_port_speed: match config.uplink_port_speed { PortSpeed::Speed0G => BaPortSpeed::Speed0G, PortSpeed::Speed1G => BaPortSpeed::Speed1G, @@ -519,7 +543,14 @@ fn validate_rack_network_config( PortFec::None => BaPortFec::None, PortFec::Rs => BaPortFec::Rs, }, - uplink_vid: config.uplink_vid, + }) + .collect(), + bgp: config + .bgp + .iter() + .map(|config| BaBgpConfig { + asn: config.asn, + originate: config.originate.clone(), }) .collect(), }) diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index b08f2612f1..3d9b195ae3 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -79,6 +79,7 @@ ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.18", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +serde_json = { version = "1.0.107", features = ["raw_value"] } sha2 = { version = "0.10.7", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } @@ -171,6 +172,7 @@ ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.18", features = ["serde"] } serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +serde_json = { version = "1.0.107", features = ["raw_value"] } sha2 = { version = "0.10.7", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } similar = { version = "2.2.1", features = ["inline", "unicode"] } From e451ca59c52a819a9c00ab6ce1703cca6040d744 Mon Sep 17 00:00:00 2001 From: Luqman Aden Date: Sat, 21 Oct 2023 00:21:43 -0700 Subject: [PATCH 3/3] bootstore: don't use hardcoded ports in tests (#4304) Fix some test flakiness by not hardcoding specific ports to use. --------- Co-authored-by: Andrew J. Stone --- bootstore/src/schemes/v0/peer.rs | 638 ++++++++++++++++++++----------- 1 file changed, 408 insertions(+), 230 deletions(-) diff --git a/bootstore/src/schemes/v0/peer.rs b/bootstore/src/schemes/v0/peer.rs index 7d29e2397a..3d273e60eb 100644 --- a/bootstore/src/schemes/v0/peer.rs +++ b/bootstore/src/schemes/v0/peer.rs @@ -91,6 +91,9 @@ pub enum NodeApiRequest { /// These are generated from DDM prefixes learned by the bootstrap agent. PeerAddresses(BTreeSet), + /// Get the local [`SocketAddrV6`] the node is listening on. + GetAddress { responder: oneshot::Sender }, + /// Get the status of this node GetStatus { responder: oneshot::Sender }, @@ -175,6 +178,17 @@ impl NodeHandle { Ok(()) } + /// Get the address of this node + pub async fn get_address(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.tx + .send(NodeApiRequest::GetAddress { responder: tx }) + .await + .map_err(|_| NodeRequestError::Send)?; + let res = rx.await?; + Ok(res) + } + /// Get the status of this node pub async fn get_status(&self) -> Result { let (tx, rx) = oneshot::channel(); @@ -361,6 +375,11 @@ impl Node { let mut interval = interval(self.config.time_per_tick); interval.set_missed_tick_behavior(MissedTickBehavior::Delay); let listener = TcpListener::bind(&self.config.addr).await.unwrap(); + // If the config didn't specify a port, let's update it + // with the actual port we binded to on our listener. + if self.config.addr.port() == 0 { + self.config.addr.set_port(listener.local_addr().unwrap().port()); + } while !self.shutdown { tokio::select! { res = listener.accept() => self.on_accept(res).await, @@ -487,6 +506,9 @@ impl Node { info!(self.log, "Updated Peer Addresses: {peers:?}"); self.manage_connections(peers).await; } + NodeApiRequest::GetAddress { responder } => { + let _ = responder.send(self.config.addr); + } NodeApiRequest::GetStatus { responder } => { let status = Status { fsm_ledger_generation: self.fsm_ledger_generation, @@ -1025,11 +1047,11 @@ mod tests { use super::*; use camino_tempfile::Utf8TempDir; use slog::Drain; - use tokio::time::sleep; + use tokio::{task::JoinHandle, time::sleep}; use uuid::Uuid; fn initial_members() -> BTreeSet { - [("a", "1"), ("b", "1"), ("c", "1")] + [("a", "0"), ("b", "1"), ("c", "2")] .iter() .map(|(id, model)| { Baseboard::new_pc(id.to_string(), model.to_string()) @@ -1037,56 +1059,10 @@ mod tests { .collect() } - fn initial_config(tempdir: &Utf8TempDir, port_start: u16) -> Vec { - initial_members() - .into_iter() - .enumerate() - .map(|(i, id)| { - let fsm_file = format!("test-{i}-fsm-state-ledger"); - let network_file = format!("test-{i}-network-config-ledger"); - Config { - id, - addr: format!("[::1]:{}{}", port_start, i).parse().unwrap(), - time_per_tick: Duration::from_millis(20), - learn_timeout: Duration::from_secs(5), - rack_init_timeout: Duration::from_secs(10), - rack_secret_request_timeout: Duration::from_secs(1), - fsm_state_ledger_paths: vec![tempdir - .path() - .join(&fsm_file)], - network_config_ledger_paths: vec![tempdir - .path() - .join(&network_file)], - } - }) - .collect() - } - fn learner_id(n: usize) -> Baseboard { Baseboard::new_pc("learner".to_string(), n.to_string()) } - fn learner_config( - tempdir: &Utf8TempDir, - n: usize, - port_start: u16, - ) -> Config { - let fsm_file = format!("test-learner-{n}-fsm-state-ledger"); - let network_file = format!("test-{n}-network-config-ledger"); - Config { - id: learner_id(n), - addr: format!("[::1]:{}{}", port_start, 3).parse().unwrap(), - time_per_tick: Duration::from_millis(20), - learn_timeout: Duration::from_secs(5), - rack_init_timeout: Duration::from_secs(10), - rack_secret_request_timeout: Duration::from_secs(1), - fsm_state_ledger_paths: vec![tempdir.path().join(&fsm_file)], - network_config_ledger_paths: vec![tempdir - .path() - .join(&network_file)], - } - } - fn log() -> slog::Logger { let decorator = slog_term::PlainDecorator::new(slog_term::TestStdoutWriter); @@ -1095,191 +1071,416 @@ mod tests { slog::Logger::root(drain, o!()) } - #[tokio::test] - async fn basic_3_nodes() { - let port_start = 3333; - let tempdir = Utf8TempDir::new().unwrap(); - let log = log(); - let config = initial_config(&tempdir, port_start); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let (mut node2, handle2) = Node::new(config[2].clone(), &log).await; - - let jh0 = tokio::spawn(async move { - node0.run().await; - }); - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let jh2 = tokio::spawn(async move { - node2.run().await; - }); + struct TestNode { + log: Logger, + config: Config, + node_handles: Option<(NodeHandle, JoinHandle<()>)>, + } + + impl TestNode { + fn new(config: Config, log: Logger) -> TestNode { + TestNode { config, log, node_handles: None } + } + + async fn start_node(&mut self) { + // Node must have previously been shutdown (or never started) + assert!( + self.node_handles.is_none(), + "node ({}) already running", + self.config.id + ); + + // Reset port to pick any available + self.config.addr.set_port(0); + + // (Re-)create node with existing config and its persistent state (if any) + let (mut node, handle) = + Node::new(self.config.clone(), &self.log).await; + let jh = tokio::spawn(async move { + node.run().await; + }); + + // Grab assigned port + let port = handle + .get_address() + .await + .unwrap_or_else(|err| { + panic!( + "failed to get local address of node ({}): {err}", + self.config.id + ) + }) + .port(); + self.config.addr.set_port(port); + + self.node_handles = Some((handle, jh)); + } + + async fn shutdown_node(&mut self) { + let (handle, jh) = self.node_handles.take().unwrap_or_else(|| { + panic!("node ({}) not active", self.config.id) + }); + // Signal to the node it should shutdown + handle.shutdown().await.unwrap_or_else(|err| { + panic!("node ({}) failed to shutdown: {err}", self.config.id) + }); + // and wait for its task to spin down. + jh.await.unwrap_or_else(|err| { + panic!("node ({}) task failed: {err}", self.config.id) + }); + } + } + + struct TestNodes { + tempdir: Utf8TempDir, + log: Logger, + nodes: Vec, + learner: Option, + addrs: BTreeSet, + } + + impl TestNodes { + /// Create test nodes for the given set of members. + fn setup(initial_members: BTreeSet) -> TestNodes { + let tempdir = Utf8TempDir::new().unwrap(); + let log = log(); + let nodes = initial_members + .into_iter() + .enumerate() + .map(|(i, id)| { + let fsm_file = format!("test-{i}-fsm-state-ledger"); + let network_file = + format!("test-{i}-network-config-ledger"); + let config = Config { + id, + addr: SocketAddrV6::new( + std::net::Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + time_per_tick: Duration::from_millis(20), + learn_timeout: Duration::from_secs(5), + rack_init_timeout: Duration::from_secs(10), + rack_secret_request_timeout: Duration::from_secs(1), + fsm_state_ledger_paths: vec![tempdir + .path() + .join(&fsm_file)], + network_config_ledger_paths: vec![tempdir + .path() + .join(&network_file)], + }; + + TestNode::new(config, log.clone()) + }) + .collect(); + TestNodes { + tempdir, + log, + nodes, + learner: None, // No initial learner node + addrs: BTreeSet::new(), + } + } + + /// (Re-)start the given node and update peer addresses for everyone + async fn start_node(&mut self, i: usize) { + let node = &mut self.nodes[i]; + node.start_node().await; + self.addrs.insert(node.config.addr); + self.load_all_peer_addresses().await; + } + + // Stop the given node and update peer addresses for everyone + async fn shutdown_node(&mut self, i: usize) { + let node = &mut self.nodes[i]; + let addr = node.config.addr; + node.shutdown_node().await; + self.addrs.remove(&addr); + self.load_all_peer_addresses().await; + } + + /// Stop all active nodes (including the learner, if present). + async fn shutdown_all(&mut self) { + let nodes = self + .nodes + .iter_mut() + .chain(&mut self.learner) + .filter(|node| node.node_handles.is_some()); + for node in nodes { + node.shutdown_node().await; + } + self.addrs.clear(); + self.learner = None; + } + + /// Configure new learner node + async fn add_learner(&mut self, n: usize) { + assert!( + self.learner.is_none(), + "learner node already configured ({})", + self.learner.as_ref().unwrap().config.id + ); + + let fsm_file = format!("test-learner-{n}-fsm-state-ledger"); + let network_file = format!("test-{n}-network-config-ledger"); + let config = Config { + id: learner_id(n), + addr: SocketAddrV6::new(std::net::Ipv6Addr::LOCALHOST, 0, 0, 0), + time_per_tick: Duration::from_millis(20), + learn_timeout: Duration::from_secs(5), + rack_init_timeout: Duration::from_secs(10), + rack_secret_request_timeout: Duration::from_secs(1), + fsm_state_ledger_paths: vec![self + .tempdir + .path() + .join(&fsm_file)], + network_config_ledger_paths: vec![self + .tempdir + .path() + .join(&network_file)], + }; + + self.learner = Some(TestNode::new(config, self.log.clone())); + } - // Inform each node about the known addresses - let mut addrs: BTreeSet<_> = config.iter().map(|c| c.addr).collect(); - for handle in [&handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; + /// Start a configured learner node and update peer addresses for everyone + async fn start_learner(&mut self) { + let learner = + self.learner.as_mut().expect("no learner node configured"); + learner.start_node().await; + let learner_addr = learner.config.addr; + + // Inform the learner and other nodes about all addresses including + // the learner. This simulates DDM discovery. + self.addrs.insert(learner_addr); + self.load_all_peer_addresses().await; } + /// Stop the learner node (but leave it configured) and update peer addresses for everyone + /// Can also optionally wipe the ledger persisted on disk. + async fn shutdown_learner(&mut self, wipe_ledger: bool) { + let learner = + self.learner.as_mut().expect("no learner node configured"); + let addr = learner.config.addr; + learner.shutdown_node().await; + + if wipe_ledger { + std::fs::remove_file(&learner.config.fsm_state_ledger_paths[0]) + .expect("failed to remove ledger"); + } + + // Update peer addresses + self.addrs.remove(&addr); + self.load_all_peer_addresses().await; + } + + /// Remove a configured learner node + async fn remove_learner(&mut self) { + // Shutdown the node if it's running + if matches!( + self.learner, + Some(TestNode { node_handles: Some(_), .. }) + ) { + self.shutdown_learner(false).await; + } + let _ = self.learner.take().expect("no learner node configured"); + } + + /// Inform each active node about its peers + async fn load_all_peer_addresses(&self) { + let nodes = + self.nodes.iter().chain(&self.learner).filter_map(|node| { + node.node_handles + .as_ref() + .map(|(h, _)| (&node.config.id, h)) + }); + for (id, node) in nodes { + node.load_peer_addresses(self.addrs.clone()).await.unwrap_or_else(|err| { + panic!("failed to update peer addresses for node ({id}): {err}") + }); + } + } + + /// Returns an iterator that yields the [`NodeHandle`]'s for all active + /// nodes (including the learner node, if present). + fn iter(&self) -> impl Iterator { + self.nodes + .iter() + .chain(&self.learner) + .filter_map(|node| node.node_handles.as_ref().map(|(h, _)| h)) + } + + /// To ensure deterministic learning of shares from node 0 which sorts first + /// we wait to ensure that the learner sees peer0 as connected before we + /// call `init_learner` + /// + /// Panics if the connection doesn't happen within `POLL_TIMEOUT` + async fn wait_for_learner_to_connect_to_node(&self, i: usize) { + const POLL_TIMEOUT: Duration = Duration::from_secs(5); + let start = Instant::now(); + loop { + let timeout = + POLL_TIMEOUT.saturating_sub(Instant::now() - start); + tokio::select! { + _ = sleep(timeout) => { + panic!("Learner not connected to node {i}"); + } + status = self[LEARNER].get_status() => { + let status = status.unwrap(); + let id = &self.nodes[i].config.id; + if status.connections.contains_key(id) { + break; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + } + } + } + + impl std::ops::Index for TestNodes { + type Output = NodeHandle; + + fn index(&self, index: usize) -> &Self::Output { + self.nodes[index] + .node_handles + .as_ref() + .map(|(handle, _)| handle) + .unwrap_or_else(|| panic!("node{index} not running")) + } + } + + // A little convenience to access the learner node in a similar + // manner as other nodes (indexing) but with a non-usize index. + const LEARNER: () = (); + impl std::ops::Index<()> for TestNodes { + type Output = NodeHandle; + + fn index(&self, _: ()) -> &Self::Output { + self.learner + .as_ref() + .expect("no learner node") + .node_handles + .as_ref() + .map(|(handle, _)| handle) + .expect("learner node not running") + } + } + + #[tokio::test] + async fn basic_3_nodes() { + // Create and start test nodes + let mut nodes = TestNodes::setup(initial_members()); + nodes.start_node(0).await; + nodes.start_node(1).await; + nodes.start_node(2).await; + let rack_uuid = RackUuid(Uuid::new_v4()); - handle0.init_rack(rack_uuid, initial_members()).await.unwrap(); + nodes[0].init_rack(rack_uuid, initial_members()).await.unwrap(); - let status = handle0.get_status().await; + let status = nodes[0].get_status().await; println!("Status = {status:?}"); // Ensure we can load the rack secret at all nodes - handle0.load_rack_secret().await.unwrap(); - handle1.load_rack_secret().await.unwrap(); - handle2.load_rack_secret().await.unwrap(); + for node in nodes.iter() { + node.load_rack_secret().await.unwrap(); + } // load the rack secret a second time on node0 - handle0.load_rack_secret().await.unwrap(); + nodes[0].load_rack_secret().await.unwrap(); // Shutdown the node2 and make sure we can still load the rack // secret (threshold=2) at node0 and node1 - handle2.shutdown().await.unwrap(); - jh2.await.unwrap(); - handle0.load_rack_secret().await.unwrap(); - handle1.load_rack_secret().await.unwrap(); - - // Add a learner node - let learner_conf = learner_config(&tempdir, 1, port_start); - let (mut learner, learner_handle) = - Node::new(learner_conf.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - // Inform the learner and node0 and node1 about all addresses including - // the learner. This simulates DDM discovery - addrs.insert(learner_conf.addr); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - let _ = handle0.load_peer_addresses(addrs.clone()).await; - let _ = handle1.load_peer_addresses(addrs.clone()).await; + nodes.shutdown_node(2).await; + nodes[0].load_rack_secret().await.unwrap(); + nodes[1].load_rack_secret().await.unwrap(); + + // Add and start a learner node + nodes.add_learner(1).await; + nodes.start_learner().await; // Tell the learner to go ahead and learn its share. - learner_handle.init_learner().await.unwrap(); + nodes[LEARNER].init_learner().await.unwrap(); // Shutdown node1 and show that we can still load the rack secret at // node0 and the learner, because threshold=2 and it never changes. - handle1.shutdown().await.unwrap(); - jh1.await.unwrap(); - handle0.load_rack_secret().await.unwrap(); - learner_handle.load_rack_secret().await.unwrap(); + nodes.shutdown_node(1).await; + nodes[0].load_rack_secret().await.unwrap(); + nodes[LEARNER].load_rack_secret().await.unwrap(); - // Now shutdown the learner and show that node0 cannot load the rack secret - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - handle0.load_rack_secret().await.unwrap_err(); + // Now shutdown and remove the learner and show that node0 cannot load the rack secret + nodes.remove_learner().await; + nodes[0].load_rack_secret().await.unwrap_err(); - // Reload an node from persistent state and successfully reload the + // Reload a node from persistent state and successfully reload the // rack secret. - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let _ = handle1.load_peer_addresses(addrs.clone()).await; - handle0.load_rack_secret().await.unwrap(); + nodes.start_node(1).await; + nodes[0].load_rack_secret().await.unwrap(); - // Add a second learner + // Grab the current generation numbers let peer0_gen = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen = - handle1.get_status().await.unwrap().fsm_ledger_generation; - let learner_config = learner_config(&tempdir, 2, port_start); - let (mut learner, learner_handle) = - Node::new(learner_config.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); + nodes[1].get_status().await.unwrap().fsm_ledger_generation; + + // Add and start a second learner + nodes.add_learner(2).await; + nodes.start_learner().await; - // Inform the learner, node0, and node1 about all addresses including - // the learner. This simulates DDM discovery - addrs.insert(learner_config.addr); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - let _ = handle0.load_peer_addresses(addrs.clone()).await; - let _ = handle1.load_peer_addresses(addrs.clone()).await; + // Wait for the learner to connect to node 0 + nodes.wait_for_learner_to_connect_to_node(0).await; // Tell the learner to go ahead and learn its share. - learner_handle.init_learner().await.unwrap(); + nodes[LEARNER].init_learner().await.unwrap(); // Get the new generation numbers let peer0_gen_new = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen_new = - handle1.get_status().await.unwrap().fsm_ledger_generation; + nodes[1].get_status().await.unwrap().fsm_ledger_generation; - // Ensure only one of the peers generation numbers gets bumped - assert!( - (peer0_gen_new == peer0_gen && peer1_gen_new == peer1_gen + 1) - || (peer0_gen_new == peer0_gen + 1 - && peer1_gen_new == peer1_gen) - ); + // Ensure only peer 0's generation number gets bumped + assert_eq!(peer0_gen_new, peer0_gen + 1); + assert_eq!(peer1_gen_new, peer1_gen); + + // Now we can stop the learner, wipe its ledger, and restart it. + nodes.shutdown_learner(true).await; + nodes.start_learner().await; // Wipe the learner ledger, restart the learner and instruct it to // relearn its share, and ensure that the neither generation number gets - // bumped because persistence doesn't occur. - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - std::fs::remove_file(&learner_config.fsm_state_ledger_paths[0]) - .unwrap(); - let (mut learner, learner_handle) = - Node::new(learner_config.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - let _ = learner_handle.load_peer_addresses(addrs.clone()).await; - learner_handle.init_learner().await.unwrap(); + // bumped because persistence doesn't occur. But for that to happen + // we need to make sure the learner asks the same peer, which is node 0 since + // it sorts first based on its id which is of type `Baseboard`. + nodes.wait_for_learner_to_connect_to_node(0).await; + nodes[LEARNER].init_learner().await.unwrap(); + + // Ensure the peers' generation numbers didn't get bumped. The learner + // should've asked the same sled for a share first, which it already + // handed out. let peer0_gen_new_2 = - handle0.get_status().await.unwrap().fsm_ledger_generation; + nodes[0].get_status().await.unwrap().fsm_ledger_generation; let peer1_gen_new_2 = - handle1.get_status().await.unwrap().fsm_ledger_generation; - - // Ensure the peer's generation numbers don't get bumped. The learner - // will ask the same sled for a share first, which it already handed - // out. - assert!( - peer0_gen_new == peer0_gen_new_2 - && peer1_gen_new == peer1_gen_new_2 - ); + nodes[1].get_status().await.unwrap().fsm_ledger_generation; + assert_eq!(peer0_gen_new, peer0_gen_new_2); + assert_eq!(peer1_gen_new, peer1_gen_new_2); - // Shutdown the new learner, node0, and node1 - learner_handle.shutdown().await.unwrap(); - learner_jh.await.unwrap(); - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - handle1.shutdown().await.unwrap(); - jh1.await.unwrap(); + // Shut it all down + nodes.shutdown_all().await; } #[tokio::test] async fn network_config() { - let port_start = 4444; - let tempdir = Utf8TempDir::new().unwrap(); - let log = log(); - let config = initial_config(&tempdir, port_start); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let (mut node1, handle1) = Node::new(config[1].clone(), &log).await; - let (mut node2, handle2) = Node::new(config[2].clone(), &log).await; - - let jh0 = tokio::spawn(async move { - node0.run().await; - }); - let jh1 = tokio::spawn(async move { - node1.run().await; - }); - let jh2 = tokio::spawn(async move { - node2.run().await; - }); - - // Inform each node about the known addresses - let mut addrs: BTreeSet<_> = config.iter().map(|c| c.addr).collect(); - for handle in [&handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; - } + // Create and start test nodes + let mut nodes = TestNodes::setup(initial_members()); + nodes.start_node(0).await; + nodes.start_node(1).await; + nodes.start_node(2).await; // Ensure there is no network config at any of the nodes - for handle in [&handle0, &handle1, &handle2] { - assert_eq!(None, handle.get_network_config().await.unwrap()); + for node in nodes.iter() { + assert_eq!(None, node.get_network_config().await.unwrap()); } // Update the network config at node0 and ensure it has taken effect @@ -1287,10 +1488,10 @@ mod tests { generation: 1, blob: b"Some network data".to_vec(), }; - handle0.update_network_config(network_config.clone()).await.unwrap(); + nodes[0].update_network_config(network_config.clone()).await.unwrap(); assert_eq!( Some(&network_config), - handle0.get_network_config().await.unwrap().as_ref() + nodes[0].get_network_config().await.unwrap().as_ref() ); // Poll node1 and node2 until the network config update shows up @@ -1305,13 +1506,13 @@ mod tests { _ = sleep(timeout) => { panic!("Network config not replicated"); } - res = handle1.get_network_config(), if !node1_done => { + res = nodes[1].get_network_config(), if !node1_done => { if res.unwrap().as_ref() == Some(&network_config) { node1_done = true; continue; } } - res = handle2.get_network_config(), if !node2_done => { + res = nodes[2].get_network_config(), if !node2_done => { if res.unwrap().as_ref() == Some(&network_config) { node2_done = true; continue; @@ -1321,18 +1522,8 @@ mod tests { } // Bring a learner online - let learner_conf = learner_config(&tempdir, 1, port_start); - let (mut learner, learner_handle) = - Node::new(learner_conf.clone(), &log).await; - let learner_jh = tokio::spawn(async move { - learner.run().await; - }); - // Inform the learner and other nodes about all addresses including - // the learner. This simulates DDM discovery. - addrs.insert(learner_conf.addr); - for handle in [&learner_handle, &handle0, &handle1, &handle2] { - let _ = handle.load_peer_addresses(addrs.clone()).await; - } + nodes.add_learner(1).await; + nodes.start_learner().await; // Poll the learner to ensure it gets the network config // Note that the learner doesn't even need to learn its share @@ -1345,7 +1536,7 @@ mod tests { _ = sleep(timeout) => { panic!("Network config not replicated"); } - res = learner_handle.get_network_config() => { + res = nodes[LEARNER].get_network_config() => { if res.unwrap().as_ref() == Some(&network_config) { done = true; } @@ -1355,34 +1546,26 @@ mod tests { // Stop node0, bring it back online and ensure it still sees the config // at generation 1 - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let jh0 = tokio::spawn(async move { - node0.run().await; - }); + nodes.shutdown_node(0).await; + nodes.start_node(0).await; assert_eq!( Some(&network_config), - handle0.get_network_config().await.unwrap().as_ref() + nodes[0].get_network_config().await.unwrap().as_ref() ); // Stop node0 again, update network config via node1, bring node0 back online, // and ensure all nodes see the latest configuration. + nodes.shutdown_node(0).await; let new_config = NetworkConfig { generation: 2, blob: b"Some more network data".to_vec(), }; - handle0.shutdown().await.unwrap(); - jh0.await.unwrap(); - handle1.update_network_config(new_config.clone()).await.unwrap(); + nodes[1].update_network_config(new_config.clone()).await.unwrap(); assert_eq!( Some(&new_config), - handle1.get_network_config().await.unwrap().as_ref() + nodes[1].get_network_config().await.unwrap().as_ref() ); - let (mut node0, handle0) = Node::new(config[0].clone(), &log).await; - let jh0 = tokio::spawn(async move { - node0.run().await; - }); + nodes.start_node(0).await; let start = Instant::now(); // These should all resolve instantly, so no real need for a select, // which is getting tedious. @@ -1392,8 +1575,8 @@ mod tests { if Instant::now() - start > POLL_TIMEOUT { panic!("network config not replicated"); } - for h in [&handle0, &handle1, &handle2, &learner_handle] { - if h.get_network_config().await.unwrap().as_ref() + for node in nodes.iter() { + if node.get_network_config().await.unwrap().as_ref() != Some(&new_config) { // We need to try again @@ -1410,16 +1593,11 @@ mod tests { current_generation: 2, }); assert_eq!( - handle0.update_network_config(network_config).await, + nodes[0].update_network_config(network_config).await, expected ); // Shut it all down - for h in [handle0, handle1, handle2, learner_handle] { - let _ = h.shutdown().await; - } - for jh in [jh0, jh1, jh2, learner_jh] { - jh.await.unwrap(); - } + nodes.shutdown_all().await; } }