From 0fd554506202946dc0efe22fce5e3a6490e076d2 Mon Sep 17 00:00:00 2001 From: James MacMahon Date: Thu, 7 Jul 2022 12:08:45 -0400 Subject: [PATCH] Update crucible and propolis revs (#1368) Bring in the changes to create all volumes with an ID. Flatten the current volume topologies a little bit. Update to the latest propolis rev (which matches this crucible rev). --- Cargo.lock | 67 ++++-------------------------- nexus/Cargo.toml | 2 +- nexus/src/app/image.rs | 41 +++++++++--------- nexus/src/app/sagas/disk_create.rs | 4 ++ openapi/sled-agent.json | 20 +++++++++ package-manifest.toml | 8 ++-- sled-agent/Cargo.toml | 4 +- 7 files changed, 58 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83cbf941f5..0f6059ad9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,15 +74,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - [[package]] name = "anyhow" version = "1.0.57" @@ -532,13 +523,9 @@ version = "2.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ - "ansi_term", - "atty", "bitflags", - "strsim 0.8.0", "textwrap 0.11.0", "unicode-width", - "vec_map", ] [[package]] @@ -553,7 +540,7 @@ dependencies = [ "clap_lex", "indexmap", "once_cell", - "strsim 0.10.0", + "strsim", "termcolor", "textwrap 0.15.0", ] @@ -823,7 +810,7 @@ dependencies = [ [[package]] name = "crucible" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=8314eeddd228ec0d76cefa40c4a41d3e2611ac18#8314eeddd228ec0d76cefa40c4a41d3e2611ac18" +source = "git+https://github.com/oxidecomputer/crucible?rev=fed3e8ca7762130ee146fc516a4ef6eed2b91629#fed3e8ca7762130ee146fc516a4ef6eed2b91629" dependencies = [ "aes-gcm-siv", "anyhow", @@ -845,7 +832,6 @@ dependencies = [ "schemars", "serde", "serde_json", - "structopt", "tokio", "tokio-rustls", "tokio-util", @@ -859,7 +845,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=8314eeddd228ec0d76cefa40c4a41d3e2611ac18#8314eeddd228ec0d76cefa40c4a41d3e2611ac18" +source = "git+https://github.com/oxidecomputer/crucible?rev=fed3e8ca7762130ee146fc516a4ef6eed2b91629#fed3e8ca7762130ee146fc516a4ef6eed2b91629" dependencies = [ "anyhow", "chrono", @@ -874,7 +860,7 @@ dependencies = [ [[package]] name = "crucible-common" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=8314eeddd228ec0d76cefa40c4a41d3e2611ac18#8314eeddd228ec0d76cefa40c4a41d3e2611ac18" +source = "git+https://github.com/oxidecomputer/crucible?rev=fed3e8ca7762130ee146fc516a4ef6eed2b91629#fed3e8ca7762130ee146fc516a4ef6eed2b91629" dependencies = [ "anyhow", "rusqlite", @@ -892,7 +878,7 @@ dependencies = [ [[package]] name = "crucible-protocol" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=8314eeddd228ec0d76cefa40c4a41d3e2611ac18#8314eeddd228ec0d76cefa40c4a41d3e2611ac18" +source = "git+https://github.com/oxidecomputer/crucible?rev=fed3e8ca7762130ee146fc516a4ef6eed2b91629#fed3e8ca7762130ee146fc516a4ef6eed2b91629" dependencies = [ "anyhow", "bincode", @@ -1041,7 +1027,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim", "syn", ] @@ -1055,7 +1041,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim 0.10.0", + "strsim", "syn", ] @@ -4137,7 +4123,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=d2b8184d11d5d5f56424fe0bee13ca03d576604e#d2b8184d11d5d5f56424fe0bee13ca03d576604e" +source = "git+https://github.com/oxidecomputer/propolis?rev=23a38f4a1df4afcba3905f94ebbf312d0d187c7d#23a38f4a1df4afcba3905f94ebbf312d0d187c7d" dependencies = [ "crucible", "reqwest", @@ -4146,7 +4132,6 @@ dependencies = [ "serde", "serde_json", "slog", - "structopt", "thiserror", "uuid", ] @@ -5454,12 +5439,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - [[package]] name = "strsim" version = "0.10.0" @@ -5489,30 +5468,6 @@ dependencies = [ "syn", ] -[[package]] -name = "structopt" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" -dependencies = [ - "clap 2.34.0", - "lazy_static", - "structopt-derive", -] - -[[package]] -name = "structopt-derive" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" -dependencies = [ - "heck 0.3.3", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "strum" version = "0.24.1" @@ -6367,12 +6322,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - [[package]] name = "version_check" version = "0.9.4" diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index a43d41bde8..cf0f3824ec 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -16,7 +16,7 @@ base64 = "0.13.0" bb8 = "0.8.0" clap = { version = "3.2", features = ["derive"] } cookie = "0.16" -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "8314eeddd228ec0d76cefa40c4a41d3e2611ac18" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "fed3e8ca7762130ee146fc516a4ef6eed2b91629" } diesel = { version = "2.0.0-rc.0", features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] } diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace" } fatfs = "0.3.5" diff --git a/nexus/src/app/image.rs b/nexus/src/app/image.rs index c3ef736afc..cd3edee548 100644 --- a/nexus/src/app/image.rs +++ b/nexus/src/app/image.rs @@ -101,16 +101,14 @@ impl super::Nexus { message: format!("block_size is invalid: {}", e), })?; - let volume_construction_request = sled_agent_client::types::VolumeConstructionRequest::Volume { - block_size: db_block_size.to_bytes().into(), - sub_volumes: vec![ - sled_agent_client::types::VolumeConstructionRequest::Url { - block_size: db_block_size.to_bytes().into(), - url: url.clone(), - } - ], - read_only_parent: None, - }; + let global_image_id = Uuid::new_v4(); + + let volume_construction_request = + sled_agent_client::types::VolumeConstructionRequest::Url { + id: global_image_id, + block_size: db_block_size.to_bytes().into(), + url: url.clone(), + }; let volume_data = serde_json::to_string(&volume_construction_request)?; @@ -196,7 +194,7 @@ impl super::Nexus { db::model::GlobalImage { identity: db::model::GlobalImageIdentity::new( - Uuid::new_v4(), + global_image_id, params.identity.clone(), ), volume_id: volume.id(), @@ -223,16 +221,15 @@ impl super::Nexus { let db_block_size = db::model::BlockSize::Traditional; let block_size: u64 = db_block_size.to_bytes() as u64; - let volume_construction_request = sled_agent_client::types::VolumeConstructionRequest::Volume { - block_size, - sub_volumes: vec![ - sled_agent_client::types::VolumeConstructionRequest::File { - block_size, - path: "/opt/oxide/propolis-server/blob/alpine.iso".into(), - } - ], - read_only_parent: None, - }; + let global_image_id = Uuid::new_v4(); + + let volume_construction_request = + sled_agent_client::types::VolumeConstructionRequest::File { + id: global_image_id, + block_size, + path: "/opt/oxide/propolis-server/blob/alpine.iso" + .into(), + }; let volume_data = serde_json::to_string(&volume_construction_request)?; @@ -255,7 +252,7 @@ impl super::Nexus { db::model::GlobalImage { identity: db::model::GlobalImageIdentity::new( - Uuid::new_v4(), + global_image_id, params.identity.clone(), ), volume_id: volume.id(), diff --git a/nexus/src/app/sagas/disk_create.rs b/nexus/src/app/sagas/disk_create.rs index b04ed57e54..6805b2ba72 100644 --- a/nexus/src/app/sagas/disk_create.rs +++ b/nexus/src/app/sagas/disk_create.rs @@ -292,6 +292,8 @@ async fn sdc_regions_ensure( sagactx: ActionContext, ) -> Result { let log = sagactx.user_data().log(); + let disk_id = sagactx.lookup::("disk_id")?; + let datasets_and_regions = sagactx .lookup::>( "datasets_and_regions", @@ -417,6 +419,7 @@ async fn sdc_regions_ensure( let mut rng = StdRng::from_entropy(); let volume_construction_request = sled_agent_client::types::VolumeConstructionRequest::Volume { + id: disk_id, block_size, sub_volumes: vec![ sled_agent_client::types::VolumeConstructionRequest::Region { @@ -424,6 +427,7 @@ async fn sdc_regions_ensure( // gen of 0 is here, these regions were just allocated. gen: 0, opts: sled_agent_client::types::CrucibleOpts { + id: disk_id, target: datasets_and_regions .iter() .map(|(dataset, region)| { diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 8dc890d68b..8b1c81c429 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -287,6 +287,10 @@ "format": "uint32", "minimum": 0 }, + "id": { + "type": "string", + "format": "uuid" + }, "key": { "nullable": true, "type": "string" @@ -318,6 +322,7 @@ } }, "required": [ + "id", "lossy", "target" ] @@ -1259,6 +1264,10 @@ "format": "uint64", "minimum": 0 }, + "id": { + "type": "string", + "format": "uuid" + }, "read_only_parent": { "nullable": true, "allOf": [ @@ -1282,6 +1291,7 @@ }, "required": [ "block_size", + "id", "sub_volumes", "type" ] @@ -1294,6 +1304,10 @@ "format": "uint64", "minimum": 0 }, + "id": { + "type": "string", + "format": "uuid" + }, "type": { "type": "string", "enum": [ @@ -1306,6 +1320,7 @@ }, "required": [ "block_size", + "id", "type", "url" ] @@ -1348,6 +1363,10 @@ "format": "uint64", "minimum": 0 }, + "id": { + "type": "string", + "format": "uuid" + }, "path": { "type": "string" }, @@ -1360,6 +1379,7 @@ }, "required": [ "block_size", + "id", "path", "type" ] diff --git a/package-manifest.toml b/package-manifest.toml index d987cb0b21..d523eb2182 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -91,10 +91,10 @@ zone = true # 3. Use type = "manual" instead of the "prebuilt" type = "prebuilt" repo = "crucible" -commit = "8314eeddd228ec0d76cefa40c4a41d3e2611ac18" +commit = "fed3e8ca7762130ee146fc516a4ef6eed2b91629" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -sha256 = "73ee7befea2d7709fe3baa358abc11faf6922d315a33b26e752363c5abcff91c" +sha256 = "050c94ff6f21bfcbaa2594f0f11922027822d8617e1a465484f92745d4b0fc12" # Refer to # https://github.com/oxidecomputer/propolis/blob/master/package/README.md @@ -105,10 +105,10 @@ zone = true [external_package.propolis-server.source] type = "prebuilt" repo = "propolis" -commit = "d2b8184d11d5d5f56424fe0bee13ca03d576604e" +commit = "23a38f4a1df4afcba3905f94ebbf312d0d187c7d" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -sha256 = "b192fbaaee850e63adaf9f7fe5ff2054a9de338d70198b1d1670285057a047be" +sha256 = "9c1f23c8db27c6fe114ae05358f76d88a8be803e622ad8d09a6c654692c2ffb8" [external_package.maghemite] service_name = "mg-ddm" diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 2e061d95a4..0c8dfeea33 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -14,7 +14,7 @@ cfg-if = "1.0" chrono = { version = "0.4", features = [ "serde" ] } clap = { version = "3.2", features = ["derive"] } # Only used by the simulated sled agent. -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "8314eeddd228ec0d76cefa40c4a41d3e2611ac18" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "fed3e8ca7762130ee146fc516a4ef6eed2b91629" } ddm-admin-client = { path = "../ddm-admin-client" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } futures = "0.3.21" @@ -27,7 +27,7 @@ omicron-common = { path = "../common" } p256 = "0.9.0" percent-encoding = "2.1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "d2b8184d11d5d5f56424fe0bee13ca03d576604e" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "23a38f4a1df4afcba3905f94ebbf312d0d187c7d" } rand = { version = "0.8.5", features = ["getrandom"] } reqwest = { version = "0.11.8", default-features = false, features = ["rustls-tls", "stream"] } schemars = { version = "0.8.10", features = [ "chrono", "uuid1" ] }