merge main into branch

.github/workflows/check-opte-ver.yml

@@ -10,6 +10,8 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
       - name: Install jq
         run: sudo apt-get install -y jq
       - name: Install toml-cli

.github/workflows/check-workspace-deps.yml

@@ -11,5 +11,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}  # see omicron#4461
       - name: Check Workspace Dependencies
         run: cargo xtask check-workspace-deps

.github/workflows/hakari.yml

@@ -18,6 +18,8 @@ jobs:
       RUSTFLAGS: -D warnings
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1
         with:
           toolchain: stable

.github/workflows/rust.yml

@@ -10,6 +10,8 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      with:
+        ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
     - name: Report cargo version
       run: cargo --version
     - name: Report rustfmt version
@@ -30,6 +32,8 @@ jobs:
     - name: Disable packages.microsoft.com repo
       run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
     - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      with:
+        ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
     - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
       if: ${{ github.ref != 'refs/heads/main' }}
     - name: Report cargo version
@@ -58,6 +62,8 @@ jobs:
     - name: Disable packages.microsoft.com repo
       run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
     - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      with:
+        ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
     - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
       if: ${{ github.ref != 'refs/heads/main' }}
     - name: Report cargo version
@@ -86,6 +92,8 @@ jobs:
     - name: Disable packages.microsoft.com repo
       run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
     - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      with:
+        ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
     - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
       if: ${{ github.ref != 'refs/heads/main' }}
     - name: Report cargo version

.github/workflows/validate-openapi-spec.yml

@@ -11,6 +11,8 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461
       - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2
         with:
           node-version: '18'

Cargo.toml

@@ -169,9 +169,9 @@ cookie = "0.16"
 criterion = { version = "0.5.1", features = [ "async_tokio" ] }
 crossbeam = "0.8"
 crossterm = { version = "0.27.0", features = ["event-stream"] }
-crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" }
-crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" }
-crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" }
+crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" }
+crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" }
+crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "51a3121c8318fc7ac97d74f917ce1d37962e785f" }
 curve25519-dalek = "4"
 datatest-stable = "0.2.3"
 display-error-chain = "0.2.0"
@@ -290,9 +290,9 @@ pretty-hex = "0.3.0"
 proc-macro2 = "1.0"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
 progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" }
-bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" }
-propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" }
-propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" }
+bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" }
+propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" }
+propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "54398875a2125227d13827d4236dce943c019b1c" }
 proptest = "1.3.1"
 quote = "1.0"
 rand = "0.8.5"

bootstore/Cargo.toml

@@ -27,7 +27,7 @@ slog.workspace = true
 thiserror.workspace = true
 tokio.workspace = true
 uuid.workspace = true
-vsss-rs = { version = "3.2.0", features = ["std", "curve25519"] }
+vsss-rs = { version = "3.3.1", features = ["std", "curve25519"] }
 zeroize.workspace = true
 
 # See omicron-rpaths for more about the "pq-sys" dependency.

clients/nexus-client/Cargo.toml

@@ -10,8 +10,6 @@ futures.workspace = true
 ipnetwork.workspace = true
 omicron-common.workspace = true
 omicron-passwords.workspace = true
-sled-hardware.workspace = true
-sled-storage.workspace = true
 progenitor.workspace = true
 regress.workspace = true
 reqwest = { workspace = true, features = ["rustls-tls", "stream"] }

clients/nexus-client/src/lib.rs

@@ -388,36 +388,3 @@ impl From<omicron_common::api::internal::shared::ExternalPortDiscovery>
         }
     }
 }
-
-impl From<sled_hardware::DiskVariant> for types::PhysicalDiskKind {
-    fn from(value: sled_hardware::DiskVariant) -> Self {
-        match value {
-            sled_hardware::DiskVariant::U2 => types::PhysicalDiskKind::U2,
-            sled_hardware::DiskVariant::M2 => types::PhysicalDiskKind::M2,
-        }
-    }
-}
-
-impl From<sled_hardware::Baseboard> for types::Baseboard {
-    fn from(b: sled_hardware::Baseboard) -> types::Baseboard {
-        types::Baseboard {
-            serial_number: b.identifier().to_string(),
-            part_number: b.model().to_string(),
-            revision: b.revision(),
-        }
-    }
-}
-
-impl From<sled_storage::dataset::DatasetKind> for types::DatasetKind {
-    fn from(k: sled_storage::dataset::DatasetKind) -> Self {
-        use sled_storage::dataset::DatasetKind::*;
-        match k {
-            CockroachDb => Self::Cockroach,
-            Crucible => Self::Crucible,
-            Clickhouse => Self::Clickhouse,
-            ClickhouseKeeper => Self::ClickhouseKeeper,
-            ExternalDns => Self::ExternalDns,
-            InternalDns => Self::InternalDns,
-        }
-    }
-}

common/src/api/external/mod.rs

@@ -750,6 +750,7 @@ pub enum ResourceType {
     UserBuiltin,
     Zpool,
     Vmm,
+    Ipv4NatEntry,
 }
 
 // IDENTITY METADATA

common/src/nexus_config.rs

@@ -335,6 +335,8 @@ pub struct BackgroundTaskConfig {
     pub dns_external: DnsTasksConfig,
     /// configuration for external endpoint list watcher
     pub external_endpoints: ExternalEndpointsConfig,
+    /// configuration for nat table garbage collector
+    pub nat_cleanup: NatCleanupConfig,
     /// configuration for inventory tasks
     pub inventory: InventoryConfig,
 }
@@ -371,6 +373,14 @@ pub struct ExternalEndpointsConfig {
     // allow/disallow wildcard certs, don't serve expired certs, etc.)
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct NatCleanupConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct InventoryConfig {
@@ -498,7 +508,7 @@ mod test {
         BackgroundTaskConfig, Config, ConfigDropshotWithTls, ConsoleConfig,
         Database, DeploymentConfig, DnsTasksConfig, DpdConfig,
         ExternalEndpointsConfig, InternalDns, InventoryConfig, LoadError,
-        LoadErrorKind, MgdConfig, PackageConfig, SchemeName,
+        LoadErrorKind, MgdConfig, NatCleanupConfig, PackageConfig, SchemeName,
         TimeseriesDbConfig, Tunables, UpdatesConfig,
     };
     use crate::address::{Ipv6Subnet, RACK_PREFIX};
@@ -649,6 +659,7 @@ mod test {
             dns_external.period_secs_propagation = 7
             dns_external.max_concurrent_server_updates = 8
             external_endpoints.period_secs = 9
+            nat_cleanup.period_secs = 30
             inventory.period_secs = 10
             inventory.nkeep = 11
             inventory.disable = false
@@ -746,6 +757,9 @@ mod test {
                         external_endpoints: ExternalEndpointsConfig {
                             period_secs: Duration::from_secs(9),
                         },
+                        nat_cleanup: NatCleanupConfig {
+                            period_secs: Duration::from_secs(30),
+                        },
                         inventory: InventoryConfig {
                             period_secs: Duration::from_secs(10),
                             nkeep: 11,
@@ -804,6 +818,7 @@ mod test {
             dns_external.period_secs_propagation = 7
             dns_external.max_concurrent_server_updates = 8
             external_endpoints.period_secs = 9
+            nat_cleanup.period_secs = 30
             inventory.period_secs = 10
             inventory.nkeep = 3
             inventory.disable = false

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -159,6 +159,7 @@ async fn cmd_nexus_background_tasks_show(
         "dns_config_external",
         "dns_servers_external",
         "dns_propagation_external",
+        "nat_v4_garbage_collector",
     ] {
         if let Some(bgtask) = tasks.remove(name) {
             print_task(&bgtask);

dev-tools/omdb/tests/env.out

@@ -61,6 +61,11 @@ task: "inventory_collection"
     collects hardware and software inventory data from the whole system
 
 
+task: "nat_v4_garbage_collector"
+    prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a
+    predetermined retention policy
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -121,6 +126,11 @@ task: "inventory_collection"
     collects hardware and software inventory data from the whole system
 
 
+task: "nat_v4_garbage_collector"
+    prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a
+    predetermined retention policy
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -168,6 +178,11 @@ task: "inventory_collection"
     collects hardware and software inventory data from the whole system
 
 
+task: "nat_v4_garbage_collector"
+    prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a
+    predetermined retention policy
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.

dev-tools/omdb/tests/successes.out

@@ -255,6 +255,11 @@ task: "inventory_collection"
     collects hardware and software inventory data from the whole system
 
 
+task: "nat_v4_garbage_collector"
+    prunes soft-deleted IPV4 NAT entries from ipv4_nat_entry table based on a
+    predetermined retention policy
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -319,6 +324,13 @@ task: "dns_propagation_external"
       [::1]:REDACTED_PORT     success     
 
 
+task: "nat_v4_garbage_collector"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: iter 2, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "nat_v4_garbage_collector" (don't know how to interpret details: Null)
+
 task: "external_endpoints"
   configured period: every 1m
   currently executing: no