Integrate SoftNPU as virtual hardware (#2089)

* Automate creation of `softnpu` zone
* Automate startup up `softnpu`
* Automate configuration of `softnpu`
* Enable communication between `switch` zone and `softnpu` zone
* Package `dendrite` with `softnpu` feature
* When `dendrite` asic type is `Softnpu`, have `dendrite` configure switch via `softnpu` socketfile
* Configure nat entries via `dendrite` upon guest VM creation
* Cleanup nat entries upon guest VM deletion
* Disable OPTE hack

Closes #1465 
Closes oxidecomputer/opte#236

---------

Co-authored-by: Ryan Goodfellow <[email protected]>

.github/buildomat/jobs/build-and-test-linux.sh

@@ -9,7 +9,6 @@
 #:	"!/var/tmp/omicron_tmp/crdb-base*",
 #:	"!/var/tmp/omicron_tmp/rustc*",
 #: ]
-#:
 
 set -o errexit
 set -o pipefail
@@ -62,6 +61,11 @@ ptime -m cargo build --locked --all-targets --verbose
 # We also don't use `--workspace` here because we're not prepared to run tests
 # from end-to-end-tests.
 #
+
+# TODO: we are bypassing calls to DPD during these tests for now. This env var
+# can be removed once we have dpd-stub added to the test setup
+export SKIP_ASIC_CONFIG=1
+
 banner test
 ptime -m cargo test --locked --verbose --no-fail-fast
 

.github/buildomat/jobs/build-and-test.sh

@@ -9,7 +9,6 @@
 #:	"!/var/tmp/omicron_tmp/crdb-base*",
 #:	"!/var/tmp/omicron_tmp/rustc*",
 #: ]
-#:
 
 set -o errexit
 set -o pipefail
@@ -62,6 +61,11 @@ ptime -m cargo build --locked --all-targets --verbose
 # We also don't use `--workspace` here because we're not prepared to run tests
 # from end-to-end-tests.
 #
+
+# TODO: we are bypassing calls to DPD during these tests for now. This env var
+# can be removed once we have dpd-stub added to the test setup
+export SKIP_ASIC_CONFIG=1
+
 banner test
 ptime -m cargo test --locked --verbose --no-fail-fast
 

.github/buildomat/jobs/build-end-to-end-tests.sh

@@ -7,7 +7,6 @@
 #: output_rules = [
 #:	"=/work/*.gz",
 #: ]
-#:
 
 set -o errexit
 set -o pipefail

.github/buildomat/jobs/clippy.sh

@@ -5,7 +5,6 @@
 #: target = "helios-latest"
 #: rust_toolchain = "1.66.1"
 #: output_rules = []
-#:
 
 # Run clippy on illumos (not just other systems) because a bunch of our code
 # (that we want to check) is conditionally-compiled on illumos only.

.github/buildomat/jobs/deploy.sh

@@ -15,7 +15,6 @@
 #:
 #: [dependencies.build-end-to-end-tests]
 #: job = "helios / build-end-to-end-tests"
-#:
 
 set -o errexit
 set -o pipefail
@@ -39,6 +38,12 @@ _exit_trap() {
 	pfexec netstat -rncva
 	pfexec netstat -anu
 	pfexec arp -an
+	pfexec ./out/softnpu/scadm \
+		--server /opt/oxide/softnpu/stuff/server \
+		--client /opt/oxide/softnpu/stuff/client \
+		standalone \
+		dump-state
+
 	pfexec zfs list
 	pfexec zpool list
 	pfexec fmdump -eVp
@@ -167,6 +172,25 @@ OMICRON_NO_UNINSTALL=1 \
     ptime -m pfexec ./target/release/omicron-package -t test install
 
 ./tests/bootstrap
+
+# NOTE: this script configures softnpu's "rack network" settings using swadm
+GATEWAY_IP=192.168.1.199 ./tools/scrimlet/softnpu-init.sh
+
+# NOTE: this command configures proxy arp for softnpu. This is needed if you want to be
+# able to reach instances from the same L2 network segment.
+# /out/softnpu/scadm standalone add-proxy-arp 192.168.1.50 192.168.1.90 a8:e1:de:01:70:1d
+pfexec ./out/softnpu/scadm \
+	--server /opt/oxide/softnpu/stuff/server \
+	--client /opt/oxide/softnpu/stuff/client \
+	standalone \
+	add-proxy-arp 192.168.1.50 192.168.1.90 a8:e1:de:01:70:1d
+
+pfexec ./out/softnpu/scadm \
+	--server /opt/oxide/softnpu/stuff/server \
+	--client /opt/oxide/softnpu/stuff/client \
+	standalone \
+	dump-state
+
 rm ./tests/bootstrap
 for test_bin in tests/*; do
 	./"$test_bin"

.github/buildomat/jobs/package.sh

@@ -29,10 +29,11 @@ cargo --version
 rustc --version
 
 ptime -m ./tools/install_builder_prerequisites.sh -yp
+ptime -m ./tools/install_softnpu_machinery.sh
 
 # Build the test target
 ptime -m cargo run --locked --release --bin omicron-package -- \
-  -t test target create -i standard -m non-gimlet -s stub
+  -t test target create -i standard -m non-gimlet -s softnpu
 ptime -m cargo run --locked --release --bin omicron-package -- \
   -t test package
 
@@ -44,10 +45,12 @@ tarball_src_dir="$(pwd)/out"
 files=(
 	out/*.tar
 	out/target/test
+	out/softnpu/*
 	package-manifest.toml
 	smf/sled-agent/non-gimlet/config.toml
 	target/release/omicron-package
 	tools/create_virtual_hardware.sh
+	tools/scrimlet/*
 )
 
 pfexec mkdir -p /work && pfexec chown $USER /work
@@ -105,7 +108,7 @@ zones=(
 	out/oximeter-collector.tar.gz
 	out/propolis-server.tar.gz
 	out/switch-asic.tar.gz
-	out/switch-stub.tar.gz
+	out/switch-softnpu.tar.gz
 )
 cp "${zones[@]}" /work/zones/
 

.gitignore

@@ -10,3 +10,5 @@ tools/cockroach*
 /cockroachdb/
 smf/nexus/root.json
 core
+*.vdev
+debug.out

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "bootstore",
     "common",
     "ddm-admin-client",
+    "dpd-client",
     "deploy",
     "dns-server",
     "dns-service-client",
@@ -56,6 +57,7 @@ members = [
 default-members = [
     "common",
     "ddm-admin-client",
+    "dpd-client",
     "deploy",
     "dns-server",
     "dns-service-client",
@@ -140,6 +142,7 @@ diesel = { version = "2.0.3" }
 diesel-dtrace = { git = "https://github.com/oxidecomputer/diesel-dtrace", rev = "309bd361d886a237fbdd5d74992bdbd783f98bff" }
 dns-server = { path = "dns-server" }
 dns-service-client = { path = "dns-service-client" }
+dpd-client = { path = "dpd-client" }
 dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] }
 expectorate = "1.0.6"
 fatfs = "0.3.6"

README.adoc

@@ -43,6 +43,18 @@ This mode of operation will be used in production.
 
 To build and run the non-simulated version of Omicron, see: xref:docs/how-to-run.adoc[].
 
+=== cargo test
+If you are running unit tests that involve ASIC configuration, such as testing
+instance creation sagas, you will need to either have a local instance of `dpd`
+running, or you will need to set `SKIP_ASIC_CONFIG=1` when running `cargo test`:
+
+----
+SKIP_ASIC_CONFIG=1 cargo test
+----
+
+This is a temporary workaround until we have a stub version of `dpd` integrated
+into the test suite.
+
 === rustfmt and clippy
 
 You can **format the code** using `cargo fmt`.  Make sure to run this before pushing changes.  The CI checks that the code is correctly formatted.

common/src/nexus_config.rs

@@ -173,6 +173,12 @@ pub struct TimeseriesDbConfig {
     pub address: Option<SocketAddr>,
 }
 
+/// Configuration for the `Dendrite` dataplane daemon.
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+pub struct DpdConfig {
+    pub address: SocketAddr,
+}
+
 // A deserializable type that does no validation on the tunable parameters.
 #[derive(Clone, Debug, Deserialize, PartialEq)]
 struct UnvalidatedTunables {
@@ -278,6 +284,8 @@ pub struct PackageConfig {
     /// Tunable configuration for testing and experimentation
     #[serde(default)]
     pub tunables: Tunables,
+    /// `Dendrite` dataplane daemon configuration
+    pub dendrite: DpdConfig,
 }
 
 #[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
@@ -350,7 +358,9 @@ mod test {
         SchemeName, TimeseriesDbConfig, UpdatesConfig,
     };
     use crate::address::{Ipv6Subnet, RACK_PREFIX};
-    use crate::nexus_config::{Database, DeploymentConfig, LoadErrorKind};
+    use crate::nexus_config::{
+        Database, DeploymentConfig, DpdConfig, LoadErrorKind,
+    };
     use dropshot::ConfigDropshot;
     use dropshot::ConfigLogging;
     use dropshot::ConfigLoggingIfExists;
@@ -360,6 +370,7 @@ mod test {
     use std::net::{Ipv6Addr, SocketAddr};
     use std::path::Path;
     use std::path::PathBuf;
+    use std::str::FromStr;
 
     /// Generates a temporary filesystem path unique for the given label.
     fn temp_path(label: &str) -> PathBuf {
@@ -479,6 +490,8 @@ mod test {
             net = "::/56"
             [deployment.database]
             type = "from_dns"
+            [dendrite]
+            address = "[::1]:12224"
             "##,
         )
         .unwrap();
@@ -528,6 +541,9 @@ mod test {
                         default_base_url: "http://example.invalid/".into(),
                     }),
                     tunables: Tunables { max_vpc_ipv4_subnet_prefix: 27 },
+                    dendrite: DpdConfig {
+                        address: SocketAddr::from_str("[::1]:12224").unwrap()
+                    },
                 },
             }
         );
@@ -562,6 +578,8 @@ mod test {
             net = "::/56"
             [deployment.database]
             type = "from_dns"
+            [dendrite]
+            address = "[::1]:12224"
             "##,
         )
         .unwrap();