Add support for publishing kernel statistics to oximeter

- Add a `kstat` module in `oximeter_instruments`. This includes a trait
  for describing how to map from one or more kstats into oximeter
  targets and metrics. It can be used to convert name-value kernel
  statistics in a pretty straightforward way into oximeter samples.
- Add a `KstatSampler`, which is used to register kstat targets, and
  will periodically poll them to generate samples. It is an
  `oximeter::Producer`, so that it can be easily hooked up to produce
  data for an `oximeter` collector.
- Add targets for tracking physical, virtual, and guest datalinks.
- Add metrics for bytes in/out, packets in/out, and errors in/out for
  the above.
- Use the `KstatStampler` in a new `MetricsManager` type in the sled
  agent, and track the physical (underlay) data links on a system. Does
  not yet track any virtual or guest links. The manager can be used to
  also collect other statistics, such as HTTP request latencies similar
  to nexus, or any kstats through the sampler.

Cargo.toml

@@ -222,6 +222,7 @@ ipcc-key-value = { path = "ipcc-key-value" }
 ipnetwork = { version = "0.20", features = ["schemars"] }
 itertools = "0.11.0"
 key-manager = { path = "key-manager" }
+kstat-rs = "0.2.3"
 lazy_static = "1.4.0"
 libc = "0.2.149"
 linear-map = "1.2.0"

oximeter/instruments/Cargo.toml

@@ -8,12 +8,22 @@ license = "MPL-2.0"
 chrono.workspace = true
 dropshot.workspace = true
 futures.workspace = true
+http = { workspace = true, optional = true }
+kstat-rs = { workspace = true, optional = true }
 oximeter.workspace = true
+serde = { workspace = true, optional = true }
+slog.workspace = true
 tokio.workspace = true
-http = { workspace = true, optional = true }
+thiserror.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true
 
 [features]
-default = ["http-instruments"]
+default = ["http-instruments", "kstat"]
 http-instruments = ["http"]
+kstat = ["kstat-rs", "serde"]
+
+[dev-dependencies]
+rand.workspace = true
+slog-async.workspace = true
+slog-term.workspace = true