diff --git a/Cargo.lock b/Cargo.lock index 5814dd101a..caddb6d8ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -462,7 +462,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=ff6c4df2e816eee6e7b2b0488777d30ef35ee217#ff6c4df2e816eee6e7b2b0488777d30ef35ee217" +source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" dependencies = [ "bhyve_api_sys", "libc", @@ -472,7 +472,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=ff6c4df2e816eee6e7b2b0488777d30ef35ee217#ff6c4df2e816eee6e7b2b0488777d30ef35ee217" +source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" dependencies = [ "libc", "strum 0.25.0", @@ -1235,20 +1235,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" -[[package]] -name = "crossbeam" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" -dependencies = [ - "cfg-if", - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - [[package]] name = "crossbeam-channel" version = "0.5.8" @@ -1283,16 +1269,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -1332,7 +1308,7 @@ dependencies = [ [[package]] name = "crucible-agent-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=2d4bc11232d53f177c286383926fa5f8c1b2a938#2d4bc11232d53f177c286383926fa5f8c1b2a938" +source = "git+https://github.com/oxidecomputer/crucible?rev=796dce526dd7ed7b52a0429a486ccba4a9da1ce5#796dce526dd7ed7b52a0429a486ccba4a9da1ce5" dependencies = [ "anyhow", "chrono", @@ -1348,7 +1324,7 @@ dependencies = [ [[package]] name = "crucible-pantry-client" version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=2d4bc11232d53f177c286383926fa5f8c1b2a938#2d4bc11232d53f177c286383926fa5f8c1b2a938" +source = "git+https://github.com/oxidecomputer/crucible?rev=796dce526dd7ed7b52a0429a486ccba4a9da1ce5#796dce526dd7ed7b52a0429a486ccba4a9da1ce5" dependencies = [ "anyhow", "chrono", @@ -1365,7 +1341,7 @@ dependencies = [ [[package]] name = "crucible-smf" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=2d4bc11232d53f177c286383926fa5f8c1b2a938#2d4bc11232d53f177c286383926fa5f8c1b2a938" +source = "git+https://github.com/oxidecomputer/crucible?rev=796dce526dd7ed7b52a0429a486ccba4a9da1ce5#796dce526dd7ed7b52a0429a486ccba4a9da1ce5" dependencies = [ "crucible-workspace-hack", "libc", @@ -4915,21 +4891,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "omicron-deploy" -version = "0.1.0" -dependencies = [ - "anyhow", - "camino", - "clap 4.4.3", - "crossbeam", - "omicron-package", - "omicron-workspace-hack", - "serde", - "serde_derive", - "thiserror", -] - [[package]] name = "omicron-dev" version = "0.1.0" @@ -6667,7 +6628,7 @@ dependencies = [ [[package]] name = "propolis-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=ff6c4df2e816eee6e7b2b0488777d30ef35ee217#ff6c4df2e816eee6e7b2b0488777d30ef35ee217" +source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" dependencies = [ "async-trait", "base64", @@ -6688,7 +6649,7 @@ dependencies = [ [[package]] name = "propolis-mock-server" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=ff6c4df2e816eee6e7b2b0488777d30ef35ee217#ff6c4df2e816eee6e7b2b0488777d30ef35ee217" +source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" dependencies = [ "anyhow", "atty", @@ -6718,7 +6679,7 @@ dependencies = [ [[package]] name = "propolis_types" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=ff6c4df2e816eee6e7b2b0488777d30ef35ee217#ff6c4df2e816eee6e7b2b0488777d30ef35ee217" +source = "git+https://github.com/oxidecomputer/propolis?rev=c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d#c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" dependencies = [ "schemars", "serde", @@ -6906,9 +6867,9 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154b85ef15a5d1719bcaa193c3c81fe645cd120c156874cd660fe49fd21d1373" +checksum = "bcb12f8fbf6c62614b0d56eb352af54f6a22410c3b079eb53ee93c7b97dd31d8" dependencies = [ "bitflags 2.4.0", "cassowary", diff --git a/Cargo.toml b/Cargo.toml index 65197da650..d33758fc06 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ members = [ "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/oxlog", - "dev-tools/thing-flinger", "dev-tools/xtask", "dns-server", "end-to-end-tests", @@ -96,7 +95,6 @@ default-members = [ "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/oxlog", - "dev-tools/thing-flinger", # Do not include xtask in the list of default members, because this causes # hakari to not work as well and build times to be longer. # See omicron#4392. @@ -180,9 +178,9 @@ cookie = "0.18" criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } -crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2d4bc11232d53f177c286383926fa5f8c1b2a938" } -crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2d4bc11232d53f177c286383926fa5f8c1b2a938" } -crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "2d4bc11232d53f177c286383926fa5f8c1b2a938" } +crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" } +crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" } +crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" } csv = "1.3.0" curve25519-dalek = "4" datatest-stable = "0.2.3" @@ -311,13 +309,13 @@ prettyplease = { version = "0.2.16", features = ["verbatim"] } proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "ff6c4df2e816eee6e7b2b0488777d30ef35ee217" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "ff6c4df2e816eee6e7b2b0488777d30ef35ee217" } -propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "ff6c4df2e816eee6e7b2b0488777d30ef35ee217" } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" } +propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" } proptest = "1.4.0" quote = "1.0" rand = "0.8.5" -ratatui = "0.26.0" +ratatui = "0.26.1" rayon = "1.8" rcgen = "0.12.1" reedline = "0.28.0" diff --git a/dev-tools/thing-flinger/.gitignore b/dev-tools/thing-flinger/.gitignore deleted file mode 100644 index ea8c4bf7f3..0000000000 --- a/dev-tools/thing-flinger/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/target diff --git a/dev-tools/thing-flinger/Cargo.toml b/dev-tools/thing-flinger/Cargo.toml deleted file mode 100644 index a427685871..0000000000 --- a/dev-tools/thing-flinger/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "omicron-deploy" -description = "Tools for deploying Omicron software to target machines" -version = "0.1.0" -edition = "2021" -license = "MPL-2.0" - -[dependencies] -anyhow.workspace = true -camino.workspace = true -clap.workspace = true -crossbeam.workspace = true -omicron-package.workspace = true -serde.workspace = true -serde_derive.workspace = true -thiserror.workspace = true -omicron-workspace-hack.workspace = true - -[[bin]] -name = "thing-flinger" -doc = false diff --git a/dev-tools/thing-flinger/README.adoc b/dev-tools/thing-flinger/README.adoc deleted file mode 100644 index 9966a7b747..0000000000 --- a/dev-tools/thing-flinger/README.adoc +++ /dev/null @@ -1,222 +0,0 @@ -Omicron is a complex piece of software consisting of many build and install-time dependencies. It's -intended to run primarily on illumos based systems, and as such is built to use runtime facilities -of illumos, such as https://illumos.org/man/5/smf[SMF]. Furthermore, Omicron is fundamentally a -distributed system, with its components intended to run on multiple servers communicating over the -network. In order to secure the system, certain cryptographic primitives, such as asymmetric key -pairs and shared secrets are required. Due to the nature of these cryptographic primitives, there is -a requirement for the distribution or creation of files unique to a specific server, such that no -other server has access to those files. Examples of this are private keys, and threshold key -shares, although other non-cryptographic unique files may also become necessary over time. - -In order to satisfy the above requirements of building and deploying a complex distributed system -consisting of unique, private files, two CLI tools have been created: - - . link:src/bin/omicron-package.rs[omicron-package] - build, package, install on local machine - . link:src/bin/thing-flinger.rs[thing-flinger] - build, package, deploy to remote machines - - -If a user is working on their local illumos based machine, and only wants to run -omicron in single node mode, they should follow the install instruction in -the link:../README.adoc[Omicron README] and use `omicron-package`. If the user -wishes for a more complete workflow, where they can code on their local laptop, -use a remote build machine, and install to multiple machines for a more realistic -deployment, they should use `thing-flinger`. - -The remainder of this document will describe a typical workflow for using -thing-flinger, pointing out room for improvement. - -== Environment and Configuration - - - +------------------+ +------------------+ - | | | | - | | | | - | Client |----------------> Builder | - | | | | - | | | | - +------------------+ +------------------+ - | - | - | - | - +---------------------------+--------------------------+ - | | | - | | | - | | | - +--------v---------+ +---------v--------+ +---------v--------+ - | | | | | | - | | | | | | - | Deployed Server | | Deployed Server | | Deployed Server | - | | | | | | - | | | | | | - +------------------+ +------------------+ +------------------+ - - -`thing-flinger` defines three types of nodes: - - * Client - Where a user typically edits their code and runs thing-flinger. This can run any OS. - * Builder - A Helios box where Omicron is built and packaged - * Deployed Server - Helios machines where Omicron will be installed and run - -It's not at all necessary for these to be separate nodes. For example, a client and builder can be -the same machine, as long as it's a Helios box. Same goes for Builder and a deployment server. The -benefit of this separation though, is that it allows editing on something like a laptop, without -having to worry about setting up a development environment on an illumos based host. - -Machine topology is configured in a `TOML` file that is passed on the command line. All illumos -machines are listed under `servers`, and just the names are used for configuring a builder and -deployment servers. An link:src/bin/deployment-example.toml[example] is provided. - -Thing flinger works over SSH, and so the user must have the public key of their client configured -for their account on all servers. SSH agent forwarding is used to prevent the need for the keys of -the builder to also be on the other servers, thus minimizing needed server configuration. - -== Typical Workflow - -=== Prerequisites - -Ensure you have an account on all illumos boxes, with the client public key in -`~/.ssh/authorized_keys`. - -.The build machine must have Rust and cargo installed, as well as -all the dependencies for Omicron installed. Following the *prerequisites* in the -https://github.com/oxidecomputer/omicron/#build-and-run[Build and run] section of the main Omicron -README is probably a good idea. - -==== Update `config-rss.toml` - -Currently rack setup is driven by a configuration file that lives at -`smf/sled-agent/non-gimlet/config-rss.toml` in the root of this repository. The committed -configuration of that file contains a single `requests` entry (with many -services inside it), which means it will start services on only one sled. To -start services (e.g., nexus) on multiple sleds, add additional entries to that -configuration file before proceeding. - -=== Command Based Workflow - -==== sync -Copy your source code to the builder. - -`+cargo run --bin thing-flinger -- -c sync+` - -==== Install Prerequisites -Install necessary build and runtime dependencies (including downloading prebuilt -binaries like Clickhouse and CockroachDB) on the builder and all deployment -targets. This step only needs to be performed once, absent any changes to the -dependencies, but is idempotent so may be run multiple times. - -`+cargo run --bin thing-flinger -- -c install-prereqs+` - -==== check (optional) -Run `cargo check` on the builder against the copy of `omicron` that was sync'd -to it in the previous step. - -`+cargo run --bin thing-flinger -- -c build check+` - -==== package -Build and package omicron using `omicron-package` on the builder. - -`+cargo run --bin thing-flinger -- -c build package+` - -==== overlay -Create files that are unique to each deployment server. - -`+cargo run --bin thing-flinger -- -c overlay+` - -==== install -Install omicron to all machines, in parallel. This consists of copying the packaged omicron tarballs -along with overlay files, and omicron-package and its manifest to a `staging` directory on each -deployment server, and then running omicron-package, installing overlay files, and restarting -services. - -`+cargo run --bin thing-flinger -- -c deploy install+` - -==== uninstall -Uninstall omicron from all machines. - -`+cargo run --bin thing-flinger -- -c deploy uninstall+` - -=== Current Limitations - -`thing-flinger` is an early prototype. It has served so far to demonstrate that unique files, -specifically secret shares, can be created and distributed over ssh, and that omicron can be -installed remotely using `omicron-package`. It is not currently complete enough to fully test a -distributed omicron setup, as the underlying dependencies are not configured yet. Specifically, -`CockroachDB` and perhaps `Clickhouse`, need to be configured to run in multiple server mode. It's -anticipated that the `overlay` feature of `thing-flinger` can be used to generate and distribute -configs for this. - -=== Design rationale - -`thing-flinger` is a command line program written in rust. It was written this way to build upon -`omicron-package`, which is also in rust, as that is our default language of choice at Oxide. -`thing-flinger` is based around SSH, as that is the minimal viable requirement for a test tool such -as this. Additionally, it provides for the most straightforward implementation, and takes the least -effort to use securely. This particular implementation wraps the openssh ssh client via -`std::process::Command`, rather than using the `ssh2` crate, because ssh2, as a wrapper around -`libssh`, does not support agent-forwarding. - -== Notes on Using VMs as Deployed Servers on a Linux Host - -TODO: This section should be fleshed out more and potentially lifted to its own -document; for now this is a collection of rough notes. - ---- - -It's possible to use a Linux libvirt host running multiple helios VMs as the -builder/deployment server targets, but it requires some additional setup beyond -`https://github.com/oxidecomputer/helios-engvm[helios-engvm]`. - -`thing-flinger` does not have any support for running the -`tools/create_virtual_hardware.sh` script; this will need to be done by hand on -each VM. - ---- - -To enable communication between the VMs over their IPv6 bootstrap networks: - -1. Enable IPv6 and DHCP on the virtual network libvirt uses for the VMs; e.g., - -```xml - - - - - -``` - -After booting the VMs with this enabled, they should be able to ping each other -over their acquired IPv6 addresses, but connecting to each other over the -`bootstrap6` interface that sled-agent creates will fail. - -2. Explicitly add routes in the Linux host for the `bootstrap6` addresses, -specifying the virtual interface libvirt created that is used by the VMs. - -``` -bash% sudo ip -6 route add fdb0:5254:13:7331::1/64 dev virbr1 -bash% sudo ip -6 route add fdb0:5254:f0:acfd::1/64 dev virbr1 -``` - -3. Once the sled-agents advance sufficiently to set up `sled6` interfaces, -routes need to be added for them both in the Linux host and in the Helios VMs. -Assuming two sleds with these interfaces: - -``` -# VM 1 -vioif0/sled6 static ok fd00:1122:3344:1::1/64 -# VM 2 -vioif0/sled6 static ok fd00:1122:3344:2::1/64 -``` - -The Linux host needs to be told to route that subnet to the appropriate virtual -interface: - -``` -bash% ip -6 route add fd00:1122:3344::1/48 dev virbr1 -``` - -and each Helios VM needs to be told to route that subnet to the host gateway: - -``` -vm% pfexec route add -inet6 fd00:1122:3344::/48 $IPV6_HOST_GATEWAY_ADDR -``` diff --git a/dev-tools/thing-flinger/src/bin/deployment-example.toml b/dev-tools/thing-flinger/src/bin/deployment-example.toml deleted file mode 100644 index 6d85de2ba6..0000000000 --- a/dev-tools/thing-flinger/src/bin/deployment-example.toml +++ /dev/null @@ -1,36 +0,0 @@ -# This manifest describes the servers that omicron will be installed to, along -# with any ancillary information specific to a given server. -# -# It is ingested by the `thing-flinger` tool. - -# This must be an absolute path. It refers to the path to Omicron on the -# machine where thing-flinger is being executed. -omicron_path = "/local/path/to/omicron" - -[builder] -# `server` must refer to one of the `servers` in the servers table -server = "foo" -# This must be an absolute path. It refers to the path to Omicron on the -# builder server. -omicron_path = "/remote/path/to/omicron" - -[deployment] -# which server is responsible for running the rack setup service; must -# refer to one of the `servers` in the servers table -rss_server = "foo" -# Location where files to install will be placed before running -# `omicron-package install` -# -# This must be an absolute path -# We specifically allow for $HOME in validating the absolute path -staging_dir = "$HOME/omicron_staging" -# which servers to deploy -servers = ["foo", "bar"] - -[servers.foo] -username = "me" -addr = "foo" - -[servers.bar] -username = "me" -addr = "bar" diff --git a/dev-tools/thing-flinger/src/bin/thing-flinger.rs b/dev-tools/thing-flinger/src/bin/thing-flinger.rs deleted file mode 100644 index 43b137790d..0000000000 --- a/dev-tools/thing-flinger/src/bin/thing-flinger.rs +++ /dev/null @@ -1,968 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Utility for deploying Omicron to remote machines - -use omicron_package::{parse, BuildCommand, DeployCommand}; - -use camino::{Utf8Path, Utf8PathBuf}; -use std::collections::{BTreeMap, BTreeSet}; -use std::process::Command; - -use anyhow::{Context, Result}; -use clap::{Parser, Subcommand}; -use crossbeam::thread::{self, ScopedJoinHandle}; -use serde_derive::Deserialize; -use thiserror::Error; - -// A server on which omicron source should be compiled into packages. -#[derive(Deserialize, Debug)] -struct Builder { - server: String, - omicron_path: Utf8PathBuf, -} - -// A server on which an omicron package is deployed. -#[derive(Deserialize, Debug, Eq, PartialEq)] -struct Server { - username: String, - addr: String, -} - -#[derive(Deserialize, Debug)] -struct Deployment { - rss_server: String, - staging_dir: Utf8PathBuf, - servers: BTreeSet, -} - -#[derive(Debug, Deserialize)] -struct Config { - omicron_path: Utf8PathBuf, - builder: Builder, - servers: BTreeMap, - deployment: Deployment, - - #[serde(default)] - rss_config_path: Option, - - #[serde(default)] - debug: bool, -} - -impl Config { - fn release_arg(&self) -> &str { - if self.debug { - "" - } else { - "--release" - } - } - - fn deployment_servers(&self) -> impl Iterator { - self.servers.iter().filter_map(|(name, s)| { - if self.deployment.servers.contains(name) { - Some(s) - } else { - None - } - }) - } -} - -fn parse_into_set(src: &str) -> Result, &'static str> { - Ok(src.split_whitespace().map(|s| s.to_owned()).collect()) -} - -#[derive(Debug, Subcommand)] -enum SubCommand { - /// Run the given command on the given servers, or all servers if none are - /// specified. - /// - /// Be careful! - Exec { - /// The command to run - #[clap(short, long, action)] - cmd: String, - - /// The servers to run the command on - #[clap(short, long, value_parser = parse_into_set)] - servers: Option>, - }, - - /// Install necessary prerequisites on the "builder" server and all "deploy" - /// servers. - InstallPrereqs, - - /// Sync our local source to the build host - Sync, - - /// Runs a command on the "builder" server. - #[clap(name = "build", subcommand)] - Builder(BuildCommand), - - /// Runs a command on all the "deploy" servers. - #[clap(subcommand)] - Deploy(DeployCommand), - - /// Create an overlay directory tree for each deployment server - /// - /// Each directory tree contains unique files for the given server that will - /// be populated in the svc/pkg dir. - /// - /// This is a separate subcommand so that we can reconstruct overlays - /// without rebuilding or repackaging. - Overlay, -} - -#[derive(Debug, Parser)] -#[clap( - name = "thing-flinger", - about = "A tool for synchronizing packages and configs between machines" -)] -struct Args { - /// The path to the deployment manifest TOML file - #[clap( - short, - long, - help = "Path to deployment manifest toml file", - action - )] - config: Utf8PathBuf, - - #[clap( - short, - long, - help = "The name of the build target to use for this command" - )] - target: String, - - /// The output directory, where artifacts should be built and staged - #[clap(long = "artifacts", default_value = "out/")] - artifact_dir: Utf8PathBuf, - - #[clap(subcommand)] - subcommand: SubCommand, -} - -/// Errors which can be returned when executing subcommands -#[derive(Error, Debug)] -enum FlingError { - #[error("Servers not listed in configuration: {0:?}")] - InvalidServers(Vec), - - /// Failed to rsync omicron to build host - #[error("Failed to sync {src} with {dst}")] - FailedSync { src: String, dst: String }, - - /// The given path must be absolute - #[error("Path for {field} must be absolute")] - NotAbsolutePath { field: &'static str }, -} - -// How should `ssh_exec` be run? -enum SshStrategy { - // Forward agent and source .profile - Forward, - - // Don't forward agent, but source .profile - NoForward, - - // Don't forward agent and don't source .profile - NoForwardNoProfile, -} - -impl SshStrategy { - fn forward_agent(&self) -> bool { - match self { - SshStrategy::Forward => true, - _ => false, - } - } - - fn source_profile(&self) -> bool { - match self { - SshStrategy::Forward | &SshStrategy::NoForward => true, - _ => false, - } - } -} - -// TODO: run in parallel when that option is given -fn do_exec( - config: &Config, - cmd: String, - servers: Option>, -) -> Result<()> { - if let Some(ref servers) = servers { - validate_servers(servers, &config.servers)?; - - for name in servers { - let server = &config.servers[name]; - ssh_exec(&server, &cmd, SshStrategy::NoForward)?; - } - } else { - for (_, server) in config.servers.iter() { - ssh_exec(&server, &cmd, SshStrategy::NoForward)?; - } - } - Ok(()) -} - -// start an `rsync` command with args common to all our uses -fn rsync_common() -> Command { - let mut cmd = Command::new("rsync"); - cmd.arg("-az") - .arg("-e") - .arg("ssh -o StrictHostKeyChecking=no") - .arg("--delete") - .arg("--progress") - .arg("--out-format") - .arg("File changed: %o %t %f"); - cmd -} - -fn do_sync(config: &Config) -> Result<()> { - let builder = - config.servers.get(&config.builder.server).ok_or_else(|| { - FlingError::InvalidServers(vec![config.builder.server.clone()]) - })?; - - // For rsync to copy from the source appropriately we must guarantee a - // trailing slash. - let src = format!( - "{}/", - config.omicron_path.canonicalize_utf8().with_context(|| format!( - "could not canonicalize {}", - config.omicron_path - ))? - ); - let dst = format!( - "{}@{}:{}", - builder.username, builder.addr, config.builder.omicron_path - ); - - println!("Synchronizing source files to: {}", dst); - let mut cmd = rsync_common(); - - // exclude build and development environment artifacts - cmd.arg("--exclude") - .arg("target/") - .arg("--exclude") - .arg("*.vdev") - .arg("--exclude") - .arg("*.swp") - .arg("--exclude") - .arg(".git/") - .arg("--exclude") - .arg("out/"); - - // exclude `config-rss.toml`, which needs to be sent to only one target - // system. we handle this in `do_overlay` below. - cmd.arg("--exclude").arg("**/config-rss.toml"); - - // finish with src/dst - cmd.arg(&src).arg(&dst); - let status = - cmd.status().context(format!("Failed to run command: ({:?})", cmd))?; - if !status.success() { - return Err(FlingError::FailedSync { src, dst }.into()); - } - - Ok(()) -} - -fn copy_to_deployment_staging_dir( - config: &Config, - src: String, - description: &str, -) -> Result<()> { - let partial_cmd = || { - let mut cmd = rsync_common(); - cmd.arg("--relative"); - cmd.arg(&src); - cmd - }; - - // A function for each deployment server to run in parallel - let fns = config.deployment_servers().map(|server| { - || { - let dst = format!( - "{}@{}:{}", - server.username, server.addr, config.deployment.staging_dir - ); - let mut cmd = partial_cmd(); - cmd.arg(&dst); - let status = cmd - .status() - .context(format!("Failed to run command: ({:?})", cmd))?; - if !status.success() { - return Err( - FlingError::FailedSync { src: src.clone(), dst }.into() - ); - } - Ok(()) - } - }); - - let named_fns = config.deployment.servers.iter().zip(fns); - run_in_parallel(description, named_fns); - - Ok(()) -} - -fn rsync_config_needed_for_tools(config: &Config) -> Result<()> { - let src = format!( - // the `./` here is load-bearing; it interacts with `--relative` to tell - // rsync to create `smf/sled-agent` but none of its parents - "{}/./smf/sled-agent/", - config.omicron_path.canonicalize_utf8().with_context(|| format!( - "could not canonicalize {}", - config.omicron_path - ))? - ); - - copy_to_deployment_staging_dir(config, src, "Copy smf/sled-agent dir") -} - -fn rsync_tools_dir_to_deployment_servers(config: &Config) -> Result<()> { - // we need to rsync `./tools/*` to each of the deployment targets (the - // "builder" already has it via `do_sync()`), and then run `pfexec - // tools/install_prerequisites.sh` on each system. - let src = format!( - // the `./` here is load-bearing; it interacts with `--relative` to tell - // rsync to create `tools` but none of its parents - "{}/./tools/", - config.omicron_path.canonicalize_utf8().with_context(|| format!( - "could not canonicalize {}", - config.omicron_path - ))? - ); - copy_to_deployment_staging_dir(config, src, "Copy tools dir") -} - -fn do_install_prereqs(config: &Config) -> Result<()> { - rsync_config_needed_for_tools(config)?; - rsync_tools_dir_to_deployment_servers(config)?; - install_rustup_on_deployment_servers(config); - create_virtual_hardware_on_deployment_servers(config); - create_external_tls_cert_on_builder(config)?; - - // Create a set of servers to install prereqs to - let builder = &config.servers[&config.builder.server]; - let build_server = (builder, &config.builder.omicron_path); - let all_servers = std::iter::once(build_server).chain( - config.deployment_servers().filter_map(|server| { - // Don't duplicate the builder - if server.addr != builder.addr { - Some((server, &config.deployment.staging_dir)) - } else { - None - } - }), - ); - - let server_names = std::iter::once(&config.builder.server).chain( - config - .deployment - .servers - .iter() - .filter(|s| **s != config.builder.server), - ); - - // Install functions to run in parallel on each server - let fns = all_servers.map(|(server, root_path)| { - || { - // -y: assume yes instead of prompting - // -p: skip check that deps end up in $PATH - let (script, script_type) = if *server == *builder { - ("install_builder_prerequisites.sh -y -p", "builder") - } else { - ("install_runner_prerequisites.sh -y", "runner") - }; - - let cmd = format!( - "cd {} && mkdir -p out && pfexec ./tools/{}", - root_path.clone(), - script - ); - println!( - "Install {} prerequisites on {}", - script_type, server.addr - ); - ssh_exec(server, &cmd, SshStrategy::NoForward) - } - }); - - let named_fns = server_names.zip(fns); - run_in_parallel("Install prerequisites", named_fns); - - Ok(()) -} - -fn create_external_tls_cert_on_builder(config: &Config) -> Result<()> { - let builder = &config.servers[&config.builder.server]; - let cmd = format!( - "cd {} && ./tools/create_self_signed_cert.sh", - config.builder.omicron_path, - ); - ssh_exec(&builder, &cmd, SshStrategy::NoForward) -} - -fn create_virtual_hardware_on_deployment_servers(config: &Config) { - let cmd = format!( - "cd {} && pfexec ./tools/create_virtual_hardware.sh", - config.deployment.staging_dir - ); - let fns = config.deployment_servers().map(|server| { - || { - println!("Create virtual hardware on {}", server.addr); - ssh_exec(server, &cmd, SshStrategy::NoForward) - } - }); - - let named_fns = config.deployment.servers.iter().zip(fns); - run_in_parallel("Create virtual hardware", named_fns); -} - -fn install_rustup_on_deployment_servers(config: &Config) { - let cmd = "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y"; - let fns = config.deployment_servers().map(|server| { - || ssh_exec(server, cmd, SshStrategy::NoForwardNoProfile) - }); - - let named_fns = config.deployment.servers.iter().zip(fns); - run_in_parallel("Install rustup", named_fns); -} - -// Build omicron-package and omicron-deploy on the builder -// -// We need to build omicron-deploy for overlay file generation -fn do_build_minimal(config: &Config) -> Result<()> { - let server = &config.servers[&config.builder.server]; - let cmd = format!( - "cd {} && cargo build {} -p {} -p {}", - config.builder.omicron_path, - config.release_arg(), - "omicron-package", - "omicron-deploy" - ); - ssh_exec(&server, &cmd, SshStrategy::NoForward) -} - -fn do_package(config: &Config, artifact_dir: Utf8PathBuf) -> Result<()> { - let builder = &config.servers[&config.builder.server]; - - // We use a bash login shell to get a proper environment, so we have a path to - // postgres, and $DEP_PQ_LIBDIRS is filled in. This is required for building - // nexus. - // - // See https://github.com/oxidecomputer/omicron/blob/8757ec542ea4ffbadd6f26094ed4ba357715d70d/rpaths/src/lib.rs - let cmd = format!( - "bash -lc \ - 'cd {} && \ - cargo run {} --bin omicron-package -- package --out {}'", - config.builder.omicron_path, - config.release_arg(), - artifact_dir, - ); - - ssh_exec(&builder, &cmd, SshStrategy::NoForward) -} - -fn do_dot(_config: &Config) -> Result<()> { - anyhow::bail!("\"dot\" command is not supported for thing-flinger"); -} - -fn do_check(config: &Config) -> Result<()> { - let builder = &config.servers[&config.builder.server]; - - let cmd = format!( - "bash -lc \ - 'cd {} && \ - cargo run {} --bin omicron-package -- check'", - config.builder.omicron_path, - config.release_arg(), - ); - - ssh_exec(&builder, &cmd, SshStrategy::NoForward) -} - -fn do_uninstall(config: &Config) -> Result<()> { - let builder = &config.servers[&config.builder.server]; - for server in config.deployment_servers() { - copy_omicron_package_binary_to_staging(config, builder, server)?; - - // Run `omicron-package uninstall` on the deployment server - let cmd = format!( - "cd {} && pfexec ./omicron-package uninstall", - config.deployment.staging_dir, - ); - println!("$ {}", cmd); - ssh_exec(&server, &cmd, SshStrategy::Forward)?; - } - Ok(()) -} - -fn do_clean( - config: &Config, - artifact_dir: Utf8PathBuf, - install_dir: Utf8PathBuf, -) -> Result<()> { - let mut deployment_src = Utf8PathBuf::from(&config.deployment.staging_dir); - deployment_src.push(&artifact_dir); - let builder = &config.servers[&config.builder.server]; - for server in config.deployment_servers() { - copy_omicron_package_binary_to_staging(config, builder, server)?; - - // Run `omicron-package uninstall` on the deployment server - let cmd = format!( - "cd {} && pfexec ./omicron-package clean --in {} --out {}", - config.deployment.staging_dir, deployment_src, install_dir, - ); - println!("$ {}", cmd); - ssh_exec(&server, &cmd, SshStrategy::Forward)?; - } - Ok(()) -} - -fn run_in_parallel<'a, F>(op: &str, cmds: impl Iterator) -where - F: FnOnce() -> Result<()> + Send, -{ - thread::scope(|s| { - let named_handles: Vec<(_, ScopedJoinHandle<'_, Result<()>>)> = cmds - .map(|(server_name, f)| (server_name, s.spawn(|_| f()))) - .collect(); - - // Join all the handles and print the install status - for (server_name, handle) in named_handles { - match handle.join() { - Ok(Ok(())) => { - println!("{} completed for server: {}", op, server_name) - } - Ok(Err(e)) => { - println!( - "{} failed for server: {} with error: {}", - op, server_name, e - ) - } - Err(_) => { - println!( - "{} failed for server: {}. Thread panicked.", - op, server_name - ) - } - } - } - }) - .unwrap(); -} - -fn do_install( - config: &Config, - artifact_dir: &Utf8Path, - install_dir: &Utf8Path, -) { - let builder = &config.servers[&config.builder.server]; - let mut pkg_dir = Utf8PathBuf::from(&config.builder.omicron_path); - pkg_dir.push(artifact_dir); - - let fns = config.deployment.servers.iter().map(|server_name| { - (server_name, || { - single_server_install( - config, - &artifact_dir, - &install_dir, - pkg_dir.as_str(), - builder, - server_name, - ) - }) - }); - - run_in_parallel("Install", fns); -} - -fn do_overlay(config: &Config) -> Result<()> { - let builder = &config.servers[&config.builder.server]; - let mut root_path = Utf8PathBuf::from(&config.builder.omicron_path); - // TODO: This needs to match the artifact_dir in `package` - root_path.push("out/overlay"); - - // Build a list of directories for each server to be deployed and tag which - // one is the server to run RSS; e.g., for servers ["foo", "bar", "baz"] - // with root_path "/my/path", we produce - // [ - // "/my/path/foo/sled-agent/pkg", - // "/my/path/bar/sled-agent/pkg", - // "/my/path/baz/sled-agent/pkg", - // ] - // As we're doing so, record which directory is the one for the server that - // will run RSS. - let mut rss_server_dir = None; - - for server_name in &config.deployment.servers { - let mut dir = root_path.clone(); - dir.push(server_name); - dir.push("sled-agent/pkg"); - if *server_name == config.deployment.rss_server { - rss_server_dir = Some(dir.clone()); - break; - } - } - - // we know exactly one of the servers matches `rss_server` from our config - // validation, so we can unwrap here - let rss_server_dir = rss_server_dir.unwrap(); - - overlay_rss_config(builder, config, &rss_server_dir)?; - - Ok(()) -} - -fn overlay_rss_config( - builder: &Server, - config: &Config, - rss_server_dir: &Utf8Path, -) -> Result<()> { - // Sync `config-rss.toml` to the directory for the RSS server on the - // builder. - let src = if let Some(src) = &config.rss_config_path { - src.clone() - } else { - config.omicron_path.join("smf/sled-agent/non-gimlet/config-rss.toml") - }; - let dst = format!( - "{}@{}:{}/config-rss.toml", - builder.username, builder.addr, rss_server_dir - ); - - let mut cmd = rsync_common(); - cmd.arg(&src).arg(&dst); - - let status = - cmd.status().context(format!("Failed to run command: ({:?})", cmd))?; - if !status.success() { - return Err(FlingError::FailedSync { src: src.to_string(), dst }.into()); - } - - Ok(()) -} - -fn single_server_install( - config: &Config, - artifact_dir: &Utf8Path, - install_dir: &Utf8Path, - pkg_dir: &str, - builder: &Server, - server_name: &str, -) -> Result<()> { - let server = &config.servers[server_name]; - - println!( - "COPYING packages from builder ({}) -> deploy server ({})", - builder.addr, server_name - ); - copy_package_artifacts_to_staging(config, pkg_dir, builder, server)?; - - println!( - "COPYING deploy tool from builder ({}) -> deploy server ({})", - builder.addr, server_name - ); - copy_omicron_package_binary_to_staging(config, builder, server)?; - - println!( - "COPYING manifest from builder ({}) -> deploy server ({})", - builder.addr, server_name - ); - copy_package_manifest_to_staging(config, builder, server)?; - - println!("UNPACKING packages on deploy server ({})", server_name); - run_omicron_package_unpack_from_staging( - config, - server, - &artifact_dir, - &install_dir, - )?; - - println!( - "COPYING overlay files from builder ({}) -> deploy server ({})", - builder.addr, server_name - ); - copy_overlay_files_to_staging( - config, - pkg_dir, - builder, - server, - server_name, - )?; - - println!("INSTALLING overlay files into the install directory of the deploy server ({})", server_name); - install_overlay_files_from_staging(config, server, &install_dir)?; - - println!("STARTING services on the deploy server ({})", server_name); - run_omicron_package_activate_from_staging(config, server, &install_dir) -} - -// Copy package artifacts as a result of `omicron-package package` from the -// builder to the deployment server staging directory. -// -// This staging directory acts as an intermediate location where -// packages may reside prior to being installed. -fn copy_package_artifacts_to_staging( - config: &Config, - pkg_dir: &str, - builder: &Server, - destination: &Server, -) -> Result<()> { - let cmd = format!( - "rsync -avz -e 'ssh -o StrictHostKeyChecking=no' \ - --include 'out/' \ - --include 'out/*.tar' \ - --include 'out/*.tar.gz' \ - --exclude '*' \ - {} {}@{}:{}", - pkg_dir, - destination.username, - destination.addr, - config.deployment.staging_dir - ); - println!("$ {}", cmd); - ssh_exec(builder, &cmd, SshStrategy::Forward) -} - -fn copy_omicron_package_binary_to_staging( - config: &Config, - builder: &Server, - destination: &Server, -) -> Result<()> { - let mut bin_path = Utf8PathBuf::from(&config.builder.omicron_path); - bin_path.push(format!( - "target/{}/omicron-package", - if config.debug { "debug" } else { "release" } - )); - let cmd = format!( - "rsync -avz {} {}@{}:{}", - bin_path, - destination.username, - destination.addr, - config.deployment.staging_dir - ); - println!("$ {}", cmd); - ssh_exec(builder, &cmd, SshStrategy::Forward) -} - -fn copy_package_manifest_to_staging( - config: &Config, - builder: &Server, - destination: &Server, -) -> Result<()> { - let mut path = Utf8PathBuf::from(&config.builder.omicron_path); - path.push("package-manifest.toml"); - let cmd = format!( - "rsync {} {}@{}:{}", - path, - destination.username, - destination.addr, - config.deployment.staging_dir - ); - println!("$ {}", cmd); - ssh_exec(builder, &cmd, SshStrategy::Forward) -} - -fn run_omicron_package_activate_from_staging( - config: &Config, - destination: &Server, - install_dir: &Utf8Path, -) -> Result<()> { - // Run `omicron-package activate` on the deployment server - let cmd = format!( - "cd {} && pfexec ./omicron-package activate --out {}", - config.deployment.staging_dir, install_dir, - ); - - println!("$ {}", cmd); - ssh_exec(destination, &cmd, SshStrategy::Forward) -} - -fn run_omicron_package_unpack_from_staging( - config: &Config, - destination: &Server, - artifact_dir: &Utf8Path, - install_dir: &Utf8Path, -) -> Result<()> { - let mut deployment_src = Utf8PathBuf::from(&config.deployment.staging_dir); - deployment_src.push(&artifact_dir); - - // Run `omicron-package unpack` on the deployment server - let cmd = format!( - "cd {} && pfexec ./omicron-package unpack --in {} --out {}", - config.deployment.staging_dir, deployment_src, install_dir, - ); - - println!("$ {}", cmd); - ssh_exec(destination, &cmd, SshStrategy::Forward) -} - -fn copy_overlay_files_to_staging( - config: &Config, - pkg_dir: &str, - builder: &Server, - destination: &Server, - destination_name: &str, -) -> Result<()> { - let cmd = format!( - "rsync -avz {}/overlay/{}/ {}@{}:{}/overlay/", - pkg_dir, - destination_name, - destination.username, - destination.addr, - config.deployment.staging_dir - ); - println!("$ {}", cmd); - ssh_exec(builder, &cmd, SshStrategy::Forward) -} - -fn install_overlay_files_from_staging( - config: &Config, - destination: &Server, - install_dir: &Utf8Path, -) -> Result<()> { - let cmd = format!( - "pfexec cp -r {}/overlay/* {}", - config.deployment.staging_dir, install_dir - ); - println!("$ {}", cmd); - ssh_exec(&destination, &cmd, SshStrategy::NoForward) -} - -fn ssh_exec( - server: &Server, - remote_cmd: &str, - strategy: SshStrategy, -) -> Result<()> { - let remote_cmd = if strategy.source_profile() { - // Source .profile, so we have access to cargo. Rustup installs knowledge - // about the cargo path here. - String::from(". $HOME/.profile && ") + remote_cmd - } else { - remote_cmd.into() - }; - - let mut cmd = Command::new("ssh"); - if strategy.forward_agent() { - cmd.arg("-A"); - } - cmd.arg("-o") - .arg("StrictHostKeyChecking=no") - .arg("-l") - .arg(&server.username) - .arg(&server.addr) - .arg(&remote_cmd); - - // If the builder is the same as the client, this will likely not be set, - // as the keys will reside on the builder. - if let Some(auth_sock) = std::env::var_os("SSH_AUTH_SOCK") { - cmd.env("SSH_AUTH_SOCK", auth_sock); - } - let exit_status = cmd - .status() - .context(format!("Failed to run {} on {}", remote_cmd, server.addr))?; - if !exit_status.success() { - anyhow::bail!("Command failed: {}", exit_status); - } - - Ok(()) -} - -fn validate_servers( - chosen: &BTreeSet, - all: &BTreeMap, -) -> Result<(), FlingError> { - let all = all.keys().cloned().collect(); - let diff: Vec = chosen.difference(&all).cloned().collect(); - if !diff.is_empty() { - Err(FlingError::InvalidServers(diff)) - } else { - Ok(()) - } -} - -fn validate_absolute_path( - path: &Utf8Path, - field: &'static str, -) -> Result<(), FlingError> { - if path.is_absolute() || path.starts_with("$HOME") { - Ok(()) - } else { - Err(FlingError::NotAbsolutePath { field }) - } -} - -fn validate(config: &Config) -> Result<(), FlingError> { - validate_absolute_path(&config.omicron_path, "omicron_path")?; - validate_absolute_path( - &config.builder.omicron_path, - "builder.omicron_path", - )?; - validate_absolute_path( - &config.deployment.staging_dir, - "deployment.staging_dir", - )?; - - validate_servers( - &BTreeSet::from([ - config.builder.server.clone(), - config.deployment.rss_server.clone(), - ]), - &config.servers, - ) -} - -fn main() -> Result<()> { - let args = Args::try_parse()?; - let config = parse::<_, Config>(args.config)?; - - validate(&config)?; - - match args.subcommand { - SubCommand::Exec { cmd, servers } => { - do_exec(&config, cmd, servers)?; - } - SubCommand::Sync => do_sync(&config)?, - SubCommand::InstallPrereqs => do_install_prereqs(&config)?, - SubCommand::Builder(BuildCommand::Target { .. }) => { - todo!("Setting target not supported through thing-flinger") - } - SubCommand::Builder(BuildCommand::Package { .. }) => { - do_package(&config, args.artifact_dir)?; - } - SubCommand::Builder(BuildCommand::Stamp { .. }) => { - anyhow::bail!("Distributed package stamping not supported") - } - SubCommand::Builder(BuildCommand::Check) => do_check(&config)?, - SubCommand::Builder(BuildCommand::Dot) => { - do_dot(&config)?; - } - SubCommand::Deploy(DeployCommand::Install { install_dir }) => { - do_build_minimal(&config)?; - do_install(&config, &args.artifact_dir, &install_dir); - } - SubCommand::Deploy(DeployCommand::Uninstall) => { - do_build_minimal(&config)?; - do_uninstall(&config)?; - } - SubCommand::Deploy(DeployCommand::Clean { install_dir }) => { - do_build_minimal(&config)?; - do_clean(&config, args.artifact_dir, install_dir)?; - } - // TODO: It doesn't really make sense to allow the user direct access - // to these low level operations in thing-flinger. Should we not use - // the DeployCommand from omicron-package directly? - SubCommand::Deploy(_) => anyhow::bail!("Unsupported action"), - SubCommand::Overlay => do_overlay(&config)?, - } - Ok(()) -} diff --git a/flake.nix b/flake.nix index 408dff5706..8897d9428d 100644 --- a/flake.nix +++ b/flake.nix @@ -17,7 +17,11 @@ system = "x86_64-linux"; }; # use the Rust toolchain defined in the `rust-toolchain.toml` file. - rustToolchain = pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml; + rustToolchain = (pkgs.pkgsBuildHost.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml).override { + extensions = [ + "rust-src" # for rust-analyzer + ]; + }; buildInputs = with pkgs; [ # libs diff --git a/nexus/db-model/src/ipv4_nat_entry.rs b/nexus/db-model/src/ipv4_nat_entry.rs index 6a74444411..c3763346c6 100644 --- a/nexus/db-model/src/ipv4_nat_entry.rs +++ b/nexus/db-model/src/ipv4_nat_entry.rs @@ -1,7 +1,10 @@ use std::net::{Ipv4Addr, Ipv6Addr}; use super::MacAddr; -use crate::{schema::ipv4_nat_entry, Ipv4Net, Ipv6Net, SqlU16, Vni}; +use crate::{ + schema::ipv4_nat_changes, schema::ipv4_nat_entry, Ipv4Net, Ipv6Net, SqlU16, + Vni, +}; use chrono::{DateTime, Utc}; use omicron_common::api::external; use schemars::JsonSchema; @@ -48,6 +51,20 @@ impl Ipv4NatEntry { } } +/// Summary of changes to ipv4 nat entries. +#[derive(Queryable, Debug, Clone, Selectable, Serialize, Deserialize)] +#[diesel(table_name = ipv4_nat_changes)] +pub struct Ipv4NatChange { + pub external_address: Ipv4Net, + pub first_port: SqlU16, + pub last_port: SqlU16, + pub sled_address: Ipv6Net, + pub vni: Vni, + pub mac: MacAddr, + pub version: i64, + pub deleted: bool, +} + /// NAT Record #[derive(Clone, Debug, Serialize, JsonSchema)] pub struct Ipv4NatEntryView { @@ -61,22 +78,17 @@ pub struct Ipv4NatEntryView { pub deleted: bool, } -impl From for Ipv4NatEntryView { - fn from(value: Ipv4NatEntry) -> Self { - let (gen, deleted) = match value.version_removed { - Some(gen) => (gen, true), - None => (value.version_added, false), - }; - +impl From for Ipv4NatEntryView { + fn from(value: Ipv4NatChange) -> Self { Self { external_address: value.external_address.ip(), - first_port: value.first_port(), - last_port: value.last_port(), + first_port: value.first_port.into(), + last_port: value.last_port.into(), sled_address: value.sled_address.ip(), vni: value.vni.0, mac: *value.mac, - gen, - deleted, + gen: value.version, + deleted: value.deleted, } } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 736442282c..7fc4f9ae45 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion; /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(32, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(33, 0, 1); table! { disk (id) { @@ -546,6 +546,20 @@ table! { } } +// View used for summarizing changes to ipv4_nat_entry +table! { + ipv4_nat_changes (version) { + external_address -> Inet, + first_port -> Int4, + last_port -> Int4, + sled_address -> Inet, + vni -> Int4, + mac -> Int8, + version -> Int8, + deleted -> Bool, + } +} + // This is the sequence used for the version number // in ipv4_nat_entry. table! { diff --git a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs index 81229162d0..27a6bad32f 100644 --- a/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs +++ b/nexus/db-queries/src/db/datastore/ipv4_nat_entry.rs @@ -9,6 +9,7 @@ use chrono::{DateTime, Utc}; use diesel::prelude::*; use diesel::sql_types::BigInt; use nexus_db_model::ExternalIp; +use nexus_db_model::Ipv4NatChange; use nexus_db_model::Ipv4NatEntryView; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; @@ -317,10 +318,19 @@ impl DataStore { version: i64, limit: u32, ) -> ListResultVec { - let nat_entries = - self.ipv4_nat_list_since_version(opctx, version, limit).await?; + use db::schema::ipv4_nat_changes::dsl; + + let nat_changes = dsl::ipv4_nat_changes + .filter(dsl::version.gt(version)) + .limit(limit as i64) + .order_by(dsl::version) + .select(Ipv4NatChange::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + let nat_entries: Vec = - nat_entries.iter().map(|e| e.clone().into()).collect(); + nat_changes.iter().map(|e| e.clone().into()).collect(); Ok(nat_entries) } @@ -367,7 +377,7 @@ fn ipv4_nat_next_version() -> diesel::expression::SqlLiteral { #[cfg(test)] mod test { - use std::str::FromStr; + use std::{net::Ipv4Addr, str::FromStr}; use crate::db::datastore::datastore_test; use chrono::Utc; @@ -375,6 +385,7 @@ mod test { use nexus_test_utils::db::test_setup_database; use omicron_common::api::external; use omicron_test_utils::dev; + use rand::seq::IteratorRandom; // Test our ability to track additions and deletions since a given version number #[tokio::test] @@ -802,4 +813,154 @@ mod test { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + // Test our ability to return all changes interleaved in the correct order + #[tokio::test] + async fn ipv4_nat_changeset() { + let logctx = dev::test_setup_log("test_nat_version_tracking"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // We should not have any NAT entries at this moment + let initial_state = + datastore.ipv4_nat_list_since_version(&opctx, 0, 10).await.unwrap(); + + assert!(initial_state.is_empty()); + assert_eq!( + datastore.ipv4_nat_current_version(&opctx).await.unwrap(), + 0 + ); + + let addresses = (0..=255).map(|i| { + let addr = Ipv4Addr::new(10, 0, 0, i); + let net = ipnetwork::Ipv4Network::new(addr, 32).unwrap(); + external::Ipv4Net(net) + }); + + let sled_address = external::Ipv6Net( + ipnetwork::Ipv6Network::try_from("fd00:1122:3344:104::1").unwrap(), + ); + + let nat_entries = addresses.map(|external_address| { + // build a bunch of nat entries + Ipv4NatValues { + external_address: external_address.into(), + first_port: u16::MIN.into(), + last_port: u16::MAX.into(), + sled_address: sled_address.into(), + vni: Vni(external::Vni::random()), + mac: MacAddr(external::MacAddr::random_guest()), + } + }); + + let mut db_records = vec![]; + + // create the nat entries + for entry in nat_entries { + let result = datastore + .ensure_ipv4_nat_entry(&opctx, entry.clone()) + .await + .unwrap(); + + db_records.push(result); + } + + // delete a subset of the entries + for entry in + db_records.iter().choose_multiple(&mut rand::thread_rng(), 50) + { + datastore.ipv4_nat_delete(&opctx, entry).await.unwrap(); + } + + // get the new state of all nat entries + // note that this is not the method under test + let db_records = datastore + .ipv4_nat_list_since_version(&opctx, 0, 300) + .await + .unwrap(); + + // Count the actual number of changes seen. + // This check is required because we _were_ getting changes in ascending order, + // but some entries were being skipped. We want to ensure we are getting + // *all* of the changes in ascending order. + let mut total_changes = 0; + + // ensure that the changeset is ordered, displaying the correct + // version numbers, and displaying the correct `deleted` status + let mut version = 0; + let limit = 100; + let mut changes = + datastore.ipv4_nat_changeset(&opctx, version, limit).await.unwrap(); + + while !changes.is_empty() { + // check ordering + assert!(changes + .windows(2) + .all(|entries| entries[0].gen < entries[1].gen)); + + // check deleted status and version numbers + changes.iter().for_each(|change| match change.deleted { + true => { + // version should match a deleted entry + let deleted_nat = db_records + .iter() + .find(|entry| entry.version_removed == Some(change.gen)) + .expect("did not find a deleted nat entry with a matching version number"); + + assert_eq!( + deleted_nat.external_address.ip(), + change.external_address + ); + assert_eq!( + deleted_nat.first_port, + change.first_port.into() + ); + assert_eq!(deleted_nat.last_port, change.last_port.into()); + assert_eq!( + deleted_nat.sled_address.ip(), + change.sled_address + ); + assert_eq!(*deleted_nat.mac, change.mac); + assert_eq!(deleted_nat.vni.0, change.vni); + } + false => { + // version should match an active nat entry + let added_nat = db_records + .iter() + .find(|entry| entry.version_added == change.gen) + .expect("did not find an active nat entry with a matching version number"); + + assert!(added_nat.version_removed.is_none()); + + assert_eq!( + added_nat.external_address.ip(), + change.external_address + ); + assert_eq!(added_nat.first_port, change.first_port.into()); + assert_eq!(added_nat.last_port, change.last_port.into()); + assert_eq!( + added_nat.sled_address.ip(), + change.sled_address + ); + assert_eq!(*added_nat.mac, change.mac); + assert_eq!(added_nat.vni.0, change.vni); + } + }); + + // bump the count of changes seen + total_changes += changes.len(); + + version = changes.last().unwrap().gen; + changes = datastore + .ipv4_nat_changeset(&opctx, version, limit) + .await + .unwrap(); + } + + // did we see everything? + assert_eq!(total_changes, db_records.len()); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } } diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 0834bf698e..27e58a298c 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -477,6 +477,12 @@ pub mod test { SocketAddr::V6(a) => a, }; + // In order to test that DNS gets propagated to a newly-added server, we + // first need to update the source of truth about DNS (the database). + // Then we need to wait for that to get propagated (by this same + // mechanism) to the existing DNS servers. Only then would we expect + // the mechanism to see the new DNS server and then propagate + // configuration to it. let update = { use nexus_params::{DnsRecord, Srv}; diff --git a/package-manifest.toml b/package-manifest.toml index 1e88ddc760..9b72dd7d18 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -405,10 +405,10 @@ only_for_targets.image = "standard" # 3. Use source.type = "manual" instead of "prebuilt" source.type = "prebuilt" source.repo = "crucible" -source.commit = "2d4bc11232d53f177c286383926fa5f8c1b2a938" +source.commit = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible.sha256.txt -source.sha256 = "88ec93657a644e8f10a32d1d22cc027db901aea81027f49ce7bee58fc4a35755" +source.sha256 = "8b654627a4250e8d444133cf3130838d224b13e53f3e48cf0d031314d6f05ee0" output.type = "zone" [package.crucible-pantry] @@ -416,10 +416,10 @@ service_name = "crucible_pantry" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "crucible" -source.commit = "2d4bc11232d53f177c286383926fa5f8c1b2a938" +source.commit = "796dce526dd7ed7b52a0429a486ccba4a9da1ce5" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/crucible/image//crucible-pantry.sha256.txt -source.sha256 = "e2c3ed2d4cd6b5da3d38dd52df6d4a259280be7d45c30a363e9c71b174ecc6f8" +source.sha256 = "8602b2d6e7beb0731ae2be481715c94795657306d6013cc6d81fd60c4784a6ed" output.type = "zone" # Refer to @@ -430,10 +430,10 @@ service_name = "propolis-server" only_for_targets.image = "standard" source.type = "prebuilt" source.repo = "propolis" -source.commit = "ff6c4df2e816eee6e7b2b0488777d30ef35ee217" +source.commit = "c7cdaf1875d259e29ca50a14b77b0bfd9dfe443d" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/propolis/image//propolis-server.sha256.txt -source.sha256 = "aa10aa245a92e657fc074bd588ef6bbddaad2d9c946a8e1b91c02dce7e057561" +source.sha256 = "0203b7f702377c877c4132851ca102d68cd8fd2c20e4fd5b59d950cbb07fd9ff" output.type = "zone" [package.mg-ddm-gz] @@ -497,8 +497,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "fd159136c552d8b4ec4d49dd9bae7e38f6a636e6" -source.sha256 = "1e24598ba77dc00682cdf54fc370696ef5aa49ed510ab7f72fcc91d61d679e7b" +source.commit = "3618dd6017b363c5d34399273453cf50b9c9a43e" +source.sha256 = "eb98985871f321411f7875ef7751dba85ae0dd3034877b63ccb78cedcb96e6e7" output.type = "zone" output.intermediate_only = true @@ -522,8 +522,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "fd159136c552d8b4ec4d49dd9bae7e38f6a636e6" -source.sha256 = "720df8aff3aaa0f8a86ec606089ebf8b5068d7f3c243bd4c868b96ef72d13485" +source.commit = "3618dd6017b363c5d34399273453cf50b9c9a43e" +source.sha256 = "cc0429f0d9ce6df94e834cea89cabbdf4d1fbfe623369dd3eb84c5b2677414be" output.type = "zone" output.intermediate_only = true @@ -540,8 +540,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "fd159136c552d8b4ec4d49dd9bae7e38f6a636e6" -source.sha256 = "5e34a10d9dca6c94f96075140d42b755dee1f5e6a3485fc239b12e12b89a30c5" +source.commit = "3618dd6017b363c5d34399273453cf50b9c9a43e" +source.sha256 = "fa25585fb3aa1a888b76133af3060b859cbea8e53287bb1cc64e70889db37679" output.type = "zone" output.intermediate_only = true diff --git a/schema/crdb/33.0.0/up01.sql b/schema/crdb/33.0.0/up01.sql new file mode 100644 index 0000000000..624aec4ea6 --- /dev/null +++ b/schema/crdb/33.0.0/up01.sql @@ -0,0 +1,42 @@ +/** + * A view of the ipv4 nat change history + * used to summarize changes for external viewing + */ +CREATE VIEW IF NOT EXISTS omicron.public.ipv4_nat_changes +AS +WITH interleaved_versions AS ( + SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + version_added AS version, + (version_removed IS NOT NULL) as deleted + FROM ipv4_nat_entry + WHERE version_removed IS NULL + + UNION + + SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + version_added AS version, + (version_removed IS NOT NULL) as deleted + FROM ipv4_nat_entry WHERE version_removed IS NOT NULL +) +SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + version, + deleted +FROM interleaved_versions; diff --git a/schema/crdb/33.0.1/up01.sql b/schema/crdb/33.0.1/up01.sql new file mode 100644 index 0000000000..354480c0c9 --- /dev/null +++ b/schema/crdb/33.0.1/up01.sql @@ -0,0 +1 @@ +DROP VIEW IF EXISTS omicron.public.ipv4_nat_changes; diff --git a/schema/crdb/33.0.1/up02.sql b/schema/crdb/33.0.1/up02.sql new file mode 100644 index 0000000000..5a2a183f4c --- /dev/null +++ b/schema/crdb/33.0.1/up02.sql @@ -0,0 +1,60 @@ +/* + * A view of the ipv4 nat change history + * used to summarize changes for external viewing + */ +CREATE VIEW IF NOT EXISTS omicron.public.ipv4_nat_changes +AS +-- Subquery: +-- We need to be able to order partial changesets. ORDER BY on separate columns +-- will not accomplish this, so we'll do this by interleaving version_added +-- and version_removed (version_removed taking priority if NOT NULL) and then sorting +-- on the appropriate version numbers at call time. +WITH interleaved_versions AS ( + -- fetch all active NAT entries (entries that have not been soft deleted) + SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + -- rename version_added to version + version_added AS version, + -- create a new virtual column, boolean value representing whether or not + -- the record has been soft deleted + (version_removed IS NOT NULL) as deleted + FROM omicron.public.ipv4_nat_entry + WHERE version_removed IS NULL + + -- combine the datasets, unifying the version_added and version_removed + -- columns to a single `version` column so we can interleave and sort the entries + UNION + + -- fetch all inactive NAT entries (entries that have been soft deleted) + SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + -- rename version_removed to version + version_removed AS version, + -- create a new virtual column, boolean value representing whether or not + -- the record has been soft deleted + (version_removed IS NOT NULL) as deleted + FROM omicron.public.ipv4_nat_entry + WHERE version_removed IS NOT NULL +) +-- this is our new "table" +-- here we select the columns from the subquery defined above +SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + version, + deleted +FROM interleaved_versions; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 103eb2e0c7..18b1b82563 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3436,6 +3436,67 @@ STORING ( time_deleted ); +/* + * A view of the ipv4 nat change history + * used to summarize changes for external viewing + */ +CREATE VIEW IF NOT EXISTS omicron.public.ipv4_nat_changes +AS +-- Subquery: +-- We need to be able to order partial changesets. ORDER BY on separate columns +-- will not accomplish this, so we'll do this by interleaving version_added +-- and version_removed (version_removed taking priority if NOT NULL) and then sorting +-- on the appropriate version numbers at call time. +WITH interleaved_versions AS ( + -- fetch all active NAT entries (entries that have not been soft deleted) + SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + -- rename version_added to version + version_added AS version, + -- create a new virtual column, boolean value representing whether or not + -- the record has been soft deleted + (version_removed IS NOT NULL) as deleted + FROM omicron.public.ipv4_nat_entry + WHERE version_removed IS NULL + + -- combine the datasets, unifying the version_added and version_removed + -- columns to a single `version` column so we can interleave and sort the entries + UNION + + -- fetch all inactive NAT entries (entries that have been soft deleted) + SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + -- rename version_removed to version + version_removed AS version, + -- create a new virtual column, boolean value representing whether or not + -- the record has been soft deleted + (version_removed IS NOT NULL) as deleted + FROM omicron.public.ipv4_nat_entry + WHERE version_removed IS NOT NULL +) +-- this is our new "table" +-- here we select the columns from the subquery defined above +SELECT + external_address, + first_port, + last_port, + sled_address, + vni, + mac, + version, + deleted +FROM interleaved_versions; + INSERT INTO omicron.public.db_metadata ( singleton, time_created, @@ -3443,7 +3504,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '32.0.0', NULL) + ( TRUE, NOW(), NOW(), '33.0.1', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index 56bcb2d9ff..6895170e02 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="fd159136c552d8b4ec4d49dd9bae7e38f6a636e6" -SHA2="e8f73a83d5c62f7efce998f821acc80a91b7995c95bd9ec2c228372829310099" +COMMIT="3618dd6017b363c5d34399273453cf50b9c9a43e" +SHA2="aa670165e5b459fab4caba36ae4d382a09264ff5cf6a2dac0dae0a0db39a378e" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 497ce5c010..74b379f359 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="1e24598ba77dc00682cdf54fc370696ef5aa49ed510ab7f72fcc91d61d679e7b" -CIDL_SHA256_LINUX_DPD="4fc43b53a048264664ede64805d4d179ec32d50cf9ab1aaa0fa4e17190e511a2" -CIDL_SHA256_LINUX_SWADM="0ab34a2063e68568aa064f7b71825a603d47b3e399f3e7f45976edb5d5283f0f" +CIDL_SHA256_ILLUMOS="eb98985871f321411f7875ef7751dba85ae0dd3034877b63ccb78cedcb96e6e7" +CIDL_SHA256_LINUX_DPD="cb9a1978d1fe3a3f2391757f80436d8cc87c0041161652ad2234e7cf83e9ae36" +CIDL_SHA256_LINUX_SWADM="b7e737be56a8a815a95624f0b5c42ce1e339b07feeae7b3d7b9b4bc17c204245"