diff --git a/.config/nextest.toml b/.config/nextest.toml index ba07186c8a7..79774e3658e 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -18,3 +18,15 @@ fail-fast = false [script.crdb-seed] command = 'cargo run -p crdb-seed' + +# The ClickHouse cluster tests currently rely on a hard-coded set of ports for +# the nodes in the cluster. We would like to relax this in the future, at which +# point this test-group configuration can be removed or at least loosened to +# support testing in parallel. For now, enforce strict serialization for all +# tests with `replicated` in the name. +[test-groups] +clickhouse-cluster = { max-threads = 1 } + +[[profile.default.overrides]] +filter = 'package(oximeter-db) and test(replicated)' +test-group = 'clickhouse-cluster' diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index d8de288239a..6fda8bb8d76 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -51,6 +51,7 @@ export RUSTFLAGS="-D warnings" export RUSTDOCFLAGS="-D warnings" export TMPDIR=$TEST_TMPDIR export RUST_BACKTRACE=1 +export CARGO_INCREMENTAL=0 ptime -m cargo test --locked --verbose --no-run # diff --git a/.github/buildomat/jobs/ci-tools.sh b/.github/buildomat/jobs/ci-tools.sh index 702561a951c..07a63af30c3 100755 --- a/.github/buildomat/jobs/ci-tools.sh +++ b/.github/buildomat/jobs/ci-tools.sh @@ -28,6 +28,7 @@ banner end-to-end-tests # export CARGO_PROFILE_DEV_DEBUG=1 export CARGO_PROFILE_TEST_DEBUG=1 +export CARGO_INCREMENTAL=0 ptime -m cargo build --locked -p end-to-end-tests --tests --bin bootstrap \ --message-format json-render-diagnostics >/tmp/output.end-to-end.json diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh index 5fd31adb764..abbcda21507 100755 --- a/.github/buildomat/jobs/clippy.sh +++ b/.github/buildomat/jobs/clippy.sh @@ -28,5 +28,6 @@ banner prerequisites ptime -m bash ./tools/install_builder_prerequisites.sh -y banner clippy +export CARGO_INCREMENTAL=0 ptime -m cargo xtask clippy ptime -m cargo doc diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index 5cfb6fcfd79..0605ab6883f 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -37,13 +37,14 @@ rustc --version # trampoline global zone images. # COMMIT=$(git rev-parse HEAD) -VERSION="1.0.3-0.ci+git${COMMIT:0:11}" +VERSION="1.0.4-0.ci+git${COMMIT:0:11}" echo "$VERSION" >/work/version.txt ptime -m ./tools/install_builder_prerequisites.sh -yp ptime -m ./tools/ci_download_softnpu_machinery # Build the test target +export CARGO_INCREMENTAL=0 ptime -m cargo run --locked --release --bin omicron-package -- \ -t test target create -i standard -m non-gimlet -s softnpu -r single-sled ptime -m cargo run --locked --release --bin omicron-package -- \ diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index 29cf7fa85ef..5e7a2d4a911 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -278,8 +278,8 @@ EOF done } # usage: SERIES ROT_DIR ROT_VERSION BOARDS... -add_hubris_artifacts rot-staging-dev staging/dev cert-staging-dev-v1.0.2 "${ALL_BOARDS[@]}" -add_hubris_artifacts rot-prod-rel prod/rel cert-prod-rel-v1.0.2 "${ALL_BOARDS[@]}" +add_hubris_artifacts rot-staging-dev staging/dev cert-staging-dev-v1.0.4 "${ALL_BOARDS[@]}" +add_hubris_artifacts rot-prod-rel prod/rel cert-prod-rel-v1.0.4 "${ALL_BOARDS[@]}" for series in "${SERIES_LIST[@]}"; do /work/tufaceous assemble --no-generate-key /work/manifest-"$series".toml /work/repo-"$series".zip diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 9d5f7444de3..6f2dc04b913 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -22,7 +22,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@11dea51b35bc2bfa42820716c6cabb14fd4c3266 # v2 + uses: taiki-e/install-action@7c4edf14345f90e1199544e41cb94c3ef67bd237 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8cc98f192f4..f2581845d98 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -19,6 +19,8 @@ jobs: check-omicron-deployment: runs-on: ${{ matrix.os }} + env: + CARGO_INCREMENTAL: 0 strategy: fail-fast: false matrix: @@ -28,7 +30,7 @@ jobs: - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 + - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version @@ -49,12 +51,14 @@ jobs: # of our code. clippy-lint: runs-on: ubuntu-22.04 + env: + CARGO_INCREMENTAL: 0 steps: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 + - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version @@ -75,12 +79,14 @@ jobs: # the separate "rustdocs" repo. build-docs: runs-on: ubuntu-22.04 + env: + CARGO_INCREMENTAL: 0 steps: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 - - uses: Swatinem/rust-cache@6fd3edff6979b79f87531400ad694fb7f2c84b1f # v2.2.1 + - uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version diff --git a/Cargo.lock b/Cargo.lock index 2d266a07fa0..f3593b873a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,10270 +18,1682 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] -name = "aead" -version = "0.5.2" +name = "aho-corasick" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ - "crypto-common", - "generic-array", + "memchr", ] [[package]] -name = "aes" -version = "0.8.3" +name = "anstream" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", ] [[package]] -name = "aes-gcm" -version = "0.10.3" +name = "anstyle" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] -name = "aes-gcm-siv" -version = "0.11.1" +name = "anstyle-parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" +checksum = "317b9a89c1868f5ea6ff1d9539a69f45dffc21ce321ac1fd1160dfa48c8e2140" dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "polyval", - "subtle", - "zeroize", + "utf8parse", ] [[package]] -name = "ahash" -version = "0.8.3" +name = "anstyle-query" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "cfg-if", - "once_cell", - "version_check", + "windows-sys", ] [[package]] -name = "aho-corasick" -version = "1.0.4" +name = "anstyle-wincon" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ - "memchr", + "anstyle", + "windows-sys", ] [[package]] -name = "allocator-api2" -version = "0.2.16" +name = "anyhow" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] -name = "android-tzdata" -version = "0.1.1" +name = "arc-swap" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] -name = "android_system_properties" -version = "0.1.5" +name = "atomic-polyfill" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ff7eb3f316534d83a8a2c3d1674ace8a5a71198eba31e2e2b597833f699b28" +dependencies = [ + "critical-section", +] + +[[package]] +name = "atty" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ + "hermit-abi 0.1.19", "libc", + "winapi", ] [[package]] -name = "anes" -version = "0.1.6" +name = "autocfg" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "ansi_term" -version = "0.12.1" +name = "backtrace" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ - "winapi", + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", ] [[package]] -name = "anstream" -version = "0.5.0" +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "utf8parse", + "generic-array", ] [[package]] -name = "anstyle" -version = "1.0.2" +name = "byteorder" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] -name = "anstyle-parse" -version = "0.2.1" +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "0f8e7c90afad890484a21653d08b6e209ae34770fb5ee298f9c699fcc1e5c856" dependencies = [ - "utf8parse", + "libc", ] [[package]] -name = "anstyle-query" +name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" -dependencies = [ - "windows-sys 0.48.0", -] +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] -name = "anstyle-wincon" -version = "2.1.0" +name = "circular" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" -dependencies = [ - "anstyle", - "windows-sys 0.48.0", -] +checksum = "b0fc239e0f6cb375d2402d48afb92f76f5404fd1df208a41930ec81eda078bea" [[package]] -name = "anyhow" -version = "1.0.75" +name = "clap" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ - "backtrace", + "clap_builder", + "clap_derive", ] [[package]] -name = "api_identity" -version = "0.1.0" +name = "clap_builder" +version = "4.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ - "omicron-workspace-hack", - "proc-macro2", - "quote", - "syn 2.0.32", + "anstream", + "anstyle", + "clap_lex", + "strsim", ] [[package]] -name = "api_identity" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ + "heck", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] -name = "approx" -version = "0.5.1" +name = "clap_lex" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" -dependencies = [ - "num-traits", -] +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] -name = "arc-swap" -version = "1.6.0" +name = "cobs" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" [[package]] -name = "argon2" -version = "0.5.2" +name = "colorchoice" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ba4cac0a46bc1d2912652a751c47f2a9f3a7fe89bcae2275d418f5270402f9" -dependencies = [ - "base64ct", - "blake2", - "cpufeatures", - "password-hash 0.5.0", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] -name = "array-init" -version = "0.0.4" +name = "colored" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" dependencies = [ - "nodrop", + "is-terminal", + "lazy_static", + "windows-sys", ] [[package]] -name = "arrayref" -version = "0.3.7" +name = "cookie-factory" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" [[package]] -name = "arrayvec" -version = "0.7.4" +name = "cpufeatures" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +dependencies = [ + "libc", +] [[package]] -name = "ascii" -version = "1.1.0" +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" [[package]] -name = "ascii-canvas" -version = "3.0.0" +name = "crossbeam-channel" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "term", + "cfg-if", + "crossbeam-utils", ] [[package]] -name = "assert_cmd" -version = "2.0.12" +name = "crossbeam-utils" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ - "anstyle", - "bstr 1.6.0", - "doc-comment", - "predicates 3.0.4", - "predicates-core", - "predicates-tree", - "wait-timeout", + "cfg-if", ] [[package]] -name = "assert_matches" -version = "1.5.0" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] [[package]] -name = "async-bb8-diesel" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/async-bb8-diesel?rev=1446f7e0c1f05f33a0581abd51fa873c7652ab61#1446f7e0c1f05f33a0581abd51fa873c7652ab61" +name = "cstr-argument" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40" dependencies = [ - "async-trait", - "bb8", - "diesel", - "thiserror", - "tokio", + "cfg-if", + "memchr", ] [[package]] -name = "async-recursion" -version = "1.0.5" +name = "ctor" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" dependencies = [ - "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] -name = "async-stream" +name = "defmt" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "a8a2d011b2fee29fb7d659b83c43fce9a2cb4df453e16d441a51448e448f3f98" dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", + "bitflags 1.3.2", + "defmt-macros", ] [[package]] -name = "async-stream-impl" -version = "0.3.5" +name = "defmt-macros" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "54f0216f6c5acb5ae1a47050a6645024e6edafc2ee32d421955eccfef12ef92e" dependencies = [ + "defmt-parser", + "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.39", ] [[package]] -name = "async-trait" -version = "0.1.74" +name = "defmt-parser" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "269924c02afd7f94bc4cecbfa5c379f6ffcf9766b3408fe63d22c728654eccd0" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", + "thiserror", ] [[package]] -name = "atomic-polyfill" -version = "0.1.11" +name = "deranged" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ff7eb3f316534d83a8a2c3d1674ace8a5a71198eba31e2e2b597833f699b28" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" dependencies = [ - "critical-section", + "powerfmt", ] [[package]] -name = "atomic-waker" -version = "1.1.1" +name = "digest" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] [[package]] -name = "atomicwrites" -version = "0.4.2" +name = "dirs-next" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4d45f362125ed144544e57b0ec6de8fd6a296d41a6252fc4a20c0cf12e9ed3a" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "rustix 0.38.9", - "tempfile", - "windows-sys 0.48.0", + "cfg-if", + "dirs-sys-next", ] [[package]] -name = "atty" -version = "0.2.14" +name = "dirs-sys-next" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ - "hermit-abi 0.1.19", "libc", + "redox_users", "winapi", ] [[package]] -name = "authz-macros" -version = "0.1.0" +name = "dlpi" +version = "0.2.0" +source = "git+https://github.com/oxidecomputer/dlpi-sys#1d587ea98cf2d36f1b1624b0b960559c76d475d2" dependencies = [ - "heck 0.4.1", - "omicron-workspace-hack", - "proc-macro2", - "quote", - "serde", - "serde_tokenstream 0.2.0", - "syn 2.0.32", + "libc", + "libdlpi-sys", + "num_enum", + "pretty-hex", + "thiserror", + "tokio", ] [[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "backoff" -version = "0.4.0" +name = "dof" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +checksum = "9e6b21a1211455e82b1245d6e1b024f30606afbb734c114515d40d0e0b34ce81" dependencies = [ - "futures-core", - "getrandom 0.2.10", - "instant", - "pin-project-lite", - "rand 0.8.5", - "tokio", + "thiserror", + "zerocopy 0.3.0", ] [[package]] -name = "backtrace" -version = "0.3.69" +name = "dtrace-parser" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "bed110893a7f9f4ceb072e166354a09f6cb4cc416eec5b5e5e8ee367442d434b" dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object 0.32.1", - "rustc-demangle", + "pest", + "pest_derive", + "thiserror", ] [[package]] -name = "base16ct" -version = "0.2.0" +name = "dyn-clone" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" [[package]] -name = "base64" -version = "0.13.1" +name = "either" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] -name = "base64" -version = "0.21.5" +name = "embedded-io" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" [[package]] -name = "base64ct" -version = "1.6.0" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "basic-toml" -version = "0.1.4" +name = "errno" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bfc506e7a2370ec239e1d072507b2a80c833083699d3c6fa176fbb4de8448c6" +checksum = "7c18ee0ed65a5f1f81cac6b1d213b69c35fa47d4252ad41f1486dbd8226fe36e" dependencies = [ - "serde", + "libc", + "windows-sys", ] [[package]] -name = "bb8" -version = "0.8.1" +name = "foreign-types" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ - "async-trait", - "futures-channel", - "futures-util", - "parking_lot 0.12.1", - "tokio", + "foreign-types-macros", + "foreign-types-shared", ] [[package]] -name = "bcrypt-pbkdf" -version = "0.10.0" +name = "foreign-types-macros" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeac2e1fe888769f34f05ac343bbef98b14d1ffb292ab69d4608b3abc86f2a2" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ - "blowfish", - "pbkdf2 0.12.2", - "sha2", + "proc-macro2", + "quote", + "syn 2.0.39", ] [[package]] -name = "bcs" -version = "0.1.6" +name = "foreign-types-shared" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b6598a2f5d564fb7855dc6b06fd1c38cff5a72bd8b863a4d021938497b440a" -dependencies = [ - "serde", - "thiserror", -] +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" [[package]] -name = "bhyve_api" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ - "bhyve_api_sys", - "libc", - "strum", + "typenum", + "version_check", ] [[package]] -name = "bhyve_api_sys" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +name = "getrandom" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ + "cfg-if", "libc", - "strum", + "wasi", ] [[package]] -name = "bincode" -version = "1.3.3" +name = "gimli" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] -name = "bindgen" -version = "0.65.1" +name = "hash32" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 2.0.32", - "which", + "byteorder", ] [[package]] -name = "bit-set" -version = "0.5.3" +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + +[[package]] +name = "heapless" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" dependencies = [ - "bit-vec", + "atomic-polyfill", + "hash32", + "rustc_version", + "spin", + "stable_deref_trait", ] [[package]] -name = "bit-vec" -version = "0.6.3" +name = "heck" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] -name = "bit_field" -version = "0.10.2" +name = "hermit-abi" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] [[package]] -name = "bitfield" -version = "0.14.0" +name = "hermit-abi" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d7e60934ceec538daadb9d8432424ed043a904d8e0243f3c6446bce549a46ac" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +name = "illumos-sys-hdrs" +version = "0.1.0" [[package]] -name = "bitflags" -version = "2.4.0" +name = "indexmap" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ - "serde", + "equivalent", + "hashbrown", ] [[package]] -name = "bitstruct" -version = "0.1.1" +name = "ipnetwork" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b10c3912af09af44ea1dafe307edb5ed374b2a32658eb610e372270c9017b4" -dependencies = [ - "bitstruct_derive", -] +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" [[package]] -name = "bitstruct_derive" -version = "0.1.0" +name = "is-terminal" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35fd19022c2b750d14eb9724c204d08ab7544570105b3b466d8a9f2f3feded27" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", + "hermit-abi 0.3.3", + "rustix", + "windows-sys", ] [[package]] -name = "bitvec" -version = "1.0.1" +name = "itertools" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ - "funty", - "radium", - "tap", - "wyz", + "either", ] [[package]] -name = "blake2" -version = "0.10.6" +name = "itertools" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" dependencies = [ - "digest", + "either", ] [[package]] -name = "blake2b_simd" -version = "1.0.1" +name = "itoa" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" -dependencies = [ - "arrayref", - "arrayvec", - "constant_time_eq 0.2.6", -] +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +name = "kstat-macro" +version = "0.1.0" dependencies = [ - "generic-array", + "quote", + "syn 2.0.39", ] [[package]] -name = "block-padding" -version = "0.3.3" +name = "lazy_static" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" -dependencies = [ - "generic-array", -] +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "blowfish" -version = "0.9.1" +name = "libc" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e412e2cd0f2b2d93e02543ceae7917b3c70331573df19ee046bcbc35e45e87d7" -dependencies = [ - "byteorder", - "cipher", -] +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" + +[[package]] +name = "libdlpi-sys" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/dlpi-sys#1d587ea98cf2d36f1b1624b0b960559c76d475d2" [[package]] -name = "bootstore" +name = "libnet" version = "0.1.0" +source = "git+https://github.com/oxidecomputer/netadm-sys?branch=main#59e69ef8fb17be233e336cf4943e31ae398aa4d1" dependencies = [ - "assert_matches", - "bytes", - "camino", - "camino-tempfile", - "chacha20poly1305", - "ciborium", - "derive_more", - "hex", - "hkdf", - "omicron-common 0.1.0", - "omicron-rpaths", - "omicron-test-utils", - "omicron-workspace-hack", - "pq-sys", - "proptest", - "rand 0.8.5", - "secrecy", - "serde", - "serde_with", - "sha3", - "sled-hardware", - "slog", - "slog-async", - "slog-term", + "anyhow", + "cfg-if", + "colored", + "dlpi", + "libc", + "num_enum", + "nvpair", + "nvpair-sys", + "rusty-doors", + "socket2 0.4.10", "thiserror", - "tokio", - "uuid", - "vsss-rs", - "zeroize", + "tracing", ] [[package]] -name = "bootstrap-agent-client" +name = "libnet" version = "0.1.0" +source = "git+https://github.com/oxidecomputer/netadm-sys#59e69ef8fb17be233e336cf4943e31ae398aa4d1" dependencies = [ - "async-trait", - "chrono", - "ipnetwork", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "progenitor", - "regress", - "reqwest", - "schemars", - "serde", - "sled-hardware", - "slog", - "uuid", + "anyhow", + "cfg-if", + "colored", + "dlpi", + "libc", + "num_enum", + "nvpair", + "nvpair-sys", + "rusty-doors", + "socket2 0.4.10", + "thiserror", + "tracing", ] [[package]] -name = "bstr" -version = "0.2.17" +name = "libredox" +version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "lazy_static", - "memchr", - "regex-automata 0.1.10", + "bitflags 2.4.1", + "libc", + "redox_syscall", ] [[package]] -name = "bstr" -version = "1.6.0" +name = "linux-raw-sys" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" -dependencies = [ - "memchr", - "regex-automata 0.3.8", - "serde", -] +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] -name = "buf-list" -version = "1.0.3" +name = "lock_api" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56bd1685d994a3e2a3ed802eb1ecee8cb500b0ad4df48cb4d5d1a2f04749c3a" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ - "bytes", - "tokio", + "autocfg", + "scopeguard", ] [[package]] -name = "bumpalo" -version = "3.13.0" +name = "log" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] -name = "bytecount" -version = "0.6.3" +name = "managed" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" [[package]] -name = "byteorder" -version = "1.5.0" +name = "memchr" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] -name = "bytes" -version = "1.5.0" +name = "minimal-lexical" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" -dependencies = [ - "serde", -] +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] -name = "bytesize" -version = "1.3.0" +name = "miniz_oxide" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ - "serde", + "adler", ] [[package]] -name = "bzip2" -version = "0.4.4" +name = "mio" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ - "bzip2-sys", "libc", + "wasi", + "windows-sys", ] [[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" +name = "nom" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ - "cc", + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.3", "libc", - "pkg-config", ] [[package]] -name = "caboose-util" -version = "0.1.0" +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" dependencies = [ - "anyhow", - "hubtools", - "omicron-workspace-hack", + "num_enum_derive", ] [[package]] -name = "camino" -version = "1.1.6" +name = "num_enum_derive" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ - "serde", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] -name = "camino-tempfile" -version = "1.0.2" +name = "num_threads" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ab15a83d13f75dbd86f082bdefd160b628476ef58d3b900a0ef74e001bb097" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ - "camino", - "tempfile", + "libc", ] [[package]] -name = "cancel-safe-futures" -version = "0.1.5" +name = "nvpair" +version = "0.5.0" +source = "git+https://github.com/jmesmon/rust-libzfs?branch=master#ecd5a922247a6c5acef55d76c5b8d115572bc850" +dependencies = [ + "cstr-argument", + "foreign-types", + "nvpair-sys", +] + +[[package]] +name = "nvpair-sys" +version = "0.4.0" +source = "git+https://github.com/jmesmon/rust-libzfs?branch=master#ecd5a922247a6c5acef55d76c5b8d115572bc850" + +[[package]] +name = "object" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97eb3167cc49e8a4c1be907817009850b8b9693b225092d0f9b00fa9de076e4e" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ - "futures-core", - "futures-sink", - "futures-util", - "pin-project-lite", - "tokio", + "memchr", ] [[package]] -name = "cargo-platform" -version = "0.1.3" +name = "once_cell" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cfa25e60aea747ec7e1124f238816749faa93759c6ff5b31f1ccdda137f4479" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "opte" +version = "0.1.0" dependencies = [ + "cfg-if", + "crc32fast", + "dyn-clone", + "heapless", + "illumos-sys-hdrs", + "itertools 0.11.0", + "kstat-macro", + "opte", + "opte-api", + "postcard", "serde", + "smoltcp", + "usdt", + "version_check", + "zerocopy 0.7.25", ] [[package]] -name = "cargo_metadata" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb9ac64500cc83ce4b9f8dafa78186aa008c8dea77a09b94cd307fd0cd5022a8" +name = "opte-api" +version = "0.1.0" dependencies = [ - "camino", - "cargo-platform", - "semver 1.0.20", + "illumos-sys-hdrs", + "ipnetwork", + "postcard", "serde", - "serde_json", - "thiserror", + "smoltcp", ] [[package]] -name = "cargo_toml" -version = "0.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3f9629bc6c4388ea699781dc988c2b99766d7679b151c81990b4fa1208fafd3" +name = "opte-ioctl" +version = "0.1.0" dependencies = [ + "libc", + "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", + "opte", + "oxide-vpc", + "postcard", "serde", - "toml 0.8.0", + "thiserror", ] [[package]] -name = "cassowary" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53" +name = "opteadm" +version = "0.2.0" +dependencies = [ + "anyhow", + "cfg-if", + "clap", + "libc", + "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", + "opte", + "opte-ioctl", + "oxide-vpc", + "postcard", + "serde", + "thiserror", +] [[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +name = "oxide-vpc" +version = "0.1.0" +dependencies = [ + "ctor", + "illumos-sys-hdrs", + "opte", + "oxide-vpc", + "pcap-parser", + "serde", + "smoltcp", + "usdt", + "zerocopy 0.7.25", +] [[package]] -name = "cbc" -version = "0.1.2" +name = "parking_lot" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "cipher", + "lock_api", + "parking_lot_core", ] [[package]] -name = "cc" -version = "1.0.83" +name = "parking_lot_core" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ - "jobserver", + "cfg-if", "libc", + "redox_syscall", + "smallvec", + "windows-targets", ] [[package]] -name = "cexpr" -version = "0.6.0" +name = "pcap-parser" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +checksum = "b79dfb40aef938ed2082c9ae9443f4eba21b79c1a9d6cfa071f5c2bd8d829491" dependencies = [ + "circular", + "cookie-factory", "nom", + "rusticata-macros", ] [[package]] -name = "cfg-if" -version = "1.0.0" +name = "pest" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] [[package]] -name = "chacha20" -version = "0.9.1" +name = "pest_derive" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", + "pest", + "pest_generator", ] [[package]] -name = "chacha20poly1305" -version = "0.10.1" +name = "pest_generator" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" dependencies = [ - "aead", - "chacha20", - "cipher", - "poly1305", - "zeroize", + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.39", ] [[package]] -name = "chrono" -version = "0.4.31" +name = "pest_meta" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-targets 0.48.5", + "once_cell", + "pest", + "sha2", ] [[package]] -name = "ciborium" -version = "0.2.1" +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "postcard" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" +checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" dependencies = [ - "ciborium-io", - "ciborium-ll", + "cobs", + "embedded-io", "serde", ] [[package]] -name = "ciborium-io" -version = "0.2.1" +name = "powerfmt" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] -name = "ciborium-ll" +name = "pretty-hex" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" -dependencies = [ - "ciborium-io", - "half", -] +checksum = "bc5c99d529f0d30937f6f4b8a86d988047327bb88d04d2c4afc356de74722131" [[package]] -name = "cipher" -version = "0.4.4" +name = "proc-macro-crate" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ - "crypto-common", - "inout", - "zeroize", + "once_cell", + "toml_edit", ] [[package]] -name = "clang-sys" -version = "1.6.1" +name = "proc-macro-error" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ - "glob", - "libc", - "libloading", + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", ] [[package]] -name = "clap" -version = "2.34.0" +name = "proc-macro-error-attr" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim 0.8.0", - "textwrap 0.11.0", - "unicode-width", - "vec_map", + "proc-macro2", + "quote", + "version_check", ] [[package]] -name = "clap" -version = "4.4.3" +name = "proc-macro2" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ - "clap_builder", - "clap_derive", + "unicode-ident", ] [[package]] -name = "clap_builder" -version = "4.4.2" +name = "quote" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim 0.10.0", - "terminal_size", + "proc-macro2", ] [[package]] -name = "clap_derive" -version = "4.4.2" +name = "redox_syscall" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 2.0.32", + "bitflags 1.3.2", ] [[package]] -name = "clap_lex" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" - -[[package]] -name = "cobs" -version = "0.2.3" +name = "redox_users" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] [[package]] -name = "colorchoice" -version = "1.0.0" +name = "regex" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] [[package]] -name = "colored" -version = "2.0.4" +name = "regex-automata" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ - "is-terminal", - "lazy_static", - "windows-sys 0.48.0", + "aho-corasick", + "memchr", + "regex-syntax", ] [[package]] -name = "console" -version = "0.15.7" +name = "regex-syntax" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.45.0", -] +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] -name = "const-oid" -version = "0.9.5" +name = "rustc-demangle" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] -name = "const_format" -version = "0.2.31" +name = "rustc_version" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c990efc7a285731f9a4378d81aff2f0e85a2c8781a05ef0f8baa8dac54d0ff48" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "const_format_proc_macros", + "semver", ] [[package]] -name = "const_format_proc_macros" -version = "0.2.31" +name = "rusticata-macros" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e026b6ce194a874cb9cf32cd5772d1ef9767cc8fcb5765948d74f37a9d8b2bf6" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", + "nom", ] [[package]] -name = "constant_time_eq" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" - -[[package]] -name = "constant_time_eq" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" - -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - -[[package]] -name = "cookie" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" -dependencies = [ - "time", - "version_check", -] - -[[package]] -name = "core-foundation" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - -[[package]] -name = "corncobs" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9236877021b66ad90f833d8a73a7acb702b985b64c5986682d9f1f1a184f0fb" - -[[package]] -name = "cpufeatures" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" -dependencies = [ - "libc", -] - -[[package]] -name = "cpuid_profile_config" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "propolis", - "serde", - "serde_derive", - "thiserror", - "toml 0.7.8", -] - -[[package]] -name = "crc" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-any" -version = "2.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774646b687f63643eb0f4bf13dc263cb581c8c9e57973b6ddf78bda3994d88df" - -[[package]] -name = "crc-catalog" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" - -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crdb-seed" -version = "0.1.0" -dependencies = [ - "anyhow", - "dropshot", - "omicron-test-utils", - "omicron-workspace-hack", - "slog", - "tokio", -] - -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap 4.4.3", - "criterion-plot", - "futures", - "is-terminal", - "itertools 0.10.5", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools 0.10.5", -] - -[[package]] -name = "critical-section" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" - -[[package]] -name = "crossbeam" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" -dependencies = [ - "cfg-if", - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset 0.9.0", - "scopeguard", -] - -[[package]] -name = "crossbeam-queue" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossterm" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" -dependencies = [ - "bitflags 1.3.2", - "crossterm_winapi", - "libc", - "mio", - "parking_lot 0.12.1", - "serde", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" -dependencies = [ - "bitflags 2.4.0", - "crossterm_winapi", - "futures-core", - "libc", - "mio", - "parking_lot 0.12.1", - "signal-hook", - "signal-hook-mio", - "winapi", -] - -[[package]] -name = "crossterm_winapi" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" -dependencies = [ - "winapi", -] - -[[package]] -name = "crucible" -version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "aes-gcm-siv", - "anyhow", - "async-recursion", - "async-trait", - "base64 0.21.5", - "bytes", - "chrono", - "crucible-client-types", - "crucible-common", - "crucible-protocol", - "crucible-workspace-hack", - "dropshot", - "futures", - "futures-core", - "itertools 0.11.0", - "libc", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter-producer 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "rand 0.8.5", - "rand_chacha 0.3.1", - "reqwest", - "ringbuffer", - "schemars", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-dtrace", - "slog-term", - "tokio", - "tokio-rustls", - "tokio-util", - "toml 0.8.0", - "tracing", - "usdt", - "uuid", - "version_check", -] - -[[package]] -name = "crucible-agent-client" -version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "anyhow", - "chrono", - "crucible-workspace-hack", - "percent-encoding", - "progenitor", - "reqwest", - "schemars", - "serde", - "serde_json", -] - -[[package]] -name = "crucible-client-types" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "base64 0.21.5", - "crucible-workspace-hack", - "schemars", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "crucible-common" -version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "anyhow", - "atty", - "crucible-workspace-hack", - "nix 0.26.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rusqlite", - "rustls-pemfile", - "schemars", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-bunyan", - "slog-dtrace", - "slog-term", - "tempfile", - "thiserror", - "tokio-rustls", - "toml 0.8.0", - "twox-hash", - "uuid", - "vergen", -] - -[[package]] -name = "crucible-pantry-client" -version = "0.0.1" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "anyhow", - "chrono", - "crucible-workspace-hack", - "percent-encoding", - "progenitor", - "reqwest", - "schemars", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "crucible-protocol" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "anyhow", - "bincode", - "bytes", - "crucible-common", - "crucible-workspace-hack", - "num_enum 0.7.0", - "schemars", - "serde", - "tokio-util", - "uuid", -] - -[[package]] -name = "crucible-smf" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/crucible?rev=da534e73380f3cc53ca0de073e1ea862ae32109b#da534e73380f3cc53ca0de073e1ea862ae32109b" -dependencies = [ - "crucible-workspace-hack", - "libc", - "num-derive", - "num-traits", - "thiserror", -] - -[[package]] -name = "crucible-workspace-hack" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbd293370c6cb9c334123675263de33fc9e53bbdfc8bdd5e329237cf0205fdc7" - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-bigint" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4c2f4e1afd912bc40bfd6fed5d9dc1f288e0ba01bfcc835cc5bc3eb13efe15" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "typenum", -] - -[[package]] -name = "cstr-argument" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bd9c8e659a473bce955ae5c35b116af38af11a7acb0b480e01f3ed348aeb40" -dependencies = [ - "cfg-if", - "memchr", -] - -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher", -] - -[[package]] -name = "curve25519-dalek" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622178105f911d937a42cdb140730ba4a3ed2becd8ae6ce39c7d28b5d75d4588" -dependencies = [ - "cfg-if", - "cpufeatures", - "curve25519-dalek-derive", - "digest", - "fiat-crypto", - "platforms", - "rand_core 0.6.4", - "rustc_version 0.4.0", - "subtle", - "zeroize", -] - -[[package]] -name = "curve25519-dalek-derive" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" -dependencies = [ - "darling_core 0.20.3", - "darling_macro 0.20.3", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", -] - -[[package]] -name = "darling_core" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 2.0.32", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core 0.14.4", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" -dependencies = [ - "darling_core 0.20.3", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "dashmap" -version = "5.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28" -dependencies = [ - "cfg-if", - "hashbrown 0.14.0", - "lock_api", - "once_cell", - "parking_lot_core 0.9.8", -] - -[[package]] -name = "data-encoding" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" - -[[package]] -name = "datatest-stable" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a384d02609f0774f4dbf0c38fc57eb2769b24c30b9185911ff657ec14837da" -dependencies = [ - "camino", - "libtest-mimic", - "regex", - "walkdir", -] - -[[package]] -name = "db-macros" -version = "0.1.0" -dependencies = [ - "heck 0.4.1", - "omicron-workspace-hack", - "proc-macro2", - "quote", - "rustfmt-wrapper", - "serde", - "serde_tokenstream 0.2.0", - "syn 2.0.32", -] - -[[package]] -name = "ddm-admin-client" -version = "0.1.0" -dependencies = [ - "anyhow", - "either", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "omicron-zone-package", - "progenitor", - "progenitor-client", - "quote", - "reqwest", - "rustfmt-wrapper", - "serde", - "serde_json", - "sled-hardware", - "slog", - "thiserror", - "tokio", - "toml 0.7.8", -] - -[[package]] -name = "debug-ignore" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe7ed1d93f4553003e20b629abe9085e1e81b1429520f897f8f8860bc6dfc21" - -[[package]] -name = "defmt" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2d011b2fee29fb7d659b83c43fce9a2cb4df453e16d441a51448e448f3f98" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54f0216f6c5acb5ae1a47050a6645024e6edafc2ee32d421955eccfef12ef92e" -dependencies = [ - "defmt-parser", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "defmt-parser" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "269924c02afd7f94bc4cecbfa5c379f6ffcf9766b3408fe63d22c728654eccd0" -dependencies = [ - "thiserror", -] - -[[package]] -name = "der" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" -dependencies = [ - "const-oid", - "der_derive", - "flagset", - "pem-rfc7468", - "zeroize", -] - -[[package]] -name = "der_derive" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "deranged" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" -dependencies = [ - "serde", -] - -[[package]] -name = "derive-where" -version = "1.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146398d62142a0f35248a608f17edf0dde57338354966d6e41d0eb2d16980ccb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "derive_builder" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" -dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_macro" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" -dependencies = [ - "derive_builder_core", - "syn 1.0.109", -] - -[[package]] -name = "derive_more" -version = "0.99.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version 0.4.0", - "syn 1.0.109", -] - -[[package]] -name = "diesel" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2268a214a6f118fce1838edba3d1561cf0e78d8de785475957a580a7f8c69d33" -dependencies = [ - "bitflags 2.4.0", - "byteorder", - "chrono", - "diesel_derives", - "ipnetwork", - "itoa", - "libc", - "pq-sys", - "r2d2", - "serde_json", - "uuid", -] - -[[package]] -name = "diesel-dtrace" -version = "0.2.0" -source = "git+https://github.com/oxidecomputer/diesel-dtrace?branch=main#c1252df734b52b4e1243e0ca2bd5f00b17730408" -dependencies = [ - "diesel", - "serde", - "usdt", - "uuid", - "version_check", -] - -[[package]] -name = "diesel_derives" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e054665eaf6d97d1e7125512bb2d35d07c73ac86cc6920174cb42d1ab697a554" -dependencies = [ - "diesel_table_macro_syntax", - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "diesel_table_macro_syntax" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" -dependencies = [ - "syn 2.0.32", -] - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "subtle", -] - -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - -[[package]] -name = "display-error-chain" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f77af9e75578c1ab34f5f04545a8b05be0c36fbd7a9bb3cf2d2a971e435fdbb9" - -[[package]] -name = "dladm" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "libc", - "strum", -] - -[[package]] -name = "dlpi" -version = "0.2.0" -source = "git+https://github.com/oxidecomputer/dlpi-sys#1d587ea98cf2d36f1b1624b0b960559c76d475d2" -dependencies = [ - "libc", - "libdlpi-sys", - "num_enum 0.5.11", - "pretty-hex 0.2.1", - "thiserror", - "tokio", -] - -[[package]] -name = "dns-server" -version = "0.1.0" -dependencies = [ - "anyhow", - "camino", - "chrono", - "clap 4.4.3", - "dns-service-client 0.1.0", - "dropshot", - "expectorate", - "http", - "omicron-test-utils", - "omicron-workspace-hack", - "openapi-lint", - "openapiv3", - "pretty-hex 0.3.0", - "schemars", - "serde", - "serde_json", - "sled", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "subprocess", - "tempdir", - "tempfile", - "thiserror", - "tokio", - "toml 0.7.8", - "trust-dns-client", - "trust-dns-proto", - "trust-dns-resolver", - "trust-dns-server", - "uuid", -] - -[[package]] -name = "dns-service-client" -version = "0.1.0" -dependencies = [ - "chrono", - "http", - "omicron-workspace-hack", - "progenitor", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "uuid", -] - -[[package]] -name = "dns-service-client" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "chrono", - "http", - "progenitor", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "uuid", -] - -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - -[[package]] -name = "dof" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e6b21a1211455e82b1245d6e1b024f30606afbb734c114515d40d0e0b34ce81" -dependencies = [ - "thiserror", - "zerocopy 0.3.0", -] - -[[package]] -name = "downcast" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" - -[[package]] -name = "dpd-client" -version = "0.1.0" -dependencies = [ - "anyhow", - "chrono", - "futures", - "http", - "ipnetwork", - "omicron-workspace-hack", - "omicron-zone-package", - "progenitor", - "progenitor-client", - "quote", - "rand 0.8.5", - "regress", - "reqwest", - "rustfmt-wrapper", - "schemars", - "serde", - "serde_json", - "slog", - "toml 0.7.8", - "uuid", -] - -[[package]] -name = "dropshot" -version = "0.9.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot?branch=main#fa728d07970824fd5f3bd57a3d4dc0fdbea09bfd" -dependencies = [ - "async-stream", - "async-trait", - "base64 0.21.5", - "bytes", - "camino", - "chrono", - "debug-ignore", - "dropshot_endpoint", - "form_urlencoded", - "futures", - "hostname", - "http", - "hyper", - "indexmap 2.0.0", - "multer", - "openapiv3", - "paste", - "percent-encoding", - "proc-macro2", - "rustls", - "rustls-pemfile", - "schemars", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "slog", - "slog-async", - "slog-bunyan", - "slog-json", - "slog-term", - "tokio", - "tokio-rustls", - "toml 0.7.8", - "usdt", - "uuid", - "version_check", - "waitgroup", -] - -[[package]] -name = "dropshot_endpoint" -version = "0.9.1-dev" -source = "git+https://github.com/oxidecomputer/dropshot?branch=main#fa728d07970824fd5f3bd57a3d4dc0fdbea09bfd" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_tokenstream 0.2.0", - "syn 2.0.32", -] - -[[package]] -name = "dtrace-parser" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bed110893a7f9f4ceb072e166354a09f6cb4cc416eec5b5e5e8ee367442d434b" -dependencies = [ - "pest", - "pest_derive", - "thiserror", -] - -[[package]] -name = "dyn-clone" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" - -[[package]] -name = "ed25519" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" -dependencies = [ - "signature 1.6.4", -] - -[[package]] -name = "ed25519" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60f6d271ca33075c88028be6f04d502853d63a5ece419d269c15315d4fc1cf1d" -dependencies = [ - "pkcs8", - "signature 2.1.0", -] - -[[package]] -name = "ed25519-dalek" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" -dependencies = [ - "curve25519-dalek", - "ed25519 2.2.2", - "rand_core 0.6.4", - "serde", - "sha2", - "zeroize", -] - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" - -[[package]] -name = "elliptic-curve" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968405c8fdc9b3bf4df0a6638858cc0b52462836ab6b1c87377785dd09cf1c0b" -dependencies = [ - "base16ct", - "crypto-bigint", - "digest", - "ff", - "generic-array", - "group", - "hkdf", - "rand_core 0.6.4", - "subtle", - "zeroize", -] - -[[package]] -name = "embedded-io" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" - -[[package]] -name = "ena" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c533630cf40e9caa44bd91aadc88a75d75a4c3a12b4cfde353cbed41daa1e1f1" -dependencies = [ - "log", -] - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "end-to-end-tests" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.5", - "camino", - "chrono", - "futures", - "http", - "omicron-sled-agent", - "omicron-test-utils", - "omicron-workspace-hack", - "oxide-client", - "rand 0.8.5", - "reqwest", - "russh", - "russh-keys", - "serde_json", - "tokio", - "toml 0.7.8", - "trust-dns-resolver", - "uuid", -] - -[[package]] -name = "endian-type" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" - -[[package]] -name = "enum-as-inner" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "env_logger" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "env_logger" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" -dependencies = [ - "is-terminal", - "log", - "termcolor", -] - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "erased-serde" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "837c0466252947ada828b975e12daf82e18bb5444e4df87be6038d4469e2a3d2" -dependencies = [ - "serde", -] - -[[package]] -name = "errno" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "expectorate" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f19b25bdfa2747ae775f37cd109c31f1272d4e4c83095be0727840aa1d75f" -dependencies = [ - "console", - "newline-converter", - "similar", -] - -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "fastrand" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" - -[[package]] -name = "fatfs" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05669f8e7e2d7badc545c513710f0eba09c2fbef683eb859fd79c46c355048e0" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "chrono", - "log", -] - -[[package]] -name = "fd-lock" -version = "3.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" -dependencies = [ - "cfg-if", - "rustix 0.38.9", - "windows-sys 0.48.0", -] - -[[package]] -name = "ff" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" -dependencies = [ - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "fiat-crypto" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0870c84016d4b481be5c9f323c24f65e31e901ae618f0e80f4308fb00de1d2d" - -[[package]] -name = "filetime" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.3.5", - "windows-sys 0.48.0", -] - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "flagset" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda653ca797810c02f7ca4b804b40b8b95ae046eb989d356bce17919a8c25499" - -[[package]] -name = "flate2" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "float-cmp" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" -dependencies = [ - "num-traits", -] - -[[package]] -name = "flume" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" -dependencies = [ - "futures-core", - "futures-sink", - "nanorand", - "spin 0.9.8", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared 0.1.1", -] - -[[package]] -name = "foreign-types" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" -dependencies = [ - "foreign-types-macros", - "foreign-types-shared 0.3.1", -] - -[[package]] -name = "foreign-types-macros" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "foreign-types-shared" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" - -[[package]] -name = "form_urlencoded" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fragile" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" - -[[package]] -name = "fs-err" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" - -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - -[[package]] -name = "funty" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - -[[package]] -name = "futures" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" - -[[package]] -name = "futures-executor" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" - -[[package]] -name = "futures-macro" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "futures-sink" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" - -[[package]] -name = "futures-task" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" - -[[package]] -name = "futures-timer" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" - -[[package]] -name = "futures-util" -version = "0.3.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "gateway-cli" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.4.3", - "futures", - "gateway-client", - "gateway-messages", - "hex", - "libc", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "reqwest", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-term", - "termios", - "tokio", - "tokio-tungstenite 0.18.0", - "uuid", -] - -[[package]] -name = "gateway-client" -version = "0.1.0" -dependencies = [ - "base64 0.21.5", - "chrono", - "omicron-workspace-hack", - "progenitor", - "rand 0.8.5", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "uuid", -] - -[[package]] -name = "gateway-messages" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/management-gateway-service?rev=2739c18e80697aa6bc235c935176d14b4d757ee9#2739c18e80697aa6bc235c935176d14b4d757ee9" -dependencies = [ - "bitflags 1.3.2", - "hubpack 0.1.2", - "serde", - "serde_repr", - "smoltcp 0.9.1", - "static_assertions", - "strum_macros 0.25.2", - "uuid", - "zerocopy 0.6.4", -] - -[[package]] -name = "gateway-sp-comms" -version = "0.1.1" -source = "git+https://github.com/oxidecomputer/management-gateway-service?rev=2739c18e80697aa6bc235c935176d14b4d757ee9#2739c18e80697aa6bc235c935176d14b4d757ee9" -dependencies = [ - "async-trait", - "backoff", - "futures", - "fxhash", - "gateway-messages", - "hex", - "hubpack 0.1.2", - "hubtools", - "lru-cache", - "nix 0.26.2 (git+https://github.com/jgallagher/nix?branch=r0.26-illumos)", - "once_cell", - "paste", - "serde", - "serde-big-array 0.5.1", - "slog", - "socket2 0.5.4", - "string_cache", - "thiserror", - "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git?branch=main)", - "tokio", - "usdt", - "uuid", - "version_check", - "zip", -] - -[[package]] -name = "gateway-test-utils" -version = "0.1.0" -dependencies = [ - "camino", - "dropshot", - "gateway-messages", - "omicron-gateway", - "omicron-test-utils", - "omicron-workspace-hack", - "slog", - "sp-sim", - "tokio", - "uuid", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", - "zeroize", -] - -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", -] - -[[package]] -name = "ghash" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" -dependencies = [ - "opaque-debug", - "polyval", -] - -[[package]] -name = "gimli" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" - -[[package]] -name = "git2" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" -dependencies = [ - "bitflags 1.3.2", - "libc", - "libgit2-sys", - "log", - "url", -] - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "globset" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" -dependencies = [ - "aho-corasick", - "bstr 1.6.0", - "fnv", - "log", - "regex", -] - -[[package]] -name = "group" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" -dependencies = [ - "ff", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "h2" -version = "0.3.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 1.9.3", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "half" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" - -[[package]] -name = "hash32" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67" -dependencies = [ - "byteorder", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" -dependencies = [ - "ahash", - "allocator-api2", -] - -[[package]] -name = "hashlink" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" -dependencies = [ - "hashbrown 0.14.0", -] - -[[package]] -name = "headers" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" -dependencies = [ - "base64 0.21.5", - "bytes", - "headers-core", - "http", - "httpdate", - "mime", - "sha1", -] - -[[package]] -name = "headers-core" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" -dependencies = [ - "http", -] - -[[package]] -name = "heapless" -version = "0.7.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db04bc24a18b9ea980628ecf00e6c0264f3c1426dac36c00cb49b6fbad8b0743" -dependencies = [ - "atomic-polyfill", - "hash32", - "rustc_version 0.4.0", - "spin 0.9.8", - "stable_deref_trait", -] - -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -dependencies = [ - "serde", -] - -[[package]] -name = "hex-literal" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" - -[[package]] -name = "highway" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ba82c000837f4e74df01a5520f0dc48735d4aed955a99eae4428bab7cf3acd" - -[[package]] -name = "hkdf" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" -dependencies = [ - "hmac", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - -[[package]] -name = "home" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" -dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - -[[package]] -name = "http" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "http-range" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573" - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "httptest" -version = "0.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f25cfb6def593d43fae1ead24861f217e93bc70768a45cc149a69b5f049df4" -dependencies = [ - "bstr 0.2.17", - "bytes", - "crossbeam-channel", - "form_urlencoded", - "futures", - "http", - "hyper", - "log", - "once_cell", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", -] - -[[package]] -name = "hubpack" -version = "0.1.0" -source = "git+https://github.com/cbiffle/hubpack?rev=df08cc3a6e1f97381cd0472ae348e310f0119e25#df08cc3a6e1f97381cd0472ae348e310f0119e25" -dependencies = [ - "hubpack_derive 0.1.0", - "serde", -] - -[[package]] -name = "hubpack" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a0b84aeae519f65e0ba3aa998327080993426024edbd5cc38dbaf5ec524303" -dependencies = [ - "hubpack_derive 0.1.1", - "serde", -] - -[[package]] -name = "hubpack_derive" -version = "0.1.0" -source = "git+https://github.com/cbiffle/hubpack?rev=df08cc3a6e1f97381cd0472ae348e310f0119e25#df08cc3a6e1f97381cd0472ae348e310f0119e25" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "hubpack_derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f928320aff16ee8818ef7309180f8b5897057fd79d9dcb8de3ed1ba6dcc125a" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "hubtools" -version = "0.4.1" -source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#73cd5a84689d59ecce9da66ad4389c540d315168" -dependencies = [ - "lpc55_areas", - "lpc55_sign", - "object 0.30.4", - "path-slash", - "rsa", - "thiserror", - "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git)", - "tlvc-text", - "toml 0.7.8", - "x509-cert", - "zerocopy 0.6.4", - "zip", -] - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.14.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.4.9", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http", - "hyper", - "log", - "rustls", - "rustls-native-certs", - "tokio", - "tokio-rustls", -] - -[[package]] -name = "hyper-staticfile" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318ca89e4827e7fe4ddd2824f52337239796ae8ecc761a663324407dc3d8d7e7" -dependencies = [ - "futures-util", - "http", - "http-range", - "httpdate", - "hyper", - "mime_guess", - "percent-encoding", - "rand 0.8.5", - "tokio", - "url", - "winapi", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "illumos-devinfo" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/illumos-devinfo?branch=main#4323b17bfdd0c94d2875ac64b47f0e60fac1d640" -dependencies = [ - "anyhow", - "libc", - "num_enum 0.5.11", -] - -[[package]] -name = "illumos-sys-hdrs" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=faaf3320e673167722c2c7e6fb2a86bd84f04a8a#faaf3320e673167722c2c7e6fb2a86bd84f04a8a" - -[[package]] -name = "illumos-utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "bhyve_api", - "byteorder", - "camino", - "cfg-if", - "crucible-smf", - "futures", - "ipnetwork", - "libc", - "macaddr", - "mockall", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "opte-ioctl", - "oxide-vpc", - "regress", - "schemars", - "serde", - "serde_json", - "slog", - "smf", - "thiserror", - "tokio", - "toml 0.7.8", - "uuid", - "zone", -] - -[[package]] -name = "impl-trait-for-tuples" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" -dependencies = [ - "equivalent", - "hashbrown 0.14.0", - "serde", -] - -[[package]] -name = "indicatif" -version = "0.17.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25" -dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic", - "rayon", - "unicode-width", -] - -[[package]] -name = "indoc" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" - -[[package]] -name = "indoc" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c785eefb63ebd0e33416dfcb8d6da0bf27ce752843a45632a67bf10d4d4b5c4" - -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "block-padding", - "generic-array", -] - -[[package]] -name = "installinator" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "buf-list", - "bytes", - "camino", - "cancel-safe-futures", - "clap 4.4.3", - "ddm-admin-client", - "display-error-chain", - "futures", - "hex", - "hex-literal", - "http", - "illumos-utils", - "installinator-artifact-client", - "installinator-common", - "ipcc-key-value", - "itertools 0.11.0", - "libc", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "once_cell", - "partial-io", - "progenitor-client", - "proptest", - "reqwest", - "serde", - "sha2", - "sled-hardware", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "smf", - "tempfile", - "test-strategy", - "thiserror", - "tokio", - "tokio-stream", - "toml 0.7.8", - "tufaceous-lib", - "update-engine", - "uuid", -] - -[[package]] -name = "installinator-artifact-client" -version = "0.1.0" -dependencies = [ - "installinator-common", - "omicron-workspace-hack", - "progenitor", - "regress", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "update-engine", - "uuid", -] - -[[package]] -name = "installinator-artifactd" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "camino", - "clap 4.4.3", - "dropshot", - "expectorate", - "hyper", - "installinator-common", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "openapi-lint", - "openapiv3", - "schemars", - "serde", - "serde_json", - "slog", - "subprocess", - "tokio", - "uuid", -] - -[[package]] -name = "installinator-common" -version = "0.1.0" -dependencies = [ - "anyhow", - "camino", - "illumos-utils", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "schemars", - "serde", - "serde_json", - "serde_with", - "thiserror", - "update-engine", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "internal-dns" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert_matches", - "chrono", - "dns-server", - "dns-service-client 0.1.0", - "dropshot", - "expectorate", - "futures", - "hyper", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "progenitor", - "reqwest", - "serde", - "serde_json", - "sled", - "slog", - "tempfile", - "thiserror", - "tokio", - "trust-dns-proto", - "trust-dns-resolver", - "uuid", -] - -[[package]] -name = "internal-dns" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "anyhow", - "chrono", - "dns-service-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "futures", - "hyper", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "reqwest", - "slog", - "thiserror", - "trust-dns-proto", - "trust-dns-resolver", - "uuid", -] - -[[package]] -name = "internal-dns-cli" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.4.3", - "dropshot", - "internal-dns 0.1.0", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "slog", - "tokio", - "trust-dns-resolver", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi 0.3.2", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "ipcc-key-value" -version = "0.1.0" -dependencies = [ - "ciborium", - "libc", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "proptest", - "serde", - "test-strategy", - "thiserror", - "uuid", -] - -[[package]] -name = "ipconfig" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" -dependencies = [ - "socket2 0.5.4", - "widestring", - "windows-sys 0.48.0", - "winreg", -] - -[[package]] -name = "ipnet" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" - -[[package]] -name = "ipnetwork" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" -dependencies = [ - "schemars", - "serde", -] - -[[package]] -name = "is-terminal" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" -dependencies = [ - "hermit-abi 0.3.2", - "rustix 0.38.9", - "windows-sys 0.48.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" - -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "keccak" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" -dependencies = [ - "cpufeatures", -] - -[[package]] -name = "key-manager" -version = "0.1.0" -dependencies = [ - "async-trait", - "hkdf", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "secrecy", - "sha3", - "slog", - "thiserror", - "tokio", - "zeroize", -] - -[[package]] -name = "kstat-macro" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=faaf3320e673167722c2c7e6fb2a86bd84f04a8a#faaf3320e673167722c2c7e6fb2a86bd84f04a8a" -dependencies = [ - "quote", - "syn 2.0.32", -] - -[[package]] -name = "lalrpop" -version = "0.19.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a1cbf952127589f2851ab2046af368fd20645491bb4b376f04b7f94d7a9837b" -dependencies = [ - "ascii-canvas", - "bit-set", - "diff", - "ena", - "is-terminal", - "itertools 0.10.5", - "lalrpop-util", - "petgraph", - "regex", - "regex-syntax 0.6.29", - "string_cache", - "term", - "tiny-keccak", - "unicode-xid", -] - -[[package]] -name = "lalrpop-util" -version = "0.19.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3c48237b9604c5a4702de6b824e02006c3214327564636aef27c1028a8fa0ed" - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -dependencies = [ - "spin 0.5.2", -] - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "libc" -version = "0.2.149" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" - -[[package]] -name = "libdlpi-sys" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/dlpi-sys#1d587ea98cf2d36f1b1624b0b960559c76d475d2" - -[[package]] -name = "libefi-illumos" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/libefi-illumos?branch=master#54c398c139f0e65252c2c0f9565d2eec7116bf02" -dependencies = [ - "libc", - "libefi-sys", - "thiserror", - "uuid", -] - -[[package]] -name = "libefi-sys" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b024e211b1b371da58cd69e4fb8fa4ed16915edcc0e2e1fb04ac4bad61959f25" - -[[package]] -name = "libgit2-sys" -version = "0.15.2+1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa" -dependencies = [ - "cc", - "libc", - "libz-sys", - "pkg-config", -] - -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - -[[package]] -name = "libm" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" - -[[package]] -name = "libnet" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/netadm-sys#f114bd0d543d886cd453932e9f0967de57289bc2" -dependencies = [ - "anyhow", - "cfg-if", - "colored", - "dlpi", - "libc", - "num_enum 0.5.11", - "nvpair", - "nvpair-sys", - "rusty-doors", - "socket2 0.4.9", - "thiserror", - "tracing", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" -dependencies = [ - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libtest-mimic" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d8de370f98a6cb8a4606618e53e802f93b094ddec0f96988eaec2c27e6e9ce7" -dependencies = [ - "clap 4.4.3", - "termcolor", - "threadpool", -] - -[[package]] -name = "libxml" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe73cdec2bcb36d25a9fe3f607ffcd44bb8907ca0100c4098d1aa342d1e7bec" -dependencies = [ - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linear-map" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfae20f6b19ad527b550c223fddc3077a547fc70cda94b9b566575423fd303ee" - -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - -[[package]] -name = "linux-raw-sys" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" - -[[package]] -name = "lock_api" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" - -[[package]] -name = "lpc55_areas" -version = "0.2.4" -source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac" -dependencies = [ - "bitfield", - "clap 4.4.3", - "packed_struct", - "serde", -] - -[[package]] -name = "lpc55_sign" -version = "0.3.3" -source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac" -dependencies = [ - "byteorder", - "const-oid", - "crc-any", - "der", - "env_logger 0.10.0", - "hex", - "log", - "lpc55_areas", - "num-traits", - "packed_struct", - "pem-rfc7468", - "rsa", - "serde", - "serde-hex", - "sha2", - "thiserror", - "x509-cert", - "zerocopy 0.6.4", -] - -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - -[[package]] -name = "macaddr" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baee0bbc17ce759db233beb01648088061bf678383130602a298e6998eedb2d8" -dependencies = [ - "serde", -] - -[[package]] -name = "managed" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" - -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - -[[package]] -name = "md-5" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" -dependencies = [ - "digest", -] - -[[package]] -name = "md5" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" - -[[package]] -name = "memchr" -version = "2.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" - -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - -[[package]] -name = "mg-admin-client" -version = "0.1.0" -dependencies = [ - "anyhow", - "either", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "omicron-zone-package", - "progenitor", - "progenitor-client", - "quote", - "reqwest", - "rustfmt-wrapper", - "serde", - "serde_json", - "sled-hardware", - "slog", - "thiserror", - "tokio", - "toml 0.7.8", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" -dependencies = [ - "libc", - "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", -] - -[[package]] -name = "mockall" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" -dependencies = [ - "cfg-if", - "downcast", - "fragile", - "lazy_static", - "mockall_derive", - "predicates 2.1.5", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "multer" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01acbdc23469fd8fe07ab135923371d5f5a422fbf9c522158677c8eb15bc51c2" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "log", - "memchr", - "mime", - "spin 0.9.8", - "version_check", -] - -[[package]] -name = "nanorand" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" -dependencies = [ - "getrandom 0.2.10", -] - -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "new_debug_unreachable" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" - -[[package]] -name = "newline-converter" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b6b097ecb1cbfed438542d16e84fd7ad9b0c76c8a65b7f9039212a3d14dc7f" -dependencies = [ - "unicode-segmentation", -] - -[[package]] -name = "newtype_derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8cd24d9f185bb7223958d8c1ff7a961b74b1953fd05dba7cc568a63b3861ec" -dependencies = [ - "rustc_version 0.1.7", -] - -[[package]] -name = "nexus-client" -version = "0.1.0" -dependencies = [ - "chrono", - "futures", - "ipnetwork", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-workspace-hack", - "progenitor", - "regress", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "uuid", -] - -[[package]] -name = "nexus-client" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "chrono", - "futures", - "ipnetwork", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "omicron-passwords 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "progenitor", - "regress", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "uuid", -] - -[[package]] -name = "nexus-db-model" -version = "0.1.0" -dependencies = [ - "anyhow", - "chrono", - "db-macros", - "diesel", - "expectorate", - "hex", - "ipnetwork", - "macaddr", - "newtype_derive", - "nexus-defaults", - "nexus-types", - "omicron-certificates", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-rpaths", - "omicron-workspace-hack", - "parse-display", - "pq-sys", - "rand 0.8.5", - "ref-cast", - "schemars", - "semver 1.0.20", - "serde", - "serde_json", - "sled-agent-client", - "steno", - "strum", - "thiserror", - "uuid", -] - -[[package]] -name = "nexus-db-queries" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert_matches", - "async-bb8-diesel", - "async-trait", - "authz-macros", - "base64 0.21.5", - "bb8", - "camino", - "chrono", - "cookie", - "db-macros", - "diesel", - "diesel-dtrace", - "dropshot", - "expectorate", - "futures", - "gateway-client", - "headers", - "hex", - "http", - "hyper", - "hyper-rustls", - "internal-dns 0.1.0", - "ipnetwork", - "itertools 0.11.0", - "lazy_static", - "macaddr", - "newtype_derive", - "nexus-db-model", - "nexus-defaults", - "nexus-inventory", - "nexus-test-utils", - "nexus-types", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-rpaths", - "omicron-sled-agent", - "omicron-test-utils", - "omicron-workspace-hack", - "once_cell", - "openapiv3", - "openssl", - "openssl-probe", - "openssl-sys", - "oso", - "oximeter 0.1.0", - "paste", - "pem 1.1.1", - "petgraph", - "pq-sys", - "rand 0.8.5", - "rcgen", - "ref-cast", - "regex", - "reqwest", - "ring 0.16.20", - "rustls", - "samael", - "serde", - "serde_json", - "serde_urlencoded", - "serde_with", - "sled-agent-client", - "slog", - "static_assertions", - "steno", - "strum", - "subprocess", - "tempfile", - "term", - "thiserror", - "tokio", - "tokio-postgres", - "toml 0.7.8", - "usdt", - "uuid", -] - -[[package]] -name = "nexus-defaults" -version = "0.1.0" -dependencies = [ - "ipnetwork", - "lazy_static", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "rand 0.8.5", - "serde_json", -] - -[[package]] -name = "nexus-inventory" -version = "0.1.0" -dependencies = [ - "anyhow", - "chrono", - "expectorate", - "futures", - "gateway-client", - "gateway-messages", - "gateway-test-utils", - "nexus-types", - "omicron-workspace-hack", - "regex", - "slog", - "strum", - "tokio", - "uuid", -] - -[[package]] -name = "nexus-test-interface" -version = "0.1.0" -dependencies = [ - "async-trait", - "dropshot", - "internal-dns 0.1.0", - "nexus-types", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "slog", - "uuid", -] - -[[package]] -name = "nexus-test-utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "bytes", - "camino", - "camino-tempfile", - "chrono", - "crucible-agent-client", - "dns-server", - "dns-service-client 0.1.0", - "dropshot", - "gateway-messages", - "gateway-test-utils", - "headers", - "http", - "hyper", - "internal-dns 0.1.0", - "nexus-db-queries", - "nexus-test-interface", - "nexus-types", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-sled-agent", - "omicron-test-utils", - "omicron-workspace-hack", - "oximeter 0.1.0", - "oximeter-client", - "oximeter-collector", - "oximeter-producer 0.1.0", - "parse-display", - "serde", - "serde_json", - "serde_urlencoded", - "slog", - "tempfile", - "trust-dns-proto", - "trust-dns-resolver", - "uuid", -] - -[[package]] -name = "nexus-test-utils-macros" -version = "0.1.0" -dependencies = [ - "omicron-workspace-hack", - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "nexus-types" -version = "0.1.0" -dependencies = [ - "anyhow", - "api_identity 0.1.0", - "base64 0.21.5", - "chrono", - "dns-service-client 0.1.0", - "futures", - "gateway-client", - "newtype_derive", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-workspace-hack", - "openssl", - "openssl-probe", - "openssl-sys", - "parse-display", - "schemars", - "serde", - "serde_json", - "steno", - "strum", - "uuid", -] - -[[package]] -name = "nibble_vec" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" -dependencies = [ - "smallvec 1.11.0", -] - -[[package]] -name = "nix" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", - "memoffset 0.7.1", - "pin-utils", - "static_assertions", -] - -[[package]] -name = "nix" -version = "0.26.2" -source = "git+https://github.com/jgallagher/nix?branch=r0.26-illumos#c1a3636db0524f194b714cfd117cd9b637b8b10e" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", - "memoffset 0.7.1", - "pin-utils", - "static_assertions", -] - -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - -[[package]] -name = "nu-ansi-term" -version = "0.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68" -dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "num" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" -dependencies = [ - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", - "rand 0.8.5", -] - -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits", - "rand 0.8.5", - "serde", - "smallvec 1.11.0", - "zeroize", -] - -[[package]] -name = "num-complex" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-derive" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e6a0fd4f737c707bd9086cc16c925f294943eb62eb71499e9fd4cf71f8b9f4e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi 0.3.2", - "libc", -] - -[[package]] -name = "num_enum" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" -dependencies = [ - "num_enum_derive 0.5.11", -] - -[[package]] -name = "num_enum" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70bf6736f74634d299d00086f02986875b3c2d924781a6a2cb6c201e73da0ceb" -dependencies = [ - "num_enum_derive 0.7.0", -] - -[[package]] -name = "num_enum_derive" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "num_enum_derive" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ea360eafe1022f7cc56cd7b869ed57330fb2453d0c7831d99b74c65d2f5597" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - -[[package]] -name = "nvpair" -version = "0.5.0" -source = "git+https://github.com/jmesmon/rust-libzfs?branch=master#ecd5a922247a6c5acef55d76c5b8d115572bc850" -dependencies = [ - "cstr-argument", - "foreign-types 0.5.0", - "nvpair-sys", -] - -[[package]] -name = "nvpair-sys" -version = "0.4.0" -source = "git+https://github.com/jmesmon/rust-libzfs?branch=master#ecd5a922247a6c5acef55d76c5b8d115572bc850" - -[[package]] -name = "object" -version = "0.30.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" -dependencies = [ - "crc32fast", - "hashbrown 0.13.2", - "indexmap 1.9.3", - "memchr", -] - -[[package]] -name = "object" -version = "0.32.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" -dependencies = [ - "memchr", -] - -[[package]] -name = "olpc-cjson" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d637c9c15b639ccff597da8f4fa968300651ad2f1e968aefc3b4927a6fb2027a" -dependencies = [ - "serde", - "serde_json", - "unicode-normalization", -] - -[[package]] -name = "omicron-certificates" -version = "0.1.0" -dependencies = [ - "display-error-chain", - "foreign-types 0.3.2", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "openssl", - "openssl-sys", - "rcgen", - "thiserror", -] - -[[package]] -name = "omicron-common" -version = "0.1.0" -dependencies = [ - "anyhow", - "api_identity 0.1.0", - "async-trait", - "backoff", - "camino", - "camino-tempfile", - "chrono", - "dropshot", - "expectorate", - "futures", - "hex", - "http", - "hyper", - "ipnetwork", - "lazy_static", - "libc", - "macaddr", - "omicron-workspace-hack", - "parse-display", - "progenitor", - "proptest", - "rand 0.8.5", - "regress", - "reqwest", - "ring 0.16.20", - "schemars", - "semver 1.0.20", - "serde", - "serde_derive", - "serde_human_bytes", - "serde_json", - "serde_urlencoded", - "serde_with", - "slog", - "strum", - "test-strategy", - "thiserror", - "tokio", - "tokio-postgres", - "toml 0.7.8", - "uuid", -] - -[[package]] -name = "omicron-common" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "anyhow", - "api_identity 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "async-trait", - "backoff", - "camino", - "chrono", - "dropshot", - "futures", - "hex", - "http", - "hyper", - "ipnetwork", - "lazy_static", - "macaddr", - "parse-display", - "progenitor", - "rand 0.8.5", - "reqwest", - "ring 0.16.20", - "schemars", - "semver 1.0.20", - "serde", - "serde_derive", - "serde_human_bytes", - "serde_json", - "serde_with", - "slog", - "strum", - "thiserror", - "tokio", - "tokio-postgres", - "toml 0.7.8", - "uuid", -] - -[[package]] -name = "omicron-deploy" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.4.3", - "crossbeam", - "omicron-package", - "omicron-workspace-hack", - "serde", - "serde_derive", - "thiserror", - "toml 0.7.8", -] - -[[package]] -name = "omicron-dev" -version = "0.1.0" -dependencies = [ - "anyhow", - "camino", - "camino-tempfile", - "clap 4.4.3", - "dropshot", - "expectorate", - "futures", - "gateway-messages", - "gateway-test-utils", - "libc", - "nexus-test-interface", - "nexus-test-utils", - "omicron-common 0.1.0", - "omicron-nexus", - "omicron-rpaths", - "omicron-sled-agent", - "omicron-test-utils", - "omicron-workspace-hack", - "openssl", - "oxide-client", - "pq-sys", - "rcgen", - "signal-hook", - "signal-hook-tokio", - "subprocess", - "tokio", - "tokio-postgres", - "toml 0.7.8", -] - -[[package]] -name = "omicron-gateway" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.5", - "ciborium", - "clap 4.4.3", - "dropshot", - "expectorate", - "futures", - "gateway-messages", - "gateway-sp-comms", - "gateway-test-utils", - "hex", - "http", - "hyper", - "illumos-utils", - "ipcc-key-value", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "once_cell", - "openapi-lint", - "openapiv3", - "schemars", - "serde", - "serde_human_bytes", - "serde_json", - "signal-hook", - "signal-hook-tokio", - "slog", - "slog-dtrace", - "sp-sim", - "subprocess", - "thiserror", - "tokio", - "tokio-stream", - "tokio-tungstenite 0.18.0", - "tokio-util", - "toml 0.7.8", - "uuid", -] - -[[package]] -name = "omicron-nexus" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert_matches", - "async-bb8-diesel", - "async-trait", - "base64 0.21.5", - "bb8", - "camino", - "cancel-safe-futures", - "chrono", - "clap 4.4.3", - "cookie", - "criterion", - "crucible-agent-client", - "crucible-pantry-client", - "diesel", - "dns-server", - "dns-service-client 0.1.0", - "dpd-client", - "dropshot", - "expectorate", - "fatfs", - "futures", - "gateway-client", - "gateway-messages", - "gateway-test-utils", - "headers", - "hex", - "http", - "httptest", - "hyper", - "hyper-rustls", - "internal-dns 0.1.0", - "ipnetwork", - "itertools 0.11.0", - "lazy_static", - "macaddr", - "mg-admin-client", - "mime_guess", - "newtype_derive", - "nexus-db-model", - "nexus-db-queries", - "nexus-defaults", - "nexus-inventory", - "nexus-test-interface", - "nexus-test-utils", - "nexus-test-utils-macros", - "nexus-types", - "num-integer", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-rpaths", - "omicron-sled-agent", - "omicron-test-utils", - "omicron-workspace-hack", - "once_cell", - "openapi-lint", - "openapiv3", - "openssl", - "openssl-probe", - "openssl-sys", - "oso", - "oxide-client", - "oximeter 0.1.0", - "oximeter-client", - "oximeter-db", - "oximeter-instruments", - "oximeter-producer 0.1.0", - "parse-display", - "paste", - "pem 1.1.1", - "petgraph", - "pq-sys", - "pretty_assertions", - "progenitor-client", - "propolis-client", - "rand 0.8.5", - "rcgen", - "ref-cast", - "regex", - "reqwest", - "ring 0.16.20", - "rustls", - "samael", - "schemars", - "semver 1.0.20", - "serde", - "serde_json", - "serde_urlencoded", - "serde_with", - "similar-asserts", - "sled-agent-client", - "slog", - "slog-async", - "slog-dtrace", - "slog-term", - "steno", - "strum", - "subprocess", - "tempfile", - "term", - "thiserror", - "tokio", - "tokio-postgres", - "toml 0.7.8", - "tough", - "trust-dns-resolver", - "usdt", - "uuid", -] - -[[package]] -name = "omicron-omdb" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-bb8-diesel", - "chrono", - "clap 4.4.3", - "diesel", - "dropshot", - "expectorate", - "futures", - "gateway-client", - "gateway-messages", - "gateway-test-utils", - "humantime", - "internal-dns 0.1.0", - "ipnetwork", - "nexus-client 0.1.0", - "nexus-db-model", - "nexus-db-queries", - "nexus-test-utils", - "nexus-test-utils-macros", - "nexus-types", - "omicron-common 0.1.0", - "omicron-nexus", - "omicron-rpaths", - "omicron-test-utils", - "omicron-workspace-hack", - "oximeter-client", - "pq-sys", - "regex", - "serde", - "serde_json", - "sled-agent-client", - "slog", - "strum", - "subprocess", - "tabled", - "textwrap 0.16.0", - "tokio", - "uuid", -] - -[[package]] -name = "omicron-package" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.4.3", - "expectorate", - "futures", - "hex", - "illumos-utils", - "indicatif", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "omicron-zone-package", - "petgraph", - "rayon", - "reqwest", - "ring 0.16.20", - "semver 1.0.20", - "serde", - "serde_derive", - "sled-hardware", - "slog", - "slog-async", - "slog-term", - "smf", - "strum", - "tar", - "tempfile", - "thiserror", - "tokio", - "toml 0.7.8", - "topological-sort", - "walkdir", -] - -[[package]] -name = "omicron-passwords" -version = "0.1.0" -dependencies = [ - "argon2", - "criterion", - "omicron-workspace-hack", - "rand 0.8.5", - "rust-argon2", - "schemars", - "serde", - "serde_with", - "thiserror", -] - -[[package]] -name = "omicron-passwords" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "argon2", - "rand 0.8.5", - "schemars", - "serde", - "serde_with", - "thiserror", -] - -[[package]] -name = "omicron-rpaths" -version = "0.1.0" -dependencies = [ - "omicron-workspace-hack", -] - -[[package]] -name = "omicron-sled-agent" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert_matches", - "async-trait", - "base64 0.21.5", - "bincode", - "bootstore", - "bootstrap-agent-client", - "bytes", - "camino", - "camino-tempfile", - "cancel-safe-futures", - "cfg-if", - "chrono", - "clap 4.4.3", - "crucible-agent-client", - "crucible-client-types", - "ddm-admin-client", - "derive_more", - "dns-server", - "dns-service-client 0.1.0", - "dpd-client", - "dropshot", - "expectorate", - "flate2", - "futures", - "gateway-client", - "glob", - "http", - "hyper", - "hyper-staticfile", - "illumos-utils", - "internal-dns 0.1.0", - "ipnetwork", - "itertools 0.11.0", - "key-manager", - "libc", - "macaddr", - "nexus-client 0.1.0", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "once_cell", - "openapi-lint", - "openapiv3", - "opte-ioctl", - "oximeter 0.1.0", - "oximeter-producer 0.1.0", - "percent-encoding", - "pretty_assertions", - "progenitor", - "propolis-client", - "propolis-server", - "rand 0.8.5", - "rcgen", - "reqwest", - "schemars", - "semver 1.0.20", - "serde", - "serde_json", - "serial_test", - "sha3", - "sled-agent-client", - "sled-hardware", - "slog", - "slog-async", - "slog-dtrace", - "slog-term", - "smf", - "static_assertions", - "subprocess", - "tar", - "tempfile", - "thiserror", - "tofino", - "tokio", - "tokio-tungstenite 0.18.0", - "toml 0.7.8", - "usdt", - "uuid", - "zeroize", - "zone", -] - -[[package]] -name = "omicron-test-utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "atomicwrites", - "camino", - "camino-tempfile", - "dropshot", - "expectorate", - "filetime", - "futures", - "headers", - "hex", - "http", - "libc", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "pem 1.1.1", - "rcgen", - "regex", - "reqwest", - "ring 0.16.20", - "rustls", - "slog", - "subprocess", - "tar", - "tempfile", - "thiserror", - "tokio", - "tokio-postgres", - "usdt", -] - -[[package]] -name = "omicron-workspace-hack" -version = "0.1.0" -dependencies = [ - "anyhow", - "bit-set", - "bit-vec", - "bitflags 1.3.2", - "bitflags 2.4.0", - "bitvec", - "bstr 0.2.17", - "bstr 1.6.0", - "bytes", - "cc", - "chrono", - "cipher", - "clap 4.4.3", - "clap_builder", - "console", - "const-oid", - "crossbeam-epoch", - "crossbeam-utils", - "crypto-common", - "diesel", - "digest", - "either", - "flate2", - "futures", - "futures-channel", - "futures-core", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", - "gateway-messages", - "generic-array", - "getrandom 0.2.10", - "hashbrown 0.13.2", - "hashbrown 0.14.0", - "hex", - "hyper", - "hyper-rustls", - "indexmap 2.0.0", - "inout", - "ipnetwork", - "itertools 0.10.5", - "lalrpop-util", - "lazy_static", - "libc", - "log", - "managed", - "memchr", - "mio", - "num-bigint", - "num-integer", - "num-iter", - "num-traits", - "once_cell", - "openapiv3", - "petgraph", - "postgres-types", - "ppv-lite86", - "predicates 3.0.4", - "rand 0.8.5", - "rand_chacha 0.3.1", - "regex", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", - "reqwest", - "ring 0.16.20", - "rustix 0.38.9", - "schemars", - "semver 1.0.20", - "serde", - "serde_json", - "sha2", - "signature 2.1.0", - "similar", - "slog", - "spin 0.9.8", - "string_cache", - "subtle", - "syn 1.0.109", - "syn 2.0.32", - "time", - "time-macros", - "tokio", - "tokio-postgres", - "tokio-stream", - "toml 0.7.8", - "toml_datetime", - "toml_edit 0.19.15", - "tracing", - "trust-dns-proto", - "unicode-bidi", - "unicode-normalization", - "unicode-xid", - "usdt", - "uuid", - "yasna", - "zeroize", - "zip", -] - -[[package]] -name = "omicron-zone-package" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dc0973625837d1c4e31d4aa60e72008f3af3aa9b0d0ebfd5b5dc67d2e721a48" -dependencies = [ - "anyhow", - "async-trait", - "chrono", - "filetime", - "flate2", - "futures-util", - "reqwest", - "semver 1.0.20", - "serde", - "serde_derive", - "tar", - "tempfile", - "thiserror", - "tokio", - "toml 0.7.8", - "walkdir", -] - -[[package]] -name = "once_cell" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" - -[[package]] -name = "oorandom" -version = "11.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" - -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - -[[package]] -name = "openapi-lint" -version = "0.4.0" -source = "git+https://github.com/oxidecomputer/openapi-lint?branch=main#bb69a3a4a184d966bac2a0df2be5c9038d9867d0" -dependencies = [ - "heck 0.4.1", - "indexmap 2.0.0", - "lazy_static", - "openapiv3", - "regex", -] - -[[package]] -name = "openapiv3" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75e56d5c441965b6425165b7e3223cc933ca469834f4a8b4786817a1f9dc4f13" -dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_json", -] - -[[package]] -name = "openssl" -version = "0.10.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" -dependencies = [ - "bitflags 2.4.0", - "cfg-if", - "foreign-types 0.3.2", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.93" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "opte" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=faaf3320e673167722c2c7e6fb2a86bd84f04a8a#faaf3320e673167722c2c7e6fb2a86bd84f04a8a" -dependencies = [ - "cfg-if", - "dyn-clone", - "heapless", - "illumos-sys-hdrs", - "kstat-macro", - "opte-api", - "postcard", - "serde", - "smoltcp 0.10.0", - "version_check", - "zerocopy 0.7.21", -] - -[[package]] -name = "opte-api" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=faaf3320e673167722c2c7e6fb2a86bd84f04a8a#faaf3320e673167722c2c7e6fb2a86bd84f04a8a" -dependencies = [ - "illumos-sys-hdrs", - "ipnetwork", - "postcard", - "serde", - "smoltcp 0.10.0", -] - -[[package]] -name = "opte-ioctl" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=faaf3320e673167722c2c7e6fb2a86bd84f04a8a#faaf3320e673167722c2c7e6fb2a86bd84f04a8a" -dependencies = [ - "libc", - "libnet", - "opte", - "oxide-vpc", - "postcard", - "serde", - "thiserror", -] - -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - -[[package]] -name = "oso" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fceecc04a9e9dcb63a42d937a4249557da8d2695cf83eb5ee78015473ab12ae2" -dependencies = [ - "impl-trait-for-tuples", - "lazy_static", - "maplit", - "oso-derive", - "polar-core", - "thiserror", - "tracing", -] - -[[package]] -name = "oso-derive" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1766857f83748ce5596ab98e1a57d64ccfe3259e71b7b53289c8c32c2cfef9a8" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "owo-colors" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" - -[[package]] -name = "oxide-client" -version = "0.1.0" -dependencies = [ - "anyhow", - "base64 0.21.5", - "chrono", - "futures", - "http", - "hyper", - "omicron-workspace-hack", - "progenitor", - "rand 0.8.5", - "regress", - "reqwest", - "serde", - "serde_json", - "thiserror", - "tokio", - "trust-dns-resolver", - "uuid", -] - -[[package]] -name = "oxide-vpc" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=faaf3320e673167722c2c7e6fb2a86bd84f04a8a#faaf3320e673167722c2c7e6fb2a86bd84f04a8a" -dependencies = [ - "illumos-sys-hdrs", - "opte", - "serde", - "smoltcp 0.10.0", - "zerocopy 0.7.21", -] - -[[package]] -name = "oximeter" -version = "0.1.0" -dependencies = [ - "approx", - "bytes", - "chrono", - "num", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "oximeter-macro-impl 0.1.0", - "rstest", - "schemars", - "serde", - "strum", - "thiserror", - "trybuild", - "uuid", -] - -[[package]] -name = "oximeter" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "bytes", - "chrono", - "num-traits", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter-macro-impl 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "schemars", - "serde", - "thiserror", - "uuid", -] - -[[package]] -name = "oximeter-client" -version = "0.1.0" -dependencies = [ - "chrono", - "futures", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "progenitor", - "reqwest", - "serde", - "slog", - "uuid", -] - -[[package]] -name = "oximeter-collector" -version = "0.1.0" -dependencies = [ - "anyhow", - "clap 4.4.3", - "dropshot", - "expectorate", - "futures", - "internal-dns 0.1.0", - "nexus-client 0.1.0", - "nexus-types", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "openapi-lint", - "openapiv3", - "oximeter 0.1.0", - "oximeter-client", - "oximeter-db", - "rand 0.8.5", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-dtrace", - "slog-term", - "subprocess", - "thiserror", - "tokio", - "toml 0.7.8", - "uuid", -] - -[[package]] -name = "oximeter-db" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "bcs", - "bytes", - "chrono", - "clap 4.4.3", - "dropshot", - "expectorate", - "highway", - "itertools 0.11.0", - "omicron-test-utils", - "omicron-workspace-hack", - "oximeter 0.1.0", - "regex", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "slog-async", - "slog-dtrace", - "slog-term", - "strum", - "thiserror", - "tokio", - "usdt", - "uuid", -] - -[[package]] -name = "oximeter-instruments" -version = "0.1.0" -dependencies = [ - "chrono", - "dropshot", - "futures", - "http", - "omicron-workspace-hack", - "oximeter 0.1.0", - "tokio", - "uuid", -] - -[[package]] -name = "oximeter-macro-impl" -version = "0.1.0" -dependencies = [ - "omicron-workspace-hack", - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "oximeter-macro-impl" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "oximeter-producer" -version = "0.1.0" -dependencies = [ - "anyhow", - "chrono", - "clap 4.4.3", - "dropshot", - "nexus-client 0.1.0", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "oximeter 0.1.0", - "reqwest", - "schemars", - "serde", - "slog", - "slog-dtrace", - "thiserror", - "tokio", - "uuid", -] - -[[package]] -name = "oximeter-producer" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/omicron?branch=main#3dcc8d2eb648c87b42454882a2ce024b409cbb8c" -dependencies = [ - "chrono", - "dropshot", - "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "reqwest", - "schemars", - "serde", - "slog", - "slog-dtrace", - "thiserror", - "tokio", - "uuid", -] - -[[package]] -name = "packed_struct" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b29691432cc9eff8b282278473b63df73bea49bc3ec5e67f31a3ae9c3ec190" -dependencies = [ - "bitvec", - "packed_struct_codegen", - "serde", -] - -[[package]] -name = "packed_struct_codegen" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd6706dfe50d53e0f6aa09e12c034c44faacd23e966ae5a209e8bdb8f179f98" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "papergrid" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ccbe15f2b6db62f9a9871642746427e297b0ceb85f9a7f1ee5ff47d184d0c8" -dependencies = [ - "bytecount", - "fnv", - "unicode-width", -] - -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core 0.9.8", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec 1.11.0", - "winapi", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.3.5", - "smallvec 1.11.0", - "windows-targets 0.48.5", -] - -[[package]] -name = "parse-display" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6b32f6c8212838b74c0f5ba412194e88897923020810d9bec72d3594c2588d" -dependencies = [ - "once_cell", - "parse-display-derive", - "regex", -] - -[[package]] -name = "parse-display-derive" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6ec9ab2477935d04fcdf7c51c9ee94a1be988938886de3239aed40980b7180" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "regex", - "regex-syntax 0.6.29", - "structmeta 0.1.6", - "syn 1.0.109", -] - -[[package]] -name = "partial-io" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af95cf22649f58b48309da6d05caeb5fab4bb335eba4a3f9ac7c3a8e176d0e16" -dependencies = [ - "futures", - "pin-project", - "proptest", - "tokio", -] - -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "password-hash" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - -[[package]] -name = "paste" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" - -[[package]] -name = "path-absolutize" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43eb3595c63a214e1b37b44f44b0a84900ef7ae0b4c5efce59e123d246d7a0de" -dependencies = [ - "path-dedot", -] - -[[package]] -name = "path-dedot" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d55e486337acb9973cdea3ec5638c1b3bcb22e573b2b7b41969e0c744d5a15e" -dependencies = [ - "once_cell", -] - -[[package]] -name = "path-slash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "498a099351efa4becc6a19c72aa9270598e8fd274ca47052e37455241c88b696" - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash 0.4.2", - "sha2", -] - -[[package]] -name = "pbkdf2" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" -dependencies = [ - "digest", -] - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - -[[package]] -name = "pem" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" -dependencies = [ - "base64 0.21.5", - "serde", -] - -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - -[[package]] -name = "percent-encoding" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" - -[[package]] -name = "pest" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" -dependencies = [ - "thiserror", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "pest_meta" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" -dependencies = [ - "once_cell", - "pest", - "sha2", -] - -[[package]] -name = "petgraph" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" -dependencies = [ - "fixedbitset", - "indexmap 2.0.0", - "serde", - "serde_derive", -] - -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_shared 0.11.2", -] - -[[package]] -name = "phf_shared" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der", - "spki", -] - -[[package]] -name = "pkg-config" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" - -[[package]] -name = "platforms" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" - -[[package]] -name = "plotters" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" - -[[package]] -name = "plotters-svg" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "polar-core" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1b77e852bec994296c8a1dddc231ab3f112bfa0a0399fc8a7fd8bddfb46b4e" -dependencies = [ - "indoc 1.0.9", - "js-sys", - "lalrpop", - "lalrpop-util", - "serde", - "serde_derive", - "strum_macros 0.24.3", - "wasm-bindgen", -] - -[[package]] -name = "poly1305" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" -dependencies = [ - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "polyval" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52cff9d1d4dee5fe6d03729099f4a310a41179e0a10dbf542039873f2e826fb" -dependencies = [ - "cfg-if", - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "portable-atomic" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" - -[[package]] -name = "postcard" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8" -dependencies = [ - "cobs", - "embedded-io", - "serde", -] - -[[package]] -name = "postgres-protocol" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" -dependencies = [ - "base64 0.21.5", - "byteorder", - "bytes", - "fallible-iterator", - "hmac", - "md-5", - "memchr", - "rand 0.8.5", - "sha2", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" -dependencies = [ - "bytes", - "chrono", - "fallible-iterator", - "postgres-protocol", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "pq-sys" -version = "0.4.6" -source = "git+https://github.com/oxidecomputer/pq-sys?branch=oxide/omicron#b1194c190f4d4a103c2280908cd1e97628c5c1cb" -dependencies = [ - "vcpkg", -] - -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" - -[[package]] -name = "predicates" -version = "2.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" -dependencies = [ - "difflib", - "float-cmp", - "itertools 0.10.5", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dfc28575c2e3f19cb3c73b93af36460ae898d426eba6fc15b9bd2a5220758a0" -dependencies = [ - "anstyle", - "difflib", - "float-cmp", - "itertools 0.11.0", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" - -[[package]] -name = "predicates-tree" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" -dependencies = [ - "predicates-core", - "termtree", -] - -[[package]] -name = "pretty-hex" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5c99d529f0d30937f6f4b8a86d988047327bb88d04d2c4afc356de74722131" - -[[package]] -name = "pretty-hex" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" - -[[package]] -name = "pretty_assertions" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" -dependencies = [ - "diff", - "yansi", -] - -[[package]] -name = "prettyplease" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" -dependencies = [ - "proc-macro2", - "syn 2.0.32", -] - -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - -[[package]] -name = "proc-macro2" -version = "1.0.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "progenitor" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" -dependencies = [ - "progenitor-client", - "progenitor-impl", - "progenitor-macro", - "serde_json", -] - -[[package]] -name = "progenitor-client" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" -dependencies = [ - "bytes", - "futures-core", - "percent-encoding", - "reqwest", - "serde", - "serde_json", - "serde_urlencoded", -] - -[[package]] -name = "progenitor-impl" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" -dependencies = [ - "getopts", - "heck 0.4.1", - "http", - "indexmap 2.0.0", - "openapiv3", - "proc-macro2", - "quote", - "regex", - "schemars", - "serde", - "serde_json", - "syn 2.0.32", - "thiserror", - "typify", - "unicode-ident", -] - -[[package]] -name = "progenitor-macro" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/progenitor?branch=main#5c941c0b41b0235031f3ade33a9c119945f1fd51" -dependencies = [ - "openapiv3", - "proc-macro2", - "progenitor-impl", - "quote", - "schemars", - "serde", - "serde_json", - "serde_tokenstream 0.2.0", - "serde_yaml", - "syn 2.0.32", -] - -[[package]] -name = "propolis" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "anyhow", - "bhyve_api", - "bitflags 2.4.0", - "bitstruct", - "byteorder", - "crucible", - "crucible-client-types", - "dladm", - "erased-serde", - "futures", - "lazy_static", - "libc", - "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "propolis_types", - "rfb", - "serde", - "serde_arrays", - "serde_json", - "slog", - "strum", - "thiserror", - "tokio", - "usdt", - "uuid", - "viona_api", -] - -[[package]] -name = "propolis-client" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "async-trait", - "base64 0.21.5", - "crucible-client-types", - "futures", - "progenitor", - "propolis_types", - "rand 0.8.5", - "reqwest", - "ring 0.16.20", - "schemars", - "serde", - "serde_json", - "slog", - "thiserror", - "tokio", - "tokio-tungstenite 0.20.1", - "uuid", -] - -[[package]] -name = "propolis-server" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "anyhow", - "async-trait", - "atty", - "base64 0.21.5", - "bit_field", - "bitvec", - "bytes", - "cfg-if", - "chrono", - "clap 4.4.3", - "const_format", - "crucible-client-types", - "dropshot", - "erased-serde", - "futures", - "http", - "hyper", - "internal-dns 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "lazy_static", - "nexus-client 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "omicron-common 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "oximeter-producer 0.1.0 (git+https://github.com/oxidecomputer/omicron?branch=main)", - "propolis", - "propolis-client", - "propolis-server-config", - "rfb", - "ron 0.7.1", - "schemars", - "serde", - "serde_derive", - "serde_json", - "slog", - "slog-async", - "slog-bunyan", - "slog-dtrace", - "slog-term", - "strum", - "thiserror", - "tokio", - "tokio-tungstenite 0.20.1", - "tokio-util", - "toml 0.7.8", - "usdt", - "uuid", -] - -[[package]] -name = "propolis-server-config" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "cpuid_profile_config", - "serde", - "serde_derive", - "thiserror", - "toml 0.7.8", -] - -[[package]] -name = "propolis_types" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" -dependencies = [ - "schemars", - "serde", -] - -[[package]] -name = "proptest" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e" -dependencies = [ - "bit-set", - "bit-vec", - "bitflags 2.4.0", - "lazy_static", - "num-traits", - "rand 0.8.5", - "rand_chacha 0.3.1", - "rand_xorshift", - "regex-syntax 0.7.5", - "rusty-fork", - "tempfile", - "unarray", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quick-xml" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11bafc859c6815fbaffbbbf4229ecb767ac913fecb27f9ad4343662e9ef099ea" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quote" -version = "1.0.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot 0.12.1", - "scheduled-thread-pool", -] - -[[package]] -name = "radium" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - -[[package]] -name = "radix_trie" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" -dependencies = [ - "endian-type", - "nibble_vec", -] - -[[package]] -name = "rand" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" -dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", -] - -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.10", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_xorshift" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" -dependencies = [ - "rand_core 0.6.4", -] - -[[package]] -name = "ratatui" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e2e4cd95294a85c3b4446e63ef054eea43e0205b1fd60120c16b74ff7ff96ad" -dependencies = [ - "bitflags 2.4.0", - "cassowary", - "crossterm 0.27.0", - "indoc 2.0.3", - "itertools 0.11.0", - "paste", - "strum", - "unicode-segmentation", - "unicode-width", -] - -[[package]] -name = "rayon" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rcgen" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6" -dependencies = [ - "pem 3.0.2", - "ring 0.16.20", - "time", - "yasna", -] - -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", - "thiserror", -] - -[[package]] -name = "reedline" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3defc4467a8909614bcb02cb304a3e8472c31f7a44e6c6c287eb9575b212bc4d" -dependencies = [ - "chrono", - "crossterm 0.26.1", - "fd-lock", - "itertools 0.10.5", - "nu-ansi-term", - "serde", - "strip-ansi-escapes", - "strum", - "strum_macros 0.25.2", - "thiserror", - "unicode-segmentation", - "unicode-width", -] - -[[package]] -name = "ref-cast" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acde58d073e9c79da00f2b5b84eed919c8326832648a5b109b3fce1bb1175280" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "regex" -version = "1.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata 0.4.3", - "regex-syntax 0.8.2", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - -[[package]] -name = "regex-automata" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" - -[[package]] -name = "regex-automata" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax 0.8.2", -] - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - -[[package]] -name = "regex-syntax" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" - -[[package]] -name = "regress" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed9969cad8051328011596bf549629f1b800cf1731e7964b1eef8dfc480d2c2" -dependencies = [ - "hashbrown 0.13.2", - "memchr", -] - -[[package]] -name = "relative-path" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - -[[package]] -name = "reqwest" -version = "0.11.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" -dependencies = [ - "base64 0.21.5", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-native-tls", - "tokio-rustls", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "webpki-roots", - "winreg", -] - -[[package]] -name = "resolv-conf" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" -dependencies = [ - "hostname", - "quick-error", -] - -[[package]] -name = "rfb" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/rfb?rev=0cac8d9c25eb27acfa35df80f3b9d371de98ab3b#0cac8d9c25eb27acfa35df80f3b9d371de98ab3b" -dependencies = [ - "ascii", - "async-trait", - "bitflags 1.3.2", - "env_logger 0.9.3", - "futures", - "log", - "thiserror", - "tokio", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - -[[package]] -name = "ring" -version = "0.17.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" -dependencies = [ - "cc", - "getrandom 0.2.10", - "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys 0.48.0", -] - -[[package]] -name = "ringbuffer" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53" - -[[package]] -name = "ron" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" -dependencies = [ - "base64 0.13.1", - "bitflags 1.3.2", - "serde", -] - -[[package]] -name = "ron" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" -dependencies = [ - "base64 0.21.5", - "bitflags 2.4.0", - "serde", - "serde_derive", -] - -[[package]] -name = "rpassword" -version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" -dependencies = [ - "libc", - "rtoolbox", - "winapi", -] - -[[package]] -name = "rsa" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab43bb47d23c1a631b4b680199a45255dce26fa9ab2fa902581f624ff13e6a8" -dependencies = [ - "byteorder", - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-iter", - "num-traits", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "serde", - "sha2", - "signature 2.1.0", - "spki", - "subtle", - "zeroize", -] - -[[package]] -name = "rstest" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" -dependencies = [ - "futures", - "futures-timer", - "rstest_macros", - "rustc_version 0.4.0", -] - -[[package]] -name = "rstest_macros" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" -dependencies = [ - "cfg-if", - "glob", - "proc-macro2", - "quote", - "regex", - "relative-path", - "rustc_version 0.4.0", - "syn 2.0.32", - "unicode-ident", -] - -[[package]] -name = "rtoolbox" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "rusqlite" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" -dependencies = [ - "bitflags 2.4.0", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec 1.11.0", -] - -[[package]] -name = "russh" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae0efcc0f4cd6c062c07e572ce4b806e3967fa029fcbfcc0aa98fb5910a37925" -dependencies = [ - "aes", - "aes-gcm", - "async-trait", - "bitflags 2.4.0", - "byteorder", - "chacha20", - "ctr", - "curve25519-dalek", - "digest", - "flate2", - "futures", - "generic-array", - "hex-literal", - "hmac", - "log", - "num-bigint", - "once_cell", - "poly1305", - "rand 0.8.5", - "russh-cryptovec", - "russh-keys", - "sha1", - "sha2", - "subtle", - "thiserror", - "tokio", - "tokio-util", -] - -[[package]] -name = "russh-cryptovec" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fdf036c2216b554053d19d4af45c1722d13b00ac494ea19825daf4beac034e" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "russh-keys" -version = "0.38.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "557ab9190022dff78116ebed5e391abbd3f424b06cd643dfe262346ab91ed8c9" -dependencies = [ - "aes", - "async-trait", - "bcrypt-pbkdf", - "bit-vec", - "block-padding", - "byteorder", - "cbc", - "ctr", - "data-encoding", - "dirs", - "ed25519-dalek", - "futures", - "hmac", - "inout", - "log", - "md5", - "num-bigint", - "num-integer", - "pbkdf2 0.11.0", - "rand 0.7.3", - "rand_core 0.6.4", - "russh-cryptovec", - "serde", - "sha2", - "thiserror", - "tokio", - "tokio-stream", - "yasna", -] - -[[package]] -name = "rust-argon2" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5885493fdf0be6cdff808d1533ce878d21cfa49c7086fa00c66355cd9141bfc" -dependencies = [ - "base64 0.21.5", - "blake2b_simd", - "constant_time_eq 0.3.0", - "crossbeam-utils", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "rustc_version" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084" -dependencies = [ - "semver 0.1.20", -] - -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver 1.0.20", -] - -[[package]] -name = "rustfmt-wrapper" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed729e3bee08ec2befd593c27e90ca9fdd25efdc83c94c3b82eaef16e4f7406e" -dependencies = [ - "serde", - "tempfile", - "thiserror", - "toml 0.5.11", - "toolchain_find", -] - -[[package]] -name = "rustix" -version = "0.37.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustix" -version = "0.38.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" -dependencies = [ - "bitflags 2.4.0", - "errno", - "libc", - "linux-raw-sys 0.4.5", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustls" -version = "0.21.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" -dependencies = [ - "log", - "ring 0.17.5", - "rustls-webpki", - "sct", -] - -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" -dependencies = [ - "base64 0.21.5", -] - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring 0.17.5", - "untrusted 0.9.0", -] - -[[package]] -name = "rustversion" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" - -[[package]] -name = "rusty-doors" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/rusty-doors#42ad0104095425eea76934b5d735b4c6a438ef66" -dependencies = [ - "libc", - "rusty-doors-macros", -] - -[[package]] -name = "rusty-doors-macros" -version = "0.1.0" -source = "git+https://github.com/oxidecomputer/rusty-doors#42ad0104095425eea76934b5d735b4c6a438ef66" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "rusty-fork" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" -dependencies = [ - "fnv", - "quick-error", - "tempfile", - "wait-timeout", -] - -[[package]] -name = "ryu" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" - -[[package]] -name = "salty" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77cdd38ed8bfe51e53ee991aae0791b94349d0a05cfdecd283835a8a965d4c37" -dependencies = [ - "ed25519 1.5.3", - "subtle", - "zeroize", -] - -[[package]] -name = "samael" -version = "0.0.10" -source = "git+https://github.com/njaremko/samael?branch=master#52028e45d11ceb7114bf0c730a9971207e965602" -dependencies = [ - "base64 0.21.5", - "bindgen", - "chrono", - "data-encoding", - "derive_builder", - "flate2", - "lazy_static", - "libc", - "libxml", - "openssl", - "openssl-probe", - "openssl-sys", - "pkg-config", - "quick-xml", - "rand 0.8.5", - "serde", - "snafu", - "url", - "uuid", -] - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" -dependencies = [ - "windows-sys 0.48.0", -] - -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot 0.12.1", -] - -[[package]] -name = "schemars" -version = "0.8.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763f8cd0d4c71ed8389c90cb8100cba87e763bd01a8e614d4f0af97bcd50a161" -dependencies = [ - "bytes", - "chrono", - "dyn-clone", - "schemars_derive", - "serde", - "serde_json", - "uuid", -] - -[[package]] -name = "schemars_derive" -version = "0.8.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0f696e21e10fa546b7ffb1c9672c6de8fbc7a81acf59524386d8639bf12737" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 1.0.109", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sct" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" -dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", -] - -[[package]] -name = "secrecy" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e" -dependencies = [ - "zeroize", -] - -[[package]] -name = "security-framework" -version = "2.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" - -[[package]] -name = "semver" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser", -] - -[[package]] -name = "semver" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" -dependencies = [ - "serde", -] - -[[package]] -name = "semver-parser" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" -dependencies = [ - "pest", -] - -[[package]] -name = "serde" -version = "1.0.188" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde-big-array" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3323f09a748af288c3dc2474ea6803ee81f118321775bffa3ac8f7e65c5e90e7" -dependencies = [ - "serde", -] - -[[package]] -name = "serde-big-array" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" -dependencies = [ - "serde", -] - -[[package]] -name = "serde-hex" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca37e3e4d1b39afd7ff11ee4e947efae85adfddf4841787bfa47c470e96dc26d" -dependencies = [ - "array-init", - "serde", - "smallvec 0.6.14", -] - -[[package]] -name = "serde_arrays" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38636132857f68ec3d5f3eb121166d2af33cb55174c4d5ff645db6165cbef0fd" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_derive" -version = "1.0.188" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "serde_derive_internals" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "serde_human_bytes" -version = "0.1.0" -source = "git+http://github.com/oxidecomputer/serde_human_bytes?branch=main#0a09794501b6208120528c3b457d5f3a8cb17424" -dependencies = [ - "hex", - "serde", -] - -[[package]] -name = "serde_json" -version = "1.0.108" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" -dependencies = [ - "itoa", - "serde", -] - -[[package]] -name = "serde_plain" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_repr" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "serde_spanned" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_tokenstream" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "797ba1d80299b264f3aac68ab5d12e5825a561749db4df7cd7c8083900c5d4e9" -dependencies = [ - "proc-macro2", - "serde", - "syn 1.0.109", -] - -[[package]] -name = "serde_tokenstream" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a00ffd23fd882d096f09fcaae2a9de8329a328628e86027e049ee051dc1621f" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "syn 2.0.32", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" -dependencies = [ - "base64 0.13.1", - "chrono", - "hex", - "indexmap 1.9.3", - "serde", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" -dependencies = [ - "darling 0.20.3", - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "serde_yaml" -version = "0.9.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" -dependencies = [ - "indexmap 2.0.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "serial_test" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c789ec87f4687d022a2405cf46e0cd6284889f1839de292cadeb6c6019506f2" -dependencies = [ - "dashmap", - "futures", - "lazy_static", - "log", - "parking_lot 0.12.1", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b64f9e531ce97c88b4778aad0ceee079216071cffec6ac9b904277f8f92e7fe3" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha3" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" -dependencies = [ - "digest", - "keccak", -] - -[[package]] -name = "shell-words" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" - -[[package]] -name = "shlex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" - -[[package]] -name = "signal-hook" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" -dependencies = [ - "libc", - "mio", - "signal-hook", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "signal-hook-tokio" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213241f76fb1e37e27de3b6aa1b068a2c333233b59cca6634f634b80a27ecf1e" -dependencies = [ - "futures-core", - "libc", - "signal-hook", - "tokio", -] - -[[package]] -name = "signature" -version = "1.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" - -[[package]] -name = "signature" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" -dependencies = [ - "digest", - "rand_core 0.6.4", -] - -[[package]] -name = "similar" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" -dependencies = [ - "bstr 0.2.17", - "unicode-segmentation", -] - -[[package]] -name = "similar-asserts" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e041bb827d1bfca18f213411d51b665309f1afb37a04a5d1464530e13779fc0f" -dependencies = [ - "console", - "similar", -] - -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] - -[[package]] -name = "sled-agent-client" -version = "0.1.0" -dependencies = [ - "async-trait", - "chrono", - "ipnetwork", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "progenitor", - "regress", - "reqwest", - "serde", - "slog", - "uuid", -] - -[[package]] -name = "sled-hardware" -version = "0.1.0" -dependencies = [ - "anyhow", - "camino", - "cfg-if", - "futures", - "illumos-devinfo", - "illumos-utils", - "key-manager", - "libc", - "libefi-illumos", - "macaddr", - "nexus-client 0.1.0", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "rand 0.8.5", - "schemars", - "serde", - "serial_test", - "slog", - "thiserror", - "tofino", - "tokio", - "uuid", -] - -[[package]] -name = "slog" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" - -[[package]] -name = "slog-async" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" -dependencies = [ - "crossbeam-channel", - "slog", - "take_mut", - "thread_local", -] - -[[package]] -name = "slog-bunyan" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "440fd32d0423c31e4f98d76c0b62ebdb847f905aa07357197e9b41ac620af97d" -dependencies = [ - "hostname", - "slog", - "slog-json", - "time", -] - -[[package]] -name = "slog-dtrace" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb79013d51afb48c5159d62068658fa672772be3aeeadee0d2710fb3903f637" -dependencies = [ - "chrono", - "serde", - "serde_json", - "slog", - "usdt", - "version_check", -] - -[[package]] -name = "slog-envlogger" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" -dependencies = [ - "log", - "regex", - "slog", - "slog-async", - "slog-scope", - "slog-stdlog", - "slog-term", -] - -[[package]] -name = "slog-json" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" -dependencies = [ - "serde", - "serde_json", - "slog", - "time", -] - -[[package]] -name = "slog-scope" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" -dependencies = [ - "arc-swap", - "lazy_static", - "slog", -] - -[[package]] -name = "slog-stdlog" -version = "4.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" -dependencies = [ - "log", - "slog", - "slog-scope", -] - -[[package]] -name = "slog-term" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" -dependencies = [ - "atty", - "slog", - "term", - "thread_local", - "time", -] - -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - -[[package]] -name = "smallvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" - -[[package]] -name = "smawk" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" - -[[package]] -name = "smf" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6015a9bbf269b84c928dc68e11680bbdfa6f065f1c6d5383ec134f55bab188b" -dependencies = [ - "thiserror", -] - -[[package]] -name = "smoltcp" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9786ac45091b96f946693e05bfa4d8ca93e2d3341237d97a380107a6b38dea" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "cfg-if", - "heapless", - "managed", -] - -[[package]] -name = "smoltcp" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d2e3a36ac8fea7b94e666dfa3871063d6e0a5c9d5d4fec9a1a6b7b6760f0229" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "cfg-if", - "defmt", - "heapless", - "managed", -] - -[[package]] -name = "snafu" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" -dependencies = [ - "doc-comment", - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" -dependencies = [ - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "sp-sim" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "clap 4.4.3", - "dropshot", - "futures", - "gateway-messages", - "hex", - "omicron-common 0.1.0", - "omicron-gateway", - "omicron-workspace-hack", - "serde", - "slog", - "slog-dtrace", - "sprockets-rot", - "thiserror", - "tokio", - "toml 0.7.8", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - -[[package]] -name = "spki" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1e996ef02c474957d681f1b05213dfb0abab947b446a62d37770b23500184a" -dependencies = [ - "base64ct", - "der", -] - -[[package]] -name = "sprockets-common" -version = "0.1.0" -source = "git+http://github.com/oxidecomputer/sprockets?rev=77df31efa5619d0767ffc837ef7468101608aee9#77df31efa5619d0767ffc837ef7468101608aee9" -dependencies = [ - "derive_more", - "hubpack 0.1.0", - "salty", - "serde", - "serde-big-array 0.4.1", -] - -[[package]] -name = "sprockets-rot" -version = "0.1.0" -source = "git+http://github.com/oxidecomputer/sprockets?rev=77df31efa5619d0767ffc837ef7468101608aee9#77df31efa5619d0767ffc837ef7468101608aee9" -dependencies = [ - "corncobs", - "derive_more", - "hubpack 0.1.0", - "rand 0.8.5", - "salty", - "serde", - "sprockets-common", - "tinyvec", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "steno" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a1e7ccea133c197729abfd16dccf91a3c4d0da1e94bb0c0aa164c2b8a227481" -dependencies = [ - "anyhow", - "async-trait", - "chrono", - "futures", - "lazy_static", - "newtype_derive", - "petgraph", - "schemars", - "serde", - "serde_json", - "slog", - "thiserror", - "tokio", - "uuid", -] - -[[package]] -name = "string_cache" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" -dependencies = [ - "new_debug_unreachable", - "once_cell", - "parking_lot 0.12.1", - "phf_shared 0.10.0", - "precomputed-hash", - "serde", -] - -[[package]] -name = "stringprep" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3737bde7edce97102e0e2b15365bf7a20bfdb5f60f4f9e8d7004258a51a8da" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "strip-ansi-escapes" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" -dependencies = [ - "vte", -] - -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "structmeta" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104842d6278bf64aa9d2f182ba4bde31e8aec7a131d29b7f444bb9b344a09e2a" -dependencies = [ - "proc-macro2", - "quote", - "structmeta-derive 0.1.6", - "syn 1.0.109", -] - -[[package]] -name = "structmeta" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" -dependencies = [ - "proc-macro2", - "quote", - "structmeta-derive 0.2.0", - "syn 2.0.32", -] - -[[package]] -name = "structmeta-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24420be405b590e2d746d83b01f09af673270cf80e9b003a5fa7b651c58c7d93" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "structmeta-derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "structopt" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" -dependencies = [ - "clap 2.34.0", - "lazy_static", - "structopt-derive", -] - -[[package]] -name = "structopt-derive" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" -dependencies = [ - "heck 0.3.3", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "strum" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" -dependencies = [ - "strum_macros 0.25.2", -] - -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", -] - -[[package]] -name = "strum_macros" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.32", -] - -[[package]] -name = "subprocess" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "unicode-xid", -] - -[[package]] -name = "tabled" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe9c3632da101aba5131ed63f9eed38665f8b3c68703a6bb18124835c1a5d22" -dependencies = [ - "papergrid", - "tabled_derive", - "unicode-width", -] - -[[package]] -name = "tabled_derive" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99f688a08b54f4f02f0a3c382aefdb7884d3d69609f785bd253dc033243e3fe4" -dependencies = [ - "heck 0.4.1", - "proc-macro-error", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - -[[package]] -name = "tap" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - -[[package]] -name = "tar" -version = "0.4.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" -dependencies = [ - "filetime", - "libc", - "xattr", -] - -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] - -[[package]] -name = "tempfile" -version = "3.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" -dependencies = [ - "cfg-if", - "fastrand", - "redox_syscall 0.3.5", - "rustix 0.38.9", - "windows-sys 0.48.0", -] - -[[package]] -name = "term" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" -dependencies = [ - "dirs-next", - "rustversion", - "winapi", -] - -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "terminal_size" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" -dependencies = [ - "rustix 0.37.23", - "windows-sys 0.48.0", -] - -[[package]] -name = "termios" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" -dependencies = [ - "libc", -] - -[[package]] -name = "termtree" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" - -[[package]] -name = "test-strategy" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8361c808554228ad09bfed70f5c823caf8a3450b6881cc3a38eb57e8c08c1d9" -dependencies = [ - "proc-macro2", - "quote", - "structmeta 0.2.0", - "syn 2.0.32", -] - -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" -dependencies = [ - "smawk", - "unicode-linebreak", - "unicode-width", -] - -[[package]] -name = "thiserror" -version = "1.0.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "thiserror-impl-no-std" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e6318948b519ba6dc2b442a6d0b904ebfb8d411a3ad3e07843615a72249758" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "thiserror-no-std" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ad459d94dd517257cc96add8a43190ee620011bb6e6cdc82dafd97dfafafea" -dependencies = [ - "thiserror-impl-no-std", -] - -[[package]] -name = "thread-id" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79474f573561cdc4871a0de34a51c92f7f5a56039113fbb5b9c9f96bdb756669" -dependencies = [ - "libc", - "redox_syscall 0.2.16", - "winapi", -] - -[[package]] -name = "thread_local" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" -dependencies = [ - "cfg-if", - "once_cell", -] - -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "time" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07" -dependencies = [ - "deranged", - "itoa", - "libc", - "num_threads", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" - -[[package]] -name = "time-macros" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9" -dependencies = [ - "time-core", -] - -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tlvc" -version = "0.3.1" -source = "git+https://github.com/oxidecomputer/tlvc.git?branch=main#e644a21a7ca973ed31499106ea926bd63ebccc6f" -dependencies = [ - "byteorder", - "crc", - "zerocopy 0.6.4", -] - -[[package]] -name = "tlvc" -version = "0.3.1" -source = "git+https://github.com/oxidecomputer/tlvc.git#e644a21a7ca973ed31499106ea926bd63ebccc6f" -dependencies = [ - "byteorder", - "crc", - "zerocopy 0.6.4", -] - -[[package]] -name = "tlvc-text" -version = "0.3.0" -source = "git+https://github.com/oxidecomputer/tlvc.git#e644a21a7ca973ed31499106ea926bd63ebccc6f" -dependencies = [ - "ron 0.8.1", - "serde", - "tlvc 0.3.1 (git+https://github.com/oxidecomputer/tlvc.git)", - "zerocopy 0.6.4", -] - -[[package]] -name = "tofino" -version = "0.1.0" -source = "git+http://github.com/oxidecomputer/tofino?branch=main#8283f8021068f055484b653f0cc6b4d5c0979dc1" -dependencies = [ - "anyhow", - "cc", - "chrono", - "illumos-devinfo", - "structopt", -] - -[[package]] -name = "tokio" -version = "1.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "parking_lot 0.12.1", - "pin-project-lite", - "signal-hook-registry", - "socket2 0.5.4", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-postgres" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures-channel", - "futures-util", - "log", - "parking_lot 0.12.1", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "rand 0.8.5", - "socket2 0.5.4", - "tokio", - "tokio-util", - "whoami", -] - -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.18.0", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.20.1", -] - -[[package]] -name = "tokio-util" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toml" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.19.15", -] - -[[package]] -name = "toml" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c226a7bba6d859b63c92c4b4fe69c5b6b72d0cb897dbc8e6012298e6154cb56e" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.20.0", -] - -[[package]] -name = "toml_datetime" -version = "0.6.3" +name = "rustix" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ - "serde", + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", ] [[package]] -name = "toml_edit" -version = "0.19.15" +name = "rustversion" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] -name = "toml_edit" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff63e60a958cefbb518ae1fd6566af80d9d4be430a33f3723dfc47d1d411d95" +name = "rusty-doors" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rusty-doors#42ad0104095425eea76934b5d735b4c6a438ef66" dependencies = [ - "indexmap 2.0.0", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", + "libc", + "rusty-doors-macros", ] [[package]] -name = "toolchain_find" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e85654a10e7a07a47c6f19d93818f3f343e22927f2fa280c84f7c8042743413" +name = "rusty-doors-macros" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/rusty-doors#42ad0104095425eea76934b5d735b4c6a438ef66" dependencies = [ - "home", - "lazy_static", - "regex", - "semver 0.11.0", - "walkdir", + "quote", + "syn 1.0.109", ] [[package]] -name = "topological-sort" -version = "0.2.2" +name = "ryu" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] -name = "tough" -version = "0.12.5" +name = "scopeguard" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc636dd1ee889a366af6731f1b63b60baf19528b46df5a7c2d4b3bf8b60bca2d" -dependencies = [ - "chrono", - "dyn-clone", - "globset", - "hex", - "log", - "olpc-cjson", - "path-absolutize", - "pem 1.1.1", - "percent-encoding", - "reqwest", - "ring 0.16.20", - "serde", - "serde_json", - "serde_plain", - "snafu", - "tempfile", - "untrusted 0.7.1", - "url", - "walkdir", -] +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] -name = "tower-service" -version = "0.3.2" +name = "semver" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] -name = "tracing" -version = "0.1.37" +name = "serde" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", + "serde_derive", ] [[package]] -name = "tracing-attributes" -version = "0.1.26" +name = "serde_derive" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", -] - -[[package]] -name = "tracing-core" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" -dependencies = [ - "once_cell", -] - -[[package]] -name = "trust-dns-client" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c408c32e6a9dbb38037cece35740f2cf23c875d8ca134d33631cec83f74d3fe" -dependencies = [ - "cfg-if", - "data-encoding", - "futures-channel", - "futures-util", - "lazy_static", - "radix_trie", - "rand 0.8.5", - "thiserror", - "time", - "tokio", - "tracing", - "trust-dns-proto", -] - -[[package]] -name = "trust-dns-proto" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.2.3", - "ipnet", - "lazy_static", - "rand 0.8.5", - "smallvec 1.11.0", - "thiserror", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" -dependencies = [ - "cfg-if", - "futures-util", - "ipconfig", - "lazy_static", - "lru-cache", - "parking_lot 0.12.1", - "resolv-conf", - "smallvec 1.11.0", - "thiserror", - "tokio", - "tracing", - "trust-dns-proto", -] - -[[package]] -name = "trust-dns-server" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99022f9befa6daec2a860be68ac28b1f0d9d7ccf441d8c5a695e35a58d88840d" -dependencies = [ - "async-trait", - "bytes", - "cfg-if", - "enum-as-inner", - "futures-executor", - "futures-util", - "serde", - "thiserror", - "time", - "tokio", - "toml 0.5.11", - "tracing", - "trust-dns-client", - "trust-dns-proto", + "syn 2.0.39", ] [[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - -[[package]] -name = "trybuild" -version = "1.0.85" +name = "serde_json" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196a58260a906cedb9bf6d8034b6379d0c11f552416960452f267402ceeddff1" -dependencies = [ - "basic-toml", - "glob", - "once_cell", - "serde", - "serde_derive", - "serde_json", - "termcolor", -] - -[[package]] -name = "tufaceous" -version = "0.1.0" -dependencies = [ - "anyhow", - "assert_cmd", - "camino", - "chrono", - "clap 4.4.3", - "console", - "datatest-stable", - "fs-err", - "humantime", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "predicates 3.0.4", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "tempfile", - "tufaceous-lib", -] - -[[package]] -name = "tufaceous-lib" -version = "0.1.0" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ - "anyhow", - "buf-list", - "bytes", - "bytesize", - "camino", - "camino-tempfile", - "chrono", - "debug-ignore", - "flate2", - "fs-err", - "hex", - "hubtools", - "itertools 0.11.0", - "omicron-common 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "rand 0.8.5", - "ring 0.16.20", + "itoa", + "ryu", "serde", - "serde_json", - "serde_path_to_error", - "sha2", - "slog", - "tar", - "toml 0.7.8", - "tough", - "url", - "zip", -] - -[[package]] -name = "tui-tree-widget" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f01f9172fb8f4f925fb1e259c2f411be14af031ab8b35d517fd05cb78c0784d5" -dependencies = [ - "ratatui", - "unicode-width", -] - -[[package]] -name = "tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror", - "url", - "utf-8", -] - -[[package]] -name = "tungstenite" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand 0.8.5", - "sha1", - "thiserror", - "url", - "utf-8", -] - -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "rand 0.8.5", - "static_assertions", ] [[package]] -name = "typenum" -version = "1.16.0" +name = "serde_tokenstream" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - -[[package]] -name = "typify" -version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" -dependencies = [ - "typify-impl", - "typify-macro", -] - -[[package]] -name = "typify-impl" -version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" -dependencies = [ - "heck 0.4.1", - "log", - "proc-macro2", - "quote", - "regress", - "schemars", - "serde_json", - "syn 2.0.32", - "thiserror", - "unicode-ident", -] - -[[package]] -name = "typify-macro" -version = "0.0.13" -source = "git+https://github.com/oxidecomputer/typify#de16c4238a2b34400d0fece086a6469951c3236b" +checksum = "797ba1d80299b264f3aac68ab5d12e5825a561749db4df7cd7c8083900c5d4e9" dependencies = [ "proc-macro2", - "quote", - "schemars", "serde", - "serde_json", - "serde_tokenstream 0.2.0", - "syn 2.0.32", - "typify-impl", -] - -[[package]] -name = "ucd-trie" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" - -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", + "syn 1.0.109", ] [[package]] -name = "unicode-bidi" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" +name = "sha2" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "tinyvec", + "cfg-if", + "cpufeatures", + "digest", ] [[package]] -name = "unicode-segmentation" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" - -[[package]] -name = "unicode-width" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" - -[[package]] -name = "unicode-xid" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" - -[[package]] -name = "universal-hash" -version = "0.5.1" +name = "signal-hook-registry" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ - "crypto-common", - "subtle", + "libc", ] [[package]] -name = "unsafe-libyaml" -version = "0.2.9" +name = "slog" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" [[package]] -name = "untrusted" -version = "0.7.1" +name = "slog-async" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] [[package]] -name = "untrusted" -version = "0.9.0" +name = "slog-envlogger" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "update-engine" -version = "0.1.0" +checksum = "906a1a0bc43fed692df4b82a5e2fbfc3733db8dad8bb514ab27a4f23ad04f5c0" dependencies = [ - "anyhow", - "buf-list", - "bytes", - "camino", - "camino-tempfile", - "cancel-safe-futures", - "debug-ignore", - "derive-where", - "either", - "futures", - "indexmap 2.0.0", - "indicatif", - "linear-map", - "omicron-test-utils", - "omicron-workspace-hack", - "owo-colors", - "petgraph", - "schemars", - "serde", - "serde_json", - "serde_with", + "log", + "regex", "slog", - "tokio", - "tokio-stream", - "uuid", + "slog-async", + "slog-scope", + "slog-stdlog", + "slog-term", ] [[package]] -name = "url" -version = "2.4.1" +name = "slog-scope" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "2f95a4b4c3274cd2869549da82b57ccc930859bdbf5bcea0424bc5f140b3c786" dependencies = [ - "form_urlencoded", - "idna 0.4.0", - "percent-encoding", + "arc-swap", + "lazy_static", + "slog", ] [[package]] -name = "usdt" -version = "0.3.5" +name = "slog-stdlog" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4c48f9e522b977bbe938a0d7c4d36633d267ba0155aaa253fb57d0531be0fb" +checksum = "6706b2ace5bbae7291d3f8d2473e2bfab073ccd7d03670946197aec98471fa3e" dependencies = [ - "dtrace-parser", - "serde", - "usdt-attr-macro", - "usdt-impl", - "usdt-macro", + "log", + "slog", + "slog-scope", ] [[package]] -name = "usdt-attr-macro" -version = "0.3.5" +name = "slog-term" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e6ae4f982ae74dcbaa8eb17baf36ca0d464a3abc8a7172b3bd74c73e9505d6" +checksum = "87d29185c55b7b258b4f120eab00f48557d4d9bc814f41713f449d35b0f8977c" dependencies = [ - "dtrace-parser", - "proc-macro2", - "quote", - "serde_tokenstream 0.1.7", - "syn 1.0.109", - "usdt-impl", + "atty", + "slog", + "term", + "thread_local", + "time", ] [[package]] -name = "usdt-impl" -version = "0.3.5" +name = "smallvec" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f53b4ca0b33aae466dc47b30b98adc4f88454928837af8010b6ed02d18474cb1" -dependencies = [ - "byteorder", - "dof", - "dtrace-parser", - "libc", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn 1.0.109", - "thiserror", - "thread-id", - "version_check", -] +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] -name = "usdt-macro" -version = "0.3.5" +name = "smoltcp" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb093f9653dc91632621c754f9ed4ee25d14e46e0239b6ccaf74a6c0c2788bd" +checksum = "8d2e3a36ac8fea7b94e666dfa3871063d6e0a5c9d5d4fec9a1a6b7b6760f0229" dependencies = [ - "dtrace-parser", - "proc-macro2", - "quote", - "serde_tokenstream 0.1.7", - "syn 1.0.109", - "usdt-impl", + "bitflags 1.3.2", + "byteorder", + "cfg-if", + "defmt", + "heapless", + "managed", ] [[package]] -name = "utf-8" -version = "0.7.6" +name = "socket2" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] [[package]] -name = "utf8parse" -version = "0.2.1" +name = "socket2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +dependencies = [ + "libc", + "windows-sys", +] [[package]] -name = "uuid" -version = "1.4.1" +name = "spin" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "getrandom 0.2.10", - "serde", + "lock_api", ] [[package]] -name = "vcpkg" -version = "0.2.15" +name = "stable_deref_trait" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] -name = "vec_map" -version = "0.8.2" +name = "strsim" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] -name = "vergen" -version = "8.2.4" +name = "syn" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbc5ad0d9d26b2c49a5ab7da76c3e79d3ee37e7821799f8223fcb8f2f391a2e7" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "anyhow", - "git2", - "rustc_version 0.4.0", - "rustversion", - "time", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] -name = "version_check" -version = "0.9.4" +name = "syn" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "viona_api" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ - "libc", - "viona_api_sys", + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] -name = "viona_api_sys" -version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=4019eb10fc2f4ba9bf210d0461dc6292b68309c2#4019eb10fc2f4ba9bf210d0461dc6292b68309c2" +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "libc", + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", ] [[package]] -name = "vsss-rs" -version = "3.3.1" +name = "take_mut" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f368a01a79af8f2fa45e20a2a478a9799c631c4a7c598563e2c94b2211f750cb" -dependencies = [ - "curve25519-dalek", - "elliptic-curve", - "hex", - "rand 0.8.5", - "rand_chacha 0.3.1", - "rand_core 0.6.4", - "serde", - "subtle", - "thiserror-no-std", - "zeroize", -] +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] -name = "vte" -version = "0.11.1" +name = "term" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ - "utf8parse", - "vte_generate_state_changes", + "dirs-next", + "rustversion", + "winapi", ] [[package]] -name = "vte_generate_state_changes" -version = "0.1.1" +name = "thiserror" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ - "proc-macro2", - "quote", + "thiserror-impl", ] [[package]] -name = "wait-timeout" -version = "0.2.0" +name = "thiserror-impl" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ - "libc", + "proc-macro2", + "quote", + "syn 2.0.39", ] [[package]] -name = "waitgroup" -version = "0.1.2" +name = "thread-id" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1f50000a783467e6c0200f9d10642f4bc424e39efc1b770203e88b488f79292" +checksum = "f0ec81c46e9eb50deaa257be2f148adf052d1fb7701cfd55ccfab2525280b70b" dependencies = [ - "atomic-waker", + "libc", + "winapi", ] [[package]] -name = "walkdir" -version = "2.4.0" +name = "thread_local" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "same-file", - "winapi-util", + "cfg-if", + "once_cell", ] [[package]] -name = "want" -version = "0.3.1" +name = "time" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" dependencies = [ - "try-lock", + "deranged", + "itoa", + "libc", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", ] [[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" +name = "time-core" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +name = "time-macros" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", +] [[package]] -name = "wasm-bindgen" -version = "0.2.87" +name = "tokio" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ - "cfg-if", - "wasm-bindgen-macro", + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.5.5", + "tokio-macros", + "windows-sys", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.87" +name = "tokio-macros" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ - "bumpalo", - "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.32", - "wasm-bindgen-shared", + "syn 2.0.39", ] [[package]] -name = "wasm-bindgen-futures" -version = "0.4.37" +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", + "indexmap", + "toml_datetime", + "winnow", ] [[package]] -name = "wasm-bindgen-macro" -version = "0.2.87" +name = "tracing" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "quote", - "wasm-bindgen-macro-support", + "pin-project-lite", + "tracing-attributes", + "tracing-core", ] [[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.87" +name = "tracing-attributes" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", - "wasm-bindgen-backend", - "wasm-bindgen-shared", + "syn 2.0.39", ] [[package]] -name = "wasm-bindgen-shared" -version = "0.2.87" +name = "tracing-core" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] [[package]] -name = "wasm-streams" -version = "0.3.0" +name = "typenum" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] -name = "web-sys" -version = "0.3.64" +name = "ucd-trie" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" -dependencies = [ - "js-sys", - "wasm-bindgen", -] +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] -name = "webpki-roots" -version = "0.25.2" +name = "unicode-ident" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] -name = "which" -version = "4.4.0" +name = "unicode-xid" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] -name = "whoami" -version = "1.4.1" +name = "usdt" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "2b4c48f9e522b977bbe938a0d7c4d36633d267ba0155aaa253fb57d0531be0fb" dependencies = [ - "wasm-bindgen", - "web-sys", + "dtrace-parser", + "serde", + "usdt-attr-macro", + "usdt-impl", + "usdt-macro", ] [[package]] -name = "wicket" -version = "0.1.0" +name = "usdt-attr-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e6ae4f982ae74dcbaa8eb17baf36ca0d464a3abc8a7172b3bd74c73e9505d6" dependencies = [ - "anyhow", - "assert_cmd", - "buf-list", - "camino", - "ciborium", - "clap 4.4.3", - "crossterm 0.27.0", - "futures", - "hex", - "humantime", - "indexmap 2.0.0", - "indicatif", - "itertools 0.11.0", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-workspace-hack", - "once_cell", - "owo-colors", - "proptest", - "ratatui", - "reqwest", - "rpassword", - "semver 1.0.20", - "serde", - "serde_json", - "shell-words", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "tempfile", - "textwrap 0.16.0", - "tokio", - "tokio-util", - "toml 0.7.8", - "toml_edit 0.19.15", - "tui-tree-widget", - "unicode-width", - "update-engine", - "wicket-common", - "wicketd-client", - "zeroize", + "dtrace-parser", + "proc-macro2", + "quote", + "serde_tokenstream", + "syn 1.0.109", + "usdt-impl", ] [[package]] -name = "wicket-common" -version = "0.1.0" +name = "usdt-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f53b4ca0b33aae466dc47b30b98adc4f88454928837af8010b6ed02d18474cb1" dependencies = [ - "anyhow", - "gateway-client", - "omicron-common 0.1.0", - "omicron-workspace-hack", - "schemars", + "byteorder", + "dof", + "dtrace-parser", + "libc", + "proc-macro2", + "quote", "serde", "serde_json", + "syn 1.0.109", "thiserror", - "update-engine", + "thread-id", + "version_check", ] [[package]] -name = "wicket-dbg" -version = "0.1.0" +name = "usdt-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cb093f9653dc91632621c754f9ed4ee25d14e46e0239b6ccaf74a6c0c2788bd" dependencies = [ - "anyhow", - "bytes", - "camino", - "ciborium", - "clap 4.4.3", - "crossterm 0.27.0", - "omicron-workspace-hack", - "ratatui", - "reedline", - "serde", - "slog", - "slog-async", - "slog-envlogger", - "slog-term", - "tokio", - "wicket", + "dtrace-parser", + "proc-macro2", + "quote", + "serde_tokenstream", + "syn 1.0.109", + "usdt-impl", ] [[package]] -name = "wicketd" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.5", - "bootstrap-agent-client", - "bytes", - "camino", - "camino-tempfile", - "clap 4.4.3", - "ddm-admin-client", - "debug-ignore", - "display-error-chain", - "dpd-client", - "dropshot", - "either", - "expectorate", - "flate2", - "flume", - "fs-err", - "futures", - "gateway-client", - "gateway-messages", - "gateway-test-utils", - "hex", - "http", - "hubtools", - "hyper", - "illumos-utils", - "installinator", - "installinator-artifact-client", - "installinator-artifactd", - "installinator-common", - "internal-dns 0.1.0", - "ipnetwork", - "itertools 0.11.0", - "omicron-certificates", - "omicron-common 0.1.0", - "omicron-passwords 0.1.0", - "omicron-test-utils", - "omicron-workspace-hack", - "openapi-lint", - "openapiv3", - "rand 0.8.5", - "reqwest", - "schemars", - "serde", - "serde_json", - "sha2", - "sled-hardware", - "slog", - "slog-dtrace", - "snafu", - "subprocess", - "tar", - "thiserror", - "tokio", - "tokio-stream", - "tokio-util", - "toml 0.7.8", - "tough", - "trust-dns-resolver", - "tufaceous", - "tufaceous-lib", - "update-engine", - "uuid", - "wicket-common", - "wicketd-client", -] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] -name = "wicketd-client" -version = "0.1.0" -dependencies = [ - "chrono", - "installinator-common", - "ipnetwork", - "omicron-workspace-hack", - "progenitor", - "regress", - "reqwest", - "schemars", - "serde", - "serde_json", - "slog", - "update-engine", - "uuid", - "wicket-common", -] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "widestring" -version = "1.0.2" +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "winapi" @@ -10299,61 +1711,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets", ] [[package]] @@ -10362,93 +1732,51 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -10457,78 +1785,43 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] [[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "wyz" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" -dependencies = [ - "tap", -] - -[[package]] -name = "x509-cert" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25eefca1d99701da3a57feb07e5079fc62abba059fc139e98c13bbb250f3ef29" +name = "xde" +version = "0.1.0" dependencies = [ - "const-oid", - "der", - "spki", + "bitflags 2.4.1", + "illumos-sys-hdrs", + "opte", + "oxide-vpc", + "postcard", + "serde", ] [[package]] -name = "xattr" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" -dependencies = [ - "libc", -] +name = "xde-link" +version = "0.1.0" [[package]] -name = "xtask" +name = "xde-tests" version = "0.1.0" dependencies = [ "anyhow", - "camino", - "cargo_metadata", - "cargo_toml", - "clap 4.4.3", -] - -[[package]] -name = "yansi" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" - -[[package]] -name = "yasna" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" -dependencies = [ - "bit-vec", - "num-bigint", - "time", + "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", + "opteadm", + "oxide-vpc", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "zone", + "ztest", ] [[package]] @@ -10543,22 +1836,12 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20707b61725734c595e840fb3704378a0cd2b9c74cc9e6e20724838fc6a1e2f9" -dependencies = [ - "byteorder", - "zerocopy-derive 0.6.4", -] - -[[package]] -name = "zerocopy" -version = "0.7.21" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686b7e407015242119c33dab17b8f61ba6843534de936d94368856528eae4dcc" +checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557" dependencies = [ "byteorder", - "zerocopy-derive 0.7.21", + "zerocopy-derive 0.7.25", ] [[package]] @@ -10574,80 +1857,48 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020f3dfe25dfc38dfea49ce62d5d45ecdd7f0d8a724fa63eb36b6eba4ec76806" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.32", -] - -[[package]] -name = "zeroize" -version = "1.6.0" +version = "0.7.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", -] - -[[package]] -name = "zip" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" -dependencies = [ - "byteorder", - "bzip2", - "crc32fast", - "crossbeam-utils", - "flate2", + "syn 2.0.39", ] [[package]] name = "zone" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0545a42fbd7a81245726d54a0146cb4fd93882ebb6da50d60acf2e37394f198" +version = "0.3.0" +source = "git+https://github.com/oxidecomputer/zone#5503b472dd87a3e6443624e1b42844882d6f769c" dependencies = [ "itertools 0.10.5", "thiserror", - "tokio", "zone_cfg_derive", ] [[package]] name = "zone_cfg_derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef224b009d070d3b1adb9e375fcf8ec2f1948a412c3bbf8755c0ef4e3f91ef94" +version = "0.3.0" +source = "git+https://github.com/oxidecomputer/zone#5503b472dd87a3e6443624e1b42844882d6f769c" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro-error", "proc-macro2", "quote", "syn 1.0.109", ] + +[[package]] +name = "ztest" +version = "0.1.0" +source = "git+https://github.com/oxidecomputer/falcon?branch=main#3546f4f5c5a366df3b02afe6f5e9803cf69996c8" +dependencies = [ + "anyhow", + "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys?branch=main)", + "slog", + "slog-async", + "slog-envlogger", + "slog-term", + "tokio", + "zone", +] diff --git a/Cargo.toml b/Cargo.toml index 1799a5f78b9..510daa65736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ members = [ "rpaths", "sled-agent", "sled-hardware", + "sled-storage", "sp-sim", "test-utils", "tufaceous-lib", @@ -122,6 +123,7 @@ default-members = [ "rpaths", "sled-agent", "sled-hardware", + "sled-storage", "sp-sim", "test-utils", "tufaceous-lib", @@ -168,7 +170,6 @@ criterion = { version = "0.5.1", features = [ "async_tokio" ] } crossbeam = "0.8" crossterm = { version = "0.27.0", features = ["event-stream"] } crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } -crucible-client-types = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "da534e73380f3cc53ca0de073e1ea862ae32109b" } curve25519-dalek = "4" @@ -206,14 +207,14 @@ hex-literal = "0.4.1" highway = "1.1.0" hkdf = "0.12.3" http = "0.2.9" -httptest = "0.15.4" +httptest = "0.15.5" hubtools = { git = "https://github.com/oxidecomputer/hubtools.git", branch = "main" } humantime = "2.1.0" hyper = "0.14" hyper-rustls = "0.24.2" hyper-staticfile = "0.9.5" illumos-utils = { path = "illumos-utils" } -indexmap = "2.0.0" +indexmap = "2.1.0" indicatif = { version = "0.17.7", features = ["rayon"] } installinator = { path = "installinator" } installinator-artifactd = { path = "installinator-artifactd" } @@ -224,10 +225,12 @@ ipcc-key-value = { path = "ipcc-key-value" } ipnetwork = { version = "0.20", features = ["schemars"] } itertools = "0.11.0" key-manager = { path = "key-manager" } +kstat-rs = "0.2.3" lazy_static = "1.4.0" -libc = "0.2.149" +libc = "0.2.150" linear-map = "1.2.0" macaddr = { version = "1.0.1", features = ["serde_std"] } +maplit = "1.0.2" mime_guess = "2.0.4" mockall = "0.11" newtype_derive = "0.1.6" @@ -274,7 +277,7 @@ oximeter-instruments = { path = "oximeter/instruments" } oximeter-macro-impl = { path = "oximeter/oximeter-macro-impl" } oximeter-producer = { path = "oximeter/producer" } p256 = "0.13" -parse-display = "0.7.0" +parse-display = "0.8.2" partial-io = { version = "0.5.4", features = ["proptest1", "tokio1"] } paste = "1.0.14" percent-encoding = "2.3.0" @@ -287,9 +290,9 @@ pretty-hex = "0.3.0" proc-macro2 = "1.0" progenitor = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } progenitor-client = { git = "https://github.com/oxidecomputer/progenitor", branch = "main" } -bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2" } -propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", features = [ "generated-migration" ] } -propolis-server = { git = "https://github.com/oxidecomputer/propolis", rev = "4019eb10fc2f4ba9bf210d0461dc6292b68309c2", default-features = false, features = ["mock-only"] } +bhyve_api = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } +propolis-client = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } +propolis-mock-server = { git = "https://github.com/oxidecomputer/propolis", rev = "5ed82315541271e2734746a9ca79e39f35c12283" } proptest = "1.3.1" quote = "1.0" rand = "0.8.5" @@ -327,6 +330,7 @@ similar-asserts = "1.5.0" sled = "0.34" sled-agent-client = { path = "clients/sled-agent-client" } sled-hardware = { path = "sled-hardware" } +sled-storage = { path = "sled-storage" } slog = { version = "2.7", features = [ "dynamic-keys", "max_level_trace", "release_max_level_debug" ] } slog-async = "2.8" slog-dtrace = "0.2" @@ -345,6 +349,8 @@ static_assertions = "1.1.0" steno = "0.4.0" strum = { version = "0.25", features = [ "derive" ] } subprocess = "0.2.9" +supports-color = "2.1.0" +swrite = "0.1.0" libsw = { version = "3.3.0", features = ["tokio"] } syn = { version = "2.0" } tabled = "0.14" @@ -362,10 +368,10 @@ tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1 tokio-stream = "0.1.14" tokio-tungstenite = "0.18" tokio-util = "0.7.10" -toml = "0.7.8" -toml_edit = "0.19.15" +toml = "0.8.8" +toml_edit = "0.21.0" topological-sort = "0.2.2" -tough = { version = "0.12", features = [ "http" ] } +tough = { version = "0.14", features = [ "http" ] } trust-dns-client = "0.22" trust-dns-proto = "0.22" trust-dns-resolver = "0.22" @@ -376,14 +382,14 @@ tufaceous-lib = { path = "tufaceous-lib" } unicode-width = "0.1.11" update-engine = { path = "update-engine" } usdt = "0.3" -uuid = { version = "1.4.1", features = ["serde", "v4"] } +uuid = { version = "1.5.0", features = ["serde", "v4"] } walkdir = "2.4" wicket = { path = "wicket" } wicket-common = { path = "wicket-common" } wicketd-client = { path = "clients/wicketd-client" } zeroize = { version = "1.6.0", features = ["zeroize_derive", "std"] } zip = { version = "0.6.6", default-features = false, features = ["deflate","bzip2"] } -zone = { version = "0.2", default-features = false, features = ["async"] } +zone = { version = "0.3", default-features = false, features = ["async"] } [profile.dev] panic = "abort" @@ -540,9 +546,9 @@ opt-level = 3 #steno = { path = "../steno" } #[patch."https://github.com/oxidecomputer/propolis"] #propolis-client = { path = "../propolis/lib/propolis-client" } +#propolis-mock-server = { path = "../propolis/bin/mock-server" } #[patch."https://github.com/oxidecomputer/crucible"] #crucible-agent-client = { path = "../crucible/agent-client" } -#crucible-client-types = { path = "../crucible/crucible-client-types" } #crucible-pantry-client = { path = "../crucible/pantry-client" } #crucible-smf = { path = "../crucible/smf" } #[patch.crates-io] diff --git a/bootstore/src/schemes/v0/peer_networking.rs b/bootstore/src/schemes/v0/peer_networking.rs index acde14d085a..33673cdab82 100644 --- a/bootstore/src/schemes/v0/peer_networking.rs +++ b/bootstore/src/schemes/v0/peer_networking.rs @@ -643,6 +643,9 @@ async fn perform_handshake( let end = INITIAL_READ + identify_len; + // Clippy for Rust 1.73 warns on this but there doesn't seem to be a + // better way to write it? + #[allow(clippy::unnecessary_unwrap)] if identify.is_some() && !out_cursor.has_remaining() { return Ok((read_sock, write_sock, identify.unwrap())); } diff --git a/clients/bootstrap-agent-client/Cargo.toml b/clients/bootstrap-agent-client/Cargo.toml index 42ae59b7aa8..3474c5814ae 100644 --- a/clients/bootstrap-agent-client/Cargo.toml +++ b/clients/bootstrap-agent-client/Cargo.toml @@ -5,8 +5,6 @@ edition = "2021" license = "MPL-2.0" [dependencies] -async-trait.workspace = true -chrono.workspace = true omicron-common.workspace = true progenitor.workspace = true ipnetwork.workspace = true diff --git a/clients/dns-service-client/Cargo.toml b/clients/dns-service-client/Cargo.toml index 681c06672f9..6132222b8a3 100644 --- a/clients/dns-service-client/Cargo.toml +++ b/clients/dns-service-client/Cargo.toml @@ -11,7 +11,5 @@ progenitor.workspace = true reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] } schemars.workspace = true serde.workspace = true -serde_json.workspace = true slog.workspace = true -uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/clients/gateway-client/Cargo.toml b/clients/gateway-client/Cargo.toml index fc331741078..9e1118cf59c 100644 --- a/clients/gateway-client/Cargo.toml +++ b/clients/gateway-client/Cargo.toml @@ -7,6 +7,7 @@ license = "MPL-2.0" [dependencies] base64.workspace = true chrono.workspace = true +gateway-messages.workspace = true progenitor.workspace = true rand.workspace = true reqwest = { workspace = true, features = ["rustls-tls", "stream"] } diff --git a/clients/gateway-client/src/lib.rs b/clients/gateway-client/src/lib.rs index b071d349758..7dbc50eea2f 100644 --- a/clients/gateway-client/src/lib.rs +++ b/clients/gateway-client/src/lib.rs @@ -6,6 +6,8 @@ //! Interface for API requests to a Management Gateway Service (MGS) instance +pub use gateway_messages::SpComponent; + // We specifically want to allow consumers, such as `wicketd`, to embed // inventory datatypes into their own APIs, rather than recreate structs. For // this purpose, we copied the `omicron_common::generate_logging_api!` macro diff --git a/clients/sled-agent-client/Cargo.toml b/clients/sled-agent-client/Cargo.toml index b2ed07caba4..e2cc737e70d 100644 --- a/clients/sled-agent-client/Cargo.toml +++ b/clients/sled-agent-client/Cargo.toml @@ -14,5 +14,6 @@ regress.workspace = true reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } serde.workspace = true slog.workspace = true +sled-storage.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 0df21d894e8..30b554a021e 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -6,6 +6,7 @@ use async_trait::async_trait; use std::convert::TryFrom; +use std::str::FromStr; use uuid::Uuid; progenitor::generate_api!( @@ -528,3 +529,27 @@ impl TestInterfaces for Client { .expect("disk_finish_transition() failed unexpectedly"); } } + +impl From for types::DatasetKind { + fn from(k: sled_storage::dataset::DatasetKind) -> Self { + use sled_storage::dataset::DatasetKind::*; + match k { + CockroachDb => Self::CockroachDb, + Crucible => Self::Crucible, + Clickhouse => Self::Clickhouse, + ClickhouseKeeper => Self::ClickhouseKeeper, + ExternalDns => Self::ExternalDns, + InternalDns => Self::InternalDns, + } + } +} + +impl From for types::DatasetName { + fn from(n: sled_storage::dataset::DatasetName) -> Self { + Self { + pool_name: types::ZpoolName::from_str(&n.pool().to_string()) + .unwrap(), + kind: n.dataset().clone().into(), + } + } +} diff --git a/clients/wicketd-client/src/lib.rs b/clients/wicketd-client/src/lib.rs index 982ec13780c..01c3b04f87d 100644 --- a/clients/wicketd-client/src/lib.rs +++ b/clients/wicketd-client/src/lib.rs @@ -59,6 +59,9 @@ progenitor::generate_api!( Ipv6Network = ipnetwork::Ipv6Network, IpNetwork = ipnetwork::IpNetwork, PutRssUserConfigInsensitive = wicket_common::rack_setup::PutRssUserConfigInsensitive, + ClearUpdateStateResponse = wicket_common::rack_update::ClearUpdateStateResponse, + SpIdentifier = wicket_common::rack_update::SpIdentifier, + SpType = wicket_common::rack_update::SpType, EventReportForWicketdEngineSpec = wicket_common::update_events::EventReport, StepEventForWicketdEngineSpec = wicket_common::update_events::StepEvent, ProgressEventForWicketdEngineSpec = wicket_common::update_events::ProgressEvent, diff --git a/common/Cargo.toml b/common/Cargo.toml index 75c1efab551..49997e619c3 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -15,18 +15,15 @@ dropshot.workspace = true futures.workspace = true hex.workspace = true http.workspace = true -hyper.workspace = true ipnetwork.workspace = true macaddr.workspace = true lazy_static.workspace = true proptest = { workspace = true, optional = true } rand.workspace = true reqwest = { workspace = true, features = ["rustls-tls", "stream"] } -ring.workspace = true schemars = { workspace = true, features = ["chrono", "uuid1"] } semver.workspace = true serde.workspace = true -serde_derive.workspace = true serde_human_bytes.workspace = true serde_json.workspace = true serde_with.workspace = true diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 784da8fcc6b..155fbf971b6 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -103,6 +103,17 @@ pub struct BgpPeerConfig { pub port: String, /// Address of the peer. pub addr: Ipv4Addr, + /// How long to keep a session alive without a keepalive in seconds. + /// Defaults to 6. + pub hold_time: Option, + /// How long to keep a peer in idle after a state machine reset in seconds. + pub idle_hold_time: Option, + /// How long to delay sending open messages to a peer. In seconds. + pub delay_open: Option, + /// The interval in seconds between peer connection retry attempts. + pub connect_retry: Option, + /// The interval to send keepalive messages at. + pub keepalive: Option, } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, JsonSchema)] diff --git a/common/src/cmd.rs b/common/src/cmd.rs index d92ebe4c984..f7ede8d1271 100644 --- a/common/src/cmd.rs +++ b/common/src/cmd.rs @@ -13,7 +13,7 @@ pub enum CmdError { /// incorrect command-line arguments Usage(String), /// all other errors - Failure(String), + Failure(anyhow::Error), } /// Exits the current process on a fatal error. @@ -26,7 +26,7 @@ pub fn fatal(cmd_error: CmdError) -> ! { .unwrap_or("command"); let (exit_code, message) = match cmd_error { CmdError::Usage(m) => (2, m), - CmdError::Failure(m) => (1, m), + CmdError::Failure(e) => (1, format!("{e:?}")), }; eprintln!("{}: {}", arg0, message); exit(exit_code); diff --git a/common/src/disk.rs b/common/src/disk.rs index 3ea80913269..3ae9c31e01f 100644 --- a/common/src/disk.rs +++ b/common/src/disk.rs @@ -5,7 +5,7 @@ //! Disk related types shared among crates /// Uniquely identifies a disk. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] pub struct DiskIdentity { pub vendor: String, pub serial: String, diff --git a/common/src/ledger.rs b/common/src/ledger.rs index c07fb5e6935..ae028998e2a 100644 --- a/common/src/ledger.rs +++ b/common/src/ledger.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use camino::{Utf8Path, Utf8PathBuf}; use serde::{de::DeserializeOwned, Serialize}; -use slog::{debug, warn, Logger}; +use slog::{error, info, warn, Logger}; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -84,8 +84,11 @@ impl Ledger { // Read all the ledgers that we can. let mut ledgers = vec![]; for path in paths.iter() { - if let Ok(ledger) = T::read_from(log, &path).await { - ledgers.push(ledger); + match T::read_from(log, &path).await { + Ok(ledger) => ledgers.push(ledger), + Err(err) => { + error!(log, "Failed to read ledger: {err}"; "path" => %path) + } } } @@ -169,8 +172,8 @@ pub trait Ledgerable: DeserializeOwned + Serialize + Send + Sync { /// Reads from `path` as a json-serialized version of `Self`. async fn read_from(log: &Logger, path: &Utf8Path) -> Result { if path.exists() { - debug!(log, "Reading ledger from {}", path); - serde_json::from_str( + info!(log, "Reading ledger from {}", path); + ::deserialize( &tokio::fs::read_to_string(&path) .await .map_err(|err| Error::io_path(&path, err))?, @@ -180,7 +183,7 @@ pub trait Ledgerable: DeserializeOwned + Serialize + Send + Sync { err, }) } else { - debug!(log, "No ledger in {path}"); + warn!(log, "No ledger in {path}"); Err(Error::NotFound) } } @@ -191,7 +194,7 @@ pub trait Ledgerable: DeserializeOwned + Serialize + Send + Sync { log: &Logger, path: &Utf8Path, ) -> Result<(), Error> { - debug!(log, "Writing ledger to {}", path); + info!(log, "Writing ledger to {}", path); let as_str = serde_json::to_string(&self).map_err(|err| { Error::JsonSerialize { path: path.to_path_buf(), err } })?; @@ -200,6 +203,10 @@ pub trait Ledgerable: DeserializeOwned + Serialize + Send + Sync { .map_err(|err| Error::io_path(&path, err))?; Ok(()) } + + fn deserialize(s: &str) -> Result { + serde_json::from_str(s) + } } #[cfg(test)] diff --git a/common/src/lib.rs b/common/src/lib.rs index ced10abb1a8..1d2ed0afdb1 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -75,3 +75,5 @@ impl slog::KV for FileKv { ) } } + +pub const OMICRON_DPD_TAG: &str = "omicron"; diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index ec7cafb559f..ce9a6ac32de 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -21,7 +21,6 @@ nexus-test-interface.workspace = true omicron-common.workspace = true omicron-nexus.workspace = true omicron-test-utils.workspace = true -omicron-sled-agent.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" rcgen.workspace = true diff --git a/dev-tools/omicron-dev/src/bin/omicron-dev.rs b/dev-tools/omicron-dev/src/bin/omicron-dev.rs index 66778d96e7a..f8deae30d03 100644 --- a/dev-tools/omicron-dev/src/bin/omicron-dev.rs +++ b/dev-tools/omicron-dev/src/bin/omicron-dev.rs @@ -4,8 +4,7 @@ //! Developer tool for easily running bits of Omicron -use anyhow::bail; -use anyhow::Context; +use anyhow::{bail, Context}; use camino::Utf8Path; use camino::Utf8PathBuf; use clap::Args; @@ -34,7 +33,7 @@ async fn main() -> Result<(), anyhow::Error> { OmicronDb::CertCreate { ref args } => cmd_cert_create(args).await, }; if let Err(error) = result { - fatal(CmdError::Failure(format!("{:#}", error))); + fatal(CmdError::Failure(error)); } Ok(()) } @@ -265,16 +264,28 @@ struct ChRunArgs { /// The HTTP port on which the server will listen #[clap(short, long, default_value = "8123", action)] port: u16, + /// Starts a ClickHouse replicated cluster of 2 replicas and 3 keeper nodes + #[clap(long, conflicts_with = "port", action)] + replicated: bool, } async fn cmd_clickhouse_run(args: &ChRunArgs) -> Result<(), anyhow::Error> { + if args.replicated { + start_replicated_cluster().await?; + } else { + start_single_node(args.port).await?; + } + Ok(()) +} + +async fn start_single_node(port: u16) -> Result<(), anyhow::Error> { // Start a stream listening for SIGINT let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); let mut signal_stream = signals.fuse(); // Start the database server process, possibly on a specific port let mut db_instance = - dev::clickhouse::ClickHouseInstance::new_single_node(args.port).await?; + dev::clickhouse::ClickHouseInstance::new_single_node(port).await?; println!( "omicron-dev: running ClickHouse with full command:\n\"clickhouse {}\"", db_instance.cmdline().join(" ") @@ -320,6 +331,118 @@ async fn cmd_clickhouse_run(args: &ChRunArgs) -> Result<(), anyhow::Error> { Ok(()) } +async fn start_replicated_cluster() -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + let mut signal_stream = signals.fuse(); + + // Start the database server and keeper processes + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let replica_config = manifest_dir + .as_path() + .join("../../oximeter/db/src/configs/replica_config.xml"); + let keeper_config = manifest_dir + .as_path() + .join("../../oximeter/db/src/configs/keeper_config.xml"); + + let mut cluster = + dev::clickhouse::ClickHouseCluster::new(replica_config, keeper_config) + .await?; + println!( + "omicron-dev: running ClickHouse cluster with configuration files:\n \ + replicas: {}\n keepers: {}", + cluster.replica_config_path().display(), + cluster.keeper_config_path().display() + ); + let pid_error_msg = "Failed to get process PID, it may not have started"; + println!( + "omicron-dev: ClickHouse cluster is running with: server PIDs = [{}, {}] \ + and keeper PIDs = [{}, {}, {}]", + cluster.replica_1 + .pid() + .expect(pid_error_msg), + cluster.replica_2 + .pid() + .expect(pid_error_msg), + cluster.keeper_1 + .pid() + .expect(pid_error_msg), + cluster.keeper_2 + .pid() + .expect(pid_error_msg), + cluster.keeper_3 + .pid() + .expect(pid_error_msg), + ); + println!( + "omicron-dev: ClickHouse HTTP servers listening on ports: {}, {}", + cluster.replica_1.port(), + cluster.replica_2.port() + ); + println!( + "omicron-dev: using {} and {} for ClickHouse data storage", + cluster.replica_1.data_path().display(), + cluster.replica_2.data_path().display() + ); + + // Wait for the replicas and keepers to exit themselves (an error), or for SIGINT + tokio::select! { + _ = cluster.replica_1.wait_for_shutdown() => { + cluster.replica_1.cleanup().await.context( + format!("clean up {} after shutdown", cluster.replica_1.data_path().display()) + )?; + bail!("omicron-dev: ClickHouse replica 1 shutdown unexpectedly"); + } + _ = cluster.replica_2.wait_for_shutdown() => { + cluster.replica_2.cleanup().await.context( + format!("clean up {} after shutdown", cluster.replica_2.data_path().display()) + )?; + bail!("omicron-dev: ClickHouse replica 2 shutdown unexpectedly"); + } + _ = cluster.keeper_1.wait_for_shutdown() => { + cluster.keeper_1.cleanup().await.context( + format!("clean up {} after shutdown", cluster.keeper_1.data_path().display()) + )?; + bail!("omicron-dev: ClickHouse keeper 1 shutdown unexpectedly"); + } + _ = cluster.keeper_2.wait_for_shutdown() => { + cluster.keeper_2.cleanup().await.context( + format!("clean up {} after shutdown", cluster.keeper_2.data_path().display()) + )?; + bail!("omicron-dev: ClickHouse keeper 2 shutdown unexpectedly"); + } + _ = cluster.keeper_3.wait_for_shutdown() => { + cluster.keeper_3.cleanup().await.context( + format!("clean up {} after shutdown", cluster.keeper_3.data_path().display()) + )?; + bail!("omicron-dev: ClickHouse keeper 3 shutdown unexpectedly"); + } + caught_signal = signal_stream.next() => { + assert_eq!(caught_signal.unwrap(), SIGINT); + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directories" + ); + + // Remove the data directories. + let mut instances = vec![ + cluster.replica_1, + cluster.replica_2, + cluster.keeper_1, + cluster.keeper_2, + cluster.keeper_3, + ]; + for instance in instances.iter_mut() { + instance + .wait_for_shutdown() + .await + .context(format!("clean up {} after SIGINT shutdown", instance.data_path().display()))?; + }; + } + } + Ok(()) +} + #[derive(Clone, Debug, Args)] struct RunAllArgs { /// Nexus external API listen port. Use `0` to request any available port. diff --git a/dev-tools/thing-flinger/Cargo.toml b/dev-tools/thing-flinger/Cargo.toml index 1a6c05a5462..2acbaf5659e 100644 --- a/dev-tools/thing-flinger/Cargo.toml +++ b/dev-tools/thing-flinger/Cargo.toml @@ -13,7 +13,6 @@ omicron-package.workspace = true serde.workspace = true serde_derive.workspace = true thiserror.workspace = true -toml.workspace = true omicron-workspace-hack.workspace = true [[bin]] diff --git a/dev-tools/xtask/Cargo.toml b/dev-tools/xtask/Cargo.toml index 30ffdc416bd..d054d856469 100644 --- a/dev-tools/xtask/Cargo.toml +++ b/dev-tools/xtask/Cargo.toml @@ -7,6 +7,6 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true camino.workspace = true -cargo_toml = "0.16" +cargo_toml = "0.17" cargo_metadata = "0.18" clap.workspace = true diff --git a/docs/how-to-run-simulated.adoc b/docs/how-to-run-simulated.adoc index 6b8efb0ed8d..de19b70f04f 100644 --- a/docs/how-to-run-simulated.adoc +++ b/docs/how-to-run-simulated.adoc @@ -159,6 +159,20 @@ $ cargo run --bin omicron-dev -- ch-run omicron-dev: running ClickHouse (PID: 2463), full command is "clickhouse server --log-file /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot/clickhouse-server.log --errorlog-file /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot/clickhouse-server.errlog -- --http_port 8123 --path /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot" omicron-dev: using /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot for ClickHouse data storage ---- ++ +If you wish to start a ClickHouse replicated cluster instead of a single node, run the following instead: +[source,text] +--- +$ cargo run --bin omicron-dev -- ch-run --replicated + Finished dev [unoptimized + debuginfo] target(s) in 0.31s + Running `target/debug/omicron-dev ch-run --replicated` +omicron-dev: running ClickHouse cluster with configuration files: + replicas: /home/{user}/src/omicron/oximeter/db/src/configs/replica_config.xml + keepers: /home/{user}/src/omicron/oximeter/db/src/configs/keeper_config.xml +omicron-dev: ClickHouse cluster is running with PIDs: 1113482, 1113681, 1113387, 1113451, 1113419 +omicron-dev: ClickHouse HTTP servers listening on ports: 8123, 8124 +omicron-dev: using /tmp/.tmpFH6v8h and /tmp/.tmpkUjDji for ClickHouse data storage +--- . `nexus` requires a configuration file to run. You can use `nexus/examples/config.toml` to start with. Build and run it like this: + diff --git a/end-to-end-tests/Cargo.toml b/end-to-end-tests/Cargo.toml index 732a4a2091f..e78a8792d3c 100644 --- a/end-to-end-tests/Cargo.toml +++ b/end-to-end-tests/Cargo.toml @@ -8,20 +8,16 @@ license = "MPL-2.0" anyhow = { workspace = true, features = ["backtrace"] } async-trait.workspace = true base64.workspace = true -camino.workspace = true chrono.workspace = true -futures.workspace = true http.workspace = true omicron-sled-agent.workspace = true omicron-test-utils.workspace = true oxide-client.workspace = true rand.workspace = true reqwest.workspace = true -russh = "0.38.0" +russh = "0.39.0" russh-keys = "0.38.0" -serde_json.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } toml.workspace = true trust-dns-resolver.workspace = true -uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/gateway-cli/Cargo.toml b/gateway-cli/Cargo.toml index ba66fa4c4f1..2412bf950fa 100644 --- a/gateway-cli/Cargo.toml +++ b/gateway-cli/Cargo.toml @@ -8,9 +8,7 @@ license = "MPL-2.0" anyhow.workspace = true clap.workspace = true futures.workspace = true -hex.workspace = true omicron-common.workspace = true -libc.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/gateway-test-utils/src/setup.rs b/gateway-test-utils/src/setup.rs index e789f8de788..46bc55805aa 100644 --- a/gateway-test-utils/src/setup.rs +++ b/gateway-test-utils/src/setup.rs @@ -62,12 +62,12 @@ pub async fn test_setup( test_name: &str, sp_port: SpPort, ) -> GatewayTestContext { - let (server_config, mut sp_sim_config) = load_test_config(); + let (server_config, sp_sim_config) = load_test_config(); test_setup_with_config( test_name, sp_port, server_config, - &mut sp_sim_config, + &sp_sim_config, None, ) .await diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 9cf41f6c2e5..75c31e99779 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -6,9 +6,7 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true -async-trait.workspace = true base64.workspace = true -ciborium.workspace = true clap.workspace = true dropshot.workspace = true futures.workspace = true @@ -23,7 +21,6 @@ omicron-common.workspace = true once_cell.workspace = true schemars.workspace = true serde.workspace = true -serde_human_bytes.workspace = true signal-hook.workspace = true signal-hook-tokio.workspace = true slog.workspace = true @@ -32,7 +29,6 @@ thiserror.workspace = true tokio = { workspace = true, features = ["full"] } tokio-stream.workspace = true tokio-tungstenite.workspace = true -tokio-util.workspace = true toml.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/gateway/src/bin/mgs.rs b/gateway/src/bin/mgs.rs index 81b10ef669b..6917d4f174f 100644 --- a/gateway/src/bin/mgs.rs +++ b/gateway/src/bin/mgs.rs @@ -4,6 +4,7 @@ //! Executable program to run gateway, the management gateway service +use anyhow::{anyhow, Context}; use clap::Parser; use futures::StreamExt; use omicron_common::cmd::{fatal, CmdError}; @@ -70,26 +71,24 @@ async fn do_run() -> Result<(), CmdError> { let args = Args::parse(); match args { - Args::Openapi => run_openapi().map_err(CmdError::Failure), + Args::Openapi => { + run_openapi().map_err(|e| CmdError::Failure(anyhow!(e))) + } Args::Run { config_file_path, id_and_address_from_smf, id, address, } => { - let config = Config::from_file(&config_file_path).map_err(|e| { - CmdError::Failure(format!( - "failed to parse {}: {}", - config_file_path.display(), - e - )) - })?; - - let mut signals = Signals::new([signal::SIGUSR1]).map_err(|e| { - CmdError::Failure(format!( - "failed to set up signal handler: {e}" - )) - })?; + let config = Config::from_file(&config_file_path) + .with_context(|| { + format!("failed to parse {}", config_file_path.display()) + }) + .map_err(CmdError::Failure)?; + + let mut signals = Signals::new([signal::SIGUSR1]) + .context("failed to set up signal handler") + .map_err(CmdError::Failure)?; let (id, addresses, rack_id) = if id_and_address_from_smf { let config = read_smf_config()?; @@ -102,8 +101,9 @@ async fn do_run() -> Result<(), CmdError> { (id.unwrap(), vec![address.unwrap()], rack_id) }; let args = MgsArguments { id, addresses, rack_id }; - let mut server = - start_server(config, args).await.map_err(CmdError::Failure)?; + let mut server = start_server(config, args) + .await + .map_err(|e| CmdError::Failure(anyhow!(e)))?; loop { tokio::select! { @@ -111,18 +111,13 @@ async fn do_run() -> Result<(), CmdError> { Some(signal::SIGUSR1) => { let new_config = read_smf_config()?; if new_config.id != id { - return Err(CmdError::Failure( - "cannot change server ID on refresh" - .to_string() - )); + return Err(CmdError::Failure(anyhow!("cannot change server ID on refresh"))); } server.set_rack_id(new_config.rack_id); server .adjust_dropshot_addresses(&new_config.addresses) .await - .map_err(|err| CmdError::Failure( - format!("config refresh failed: {err}") - ))?; + .map_err(|err| CmdError::Failure(anyhow!("config refresh failed: {err}")))?; } // We only register `SIGUSR1` and never close the // handle, so we never expect `None` or any other @@ -130,7 +125,7 @@ async fn do_run() -> Result<(), CmdError> { _ => unreachable!("invalid signal: {signal:?}"), }, result = server.wait_for_finish() => { - return result.map_err(CmdError::Failure) + return result.map_err(|err| CmdError::Failure(anyhow!(err))) } } } @@ -141,7 +136,7 @@ async fn do_run() -> Result<(), CmdError> { #[cfg(target_os = "illumos")] fn read_smf_config() -> Result { fn scf_to_cmd_err(err: illumos_utils::scf::ScfError) -> CmdError { - CmdError::Failure(err.to_string()) + CmdError::Failure(anyhow!(err)) } use illumos_utils::scf::ScfHandle; @@ -165,12 +160,14 @@ fn read_smf_config() -> Result { let prop_id = config.value_as_string(PROP_ID).map_err(scf_to_cmd_err)?; - let prop_id = Uuid::try_parse(&prop_id).map_err(|err| { - CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_ID}` \ - ({prop_id:?}) as a UUID: {err}" - )) - })?; + let prop_id = Uuid::try_parse(&prop_id) + .with_context(|| { + format!( + "failed to parse `{CONFIG_PG}/{PROP_ID}` ({prop_id:?}) as a \ + UUID" + ) + }) + .map_err(CmdError::Failure)?; let prop_rack_id = config.value_as_string(PROP_RACK_ID).map_err(scf_to_cmd_err)?; @@ -178,12 +175,16 @@ fn read_smf_config() -> Result { let rack_id = if prop_rack_id == "unknown" { None } else { - Some(Uuid::try_parse(&prop_rack_id).map_err(|err| { - CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` \ - ({prop_rack_id:?}) as a UUID: {err}" - )) - })?) + Some( + Uuid::try_parse(&prop_rack_id) + .with_context(|| { + format!( + "failed to parse `{CONFIG_PG}/{PROP_RACK_ID}` \ + ({prop_rack_id:?}) as a UUID" + ) + }) + .map_err(CmdError::Failure)?, + ) }; let prop_addr = @@ -192,16 +193,20 @@ fn read_smf_config() -> Result { let mut addresses = Vec::with_capacity(prop_addr.len()); for addr in prop_addr { - addresses.push(addr.parse().map_err(|err| { - CmdError::Failure(format!( - "failed to parse `{CONFIG_PG}/{PROP_ADDR}` \ - ({addr:?}) as a socket address: {err}" - )) - })?); + addresses.push( + addr.parse() + .with_context(|| { + format!( + "failed to parse `{CONFIG_PG}/{PROP_ADDR}` ({addr:?}) \ + as a socket address" + ) + }) + .map_err(CmdError::Failure)?, + ); } if addresses.is_empty() { - Err(CmdError::Failure(format!( + Err(CmdError::Failure(anyhow!( "no addresses specified by `{CONFIG_PG}/{PROP_ADDR}`" ))) } else { @@ -211,7 +216,7 @@ fn read_smf_config() -> Result { #[cfg(not(target_os = "illumos"))] fn read_smf_config() -> Result { - Err(CmdError::Failure( - "SMF configuration only available on illumos".to_string(), - )) + Err(CmdError::Failure(anyhow!( + "SMF configuration only available on illumos" + ))) } diff --git a/illumos-utils/Cargo.toml b/illumos-utils/Cargo.toml index a291a15e78c..497454e047c 100644 --- a/illumos-utils/Cargo.toml +++ b/illumos-utils/Cargo.toml @@ -44,3 +44,6 @@ toml.workspace = true [features] # Enable to generate MockZones testing = ["mockall"] +# Useful for tests that want real functionality and ability to run without +# pfexec +tmp_keypath = [] diff --git a/illumos-utils/src/lib.rs b/illumos-utils/src/lib.rs index 345f097ae25..1faa4c5c37e 100644 --- a/illumos-utils/src/lib.rs +++ b/illumos-utils/src/lib.rs @@ -4,6 +4,9 @@ //! Wrappers around illumos-specific commands. +#[allow(unused)] +use std::sync::atomic::{AtomicBool, Ordering}; + use cfg_if::cfg_if; pub mod addrobj; @@ -93,7 +96,7 @@ mod inner { // Helper function for starting the process and checking the // exit code result. - pub fn execute( + pub fn execute_helper( command: &mut std::process::Command, ) -> Result { let output = command.output().map_err(|err| { @@ -108,6 +111,34 @@ mod inner { } } +// Due to feature unification, the `testing` feature is enabled when some tests +// don't actually want to use it. We allow them to opt out of the use of the +// free function here. We also explicitly opt-in where mocks are used. +// +// Note that this only works if the tests that use mocks and those that don't +// are run sequentially. However, this is how we do things in CI with nextest, +// so there is no problem currently. +// +// We can remove all this when we get rid of the mocks. +#[cfg(any(test, feature = "testing"))] +pub static USE_MOCKS: AtomicBool = AtomicBool::new(false); + +pub fn execute( + command: &mut std::process::Command, +) -> Result { + cfg_if! { + if #[cfg(any(test, feature = "testing"))] { + if USE_MOCKS.load(Ordering::SeqCst) { + mock_inner::execute_helper(command) + } else { + inner::execute_helper(command) + } + } else { + inner::execute_helper(command) + } + } +} + cfg_if! { if #[cfg(any(test, feature = "testing"))] { pub use mock_inner::*; diff --git a/illumos-utils/src/running_zone.rs b/illumos-utils/src/running_zone.rs index 805419cb5d9..bdf7ed0cbf3 100644 --- a/illumos-utils/src/running_zone.rs +++ b/illumos-utils/src/running_zone.rs @@ -517,12 +517,12 @@ impl RunningZone { // services are up, so future requests to create network addresses // or manipulate services will work. let fmri = "svc:/milestone/single-user:default"; - wait_for_service(Some(&zone.name), fmri).await.map_err(|_| { - BootError::Timeout { + wait_for_service(Some(&zone.name), fmri, zone.log.clone()) + .await + .map_err(|_| BootError::Timeout { service: fmri.to_string(), zone: zone.name.to_string(), - } - })?; + })?; // If the zone is self-assembling, then SMF service(s) inside the zone // will be creating the listen address for the zone's service(s), diff --git a/illumos-utils/src/svc.rs b/illumos-utils/src/svc.rs index b08679b7201..a16795771cd 100644 --- a/illumos-utils/src/svc.rs +++ b/illumos-utils/src/svc.rs @@ -12,6 +12,7 @@ use omicron_common::backoff; #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))] mod inner { use super::*; + use slog::{warn, Logger}; // TODO(https://www.illumos.org/issues/13837): This is a hack; // remove me when when fixed. Ideally, the ".synchronous()" argument @@ -27,10 +28,19 @@ mod inner { pub async fn wait_for_service<'a, 'b>( zone: Option<&'a str>, fmri: &'b str, + log: Logger, ) -> Result<(), Error> { let name = smf::PropertyName::new("restarter", "state").unwrap(); - let log_notification_failure = |_error, _delay| {}; + let log_notification_failure = |error, delay| { + warn!( + log, + "wait for service {:?} failed: {}. retry in {:?}", + zone, + error, + delay + ); + }; backoff::retry_notify( backoff::retry_policy_local(), || async { @@ -47,6 +57,26 @@ mod inner { == &smf::PropertyValue::Astring("online".to_string()) { return Ok(()); + } else { + // This is helpful in virtual environments where + // services take a few tries to come up. To enable, + // compile with RUSTFLAGS="--cfg svcadm_autoclear" + #[cfg(svcadm_autoclear)] + if let Some(zname) = zone { + if let Err(out) = + tokio::process::Command::new(crate::PFEXEC) + .env_clear() + .arg("svcadm") + .arg("-z") + .arg(zname) + .arg("clear") + .arg("*") + .output() + .await + { + warn!(log, "clearing service maintenance failed: {out}"); + }; + } } } return Err(backoff::BackoffError::transient( diff --git a/illumos-utils/src/zfs.rs b/illumos-utils/src/zfs.rs index a6af9976192..e9554100af1 100644 --- a/illumos-utils/src/zfs.rs +++ b/illumos-utils/src/zfs.rs @@ -20,7 +20,16 @@ pub const ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT: &str = "/zone"; pub const ZONE_ZFS_RAMDISK_DATASET: &str = "rpool/zone"; pub const ZFS: &str = "/usr/sbin/zfs"; + +/// This path is intentionally on a `tmpfs` to prevent copy-on-write behavior +/// and to ensure it goes away on power off. +/// +/// We want minimize the time the key files are in memory, and so we rederive +/// the keys and recreate the files on demand when creating and mounting +/// encrypted filesystems. We then zero them and unlink them. pub const KEYPATH_ROOT: &str = "/var/run/oxide/"; +// Use /tmp so we don't have to worry about running tests with pfexec +pub const TEST_KEYPATH_ROOT: &str = "/tmp"; /// Error returned by [`Zfs::list_datasets`]. #[derive(thiserror::Error, Debug)] @@ -158,19 +167,27 @@ impl fmt::Display for Keypath { } } +#[cfg(not(feature = "tmp_keypath"))] +impl From<&DiskIdentity> for Keypath { + fn from(id: &DiskIdentity) -> Self { + build_keypath(id, KEYPATH_ROOT) + } +} + +#[cfg(feature = "tmp_keypath")] impl From<&DiskIdentity> for Keypath { fn from(id: &DiskIdentity) -> Self { - let filename = format!( - "{}-{}-{}-zfs-aes-256-gcm.key", - id.vendor, id.serial, id.model - ); - let mut path = Utf8PathBuf::new(); - path.push(KEYPATH_ROOT); - path.push(filename); - Keypath(path) + build_keypath(id, TEST_KEYPATH_ROOT) } } +fn build_keypath(id: &DiskIdentity, root: &str) -> Keypath { + let filename = + format!("{}-{}-{}-zfs-aes-256-gcm.key", id.vendor, id.serial, id.model); + let path: Utf8PathBuf = [root, &filename].iter().collect(); + Keypath(path) +} + #[derive(Debug)] pub struct EncryptionDetails { pub keypath: Keypath, diff --git a/illumos-utils/src/zpool.rs b/illumos-utils/src/zpool.rs index 81ded2655ec..f2c395e22b8 100644 --- a/illumos-utils/src/zpool.rs +++ b/illumos-utils/src/zpool.rs @@ -39,6 +39,13 @@ pub struct CreateError { err: Error, } +#[derive(thiserror::Error, Debug)] +#[error("Failed to destroy zpool: {err}")] +pub struct DestroyError { + #[from] + err: Error, +} + #[derive(thiserror::Error, Debug)] #[error("Failed to list zpools: {err}")] pub struct ListError { @@ -89,7 +96,7 @@ impl FromStr for ZpoolHealth { } /// Describes a Zpool. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct ZpoolInfo { name: String, size: u64, @@ -121,6 +128,17 @@ impl ZpoolInfo { pub fn health(&self) -> ZpoolHealth { self.health } + + #[cfg(any(test, feature = "testing"))] + pub fn new_hardcoded(name: String) -> ZpoolInfo { + ZpoolInfo { + name, + size: 1024 * 1024 * 64, + allocated: 1024, + free: 1024 * 1023 * 64, + health: ZpoolHealth::Online, + } + } } impl FromStr for ZpoolInfo { @@ -167,7 +185,10 @@ pub struct Zpool {} #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))] impl Zpool { - pub fn create(name: ZpoolName, vdev: &Utf8Path) -> Result<(), CreateError> { + pub fn create( + name: &ZpoolName, + vdev: &Utf8Path, + ) -> Result<(), CreateError> { let mut cmd = std::process::Command::new(PFEXEC); cmd.env_clear(); cmd.env("LC_ALL", "C.UTF-8"); @@ -189,7 +210,17 @@ impl Zpool { Ok(()) } - pub fn import(name: ZpoolName) -> Result<(), Error> { + pub fn destroy(name: &ZpoolName) -> Result<(), DestroyError> { + let mut cmd = std::process::Command::new(PFEXEC); + cmd.env_clear(); + cmd.env("LC_ALL", "C.UTF-8"); + cmd.arg(ZPOOL).arg("destroy"); + cmd.arg(&name.to_string()); + execute(&mut cmd).map_err(Error::from)?; + Ok(()) + } + + pub fn import(name: &ZpoolName) -> Result<(), Error> { let mut cmd = std::process::Command::new(PFEXEC); cmd.env_clear(); cmd.env("LC_ALL", "C.UTF-8"); diff --git a/installinator-artifactd/Cargo.toml b/installinator-artifactd/Cargo.toml index b14ca4002f6..e9ddc222cd7 100644 --- a/installinator-artifactd/Cargo.toml +++ b/installinator-artifactd/Cargo.toml @@ -7,7 +7,6 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true async-trait.workspace = true -camino.workspace = true clap.workspace = true dropshot.workspace = true hyper.workspace = true @@ -15,7 +14,6 @@ schemars.workspace = true serde.workspace = true serde_json.workspace = true slog.workspace = true -tokio.workspace = true uuid.workspace = true installinator-common.workspace = true diff --git a/installinator-artifactd/src/bin/installinator-artifactd.rs b/installinator-artifactd/src/bin/installinator-artifactd.rs index b09dc82acd8..abe63bbe31e 100644 --- a/installinator-artifactd/src/bin/installinator-artifactd.rs +++ b/installinator-artifactd/src/bin/installinator-artifactd.rs @@ -29,10 +29,9 @@ fn do_run() -> Result<(), CmdError> { match args { Args::Openapi => { installinator_artifactd::run_openapi().map_err(|error| { - CmdError::Failure(format!( - "failed to generate OpenAPI spec: {:?}", - error - )) + CmdError::Failure( + error.context("failed to generate OpenAPI spec"), + ) }) } } diff --git a/installinator-common/Cargo.toml b/installinator-common/Cargo.toml index 8fea234e200..4381de74ebf 100644 --- a/installinator-common/Cargo.toml +++ b/installinator-common/Cargo.toml @@ -8,7 +8,6 @@ license = "MPL-2.0" anyhow.workspace = true camino.workspace = true illumos-utils.workspace = true -omicron-common.workspace = true schemars.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/installinator/Cargo.toml b/installinator/Cargo.toml index a4f170ddba4..d489e73ec16 100644 --- a/installinator/Cargo.toml +++ b/installinator/Cargo.toml @@ -23,13 +23,11 @@ installinator-common.workspace = true ipcc-key-value.workspace = true itertools.workspace = true libc.workspace = true -once_cell.workspace = true omicron-common.workspace = true -progenitor-client.workspace = true reqwest.workspace = true -serde.workspace = true sha2.workspace = true sled-hardware.workspace = true +sled-storage.workspace = true slog.workspace = true slog-async.workspace = true slog-envlogger.workspace = true @@ -38,7 +36,6 @@ smf.workspace = true tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["full"] } -toml.workspace = true tufaceous-lib.workspace = true update-engine.workspace = true uuid.workspace = true diff --git a/installinator/src/hardware.rs b/installinator/src/hardware.rs index ffa0b74739e..b037384cbee 100644 --- a/installinator/src/hardware.rs +++ b/installinator/src/hardware.rs @@ -6,10 +6,11 @@ use anyhow::anyhow; use anyhow::ensure; use anyhow::Context; use anyhow::Result; -use sled_hardware::Disk; use sled_hardware::DiskVariant; use sled_hardware::HardwareManager; use sled_hardware::SledMode; +use sled_storage::disk::Disk; +use sled_storage::disk::RawDisk; use slog::info; use slog::Logger; @@ -28,7 +29,8 @@ impl Hardware { anyhow!("failed to create HardwareManager: {err}") })?; - let disks = hardware.disks(); + let disks: Vec = + hardware.disks().into_iter().map(|disk| disk.into()).collect(); info!( log, "found gimlet hardware"; diff --git a/installinator/src/write.rs b/installinator/src/write.rs index 6c0c1f63c7b..22dd2adbf68 100644 --- a/installinator/src/write.rs +++ b/installinator/src/write.rs @@ -122,8 +122,9 @@ impl WriteDestination { ); let zpool_name = disk.zpool_name().clone(); - let control_plane_dir = zpool_name - .dataset_mountpoint(sled_hardware::INSTALL_DATASET); + let control_plane_dir = zpool_name.dataset_mountpoint( + sled_storage::dataset::INSTALL_DATASET, + ); match drives.entry(slot) { Entry::Vacant(entry) => { diff --git a/internal-dns/Cargo.toml b/internal-dns/Cargo.toml index ecb2d48bda1..96993ce6a21 100644 --- a/internal-dns/Cargo.toml +++ b/internal-dns/Cargo.toml @@ -14,7 +14,6 @@ omicron-common.workspace = true reqwest = { workspace = true, features = ["rustls-tls", "stream"] } slog.workspace = true thiserror.workspace = true -trust-dns-proto.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/internal-dns/tests/output/test-server.json b/internal-dns/tests/output/test-server.json index 5f4d6d155e8..5720dec19fd 100644 --- a/internal-dns/tests/output/test-server.json +++ b/internal-dns/tests/output/test-server.json @@ -33,18 +33,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "Error": { "description": "Error information from a response.", @@ -65,6 +53,18 @@ "request_id" ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index feb25eb1f14..4fc13a31d8b 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -12,12 +12,10 @@ anyhow.workspace = true assert_matches.workspace = true async-trait.workspace = true base64.workspace = true -bb8.workspace = true cancel-safe-futures.workspace = true camino.workspace = true clap.workspace = true chrono.workspace = true -cookie.workspace = true crucible-agent-client.workspace = true crucible-pantry-client.workspace = true dns-service-client.workspace = true @@ -36,16 +34,12 @@ ipnetwork.workspace = true lazy_static.workspace = true macaddr.workspace = true mime_guess.workspace = true -newtype_derive.workspace = true # Not under "dev-dependencies"; these also need to be implemented for # integration tests. nexus-test-interface.workspace = true num-integer.workspace = true once_cell.workspace = true openssl.workspace = true -openssl-sys.workspace = true -openssl-probe.workspace = true -oso.workspace = true oximeter-client.workspace = true oximeter-db.workspace = true parse-display.workspace = true @@ -75,11 +69,9 @@ tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["full"] } tokio-postgres = { workspace = true, features = ["with-serde_json-1"] } -toml.workspace = true tough.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true -usdt.workspace = true nexus-defaults.workspace = true nexus-db-model.workspace = true @@ -104,6 +96,7 @@ hyper-rustls.workspace = true itertools.workspace = true gateway-messages.workspace = true gateway-test-utils.workspace = true +hubtools.workspace = true nexus-test-utils-macros.workspace = true nexus-test-utils.workspace = true omicron-sled-agent.workspace = true @@ -117,6 +110,7 @@ pretty_assertions.workspace = true rcgen.workspace = true regex.workspace = true similar-asserts.workspace = true +sp-sim.workspace = true rustls = { workspace = true } subprocess.workspace = true term.workspace = true diff --git a/nexus/db-model/Cargo.toml b/nexus/db-model/Cargo.toml index a5cb9a06bef..b7514c4806a 100644 --- a/nexus/db-model/Cargo.toml +++ b/nexus/db-model/Cargo.toml @@ -20,7 +20,6 @@ parse-display.workspace = true pq-sys = "*" rand.workspace = true ref-cast.workspace = true -thiserror.workspace = true schemars = { workspace = true, features = ["chrono", "uuid1"] } semver.workspace = true serde.workspace = true diff --git a/nexus/db-model/src/oximeter_info.rs b/nexus/db-model/src/oximeter_info.rs index ac30384c59a..39bde98ea80 100644 --- a/nexus/db-model/src/oximeter_info.rs +++ b/nexus/db-model/src/oximeter_info.rs @@ -8,7 +8,7 @@ use chrono::{DateTime, Utc}; use nexus_types::internal_api; use uuid::Uuid; -/// Message used to notify Nexus that this oximeter instance is up and running. +/// A record representing a registered `oximeter` collector. #[derive(Queryable, Insertable, Debug, Clone, Copy)] #[diesel(table_name = oximeter)] pub struct OximeterInfo { @@ -18,8 +18,9 @@ pub struct OximeterInfo { pub time_created: DateTime, /// When this resource was last modified. pub time_modified: DateTime, - /// The address on which this oximeter instance listens for requests + /// The address on which this `oximeter` instance listens for requests. pub ip: ipnetwork::IpNetwork, + /// The port on which this `oximeter` instance listens for requests. pub port: SqlU16, } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index cff261e01a4..7c6b8bbd0a2 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1243,7 +1243,7 @@ table! { /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(9, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(10, 0, 0); allow_tables_to_appear_in_same_query!( system_update, diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index 5edf4f1e896..b1b8f3b28f7 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -21,25 +21,18 @@ diesel-dtrace.workspace = true dropshot.workspace = true futures.workspace = true headers.workspace = true -hex.workspace = true http.workspace = true hyper.workspace = true ipnetwork.workspace = true lazy_static.workspace = true macaddr.workspace = true newtype_derive.workspace = true -once_cell.workspace = true openssl.workspace = true -openssl-sys.workspace = true -openssl-probe.workspace = true oso.workspace = true paste.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" -rand.workspace = true ref-cast.workspace = true -reqwest = { workspace = true, features = [ "json" ] } -ring.workspace = true samael.workspace = true serde.workspace = true serde_json.workspace = true @@ -51,14 +44,11 @@ static_assertions.workspace = true steno.workspace = true thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } -tokio-postgres = { workspace = true, features = [ "with-serde_json-1" ] } -toml.workspace = true uuid.workspace = true usdt.workspace = true authz-macros.workspace = true db-macros.workspace = true -nexus-defaults.workspace = true nexus-db-model.workspace = true nexus-types.workspace = true omicron-common.workspace = true diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 6b7d97754ae..114b9dbe314 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -685,7 +685,7 @@ impl DataStore { .rev() .find(|(_i, (_collection_id, nerrors))| *nerrors == 0); let candidate = match last_completed_idx { - Some((i, _)) if i == 0 => candidates.iter().skip(1).next(), + Some((0, _)) => candidates.iter().skip(1).next(), _ => candidates.iter().next(), } .map(|(collection_id, _nerrors)| *collection_id); diff --git a/nexus/db-queries/src/db/datastore/oximeter.rs b/nexus/db-queries/src/db/datastore/oximeter.rs index c9b3a59b051..55b650ea538 100644 --- a/nexus/db-queries/src/db/datastore/oximeter.rs +++ b/nexus/db-queries/src/db/datastore/oximeter.rs @@ -21,7 +21,20 @@ use omicron_common::api::external::ResourceType; use uuid::Uuid; impl DataStore { - // Create a record for a new Oximeter instance + /// Lookup an oximeter instance by its ID. + pub async fn oximeter_lookup( + &self, + id: &Uuid, + ) -> Result { + use db::schema::oximeter::dsl; + dsl::oximeter + .find(*id) + .first_async(&*self.pool_connection_unauthorized().await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// Create a record for a new Oximeter instance pub async fn oximeter_create( &self, info: &OximeterInfo, @@ -55,7 +68,7 @@ impl DataStore { Ok(()) } - // List the oximeter collector instances + /// List the oximeter collector instances pub async fn oximeter_list( &self, page_params: &DataPageParams<'_, Uuid>, @@ -69,7 +82,7 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } - // Create a record for a new producer endpoint + /// Create a record for a new producer endpoint pub async fn producer_endpoint_create( &self, producer: &ProducerEndpoint, @@ -102,7 +115,27 @@ impl DataStore { Ok(()) } - // List the producer endpoint records by the oximeter instance to which they're assigned. + /// Delete a record for a producer endpoint, by its ID. + /// + /// This is idempotent, and deleting a record that is already removed is a + /// no-op. If the record existed, then the ID of the `oximeter` collector is + /// returned. If there was no record, `None` is returned. + pub async fn producer_endpoint_delete( + &self, + id: &Uuid, + ) -> Result, Error> { + use db::schema::metric_producer::dsl; + diesel::delete(dsl::metric_producer.find(*id)) + .returning(dsl::oximeter_id) + .get_result_async::( + &*self.pool_connection_unauthorized().await?, + ) + .await + .optional() + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// List the producer endpoint records by the oximeter instance to which they're assigned. pub async fn producers_list_by_oximeter_id( &self, oximeter_id: Uuid, diff --git a/nexus/inventory/Cargo.toml b/nexus/inventory/Cargo.toml index 965ff3f02a2..202aff49b2e 100644 --- a/nexus/inventory/Cargo.toml +++ b/nexus/inventory/Cargo.toml @@ -7,7 +7,6 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true chrono.workspace = true -futures.workspace = true gateway-client.workspace = true gateway-messages.workspace = true nexus-types.workspace = true diff --git a/nexus/inventory/src/collector.rs b/nexus/inventory/src/collector.rs index d40b09d2be5..1676f44083f 100644 --- a/nexus/inventory/src/collector.rs +++ b/nexus/inventory/src/collector.rs @@ -280,6 +280,15 @@ mod test { let message = regex::Regex::new(r"os error \d+") .unwrap() .replace_all(&e, "os error <>"); + // Communication errors differ based on the configuration of the + // machine running the test. For example whether or not the machine + // has IPv6 configured will determine if an error is network + // unreachable or a timeout due to sending a packet to a known + // discard prefix. So just key in on the communication error in a + // general sense. + let message = regex::Regex::new(r"Communication Error.*") + .unwrap() + .replace_all(&message, "Communication Error <>"); write!(&mut s, "error: {}\n", message).unwrap(); } diff --git a/nexus/inventory/tests/output/collector_errors.txt b/nexus/inventory/tests/output/collector_errors.txt index f231cc7d97b..44040462530 100644 --- a/nexus/inventory/tests/output/collector_errors.txt +++ b/nexus/inventory/tests/output/collector_errors.txt @@ -41,4 +41,4 @@ cabooses found: RotSlotB baseboard part "FAKE_SIM_SIDECAR" serial "SimSidecar1": board "SimSidecarRot" errors: -error: MGS "http://[100::1]:12345": listing ignition targets: Communication Error: error sending request for url (http://[100::1]:12345/ignition): error trying to connect: tcp connect error: Network is unreachable (os error <>): error sending request for url (http://[100::1]:12345/ignition): error trying to connect: tcp connect error: Network is unreachable (os error <>): error trying to connect: tcp connect error: Network is unreachable (os error <>): tcp connect error: Network is unreachable (os error <>): Network is unreachable (os error <>) +error: MGS "http://[100::1]:12345": listing ignition targets: Communication Error <> diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 17d033c5a0d..923bb1777ec 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1320,7 +1320,9 @@ impl super::Nexus { .await?; // If the supplied instance state indicates that the instance no longer - // has an active VMM, attempt to delete the virtual provisioning record + // has an active VMM, attempt to delete the virtual provisioning record, + // and the assignment of the Propolis metric producer to an oximeter + // collector. // // As with updating networking state, this must be done before // committing the new runtime state to the database: once the DB is @@ -1338,6 +1340,21 @@ impl super::Nexus { (&new_runtime_state.instance_state.gen).into(), ) .await?; + + // TODO-correctness: The `notify_instance_updated` method can run + // concurrently with itself in some situations, such as where a + // sled-agent attempts to update Nexus about a stopped instance; + // that times out; and it makes another request to a different + // Nexus. The call to `unassign_producer` is racy in those + // situations, and we may end with instances with no metrics. + // + // This unfortunate case should be handled as part of + // instance-lifecycle improvements, notably using a reliable + // persistent workflow to correctly update the oximete assignment as + // an instance's state changes. + // + // Tracked in https://github.com/oxidecomputer/omicron/issues/3742. + self.unassign_producer(instance_id).await?; } // Write the new instance and VMM states back to CRDB. This needs to be diff --git a/nexus/src/app/oximeter.rs b/nexus/src/app/oximeter.rs index bc947cf4bca..7dfa2fb68b6 100644 --- a/nexus/src/app/oximeter.rs +++ b/nexus/src/app/oximeter.rs @@ -87,32 +87,43 @@ impl super::Nexus { "address" => oximeter_info.address, ); - // Regardless, notify the collector of any assigned metric producers. This should be empty - // if this Oximeter collector is registering for the first time, but may not be if the - // service is re-registering after failure. - let pagparams = DataPageParams { - marker: None, - direction: PaginationOrder::Ascending, - limit: std::num::NonZeroU32::new(100).unwrap(), - }; - let producers = self - .db_datastore - .producers_list_by_oximeter_id( - oximeter_info.collector_id, - &pagparams, - ) - .await?; - if !producers.is_empty() { + // Regardless, notify the collector of any assigned metric producers. + // + // This should be empty if this Oximeter collector is registering for + // the first time, but may not be if the service is re-registering after + // failure. + let client = self.build_oximeter_client( + &oximeter_info.collector_id, + oximeter_info.address, + ); + let mut last_producer_id = None; + loop { + let pagparams = DataPageParams { + marker: last_producer_id.as_ref(), + direction: PaginationOrder::Ascending, + limit: std::num::NonZeroU32::new(100).unwrap(), + }; + let producers = self + .db_datastore + .producers_list_by_oximeter_id( + oximeter_info.collector_id, + &pagparams, + ) + .await?; + if producers.is_empty() { + return Ok(()); + } debug!( self.log, - "registered oximeter collector that is already assigned producers, re-assigning them to the collector"; + "re-assigning existing metric producers to a collector"; "n_producers" => producers.len(), "collector_id" => ?oximeter_info.collector_id, ); - let client = self.build_oximeter_client( - &oximeter_info.collector_id, - oximeter_info.address, - ); + // Be sure to continue paginating from the last producer. + // + // Safety: We check just above if the list is empty, so there is a + // last element. + last_producer_id.replace(producers.last().unwrap().id()); for producer in producers.into_iter() { let producer_info = oximeter_client::types::ProducerEndpoint { id: producer.id(), @@ -132,7 +143,6 @@ impl super::Nexus { .map_err(Error::from)?; } } - Ok(()) } /// Register as a metric producer with the oximeter metric collection server. @@ -187,6 +197,58 @@ impl super::Nexus { Ok(()) } + /// Idempotently un-assign a producer from an oximeter collector. + pub(crate) async fn unassign_producer( + &self, + id: &Uuid, + ) -> Result<(), Error> { + if let Some(collector_id) = + self.db_datastore.producer_endpoint_delete(id).await? + { + debug!( + self.log, + "deleted metric producer assignment"; + "producer_id" => %id, + "collector_id" => %collector_id, + ); + let oximeter_info = + self.db_datastore.oximeter_lookup(&collector_id).await?; + let address = + SocketAddr::new(oximeter_info.ip.ip(), *oximeter_info.port); + let client = self.build_oximeter_client(&id, address); + if let Err(e) = client.producer_delete(&id).await { + error!( + self.log, + "failed to delete producer from collector"; + "producer_id" => %id, + "collector_id" => %collector_id, + "address" => %address, + "error" => ?e, + ); + return Err(Error::internal_error( + format!("failed to delete producer from collector: {e:?}") + .as_str(), + )); + } else { + debug!( + self.log, + "successfully deleted producer from collector"; + "producer_id" => %id, + "collector_id" => %collector_id, + "address" => %address, + ); + Ok(()) + } + } else { + trace!( + self.log, + "un-assigned non-existent metric producer"; + "producer_id" => %id, + ); + Ok(()) + } + } + /// Returns a results from the timeseries DB based on the provided query /// parameters. /// diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index bed690f839d..163f3bd5bb3 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -675,10 +675,15 @@ impl super::Nexus { addresses: info.addresses.iter().map(|a| a.address).collect(), bgp_peers: peer_info .iter() - .map(|(_p, asn, addr)| BgpPeerConfig { + .map(|(p, asn, addr)| BgpPeerConfig { addr: *addr, asn: *asn, port: port.port_name.clone(), + hold_time: Some(p.hold_time.0.into()), + connect_retry: Some(p.connect_retry.0.into()), + delay_open: Some(p.delay_open.0.into()), + idle_hold_time: Some(p.idle_hold_time.0.into()), + keepalive: Some(p.keepalive.0.into()), }) .collect(), switch: port.switch_location.parse().unwrap(), diff --git a/nexus/src/app/sagas/mod.rs b/nexus/src/app/sagas/mod.rs index 83e0e9b8b4e..5b1843be3d0 100644 --- a/nexus/src/app/sagas/mod.rs +++ b/nexus/src/app/sagas/mod.rs @@ -313,7 +313,7 @@ macro_rules! declare_saga_actions { }; } -pub(crate) const NEXUS_DPD_TAG: &str = "nexus"; +use omicron_common::OMICRON_DPD_TAG as NEXUS_DPD_TAG; pub(crate) use __action_name; pub(crate) use __emit_action; diff --git a/nexus/src/app/sagas/switch_port_settings_apply.rs b/nexus/src/app/sagas/switch_port_settings_apply.rs index fb06dc5fc03..0c06d6ff832 100644 --- a/nexus/src/app/sagas/switch_port_settings_apply.rs +++ b/nexus/src/app/sagas/switch_port_settings_apply.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::{NexusActionContext, NEXUS_DPD_TAG}; +use crate::app::map_switch_zone_addrs; use crate::app::sagas::retry_until_known_result; use crate::app::sagas::{ declare_saga_actions, ActionRegistry, NexusSaga, SagaInitError, @@ -15,15 +16,17 @@ use dpd_client::types::{ RouteSettingsV4, RouteSettingsV6, }; use dpd_client::{Ipv4Cidr, Ipv6Cidr}; +use internal_dns::ServiceName; use ipnetwork::IpNetwork; use mg_admin_client::types::Prefix4; -use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, BgpRoute}; +use mg_admin_client::types::{ApplyRequest, BgpPeerConfig}; use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed, NETWORK_KEY}; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::UpdatePrecondition; use nexus_db_queries::{authn, db}; use nexus_types::external_api::params; -use omicron_common::api::external::{self, DataPageParams, NameOrId}; +use omicron_common::address::SLED_AGENT_PORT; +use omicron_common::api::external::{self, NameOrId}; use omicron_common::api::internal::shared::{ ParseSwitchLocationError, SwitchLocation, }; @@ -35,8 +38,8 @@ use sled_agent_client::types::{ BgpPeerConfig as OmicronBgpPeerConfig, HostPortConfig, }; use std::collections::HashMap; -use std::net::IpAddr; use std::net::SocketAddrV6; +use std::net::{IpAddr, Ipv6Addr}; use std::str::FromStr; use std::sync::Arc; use steno::ActionError; @@ -177,7 +180,6 @@ pub(crate) fn api_to_dpd_port_settings( settings: &SwitchPortSettingsCombinedResult, ) -> Result { let mut dpd_port_settings = PortSettings { - tag: NEXUS_DPD_TAG.into(), links: HashMap::new(), v4_routes: HashMap::new(), v6_routes: HashMap::new(), @@ -192,6 +194,7 @@ pub(crate) fn api_to_dpd_port_settings( LinkSettings { params: LinkCreate { autoneg: false, + lane: Some(LinkId(0)), kr: false, fec: match l.fec { SwitchLinkFec::Firecode => PortFec::Firecode, @@ -283,7 +286,13 @@ async fn spa_ensure_switch_port_settings( })?; retry_until_known_result(log, || async { - dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await + dpd_client + .port_settings_apply( + &port_id, + Some(NEXUS_DPD_TAG), + &dpd_port_settings, + ) + .await }) .await .map_err(|e| match e { @@ -331,7 +340,9 @@ async fn spa_undo_ensure_switch_port_settings( Some(id) => id, None => { retry_until_known_result(log, || async { - dpd_client.port_settings_clear(&port_id).await + dpd_client + .port_settings_clear(&port_id, Some(NEXUS_DPD_TAG)) + .await }) .await .map_err(|e| external::Error::internal_error(&e.to_string()))?; @@ -355,7 +366,13 @@ async fn spa_undo_ensure_switch_port_settings( })?; retry_until_known_result(log, || async { - dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await + dpd_client + .port_settings_apply( + &port_id, + Some(NEXUS_DPD_TAG), + &dpd_port_settings, + ) + .await }) .await .map_err(|e| external::Error::internal_error(&e.to_string()))?; @@ -418,22 +435,6 @@ pub(crate) async fn ensure_switch_port_bgp_settings( )) })?; - // TODO picking the first configured address by default, but this needs - // to be something that can be specified in the API. - let nexthop = match settings.addresses.get(0) { - Some(switch_port_addr) => Ok(switch_port_addr.address.ip()), - None => Err(ActionError::action_failed( - "at least one address required for bgp peering".to_string(), - )), - }?; - - let nexthop = match nexthop { - IpAddr::V4(nexthop) => Ok(nexthop), - IpAddr::V6(_) => Err(ActionError::action_failed( - "IPv6 nexthop not yet supported".to_string(), - )), - }?; - let mut prefixes = Vec::new(); for a in &announcements { let value = match a.network.ip() { @@ -455,7 +456,7 @@ pub(crate) async fn ensure_switch_port_bgp_settings( connect_retry: peer.connect_retry.0.into(), keepalive: peer.keepalive.0.into(), resolution: BGP_SESSION_RESOLUTION, - routes: vec![BgpRoute { nexthop, prefixes }], + originate: prefixes, }; bgp_peer_configs.push(bpc); @@ -809,7 +810,7 @@ pub(crate) async fn select_mg_client( } pub(crate) async fn get_scrimlet_address( - _location: SwitchLocation, + location: SwitchLocation, nexus: &Arc, ) -> Result { /* TODO this depends on DNS entries only coming from RSS, it's broken @@ -826,21 +827,41 @@ pub(crate) async fn get_scrimlet_address( )) }) */ - let opctx = &nexus.opctx_for_internal_api(); - Ok(nexus - .sled_list(opctx, &DataPageParams::max_page()) + let result = nexus + .resolver() + .await + .lookup_all_ipv6(ServiceName::Dendrite) .await .map_err(|e| { ActionError::action_failed(format!( - "get_scrimlet_address: failed to list sleds: {e}" + "scrimlet dns lookup failed {e}", )) - })? - .into_iter() - .find(|x| x.is_scrimlet()) - .ok_or(ActionError::action_failed( - "get_scrimlet_address: no scrimlets found".to_string(), - ))? - .address()) + }); + + let mappings = match result { + Ok(addrs) => map_switch_zone_addrs(&nexus.log, addrs).await, + Err(e) => { + warn!(nexus.log, "Failed to lookup Dendrite address: {e}"); + return Err(ActionError::action_failed(format!( + "switch mapping failed {e}", + ))); + } + }; + + let addr = match mappings.get(&location) { + Some(addr) => addr, + None => { + return Err(ActionError::action_failed(format!( + "address for switch at location: {location} not found", + ))); + } + }; + + let mut segments = addr.segments(); + segments[7] = 1; + let addr = Ipv6Addr::from(segments); + + Ok(SocketAddrV6::new(addr, SLED_AGENT_PORT, 0, 0)) } #[derive(Clone, Debug)] @@ -941,6 +962,11 @@ pub(crate) async fn bootstore_update( asn: *asn, port: switch_port_name.into(), addr, + hold_time: Some(p.hold_time.0.into()), + connect_retry: Some(p.connect_retry.0.into()), + delay_open: Some(p.delay_open.0.into()), + idle_hold_time: Some(p.idle_hold_time.0.into()), + keepalive: Some(p.keepalive.0.into()), }), IpAddr::V6(_) => { warn!(opctx.log, "IPv6 peers not yet supported"); diff --git a/nexus/src/app/sagas/switch_port_settings_clear.rs b/nexus/src/app/sagas/switch_port_settings_clear.rs index 14544b0f552..1ab2f6be0c6 100644 --- a/nexus/src/app/sagas/switch_port_settings_clear.rs +++ b/nexus/src/app/sagas/switch_port_settings_clear.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::switch_port_settings_apply::select_dendrite_client; -use super::NexusActionContext; +use super::{NexusActionContext, NEXUS_DPD_TAG}; use crate::app::sagas::retry_until_known_result; use crate::app::sagas::switch_port_settings_apply::{ api_to_dpd_port_settings, apply_bootstore_update, bootstore_update, @@ -154,7 +154,7 @@ async fn spa_clear_switch_port_settings( let dpd_client = select_dendrite_client(&sagactx).await?; retry_until_known_result(log, || async { - dpd_client.port_settings_clear(&port_id).await + dpd_client.port_settings_clear(&port_id, Some(NEXUS_DPD_TAG)).await }) .await .map_err(|e| ActionError::action_failed(e.to_string()))?; @@ -197,7 +197,13 @@ async fn spa_undo_clear_switch_port_settings( .map_err(ActionError::action_failed)?; retry_until_known_result(log, || async { - dpd_client.port_settings_apply(&port_id, &dpd_port_settings).await + dpd_client + .port_settings_apply( + &port_id, + Some(NEXUS_DPD_TAG), + &dpd_port_settings, + ) + .await }) .await .map_err(|e| external::Error::internal_error(&e.to_string()))?; diff --git a/nexus/src/app/test_interfaces.rs b/nexus/src/app/test_interfaces.rs index c7a6165998c..ad2ea50e071 100644 --- a/nexus/src/app/test_interfaces.rs +++ b/nexus/src/app/test_interfaces.rs @@ -10,6 +10,11 @@ use sled_agent_client::Client as SledAgentClient; use std::sync::Arc; use uuid::Uuid; +pub use super::update::SpUpdateError; +pub use super::update::SpUpdater; +pub use super::update::UpdateProgress; +pub use gateway_client::types::SpType; + /// Exposes additional [`super::Nexus`] interfaces for use by the test suite #[async_trait] pub trait TestInterfaces { diff --git a/nexus/src/app/update.rs b/nexus/src/app/update/mod.rs similarity index 99% rename from nexus/src/app/update.rs rename to nexus/src/app/update/mod.rs index 12f3c1016e0..4196cd8a71e 100644 --- a/nexus/src/app/update.rs +++ b/nexus/src/app/update/mod.rs @@ -26,6 +26,10 @@ use std::path::Path; use tokio::io::AsyncWriteExt; use uuid::Uuid; +mod sp_updater; + +pub use sp_updater::{SpUpdateError, SpUpdater, UpdateProgress}; + static BASE_ARTIFACT_DIR: &str = "/var/tmp/oxide_artifacts"; impl super::Nexus { diff --git a/nexus/src/app/update/sp_updater.rs b/nexus/src/app/update/sp_updater.rs new file mode 100644 index 00000000000..9abb2ad222c --- /dev/null +++ b/nexus/src/app/update/sp_updater.rs @@ -0,0 +1,345 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Module containing types for updating SPs via MGS. + +use futures::Future; +use gateway_client::types::SpType; +use gateway_client::types::SpUpdateStatus; +use gateway_client::SpComponent; +use slog::Logger; +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::watch; +use uuid::Uuid; + +type GatewayClientError = gateway_client::Error; + +#[derive(Debug, thiserror::Error)] +pub enum SpUpdateError { + #[error("error communicating with MGS")] + MgsCommunication(#[from] GatewayClientError), + + // Error returned when we successfully start an update but it fails to + // complete successfully. + #[error("update failed to complete: {0}")] + FailedToComplete(String), +} + +// TODO-cleanup Probably share this with other update implementations? +#[derive(Debug, PartialEq, Clone)] +pub enum UpdateProgress { + Started, + Preparing { progress: Option }, + InProgress { progress: Option }, + Complete, + Failed(String), +} + +pub struct SpUpdater { + log: Logger, + progress: watch::Sender>, + sp_type: SpType, + sp_slot: u32, + update_id: Uuid, + // TODO-clarity maybe a newtype for this? TBD how we get this from + // wherever it's stored, which might give us a stronger type already. + sp_hubris_archive: Vec, +} + +impl SpUpdater { + pub fn new( + sp_type: SpType, + sp_slot: u32, + update_id: Uuid, + sp_hubris_archive: Vec, + log: &Logger, + ) -> Self { + let log = log.new(slog::o!( + "sp_type" => format!("{sp_type:?}"), + "sp_slot" => sp_slot, + "update_id" => format!("{update_id}"), + )); + let progress = watch::Sender::new(None); + Self { log, progress, sp_type, sp_slot, update_id, sp_hubris_archive } + } + + pub fn progress_watcher(&self) -> watch::Receiver> { + self.progress.subscribe() + } + + /// Drive this SP update to completion (or failure). + /// + /// Only one MGS instance is required to drive an update; however, if + /// multiple MGS instances are available and passed to this method and an + /// error occurs communicating with one instance, `SpUpdater` will try the + /// remaining instances before failing. + /// + /// # Panics + /// + /// If `mgs_clients` is empty. + pub async fn update>>>( + self, + mgs_clients: T, + ) -> Result<(), SpUpdateError> { + let mut mgs_clients = mgs_clients.into(); + assert!(!mgs_clients.is_empty()); + + // The async blocks below want `&self` references, but we take `self` + // for API clarity (to start a new SP update, the caller should + // construct a new `SpUpdater`). Create a `&self` ref that we use + // through the remainder of this method. + let me = &self; + + me.try_all_mgs_clients(&mut mgs_clients, |client| async move { + me.start_update_one_mgs(&client).await + }) + .await?; + + // `wait_for_update_completion` uses `try_all_mgs_clients` internally, + // so we don't wrap it here. + me.wait_for_update_completion(&mut mgs_clients).await?; + + me.try_all_mgs_clients(&mut mgs_clients, |client| async move { + me.finalize_update_via_reset_one_mgs(&client).await + }) + .await?; + + Ok(()) + } + + // Helper method to run `op` against all clients. If `op` returns + // successfully for one client, that client will be rotated to the front of + // the list (so any subsequent operations can start with the first client). + async fn try_all_mgs_clients( + &self, + mgs_clients: &mut VecDeque>, + op: F, + ) -> Result + where + F: Fn(Arc) -> Fut, + Fut: Future>, + { + let mut last_err = None; + for (i, client) in mgs_clients.iter().enumerate() { + match op(Arc::clone(client)).await { + Ok(val) => { + // Shift our list of MGS clients such that the one we just + // used is at the front for subsequent requests. + mgs_clients.rotate_left(i); + return Ok(val); + } + // If we have an error communicating with an MGS instance + // (timeout, unexpected connection close, etc.), we'll move on + // and try the next MGS client. If this was the last client, + // we'll stash the error in `last_err` and return it below the + // loop. + Err(GatewayClientError::CommunicationError(err)) => { + last_err = Some(err); + continue; + } + Err(err) => return Err(err), + } + } + + // We know we have at least one `mgs_client`, so the only way to get + // here is if all clients failed with connection errors. Return the + // error from the last MGS we tried. + Err(GatewayClientError::CommunicationError(last_err.unwrap())) + } + + async fn start_update_one_mgs( + &self, + client: &gateway_client::Client, + ) -> Result<(), GatewayClientError> { + // The SP has two firmware slots, but they're aren't individually + // labled. We always request an update to slot 0, which means "the + // inactive slot". + let firmware_slot = 0; + + // Start the update. + client + .sp_component_update( + self.sp_type, + self.sp_slot, + SpComponent::SP_ITSELF.const_as_str(), + firmware_slot, + &self.update_id, + reqwest::Body::from(self.sp_hubris_archive.clone()), + ) + .await?; + + self.progress.send_replace(Some(UpdateProgress::Started)); + + info!( + self.log, "SP update started"; + "mgs_addr" => client.baseurl(), + ); + + Ok(()) + } + + async fn wait_for_update_completion( + &self, + mgs_clients: &mut VecDeque>, + ) -> Result<(), SpUpdateError> { + // How frequently do we poll MGS for the update progress? + const STATUS_POLL_INTERVAL: Duration = Duration::from_secs(3); + + loop { + let update_status = self + .try_all_mgs_clients(mgs_clients, |client| async move { + let update_status = client + .sp_component_update_status( + self.sp_type, + self.sp_slot, + SpComponent::SP_ITSELF.const_as_str(), + ) + .await?; + + info!( + self.log, "got SP update status"; + "mgs_addr" => client.baseurl(), + "status" => ?update_status, + ); + + Ok(update_status) + }) + .await? + .into_inner(); + + // The majority of possible update statuses indicate failure; we'll + // handle the small number of non-failure cases by either + // `continue`ing or `return`ing; all other branches will give us an + // error string that we can report. + let error_message = match update_status { + // For `Preparing` and `InProgress`, we could check the progress + // information returned by these steps and try to check that + // we're still _making_ progress, but every Nexus instance needs + // to do that anyway in case we (or the MGS instance delivering + // the update) crash, so we'll omit that check here. Instead, we + // just sleep and we'll poll again shortly. + SpUpdateStatus::Preparing { id, progress } => { + if id == self.update_id { + let progress = progress.and_then(|progress| { + if progress.current > progress.total { + warn!( + self.log, "nonsense SP preparing progress"; + "current" => progress.current, + "total" => progress.total, + ); + None + } else if progress.total == 0 { + None + } else { + Some( + f64::from(progress.current) + / f64::from(progress.total), + ) + } + }); + self.progress.send_replace(Some( + UpdateProgress::Preparing { progress }, + )); + tokio::time::sleep(STATUS_POLL_INTERVAL).await; + continue; + } else { + format!("different update is now preparing ({id})") + } + } + SpUpdateStatus::InProgress { + id, + bytes_received, + total_bytes, + } => { + if id == self.update_id { + let progress = if bytes_received > total_bytes { + warn!( + self.log, "nonsense SP progress"; + "bytes_received" => bytes_received, + "total_bytes" => total_bytes, + ); + None + } else if total_bytes == 0 { + None + } else { + Some( + f64::from(bytes_received) + / f64::from(total_bytes), + ) + }; + self.progress.send_replace(Some( + UpdateProgress::InProgress { progress }, + )); + tokio::time::sleep(STATUS_POLL_INTERVAL).await; + continue; + } else { + format!("different update is now in progress ({id})") + } + } + SpUpdateStatus::Complete { id } => { + if id == self.update_id { + self.progress.send_replace(Some( + UpdateProgress::InProgress { progress: Some(1.0) }, + )); + return Ok(()); + } else { + format!("different update is now in complete ({id})") + } + } + SpUpdateStatus::None => { + "update status lost (did the SP reset?)".to_string() + } + SpUpdateStatus::Aborted { id } => { + if id == self.update_id { + "update was aborted".to_string() + } else { + format!("different update is now in complete ({id})") + } + } + SpUpdateStatus::Failed { code, id } => { + if id == self.update_id { + format!("update failed (error code {code})") + } else { + format!("different update failed ({id})") + } + } + SpUpdateStatus::RotError { id, message } => { + if id == self.update_id { + format!("update failed (rot error: {message})") + } else { + format!("different update failed with rot error ({id})") + } + } + }; + + self.progress.send_replace(Some(UpdateProgress::Failed( + error_message.clone(), + ))); + return Err(SpUpdateError::FailedToComplete(error_message)); + } + } + + async fn finalize_update_via_reset_one_mgs( + &self, + client: &gateway_client::Client, + ) -> Result<(), GatewayClientError> { + client + .sp_component_reset( + self.sp_type, + self.sp_slot, + SpComponent::SP_ITSELF.const_as_str(), + ) + .await?; + + self.progress.send_replace(Some(UpdateProgress::Complete)); + info!( + self.log, "SP update complete"; + "mgs_addr" => client.baseurl(), + ); + + Ok(()) + } +} diff --git a/nexus/src/bin/nexus.rs b/nexus/src/bin/nexus.rs index 76e4f7aacaf..b67085db2c3 100644 --- a/nexus/src/bin/nexus.rs +++ b/nexus/src/bin/nexus.rs @@ -10,6 +10,7 @@ // - General networking and runtime tuning for availability and security: see // omicron#2184, omicron#2414. +use anyhow::anyhow; use clap::Parser; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; @@ -26,6 +27,7 @@ struct Args { short = 'O', long = "openapi", help = "Print the external OpenAPI Spec document and exit", + conflicts_with = "openapi_internal", action )] openapi: bool, @@ -39,7 +41,7 @@ struct Args { openapi_internal: bool, #[clap(name = "CONFIG_FILE_PATH", action)] - config_file_path: PathBuf, + config_file_path: Option, } #[tokio::main] @@ -52,14 +54,25 @@ async fn main() { async fn do_run() -> Result<(), CmdError> { let args = Args::parse(); - let config = Config::from_file(args.config_file_path) - .map_err(|e| CmdError::Failure(e.to_string()))?; - if args.openapi { - run_openapi_external().map_err(CmdError::Failure) + run_openapi_external().map_err(|err| CmdError::Failure(anyhow!(err))) } else if args.openapi_internal { - run_openapi_internal().map_err(CmdError::Failure) + run_openapi_internal().map_err(|err| CmdError::Failure(anyhow!(err))) } else { - run_server(&config).await.map_err(CmdError::Failure) + let config_path = match args.config_file_path { + Some(path) => path, + None => { + use clap::CommandFactory; + + eprintln!("{}", Args::command().render_help()); + return Err(CmdError::Usage( + "CONFIG_FILE_PATH is required".to_string(), + )); + } + }; + let config = Config::from_file(config_path) + .map_err(|e| CmdError::Failure(anyhow!(e)))?; + + run_server(&config).await.map_err(|err| CmdError::Failure(anyhow!(err))) } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 48be2de6b0a..eba97a88ec7 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -3992,6 +3992,7 @@ async fn vpc_firewall_rules_update( method = GET, path = "/v1/vpc-routers", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_list( rqctx: RequestContext>, @@ -4027,6 +4028,7 @@ async fn vpc_router_list( method = GET, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_view( rqctx: RequestContext>, @@ -4056,6 +4058,7 @@ async fn vpc_router_view( method = POST, path = "/v1/vpc-routers", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_create( rqctx: RequestContext>, @@ -4087,6 +4090,7 @@ async fn vpc_router_create( method = DELETE, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_delete( rqctx: RequestContext>, @@ -4116,6 +4120,7 @@ async fn vpc_router_delete( method = PUT, path = "/v1/vpc-routers/{router}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_update( rqctx: RequestContext>, @@ -4151,6 +4156,7 @@ async fn vpc_router_update( method = GET, path = "/v1/vpc-router-routes", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_list( rqctx: RequestContext>, @@ -4188,6 +4194,7 @@ async fn vpc_router_route_list( method = GET, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_view( rqctx: RequestContext>, @@ -4220,6 +4227,7 @@ async fn vpc_router_route_view( method = POST, path = "/v1/vpc-router-routes", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_create( rqctx: RequestContext>, @@ -4251,6 +4259,7 @@ async fn vpc_router_route_create( method = DELETE, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_delete( rqctx: RequestContext>, @@ -4282,6 +4291,7 @@ async fn vpc_router_route_delete( method = PUT, path = "/v1/vpc-router-routes/{route}", tags = ["vpcs"], + unpublished = true, }] async fn vpc_router_route_update( rqctx: RequestContext>, diff --git a/nexus/test-interface/Cargo.toml b/nexus/test-interface/Cargo.toml index 0071ffaa28b..b96afa6dbf1 100644 --- a/nexus/test-interface/Cargo.toml +++ b/nexus/test-interface/Cargo.toml @@ -6,8 +6,6 @@ license = "MPL-2.0" [dependencies] async-trait.workspace = true -dropshot.workspace = true -internal-dns.workspace = true nexus-types.workspace = true omicron-common.workspace = true slog.workspace = true diff --git a/nexus/test-utils-macros/Cargo.toml b/nexus/test-utils-macros/Cargo.toml index d3d28a76401..5ed57b9c4a2 100644 --- a/nexus/test-utils-macros/Cargo.toml +++ b/nexus/test-utils-macros/Cargo.toml @@ -8,7 +8,6 @@ license = "MPL-2.0" proc-macro = true [dependencies] -proc-macro2.workspace = true quote.workspace = true syn = { workspace = true, features = [ "fold", "parsing" ] } omicron-workspace-hack.workspace = true diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index 56cee27b377..024cba958b4 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -28,16 +28,12 @@ omicron-passwords.workspace = true omicron-sled-agent.workspace = true omicron-test-utils.workspace = true oximeter.workspace = true -oximeter-client.workspace = true oximeter-collector.workspace = true oximeter-producer.workspace = true -parse-display.workspace = true serde.workspace = true serde_json.workspace = true serde_urlencoded.workspace = true slog.workspace = true -tempfile.workspace = true -trust-dns-proto.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/nexus/tests/integration_tests/commands.rs b/nexus/tests/integration_tests/commands.rs index 66006e0bdfe..02d938b2ace 100644 --- a/nexus/tests/integration_tests/commands.rs +++ b/nexus/tests/integration_tests/commands.rs @@ -56,10 +56,9 @@ fn test_nexus_bad_config() { let (exit_status, stdout_text, stderr_text) = run_command(exec); assert_exit_code(exit_status, EXIT_FAILURE, &stderr_text); assert_contents("tests/output/cmd-nexus-badconfig-stdout", &stdout_text); - assert_eq!( - stderr_text, - format!("nexus: read \"nonexistent\": {}\n", error_for_enoent()) - ); + let expected_err = + format!("nexus: read \"nonexistent\": {}\n", error_for_enoent()); + assert!(&stderr_text.starts_with(&expected_err)); } #[test] @@ -73,13 +72,11 @@ fn test_nexus_invalid_config() { "tests/output/cmd-nexus-invalidconfig-stdout", &stdout_text, ); - assert_eq!( - stderr_text, - format!( - "nexus: parse \"{}\": missing field `deployment`\n", - config_path.display() - ), + let expected_err = format!( + "nexus: parse \"{}\": missing field `deployment`\n", + config_path.display() ); + assert!(&stderr_text.starts_with(&expected_err)); } #[track_caller] diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index 98303995628..e0bb09de4fb 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -32,6 +32,7 @@ mod silo_users; mod silos; mod sleds; mod snapshots; +mod sp_updater; mod ssh_keys; mod subnet_allocation; mod switch_port; diff --git a/nexus/tests/integration_tests/oximeter.rs b/nexus/tests/integration_tests/oximeter.rs index 2cda594e183..65aaa186426 100644 --- a/nexus/tests/integration_tests/oximeter.rs +++ b/nexus/tests/integration_tests/oximeter.rs @@ -4,11 +4,17 @@ //! Integration tests for oximeter collectors and producers. +use dropshot::Method; +use http::StatusCode; use nexus_test_interface::NexusServer; use nexus_test_utils_macros::nexus_test; +use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use oximeter_db::DbWrite; +use std::collections::BTreeSet; use std::net; +use std::net::Ipv6Addr; +use std::net::SocketAddr; use std::time::Duration; use uuid::Uuid; @@ -332,3 +338,87 @@ async fn test_oximeter_reregistration() { ); context.teardown().await; } + +// A regression test for https://github.com/oxidecomputer/omicron/issues/4498 +#[tokio::test] +async fn test_oximeter_collector_reregistration_gets_all_assignments() { + let mut context = nexus_test_utils::test_setup::( + "test_oximeter_collector_reregistration_gets_all_assignments", + ) + .await; + let oximeter_id = nexus_test_utils::OXIMETER_UUID.parse().unwrap(); + + // Create a bunch of producer records. + // + // Note that the actual count is arbitrary, but it should be larger than the + // internal pagination limit used in `Nexus::upsert_oximeter_collector()`, + // which is currently 100. + const N_PRODUCERS: usize = 150; + let mut ids = BTreeSet::new(); + for _ in 0..N_PRODUCERS { + let id = Uuid::new_v4(); + ids.insert(id); + let info = ProducerEndpoint { + id, + address: SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 12345), + base_route: String::from("/collect"), + interval: Duration::from_secs(1), + }; + context + .internal_client + .make_request( + Method::POST, + "/metrics/producers", + Some(&info), + StatusCode::NO_CONTENT, + ) + .await + .expect("failed to register test producer"); + } + + // Check that `oximeter` has these registered. + let producers = + context.oximeter.list_producers(None, N_PRODUCERS * 2).await; + let actual_ids: BTreeSet<_> = + producers.iter().map(|info| info.id).collect(); + + // There is an additional producer that's created as part of the normal test + // setup, so we'll check that all of the new producers exist, and that + // there's exactly 1 additional one. + assert!( + ids.is_subset(&actual_ids), + "oximeter did not get the right set of producers" + ); + assert_eq!( + ids.len(), + actual_ids.len() - 1, + "oximeter did not get the right set of producers" + ); + + // Drop and restart oximeter, which should result in the exact same set of + // producers again. + drop(context.oximeter); + context.oximeter = nexus_test_utils::start_oximeter( + context.logctx.log.new(o!("component" => "oximeter")), + context.server.get_http_server_internal_address().await, + context.clickhouse.port(), + oximeter_id, + ) + .await + .expect("failed to restart oximeter"); + + let producers = + context.oximeter.list_producers(None, N_PRODUCERS * 2).await; + let actual_ids: BTreeSet<_> = + producers.iter().map(|info| info.id).collect(); + assert!( + ids.is_subset(&actual_ids), + "oximeter did not get the right set of producers after re-registering" + ); + assert_eq!( + ids.len(), + actual_ids.len() - 1, + "oximeter did not get the right set of producers after re-registering" + ); + context.teardown().await; +} diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index d79dd09fc1f..213e7f9e4ff 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -168,13 +168,15 @@ async fn query_crdb_schema_version(crdb: &CockroachInstance) -> String { // // Note that for the purposes of schema comparisons, we don't care about parsing // the contents of the database, merely the schema and equality of contained data. -#[derive(Eq, PartialEq, Clone, Debug)] +#[derive(PartialEq, Clone, Debug)] enum AnySqlType { DateTime, String(String), Bool(bool), Uuid(Uuid), Int8(i64), + Float4(f32), + TextArray(Vec), // TODO: This isn't exhaustive, feel free to add more. // // These should only be necessary for rows where the database schema changes also choose to @@ -213,6 +215,14 @@ impl<'a> tokio_postgres::types::FromSql<'a> for AnySqlType { if i64::accepts(ty) { return Ok(AnySqlType::Int8(i64::from_sql(ty, raw)?)); } + if f32::accepts(ty) { + return Ok(AnySqlType::Float4(f32::from_sql(ty, raw)?)); + } + if Vec::::accepts(ty) { + return Ok(AnySqlType::TextArray(Vec::::from_sql( + ty, raw, + )?)); + } Err(anyhow::anyhow!( "Cannot parse type {ty}. If you're trying to use this type in a table which is populated \ during a schema migration, consider adding it to `AnySqlType`." @@ -224,7 +234,7 @@ during a schema migration, consider adding it to `AnySqlType`." } } -#[derive(Eq, PartialEq, Debug)] +#[derive(PartialEq, Debug)] struct NamedSqlValue { // It's a little redunant to include the column name alongside each value, // but it results in a prettier diff. @@ -240,7 +250,7 @@ impl NamedSqlValue { } // A generic representation of a row of SQL data -#[derive(Eq, PartialEq, Debug)] +#[derive(PartialEq, Debug)] struct Row { values: Vec, } @@ -262,19 +272,7 @@ impl<'a> From<&'a [&'static str]> for ColumnSelector<'a> { } } -async fn crdb_show_constraints( - crdb: &CockroachInstance, - table: &str, -) -> Vec { - let client = crdb.connect().await.expect("failed to connect"); - - let sql = format!("SHOW CONSTRAINTS FROM {table}"); - let rows = client - .query(&sql, &[]) - .await - .unwrap_or_else(|_| panic!("failed to query {table}")); - client.cleanup().await.expect("cleaning up after wipe"); - +fn process_rows(rows: &Vec) -> Vec { let mut result = vec![]; for row in rows { let mut row_result = Row::new(); @@ -290,6 +288,22 @@ async fn crdb_show_constraints( result } +async fn crdb_show_constraints( + crdb: &CockroachInstance, + table: &str, +) -> Vec { + let client = crdb.connect().await.expect("failed to connect"); + + let sql = format!("SHOW CONSTRAINTS FROM {table}"); + let rows = client + .query(&sql, &[]) + .await + .unwrap_or_else(|_| panic!("failed to query {table}")); + client.cleanup().await.expect("cleaning up after wipe"); + + process_rows(&rows) +} + async fn crdb_select( crdb: &CockroachInstance, columns: ColumnSelector<'_>, @@ -324,19 +338,20 @@ async fn crdb_select( .unwrap_or_else(|_| panic!("failed to query {table}")); client.cleanup().await.expect("cleaning up after wipe"); - let mut result = vec![]; - for row in rows { - let mut row_result = Row::new(); - for i in 0..row.len() { - let column_name = row.columns()[i].name(); - row_result.values.push(NamedSqlValue { - column: column_name.to_string(), - value: row.get(i), - }); - } - result.push(row_result); - } - result + process_rows(&rows) +} + +async fn crdb_list_enums(crdb: &CockroachInstance) -> Vec { + let client = crdb.connect().await.expect("failed to connect"); + + // https://www.cockroachlabs.com/docs/stable/show-enums + let rows = client + .query("show enums;", &[]) + .await + .unwrap_or_else(|_| panic!("failed to list enums")); + client.cleanup().await.expect("cleaning up after wipe"); + + process_rows(&rows) } async fn read_all_schema_versions() -> BTreeSet { @@ -569,10 +584,11 @@ const PG_INDEXES: [&'static str; 5] = const TABLES: [&'static str; 4] = ["table_catalog", "table_schema", "table_name", "table_type"]; -#[derive(Eq, PartialEq, Debug)] +#[derive(PartialEq, Debug)] struct InformationSchema { columns: Vec, constraint_column_usage: Vec, + enums: Vec, key_column_usage: Vec, referential_constraints: Vec, views: Vec, @@ -589,6 +605,13 @@ impl InformationSchema { // the columns diff especially needs this: it can be 20k lines otherwise similar_asserts::assert_eq!(self.tables, other.tables); similar_asserts::assert_eq!(self.columns, other.columns); + similar_asserts::assert_eq!( + self.enums, + other.enums, + "Enums did not match. Members must have the same order in dbinit.sql and \ + migrations. If a migration adds a member, it should use BEFORE or AFTER \ + to add it in the same order as dbinit.sql." + ); similar_asserts::assert_eq!(self.views, other.views); similar_asserts::assert_eq!( self.table_constraints, @@ -624,6 +647,8 @@ impl InformationSchema { ) .await; + let enums = crdb_list_enums(crdb).await; + let constraint_column_usage = crdb_select( crdb, CONSTRAINT_COLUMN_USAGE.as_slice().into(), @@ -694,6 +719,7 @@ impl InformationSchema { Self { columns, constraint_column_usage, + enums, key_column_usage, referential_constraints, views, diff --git a/nexus/tests/integration_tests/sp_updater.rs b/nexus/tests/integration_tests/sp_updater.rs new file mode 100644 index 00000000000..351c28ad9c8 --- /dev/null +++ b/nexus/tests/integration_tests/sp_updater.rs @@ -0,0 +1,609 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tests `SpUpdater`'s delivery of updates to SPs via MGS + +use gateway_client::types::SpType; +use gateway_messages::{SpPort, UpdateInProgressStatus, UpdateStatus}; +use gateway_test_utils::setup as mgs_setup; +use hubtools::RawHubrisArchive; +use hubtools::{CabooseBuilder, HubrisArchiveBuilder}; +use omicron_nexus::app::test_interfaces::{SpUpdater, UpdateProgress}; +use sp_sim::SimulatedSp; +use sp_sim::SIM_GIMLET_BOARD; +use sp_sim::SIM_SIDECAR_BOARD; +use std::mem; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use tokio::sync::mpsc; +use uuid::Uuid; + +fn make_fake_sp_image(board: &str) -> Vec { + let caboose = CabooseBuilder::default() + .git_commit("fake-git-commit") + .board(board) + .version("0.0.0") + .name("fake-name") + .build(); + + let mut builder = HubrisArchiveBuilder::with_fake_image(); + builder.write_caboose(caboose.as_slice()).unwrap(); + builder.build_to_vec().unwrap() +} + +#[tokio::test] +async fn test_sp_updater_updates_sled() { + // Start MGS + Sim SP. + let mgstestctx = + mgs_setup::test_setup("test_sp_updater_updates_sled", SpPort::One) + .await; + + // Configure an MGS client. + let mgs_client = Arc::new(gateway_client::Client::new( + &mgstestctx.client.url("/").to_string(), + mgstestctx.logctx.log.new(slog::o!("component" => "MgsClient")), + )); + + // Configure and instantiate an `SpUpdater`. + let sp_type = SpType::Sled; + let sp_slot = 0; + let update_id = Uuid::new_v4(); + let hubris_archive = make_fake_sp_image(SIM_GIMLET_BOARD); + + let sp_updater = SpUpdater::new( + sp_type, + sp_slot, + update_id, + hubris_archive.clone(), + &mgstestctx.logctx.log, + ); + + // Run the update. + sp_updater.update([mgs_client]).await.expect("update failed"); + + // Ensure the SP received the complete update. + let last_update_image = mgstestctx.simrack.gimlets[sp_slot as usize] + .last_update_data() + .await + .expect("simulated SP did not receive an update"); + + let hubris_archive = RawHubrisArchive::from_vec(hubris_archive).unwrap(); + + assert_eq!( + hubris_archive.image.data.as_slice(), + &*last_update_image, + "simulated SP update contents (len {}) \ + do not match test generated fake image (len {})", + last_update_image.len(), + hubris_archive.image.data.len() + ); + + mgstestctx.teardown().await; +} + +#[tokio::test] +async fn test_sp_updater_updates_switch() { + // Start MGS + Sim SP. + let mgstestctx = + mgs_setup::test_setup("test_sp_updater_updates_switch", SpPort::One) + .await; + + // Configure an MGS client. + let mgs_client = Arc::new(gateway_client::Client::new( + &mgstestctx.client.url("/").to_string(), + mgstestctx.logctx.log.new(slog::o!("component" => "MgsClient")), + )); + + let sp_type = SpType::Switch; + let sp_slot = 0; + let update_id = Uuid::new_v4(); + let hubris_archive = make_fake_sp_image(SIM_SIDECAR_BOARD); + + let sp_updater = SpUpdater::new( + sp_type, + sp_slot, + update_id, + hubris_archive.clone(), + &mgstestctx.logctx.log, + ); + + sp_updater.update([mgs_client]).await.expect("update failed"); + + let last_update_image = mgstestctx.simrack.sidecars[sp_slot as usize] + .last_update_data() + .await + .expect("simulated SP did not receive an update"); + + let hubris_archive = RawHubrisArchive::from_vec(hubris_archive).unwrap(); + + assert_eq!( + hubris_archive.image.data.as_slice(), + &*last_update_image, + "simulated SP update contents (len {}) \ + do not match test generated fake image (len {})", + last_update_image.len(), + hubris_archive.image.data.len() + ); + + mgstestctx.teardown().await; +} + +#[tokio::test] +async fn test_sp_updater_remembers_successful_mgs_instance() { + // Start MGS + Sim SP. + let mgstestctx = mgs_setup::test_setup( + "test_sp_updater_remembers_successful_mgs_instance", + SpPort::One, + ) + .await; + + // Also start a local TCP server that we will claim is an MGS instance, but + // it will close connections immediately after accepting them. This will + // allow us to count how many connections it receives, while simultaneously + // causing errors in the SpUpdater when it attempts to use this "MGS". + let (failing_mgs_task, failing_mgs_addr, failing_mgs_conn_counter) = { + let socket = TcpListener::bind("[::1]:0").await.unwrap(); + let addr = socket.local_addr().unwrap(); + let conn_count = Arc::new(AtomicUsize::new(0)); + + let task = { + let conn_count = Arc::clone(&conn_count); + tokio::spawn(async move { + loop { + let (mut stream, _peer) = socket.accept().await.unwrap(); + conn_count.fetch_add(1, Ordering::SeqCst); + stream.shutdown().await.unwrap(); + } + }) + }; + + (task, addr, conn_count) + }; + + // Order the MGS clients such that the bogus MGS that immediately closes + // connections comes first. `SpUpdater` should remember that the second MGS + // instance succeeds, and only send subsequent requests to it: we should + // only see a single attempted connection to the bogus MGS, even though + // delivering an update requires a bare minimum of three requests (start the + // update, query the status, reset the SP) and often more (if repeated + // queries are required to wait for completion). + let mgs_clients = [ + Arc::new(gateway_client::Client::new( + &format!("http://{failing_mgs_addr}"), + mgstestctx.logctx.log.new(slog::o!("component" => "MgsClient1")), + )), + Arc::new(gateway_client::Client::new( + &mgstestctx.client.url("/").to_string(), + mgstestctx.logctx.log.new(slog::o!("component" => "MgsClient")), + )), + ]; + + let sp_type = SpType::Sled; + let sp_slot = 0; + let update_id = Uuid::new_v4(); + let hubris_archive = make_fake_sp_image(SIM_GIMLET_BOARD); + + let sp_updater = SpUpdater::new( + sp_type, + sp_slot, + update_id, + hubris_archive.clone(), + &mgstestctx.logctx.log, + ); + + sp_updater.update(mgs_clients).await.expect("update failed"); + + let last_update_image = mgstestctx.simrack.gimlets[sp_slot as usize] + .last_update_data() + .await + .expect("simulated SP did not receive an update"); + + let hubris_archive = RawHubrisArchive::from_vec(hubris_archive).unwrap(); + + assert_eq!( + hubris_archive.image.data.as_slice(), + &*last_update_image, + "simulated SP update contents (len {}) \ + do not match test generated fake image (len {})", + last_update_image.len(), + hubris_archive.image.data.len() + ); + + // Check that our bogus MGS only received a single connection attempt. + // (After SpUpdater failed to talk to this instance, it should have fallen + // back to the valid one for all further requests.) + assert_eq!( + failing_mgs_conn_counter.load(Ordering::SeqCst), + 1, + "bogus MGS instance didn't receive the expected number of connections" + ); + failing_mgs_task.abort(); + + mgstestctx.teardown().await; +} + +#[tokio::test] +async fn test_sp_updater_switches_mgs_instances_on_failure() { + enum MgsProxy { + One(TcpStream), + Two(TcpStream), + } + + // Start MGS + Sim SP. + let mgstestctx = mgs_setup::test_setup( + "test_sp_updater_switches_mgs_instances_on_failure", + SpPort::One, + ) + .await; + let mgs_bind_addr = mgstestctx.client.bind_address; + + let spawn_mgs_proxy_task = |mut stream: TcpStream| { + tokio::spawn(async move { + let mut mgs_stream = TcpStream::connect(mgs_bind_addr) + .await + .expect("failed to connect to MGS"); + tokio::io::copy_bidirectional(&mut stream, &mut mgs_stream) + .await + .expect("failed to proxy connection to MGS"); + }) + }; + + // Start two MGS proxy tasks; when each receives an incoming TCP connection, + // it forwards that `TcpStream` along the `mgs_proxy_connections` channel + // along with a tag of which proxy it is. We'll use this below to flip flop + // between MGS "instances" (really these two proxies). + let (mgs_proxy_connections_tx, mut mgs_proxy_connections_rx) = + mpsc::unbounded_channel(); + let (mgs_proxy_one_task, mgs_proxy_one_addr) = { + let socket = TcpListener::bind("[::1]:0").await.unwrap(); + let addr = socket.local_addr().unwrap(); + let mgs_proxy_connections_tx = mgs_proxy_connections_tx.clone(); + let task = tokio::spawn(async move { + loop { + let (stream, _peer) = socket.accept().await.unwrap(); + mgs_proxy_connections_tx.send(MgsProxy::One(stream)).unwrap(); + } + }); + (task, addr) + }; + let (mgs_proxy_two_task, mgs_proxy_two_addr) = { + let socket = TcpListener::bind("[::1]:0").await.unwrap(); + let addr = socket.local_addr().unwrap(); + let task = tokio::spawn(async move { + loop { + let (stream, _peer) = socket.accept().await.unwrap(); + mgs_proxy_connections_tx.send(MgsProxy::Two(stream)).unwrap(); + } + }); + (task, addr) + }; + + // Disable connection pooling so each request gets a new TCP connection. + let client = + reqwest::Client::builder().pool_max_idle_per_host(0).build().unwrap(); + + // Configure two MGS clients pointed at our two proxy tasks. + let mgs_clients = [ + Arc::new(gateway_client::Client::new_with_client( + &format!("http://{mgs_proxy_one_addr}"), + client.clone(), + mgstestctx.logctx.log.new(slog::o!("component" => "MgsClient1")), + )), + Arc::new(gateway_client::Client::new_with_client( + &format!("http://{mgs_proxy_two_addr}"), + client, + mgstestctx.logctx.log.new(slog::o!("component" => "MgsClient2")), + )), + ]; + + let sp_type = SpType::Sled; + let sp_slot = 0; + let update_id = Uuid::new_v4(); + let hubris_archive = make_fake_sp_image(SIM_GIMLET_BOARD); + + let sp_updater = SpUpdater::new( + sp_type, + sp_slot, + update_id, + hubris_archive.clone(), + &mgstestctx.logctx.log, + ); + + // Spawn the actual update task. + let mut update_task = tokio::spawn(sp_updater.update(mgs_clients)); + + // Loop over incoming requests. We expect this sequence: + // + // 1. Connection arrives on the first proxy + // 2. We spawn a task to service that request, and set `should_swap` + // 3. Connection arrives on the first proxy + // 4. We drop that connection, flip `expected_proxy`, and clear + // `should_swap` + // 5. Connection arrives on the second proxy + // 6. We spawn a task to service that request, and set `should_swap` + // 7. Connection arrives on the second proxy + // 8. We drop that connection, flip `expected_proxy`, and clear + // `should_swap` + // + // ... repeat until the update is complete. + let mut expected_proxy = 0; + let mut proxy_one_count = 0; + let mut proxy_two_count = 0; + let mut total_requests_handled = 0; + let mut should_swap = false; + loop { + tokio::select! { + Some(proxy_stream) = mgs_proxy_connections_rx.recv() => { + let stream = match proxy_stream { + MgsProxy::One(stream) => { + assert_eq!(expected_proxy, 0); + proxy_one_count += 1; + stream + } + MgsProxy::Two(stream) => { + assert_eq!(expected_proxy, 1); + proxy_two_count += 1; + stream + } + }; + + // Should we trigger `SpUpdater` to swap to the other MGS + // (proxy)? If so, do that by dropping this connection (which + // will cause a client failure) and note that we expect the next + // incoming request to come on the other proxy. + if should_swap { + mem::drop(stream); + expected_proxy ^= 1; + should_swap = false; + } else { + // Otherwise, handle this connection. + total_requests_handled += 1; + spawn_mgs_proxy_task(stream); + should_swap = true; + } + } + + result = &mut update_task => { + match result { + Ok(Ok(())) => { + mgs_proxy_one_task.abort(); + mgs_proxy_two_task.abort(); + break; + } + Ok(Err(err)) => panic!("update failed: {err}"), + Err(err) => panic!("update task panicked: {err}"), + } + } + } + } + + // An SP update requires a minimum of 3 requests to MGS: post the update, + // check the status, and post an SP reset. There may be more requests if the + // update is not yet complete when the status is checked, but we can just + // check that each of our proxies received at least 2 incoming requests; + // based on our outline above, if we got the minimum of 3 requests, it would + // look like this: + // + // 1. POST update -> first proxy (success) + // 2. GET status -> first proxy (fail) + // 3. GET status retry -> second proxy (success) + // 4. POST reset -> second proxy (fail) + // 5. POST reset -> first proxy (success) + // + // This pattern would repeat if multiple status requests were required, so + // we always expect the first proxy to see exactly one more connection + // attempt than the second (because it went first before they started + // swapping), and the two together should see a total of one less than + // double the number of successful requests required. + assert!(total_requests_handled >= 3); + assert_eq!(proxy_one_count, proxy_two_count + 1); + assert_eq!( + (proxy_one_count + proxy_two_count + 1) / 2, + total_requests_handled + ); + + let last_update_image = mgstestctx.simrack.gimlets[sp_slot as usize] + .last_update_data() + .await + .expect("simulated SP did not receive an update"); + + let hubris_archive = RawHubrisArchive::from_vec(hubris_archive).unwrap(); + + assert_eq!( + hubris_archive.image.data.as_slice(), + &*last_update_image, + "simulated SP update contents (len {}) \ + do not match test generated fake image (len {})", + last_update_image.len(), + hubris_archive.image.data.len() + ); + + mgstestctx.teardown().await; +} + +#[tokio::test] +async fn test_sp_updater_delivers_progress() { + // Start MGS + Sim SP. + let mgstestctx = + mgs_setup::test_setup("test_sp_updater_delivers_progress", SpPort::One) + .await; + + // Configure an MGS client. + let mgs_client = Arc::new(gateway_client::Client::new( + &mgstestctx.client.url("/").to_string(), + mgstestctx.logctx.log.new(slog::o!("component" => "MgsClient")), + )); + + let sp_type = SpType::Sled; + let sp_slot = 0; + let update_id = Uuid::new_v4(); + let hubris_archive = make_fake_sp_image(SIM_GIMLET_BOARD); + + let sp_updater = SpUpdater::new( + sp_type, + sp_slot, + update_id, + hubris_archive.clone(), + &mgstestctx.logctx.log, + ); + + let hubris_archive = RawHubrisArchive::from_vec(hubris_archive).unwrap(); + let sp_image_len = hubris_archive.image.data.len() as u32; + + // Subscribe to update progress, and check that there is no status yet; we + // haven't started the update. + let mut progress = sp_updater.progress_watcher(); + assert_eq!(*progress.borrow_and_update(), None); + + // Install a semaphore on the requests our target SP will receive so we can + // inspect progress messages without racing. + let target_sp = &mgstestctx.simrack.gimlets[sp_slot as usize]; + let sp_accept_sema = target_sp.install_udp_accept_semaphore().await; + let mut sp_responses = target_sp.responses_sent_count().unwrap(); + + // Spawn the update on a background task so we can watch `progress` as it is + // applied. + let do_update_task = tokio::spawn(sp_updater.update([mgs_client])); + + // Allow the SP to respond to 2 messages: the caboose check and the "prepare + // update" messages that trigger the start of an update, then ensure we see + // the "started" progress. + sp_accept_sema.send(2).unwrap(); + progress.changed().await.unwrap(); + assert_eq!(*progress.borrow_and_update(), Some(UpdateProgress::Started)); + + // Ensure our simulated SP is in the state we expect: it's prepared for an + // update but has not yet received any data. + assert_eq!( + target_sp.current_update_status().await, + UpdateStatus::InProgress(UpdateInProgressStatus { + id: update_id.into(), + bytes_received: 0, + total_size: sp_image_len, + }) + ); + + // Record the number of responses the SP has sent; we'll use + // `sp_responses.changed()` in the loop below, and want to mark whatever + // value this watch channel currently has as seen. + sp_responses.borrow_and_update(); + + // At this point, there are two clients racing each other to talk to our + // simulated SP: + // + // 1. MGS is trying to deliver the update + // 2. `sp_updater` is trying to poll (via MGS) for update status + // + // and we want to ensure that we see any relevant progress reports from + // `sp_updater`. We'll let one MGS -> SP message through at a time (waiting + // until our SP has responded by waiting for a change to `sp_responses`) + // then check its update state: if it changed, the packet we let through was + // data from MGS; otherwise, it was a status request from `sp_updater`. + // + // This loop will continue until either: + // + // 1. We see an `UpdateStatus::InProgress` message indicating 100% delivery, + // at which point we break out of the loop + // 2. We time out waiting for the previous step (by timing out for either + // the SP to process a request or `sp_updater` to realize there's been + // progress), at which point we panic and fail this test. + let mut prev_bytes_received = 0; + let mut expect_progress_change = false; + loop { + // Allow the SP to accept and respond to a single UDP packet. + sp_accept_sema.send(1).unwrap(); + + // Wait until the SP has sent a response, with a safety rail that we + // haven't screwed up our untangle-the-race logic: if we don't see the + // SP process any new messages after several seconds, our test is + // broken, so fail. + tokio::time::timeout(Duration::from_secs(10), sp_responses.changed()) + .await + .expect("timeout waiting for SP response count to change") + .expect("sp response count sender dropped"); + + // Inspec the SP's in-memory update state; we expect only `InProgress` + // or `Complete`, and in either case we note whether we expect to see + // status changes from `sp_updater`. + match target_sp.current_update_status().await { + UpdateStatus::InProgress(sp_progress) => { + if sp_progress.bytes_received > prev_bytes_received { + prev_bytes_received = sp_progress.bytes_received; + expect_progress_change = true; + continue; + } + } + UpdateStatus::Complete(_) => { + if prev_bytes_received < sp_image_len { + prev_bytes_received = sp_image_len; + continue; + } + } + status @ (UpdateStatus::None + | UpdateStatus::Preparing(_) + | UpdateStatus::SpUpdateAuxFlashChckScan { .. } + | UpdateStatus::Aborted(_) + | UpdateStatus::Failed { .. } + | UpdateStatus::RotError { .. }) => { + panic!("unexpected status {status:?}"); + } + } + + // If we get here, the most recent packet did _not_ change the SP's + // internal update state, so it was a status request from `sp_updater`. + // If we expect the updater to see new progress, wait for that change + // here. + if expect_progress_change || prev_bytes_received == sp_image_len { + // Safety rail that we haven't screwed up our untangle-the-race + // logic: if we don't see a new progress after several seconds, our + // test is broken, so fail. + tokio::time::timeout(Duration::from_secs(10), progress.changed()) + .await + .expect("progress timeout") + .expect("progress watch sender dropped"); + let status = progress.borrow_and_update().clone().unwrap(); + expect_progress_change = false; + + // We're done if we've observed the final progress message. + if let UpdateProgress::InProgress { progress: Some(value) } = status + { + if value == 1.0 { + break; + } + } else { + panic!("unexpected progerss status {status:?}"); + } + } + } + + // The update has been fully delivered to the SP, but we don't see an + // `UpdateStatus::Complete` message until the SP is reset. Release the SP + // semaphore since we're no longer racing to observe intermediate progress, + // and wait for the completion message. + sp_accept_sema.send(usize::MAX).unwrap(); + progress.changed().await.unwrap(); + assert_eq!(*progress.borrow_and_update(), Some(UpdateProgress::Complete)); + + do_update_task.await.expect("update task panicked").expect("update failed"); + + let last_update_image = target_sp + .last_update_data() + .await + .expect("simulated SP did not receive an update"); + + assert_eq!( + hubris_archive.image.data.as_slice(), + &*last_update_image, + "simulated SP update contents (len {}) \ + do not match test generated fake image (len {})", + last_update_image.len(), + hubris_archive.image.data.len() + ); + + mgstestctx.teardown().await; +} diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index fada45694dc..ccd0b50fbee 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -24,6 +24,10 @@ use omicron_common::api::external::{ type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; +// TODO: unfortunately this test can no longer be run in the integration test +// suite because it depends on communicating with MGS which is not part +// of the infrastructure available in the integration test context. +#[ignore] #[nexus_test] async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { let client = &ctx.external_client; diff --git a/nexus/tests/output/cmd-nexus-noargs-stderr b/nexus/tests/output/cmd-nexus-noargs-stderr index f371553325d..8dff679340b 100644 --- a/nexus/tests/output/cmd-nexus-noargs-stderr +++ b/nexus/tests/output/cmd-nexus-noargs-stderr @@ -1,6 +1,13 @@ -error: the following required arguments were not provided: - +See README.adoc for more information -Usage: nexus +Usage: nexus [OPTIONS] [CONFIG_FILE_PATH] -For more information, try '--help'. +Arguments: + [CONFIG_FILE_PATH] + +Options: + -O, --openapi Print the external OpenAPI Spec document and exit + -I, --openapi-internal Print the internal OpenAPI Spec document and exit + -h, --help Print help + +nexus: CONFIG_FILE_PATH is required diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index e55eaa4df6a..8c5fe953e32 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -191,16 +191,6 @@ vpc_delete DELETE /v1/vpcs/{vpc} vpc_firewall_rules_update PUT /v1/vpc-firewall-rules vpc_firewall_rules_view GET /v1/vpc-firewall-rules vpc_list GET /v1/vpcs -vpc_router_create POST /v1/vpc-routers -vpc_router_delete DELETE /v1/vpc-routers/{router} -vpc_router_list GET /v1/vpc-routers -vpc_router_route_create POST /v1/vpc-router-routes -vpc_router_route_delete DELETE /v1/vpc-router-routes/{route} -vpc_router_route_list GET /v1/vpc-router-routes -vpc_router_route_update PUT /v1/vpc-router-routes/{route} -vpc_router_route_view GET /v1/vpc-router-routes/{route} -vpc_router_update PUT /v1/vpc-routers/{router} -vpc_router_view GET /v1/vpc-routers/{router} vpc_subnet_create POST /v1/vpc-subnets vpc_subnet_delete DELETE /v1/vpc-subnets/{subnet} vpc_subnet_list GET /v1/vpc-subnets diff --git a/nexus/tests/output/unexpected-authz-endpoints.txt b/nexus/tests/output/unexpected-authz-endpoints.txt index b034ac3869a..1cd87a75e52 100644 --- a/nexus/tests/output/unexpected-authz-endpoints.txt +++ b/nexus/tests/output/unexpected-authz-endpoints.txt @@ -1,4 +1,14 @@ API endpoints tested by unauthorized.rs but not found in the OpenAPI spec: +GET "/v1/vpc-routers?project=demo-project&vpc=demo-vpc" +POST "/v1/vpc-routers?project=demo-project&vpc=demo-vpc" +GET "/v1/vpc-routers/demo-vpc-router?project=demo-project&vpc=demo-vpc" +PUT "/v1/vpc-routers/demo-vpc-router?project=demo-project&vpc=demo-vpc" +DELETE "/v1/vpc-routers/demo-vpc-router?project=demo-project&vpc=demo-vpc" +GET "/v1/vpc-router-routes?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" +POST "/v1/vpc-router-routes?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" +GET "/v1/vpc-router-routes/demo-router-route?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" +PUT "/v1/vpc-router-routes/demo-router-route?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" +DELETE "/v1/vpc-router-routes/demo-router-route?project=demo-project&vpc=demo-vpc&router=demo-vpc-router" POST "/v1/system/update/refresh" GET "/v1/system/update/version" GET "/v1/system/update/components" diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index 5722b065cf3..9cb94a8484b 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -9,10 +9,7 @@ anyhow.workspace = true chrono.workspace = true base64.workspace = true futures.workspace = true -newtype_derive.workspace = true openssl.workspace = true -openssl-sys.workspace = true -openssl-probe.workspace = true parse-display.workspace = true schemars = { workspace = true, features = ["chrono", "uuid1"] } serde.workspace = true diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 6dcf756737a..2c7ffbc337a 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -160,18 +160,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "Baseboard": { "description": "Describes properties that should uniquely identify a Gimlet.", @@ -277,6 +265,41 @@ "format": "uint32", "minimum": 0 }, + "connect_retry": { + "nullable": true, + "description": "The interval in seconds between peer connection retry attempts.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "delay_open": { + "nullable": true, + "description": "How long to delay sending open messages to a peer. In seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "hold_time": { + "nullable": true, + "description": "How long to keep a session alive without a keepalive in seconds. Defaults to 6.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "idle_hold_time": { + "nullable": true, + "description": "How long to keep a peer in idle after a state machine reset in seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "keepalive": { + "nullable": true, + "description": "The interval to send keepalive messages at.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, "port": { "description": "Switch port the peer is reachable on.", "type": "string" @@ -942,6 +965,18 @@ "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", "type": "string" } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/openapi/dns-server.json b/openapi/dns-server.json index 7ffd21eb246..41b351d4c1e 100644 --- a/openapi/dns-server.json +++ b/openapi/dns-server.json @@ -54,18 +54,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "DnsConfig": { "type": "object", @@ -251,6 +239,18 @@ "weight" ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/openapi/gateway.json b/openapi/gateway.json index 97cb7994aa8..9eacbe122d0 100644 --- a/openapi/gateway.json +++ b/openapi/gateway.json @@ -1393,18 +1393,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "Duration": { "type": "object", @@ -3109,6 +3097,18 @@ "power_reset" ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/openapi/installinator-artifactd.json b/openapi/installinator-artifactd.json index 3132af6ff64..136e60a8c40 100644 --- a/openapi/installinator-artifactd.json +++ b/openapi/installinator-artifactd.json @@ -95,18 +95,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "Duration": { "type": "object", @@ -2323,6 +2311,18 @@ "slots_written" ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 411c52ddffc..f83cf68a8a0 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -679,18 +679,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "ActivationReason": { "description": "Describes why a background task was activated\n\nThis is only used for debugging. This is deliberately not made available to the background task itself. See \"Design notes\" in the module-level documentation for details.", @@ -803,6 +791,41 @@ "format": "uint32", "minimum": 0 }, + "connect_retry": { + "nullable": true, + "description": "The interval in seconds between peer connection retry attempts.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "delay_open": { + "nullable": true, + "description": "How long to delay sending open messages to a peer. In seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "hold_time": { + "nullable": true, + "description": "How long to keep a session alive without a keepalive in seconds. Defaults to 6.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "idle_hold_time": { + "nullable": true, + "description": "How long to keep a peer in idle after a state machine reset in seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "keepalive": { + "nullable": true, + "description": "The interval to send keepalive messages at.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, "port": { "description": "Switch port the peer is reachable on.", "type": "string" @@ -5375,6 +5398,18 @@ } ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/openapi/nexus.json b/openapi/nexus.json index f1bfa4351fd..74162a9b2bc 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -6582,14 +6582,13 @@ } } }, - "/v1/vpc-router-routes": { + "/v1/vpc-subnets": { "get": { "tags": [ "vpcs" ], - "summary": "List routes", - "description": "List the routes associated with a router in a particular VPC.", - "operationId": "vpc_router_route_list", + "summary": "List subnets", + "operationId": "vpc_subnet_list", "parameters": [ { "in": "query", @@ -6619,14 +6618,6 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "sort_by", @@ -6637,7 +6628,7 @@ { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -6649,7 +6640,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteResultsPage" + "$ref": "#/components/schemas/VpcSubnetResultsPage" } } } @@ -6663,7 +6654,7 @@ }, "x-dropshot-pagination": { "required": [ - "router" + "vpc" ] } }, @@ -6671,8 +6662,8 @@ "tags": [ "vpcs" ], - "summary": "Create a router", - "operationId": "vpc_router_route_create", + "summary": "Create a subnet", + "operationId": "vpc_subnet_create", "parameters": [ { "in": "query", @@ -6682,19 +6673,11 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -6704,7 +6687,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteCreate" + "$ref": "#/components/schemas/VpcSubnetCreate" } } }, @@ -6716,7 +6699,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -6730,18 +6713,18 @@ } } }, - "/v1/vpc-router-routes/{route}": { + "/v1/vpc-subnets/{subnet}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch a route", - "operationId": "vpc_router_route_view", + "summary": "Fetch a subnet", + "operationId": "vpc_subnet_view", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -6755,19 +6738,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -6779,7 +6753,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -6796,13 +6770,13 @@ "tags": [ "vpcs" ], - "summary": "Update a route", - "operationId": "vpc_router_route_update", + "summary": "Update a subnet", + "operationId": "vpc_subnet_update", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -6816,18 +6790,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -6837,7 +6803,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteUpdate" + "$ref": "#/components/schemas/VpcSubnetUpdate" } } }, @@ -6849,7 +6815,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -6866,13 +6832,13 @@ "tags": [ "vpcs" ], - "summary": "Delete a route", - "operationId": "vpc_router_route_delete", + "summary": "Delete a subnet", + "operationId": "vpc_subnet_delete", "parameters": [ { "in": "path", - "name": "route", - "description": "Name or ID of the route", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -6886,18 +6852,10 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `subnet` is provided as a `Name`", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -6916,14 +6874,23 @@ } } }, - "/v1/vpc-routers": { + "/v1/vpc-subnets/{subnet}/network-interfaces": { "get": { "tags": [ "vpcs" ], - "summary": "List routers", - "operationId": "vpc_router_list", + "summary": "List network interfaces", + "operationId": "vpc_subnet_list_network_interfaces", "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -6974,7 +6941,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterResultsPage" + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" } } } @@ -6987,30 +6954,89 @@ } }, "x-dropshot-pagination": { - "required": [ - "vpc" - ] + "required": [] } - }, - "post": { + } + }, + "/v1/vpcs": { + "get": { "tags": [ "vpcs" ], - "summary": "Create a VPC router", - "operationId": "vpc_router_create", + "summary": "List VPCs", + "operationId": "vpc_list", "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create a VPC", + "operationId": "vpc_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -7021,7 +7047,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterCreate" + "$ref": "#/components/schemas/VpcCreate" } } }, @@ -7033,7 +7059,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -7047,18 +7073,18 @@ } } }, - "/v1/vpc-routers/{router}": { + "/v1/vpcs/{vpc}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch a router", - "operationId": "vpc_router_view", + "summary": "Fetch a VPC", + "operationId": "vpc_view", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -7067,15 +7093,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -7087,7 +7105,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -7104,13 +7122,13 @@ "tags": [ "vpcs" ], - "summary": "Update a router", - "operationId": "vpc_router_update", + "summary": "Update a VPC", + "operationId": "vpc_update", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -7119,15 +7137,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -7137,7 +7147,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterUpdate" + "$ref": "#/components/schemas/VpcUpdate" } } }, @@ -7149,7 +7159,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/Vpc" } } } @@ -7166,13 +7176,13 @@ "tags": [ "vpcs" ], - "summary": "Delete a router", - "operationId": "vpc_router_delete", + "summary": "Delete a VPC", + "operationId": "vpc_delete", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "vpc", + "description": "Name or ID of the VPC", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -7181,15 +7191,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the project", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -7207,661 +7209,21 @@ } } } - }, - "/v1/vpc-subnets": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List subnets", - "operationId": "vpc_subnet_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "vpc" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create a subnet", - "operationId": "vpc_subnet_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetCreate" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpc-subnets/{subnet}": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "Fetch a subnet", - "operationId": "vpc_subnet_view", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "vpcs" - ], - "summary": "Update a subnet", - "operationId": "vpc_subnet_update", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnetUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcSubnet" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "vpcs" - ], - "summary": "Delete a subnet", - "operationId": "vpc_subnet_delete", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpc-subnets/{subnet}/network-interfaces": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List network interfaces", - "operationId": "vpc_subnet_list_network_interfaces", - "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [] - } - } - }, - "/v1/vpcs": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "List VPCs", - "operationId": "vpc_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create a VPC", - "operationId": "vpc_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcCreate" - } - } - }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/vpcs/{vpc}": { - "get": { - "tags": [ - "vpcs" - ], - "summary": "Fetch a VPC", - "operationId": "vpc_view", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "vpcs" - ], - "summary": "Update a VPC", - "operationId": "vpc_update", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Vpc" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "vpcs" - ], - "summary": "Delete a VPC", - "operationId": "vpc_delete", - "parameters": [ - { - "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - } - }, - "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, - "schemas": { - "Address": { - "description": "An address tied to an address lot.", - "type": "object", - "properties": { - "address": { - "description": "The address and prefix length of this address.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + } + }, + "components": { + "schemas": { + "Address": { + "description": "An address tied to an address lot.", + "type": "object", + "properties": { + "address": { + "description": "The address and prefix length of this address.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] }, "address_lot": { "description": "The address lot this address is drawn from.", @@ -12935,181 +12297,34 @@ } ] }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "description", - "id", - "name", - "time_created", - "time_modified" - ] - }, - "ProjectCreate": { - "description": "Create-time parameters for a `Project`", - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "name": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "description", - "name" - ] - }, - "ProjectResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Project" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, - "ProjectRole": { - "type": "string", - "enum": [ - "admin", - "collaborator", - "viewer" - ] - }, - "ProjectRolePolicy": { - "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", - "type": "object", - "properties": { - "role_assignments": { - "description": "Roles directly assigned on this resource", - "type": "array", - "items": { - "$ref": "#/components/schemas/ProjectRoleRoleAssignment" - } - } - }, - "required": [ - "role_assignments" - ] - }, - "ProjectRoleRoleAssignment": { - "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", - "type": "object", - "properties": { - "identity_id": { - "type": "string", - "format": "uuid" - }, - "identity_type": { - "$ref": "#/components/schemas/IdentityType" - }, - "role_name": { - "$ref": "#/components/schemas/ProjectRole" - } - }, - "required": [ - "identity_id", - "identity_type", - "role_name" - ] - }, - "ProjectUpdate": { - "description": "Updateable properties of a `Project`", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "name": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - } - } - }, - "Rack": { - "description": "View of an Rack", - "type": "object", - "properties": { - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - } - }, - "required": [ - "id", - "time_created", - "time_modified" - ] - }, - "RackResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/Rack" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" + "time_created": { + "description": "timestamp when this resource was created", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "timestamp when this resource was last modified", + "type": "string", + "format": "date-time" } }, "required": [ - "items" + "description", + "id", + "name", + "time_created", + "time_modified" ] }, - "Role": { - "description": "View of a Role", + "ProjectCreate": { + "description": "Create-time parameters for a `Project`", "type": "object", "properties": { "description": { "type": "string" }, "name": { - "$ref": "#/components/schemas/RoleName" + "$ref": "#/components/schemas/Name" } }, "required": [ @@ -13117,14 +12332,7 @@ "name" ] }, - "RoleName": { - "title": "A name for a built-in role", - "description": "Role names consist of two string components separated by dot (\".\").", - "type": "string", - "pattern": "[a-z-]+\\.[a-z-]+", - "maxLength": 63 - }, - "RoleResultsPage": { + "ProjectResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -13132,7 +12340,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/Role" + "$ref": "#/components/schemas/Project" } }, "next_page": { @@ -13145,269 +12353,77 @@ "items" ] }, - "Route": { - "description": "A route to a destination network through a gateway address.", - "type": "object", - "properties": { - "dst": { - "description": "The route destination.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] - }, - "gw": { - "description": "The route gateway.", - "type": "string", - "format": "ip" - }, - "vid": { - "nullable": true, - "description": "VLAN id the gateway is reachable over.", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "dst", - "gw" + "ProjectRole": { + "type": "string", + "enum": [ + "admin", + "collaborator", + "viewer" ] }, - "RouteConfig": { - "description": "Route configuration data associated with a switch port configuration.", + "ProjectRolePolicy": { + "description": "Policy for a particular resource\n\nNote that the Policy only describes access granted explicitly for this resource. The policies of parent resources can also cause a user to have access to this resource.", "type": "object", "properties": { - "routes": { - "description": "The set of routes assigned to a switch port.", + "role_assignments": { + "description": "Roles directly assigned on this resource", "type": "array", "items": { - "$ref": "#/components/schemas/Route" + "$ref": "#/components/schemas/ProjectRoleRoleAssignment" } } }, "required": [ - "routes" - ] - }, - "RouteDestination": { - "description": "A `RouteDestination` is used to match traffic with a routing rule, on the destination of that traffic.\n\nWhen traffic is to be sent to a destination that is within a given `RouteDestination`, the corresponding `RouterRoute` applies, and traffic will be forward to the `RouteTarget` for that rule.", - "oneOf": [ - { - "description": "Route applies to traffic destined for a specific IP address", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip" - ] - }, - "value": { - "type": "string", - "format": "ip" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic destined for a specific IP subnet", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip_net" - ] - }, - "value": { - "$ref": "#/components/schemas/IpNet" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic destined for the given VPC.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "vpc" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Route applies to traffic", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "subnet" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - } + "role_assignments" ] }, - "RouteTarget": { - "description": "A `RouteTarget` describes the possible locations that traffic matching a route destination can be sent.", - "oneOf": [ - { - "description": "Forward traffic to a particular IP address.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ip" - ] - }, - "value": { - "type": "string", - "format": "ip" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a VPC", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "vpc" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] - }, - { - "description": "Forward traffic to a VPC Subnet", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "subnet" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] + "ProjectRoleRoleAssignment": { + "description": "Describes the assignment of a particular role on a particular resource to a particular identity (user, group, etc.)\n\nThe resource is not part of this structure. Rather, `RoleAssignment`s are put into a `Policy` and that Policy is applied to a particular resource.", + "type": "object", + "properties": { + "identity_id": { + "type": "string", + "format": "uuid" }, - { - "description": "Forward traffic to a specific instance", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "instance" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] + "identity_type": { + "$ref": "#/components/schemas/IdentityType" }, - { - "description": "Forward traffic to an internet gateway", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "internet_gateway" - ] - }, - "value": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "type", - "value" - ] + "role_name": { + "$ref": "#/components/schemas/ProjectRole" } + }, + "required": [ + "identity_id", + "identity_type", + "role_name" ] }, - "RouterRoute": { - "description": "A route defines a rule that governs where traffic should be sent based on its destination.", + "ProjectUpdate": { + "description": "Updateable properties of a `Project`", "type": "object", "properties": { "description": { - "description": "human-readable free-form text about a resource", + "nullable": true, "type": "string" }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "kind": { - "description": "Describes the kind of router. Set at creation. `read-only`", - "allOf": [ - { - "$ref": "#/components/schemas/RouterRouteKind" - } - ] - }, "name": { - "description": "unique, mutable, user-controlled identifier for each resource", + "nullable": true, "allOf": [ { "$ref": "#/components/schemas/Name" } ] - }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + } + } + }, + "Rack": { + "description": "View of an Rack", + "type": "object", + "properties": { + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" }, "time_created": { "description": "timestamp when this resource was created", @@ -13418,83 +12434,59 @@ "description": "timestamp when this resource was last modified", "type": "string", "format": "date-time" - }, - "vpc_router_id": { - "description": "The ID of the VPC Router to which the route belongs", - "type": "string", - "format": "uuid" } }, "required": [ - "description", - "destination", "id", - "kind", - "name", - "target", "time_created", - "time_modified", - "vpc_router_id" + "time_modified" + ] + }, + "RackResultsPage": { + "description": "A single page of results", + "type": "object", + "properties": { + "items": { + "description": "list of items on this page of results", + "type": "array", + "items": { + "$ref": "#/components/schemas/Rack" + } + }, + "next_page": { + "nullable": true, + "description": "token used to fetch the next page of results (if any)", + "type": "string" + } + }, + "required": [ + "items" ] }, - "RouterRouteCreate": { - "description": "Create-time parameters for a `RouterRoute`", + "Role": { + "description": "View of a Role", "type": "object", "properties": { "description": { "type": "string" }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, "name": { - "$ref": "#/components/schemas/Name" - }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + "$ref": "#/components/schemas/RoleName" } }, "required": [ "description", - "destination", - "name", - "target" + "name" ] }, - "RouterRouteKind": { - "description": "The kind of a `RouterRoute`\n\nThe kind determines certain attributes such as if the route is modifiable and describes how or where the route was created.", - "oneOf": [ - { - "description": "Determines the default destination of traffic, such as whether it goes to the internet or not.\n\n`Destination: An Internet Gateway` `Modifiable: true`", - "type": "string", - "enum": [ - "default" - ] - }, - { - "description": "Automatically added for each VPC Subnet in the VPC\n\n`Destination: A VPC Subnet` `Modifiable: false`", - "type": "string", - "enum": [ - "vpc_subnet" - ] - }, - { - "description": "Automatically added when VPC peering is established\n\n`Destination: A different VPC` `Modifiable: false`", - "type": "string", - "enum": [ - "vpc_peering" - ] - }, - { - "description": "Created by a user; see `RouteTarget`\n\n`Destination: User defined` `Modifiable: true`", - "type": "string", - "enum": [ - "custom" - ] - } - ] + "RoleName": { + "title": "A name for a built-in role", + "description": "Role names consist of two string components separated by dot (\".\").", + "type": "string", + "pattern": "[a-z-]+\\.[a-z-]+", + "maxLength": 63 }, - "RouterRouteResultsPage": { + "RoleResultsPage": { "description": "A single page of results", "type": "object", "properties": { @@ -13502,7 +12494,7 @@ "description": "list of items on this page of results", "type": "array", "items": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/Role" } }, "next_page": { @@ -13515,32 +12507,50 @@ "items" ] }, - "RouterRouteUpdate": { - "description": "Updateable properties of a `RouterRoute`", + "Route": { + "description": "A route to a destination network through a gateway address.", "type": "object", "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "destination": { - "$ref": "#/components/schemas/RouteDestination" - }, - "name": { - "nullable": true, + "dst": { + "description": "The route destination.", "allOf": [ { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/IpNet" } ] }, - "target": { - "$ref": "#/components/schemas/RouteTarget" + "gw": { + "description": "The route gateway.", + "type": "string", + "format": "ip" + }, + "vid": { + "nullable": true, + "description": "VLAN id the gateway is reachable over.", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "dst", + "gw" + ] + }, + "RouteConfig": { + "description": "Route configuration data associated with a switch port configuration.", + "type": "object", + "properties": { + "routes": { + "description": "The set of routes assigned to a switch port.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Route" + } } }, "required": [ - "destination", - "target" + "routes" ] }, "SamlIdentityProvider": { @@ -15706,118 +14716,6 @@ "items" ] }, - "VpcRouter": { - "description": "A VPC router defines a series of rules that indicate where traffic should be sent depending on its destination.", - "type": "object", - "properties": { - "description": { - "description": "human-readable free-form text about a resource", - "type": "string" - }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" - }, - "kind": { - "$ref": "#/components/schemas/VpcRouterKind" - }, - "name": { - "description": "unique, mutable, user-controlled identifier for each resource", - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - }, - "time_created": { - "description": "timestamp when this resource was created", - "type": "string", - "format": "date-time" - }, - "time_modified": { - "description": "timestamp when this resource was last modified", - "type": "string", - "format": "date-time" - }, - "vpc_id": { - "description": "The VPC to which the router belongs.", - "type": "string", - "format": "uuid" - } - }, - "required": [ - "description", - "id", - "kind", - "name", - "time_created", - "time_modified", - "vpc_id" - ] - }, - "VpcRouterCreate": { - "description": "Create-time parameters for a `VpcRouter`", - "type": "object", - "properties": { - "description": { - "type": "string" - }, - "name": { - "$ref": "#/components/schemas/Name" - } - }, - "required": [ - "description", - "name" - ] - }, - "VpcRouterKind": { - "type": "string", - "enum": [ - "system", - "custom" - ] - }, - "VpcRouterResultsPage": { - "description": "A single page of results", - "type": "object", - "properties": { - "items": { - "description": "list of items on this page of results", - "type": "array", - "items": { - "$ref": "#/components/schemas/VpcRouter" - } - }, - "next_page": { - "nullable": true, - "description": "token used to fetch the next page of results (if any)", - "type": "string" - } - }, - "required": [ - "items" - ] - }, - "VpcRouterUpdate": { - "description": "Updateable properties of a `VpcRouter`", - "type": "object", - "properties": { - "description": { - "nullable": true, - "type": "string" - }, - "name": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/Name" - } - ] - } - } - }, "VpcSubnet": { "description": "A VPC subnet represents a logical grouping for instances that allows network traffic between them, within a IPv4 subnetwork or optionall an IPv6 subnetwork.", "type": "object", @@ -16058,6 +14956,18 @@ } ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } }, "tags": [ diff --git a/openapi/oximeter.json b/openapi/oximeter.json index ebc7957c2ee..529d20e921c 100644 --- a/openapi/oximeter.json +++ b/openapi/oximeter.json @@ -134,18 +134,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "CollectorInfo": { "type": "object", @@ -244,6 +232,18 @@ "items" ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 486662853cc..ed202ddbdb4 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -289,6 +289,45 @@ } } }, + "/metrics/collect/{producer_id}": { + "get": { + "summary": "Collect oximeter samples from the sled agent.", + "operationId": "metrics_collect", + "parameters": [ + { + "in": "path", + "name": "producer_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_ProducerResultsItem", + "type": "array", + "items": { + "$ref": "#/components/schemas/ProducerResultsItem" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/network-bootstore-config": { "get": { "summary": "This API endpoint is only reading the local sled agent's view of the", @@ -387,6 +426,33 @@ } } }, + "/sleds": { + "put": { + "summary": "Add a sled to a rack that was already initialized via RSS", + "operationId": "add_sled_to_initialized_rack", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddSledRequest" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/switch-ports": { "post": { "operationId": "uplink_ensure", @@ -925,19 +991,91 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { + "AddSledRequest": { + "description": "A request to Add a given sled after rack initialization has occurred", + "type": "object", + "properties": { + "sled_id": { + "$ref": "#/components/schemas/Baseboard" + }, + "start_request": { + "$ref": "#/components/schemas/StartSledAgentRequest" + } + }, + "required": [ + "sled_id", + "start_request" + ] + }, + "Baseboard": { + "description": "Describes properties that should uniquely identify a Gimlet.", + "oneOf": [ + { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "model": { + "type": "string" + }, + "revision": { + "type": "integer", + "format": "int64" + }, + "type": { + "type": "string", + "enum": [ + "gimlet" + ] + } + }, + "required": [ + "identifier", + "model", + "revision", + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "unknown" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "model": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "pc" + ] + } + }, + "required": [ + "identifier", + "model", + "type" + ] + } + ] + }, "BgpConfig": { "type": "object", "properties": { @@ -974,827 +1112,3068 @@ "format": "uint32", "minimum": 0 }, - "port": { - "description": "Switch port the peer is reachable on.", - "type": "string" - } - }, - "required": [ - "addr", - "asn", - "port" - ] - }, - "BundleUtilization": { - "description": "The portion of a debug dataset used for zone bundles.", - "type": "object", - "properties": { - "bytes_available": { - "description": "The total number of bytes available for zone bundles.\n\nThis is `dataset_quota` multiplied by the context's storage limit.", + "connect_retry": { + "nullable": true, + "description": "The interval in seconds between peer connection retry attempts.", "type": "integer", "format": "uint64", "minimum": 0 }, - "bytes_used": { - "description": "Total bundle usage, in bytes.", + "delay_open": { + "nullable": true, + "description": "How long to delay sending open messages to a peer. In seconds.", "type": "integer", "format": "uint64", "minimum": 0 }, - "dataset_quota": { - "description": "The total dataset quota, in bytes.", + "hold_time": { + "nullable": true, + "description": "How long to keep a session alive without a keepalive in seconds. Defaults to 6.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "idle_hold_time": { + "nullable": true, + "description": "How long to keep a peer in idle after a state machine reset in seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "keepalive": { + "nullable": true, + "description": "The interval to send keepalive messages at.", "type": "integer", "format": "uint64", "minimum": 0 + }, + "port": { + "description": "Switch port the peer is reachable on.", + "type": "string" } }, "required": [ - "bytes_available", - "bytes_used", - "dataset_quota" + "addr", + "asn", + "port" ] }, - "ByteCount": { - "description": "Byte count to express memory or storage capacity.", - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "CleanupContext": { - "description": "Context provided for the zone bundle cleanup task.", - "type": "object", - "properties": { - "period": { - "description": "The period on which automatic checks and cleanup is performed.", - "allOf": [ - { - "$ref": "#/components/schemas/CleanupPeriod" + "BinRangedouble": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "number", + "format": "double" + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] } + }, + "required": [ + "end", + "type" ] }, - "priority": { - "description": "The priority ordering for keeping old bundles.", - "allOf": [ - { - "$ref": "#/components/schemas/PriorityOrder" + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "number", + "format": "double" + }, + "start": { + "type": "number", + "format": "double" + }, + "type": { + "type": "string", + "enum": [ + "range" + ] } + }, + "required": [ + "end", + "start", + "type" ] }, - "storage_limit": { - "description": "The limit on the dataset quota available for zone bundles.", - "allOf": [ - { - "$ref": "#/components/schemas/StorageLimit" + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "number", + "format": "double" + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] } + }, + "required": [ + "start", + "type" ] } - }, - "required": [ - "period", - "priority", - "storage_limit" ] }, - "CleanupContextUpdate": { - "description": "Parameters used to update the zone bundle cleanup context.", - "type": "object", - "properties": { - "period": { - "nullable": true, - "description": "The new period on which automatic cleanups are run.", - "allOf": [ - { - "$ref": "#/components/schemas/Duration" + "BinRangefloat": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "number", + "format": "float" + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "number", + "format": "float" + }, + "start": { + "type": "number", + "format": "float" + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "number", + "format": "float" + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeint16": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int16" + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int16" + }, + "start": { + "type": "integer", + "format": "int16" + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "int16" + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeint32": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int32" + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int32" + }, + "start": { + "type": "integer", + "format": "int32" + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "int32" + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeint64": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int64" + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int64" + }, + "start": { + "type": "integer", + "format": "int64" + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "int64" + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeint8": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int8" + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "int8" + }, + "start": { + "type": "integer", + "format": "int8" + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "int8" + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeuint16": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "start": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeuint32": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "start": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeuint64": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "BinRangeuint8": { + "description": "A type storing a range over `T`.\n\nThis type supports ranges similar to the `RangeTo`, `Range` and `RangeFrom` types in the standard library. Those cover `(..end)`, `(start..end)`, and `(start..)` respectively.", + "oneOf": [ + { + "description": "A range unbounded below and exclusively above, `..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_to" + ] + } + }, + "required": [ + "end", + "type" + ] + }, + { + "description": "A range bounded inclusively below and exclusively above, `start..end`.", + "type": "object", + "properties": { + "end": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "start": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range" + ] + } + }, + "required": [ + "end", + "start", + "type" + ] + }, + { + "description": "A range bounded inclusively below and unbounded above, `start..`.", + "type": "object", + "properties": { + "start": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "range_from" + ] + } + }, + "required": [ + "start", + "type" + ] + } + ] + }, + "Bindouble": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangedouble" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binfloat": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangefloat" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binint16": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeint16" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binint32": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeint32" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binint64": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeint64" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binint8": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeint8" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binuint16": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeuint16" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binuint32": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeuint32" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binuint64": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeuint64" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "Binuint8": { + "description": "Type storing bin edges and a count of samples within it.", + "type": "object", + "properties": { + "count": { + "description": "The total count of samples in this bin.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "range": { + "description": "The range of the support covered by this bin.", + "allOf": [ + { + "$ref": "#/components/schemas/BinRangeuint8" + } + ] + } + }, + "required": [ + "count", + "range" + ] + }, + "BundleUtilization": { + "description": "The portion of a debug dataset used for zone bundles.", + "type": "object", + "properties": { + "bytes_available": { + "description": "The total number of bytes available for zone bundles.\n\nThis is `dataset_quota` multiplied by the context's storage limit.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes_used": { + "description": "Total bundle usage, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "dataset_quota": { + "description": "The total dataset quota, in bytes.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "bytes_available", + "bytes_used", + "dataset_quota" + ] + }, + "ByteCount": { + "description": "Byte count to express memory or storage capacity.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "CleanupContext": { + "description": "Context provided for the zone bundle cleanup task.", + "type": "object", + "properties": { + "period": { + "description": "The period on which automatic checks and cleanup is performed.", + "allOf": [ + { + "$ref": "#/components/schemas/CleanupPeriod" + } + ] + }, + "priority": { + "description": "The priority ordering for keeping old bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/PriorityOrder" + } + ] + }, + "storage_limit": { + "description": "The limit on the dataset quota available for zone bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/StorageLimit" + } + ] + } + }, + "required": [ + "period", + "priority", + "storage_limit" + ] + }, + "CleanupContextUpdate": { + "description": "Parameters used to update the zone bundle cleanup context.", + "type": "object", + "properties": { + "period": { + "nullable": true, + "description": "The new period on which automatic cleanups are run.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "priority": { + "nullable": true, + "description": "The priority ordering for preserving old zone bundles.", + "allOf": [ + { + "$ref": "#/components/schemas/PriorityOrder" + } + ] + }, + "storage_limit": { + "nullable": true, + "description": "The new limit on the underlying dataset quota allowed for bundles.", + "type": "integer", + "format": "uint8", + "minimum": 0 + } + } + }, + "CleanupCount": { + "description": "The count of bundles / bytes removed during a cleanup operation.", + "type": "object", + "properties": { + "bundles": { + "description": "The number of bundles removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "bytes": { + "description": "The number of bytes removed.", + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "bundles", + "bytes" + ] + }, + "CleanupPeriod": { + "description": "A period on which bundles are automatically cleaned up.", + "allOf": [ + { + "$ref": "#/components/schemas/Duration" + } + ] + }, + "CrucibleOpts": { + "type": "object", + "properties": { + "cert_pem": { + "nullable": true, + "type": "string" + }, + "control": { + "nullable": true, + "type": "string" + }, + "flush_timeout": { + "nullable": true, + "type": "number", + "format": "float" + }, + "id": { + "type": "string", + "format": "uuid" + }, + "key": { + "nullable": true, + "type": "string" + }, + "key_pem": { + "nullable": true, + "type": "string" + }, + "lossy": { + "type": "boolean" + }, + "read_only": { + "type": "boolean" + }, + "root_cert_pem": { + "nullable": true, + "type": "string" + }, + "target": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "id", + "lossy", + "read_only", + "target" + ] + }, + "Cumulativedouble": { + "description": "A cumulative or counter data type.", + "type": "object", + "properties": { + "start_time": { + "type": "string", + "format": "date-time" + }, + "value": { + "type": "number", + "format": "double" + } + }, + "required": [ + "start_time", + "value" + ] + }, + "Cumulativefloat": { + "description": "A cumulative or counter data type.", + "type": "object", + "properties": { + "start_time": { + "type": "string", + "format": "date-time" + }, + "value": { + "type": "number", + "format": "float" + } + }, + "required": [ + "start_time", + "value" + ] + }, + "Cumulativeint64": { + "description": "A cumulative or counter data type.", + "type": "object", + "properties": { + "start_time": { + "type": "string", + "format": "date-time" + }, + "value": { + "type": "integer", + "format": "int64" + } + }, + "required": [ + "start_time", + "value" + ] + }, + "Cumulativeuint64": { + "description": "A cumulative or counter data type.", + "type": "object", + "properties": { + "start_time": { + "type": "string", + "format": "date-time" + }, + "value": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "start_time", + "value" + ] + }, + "DatasetKind": { + "description": "The type of a dataset, and an auxiliary information necessary to successfully launch a zone managing the associated data.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "cockroach_db" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "crucible" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "clickhouse" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "clickhouse_keeper" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "external_dns" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internal_dns" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, + "DatasetName": { + "type": "object", + "properties": { + "kind": { + "$ref": "#/components/schemas/DatasetKind" + }, + "pool_name": { + "$ref": "#/components/schemas/ZpoolName" + } + }, + "required": [ + "kind", + "pool_name" + ] + }, + "DatasetRequest": { + "description": "Describes a request to provision a specific dataset", + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "name": { + "$ref": "#/components/schemas/DatasetName" + }, + "service_address": { + "type": "string" + } + }, + "required": [ + "id", + "name", + "service_address" + ] + }, + "Datum": { + "description": "A `Datum` is a single sampled data point from a metric.", + "oneOf": [ + { + "type": "object", + "properties": { + "datum": { + "type": "boolean" + }, + "type": { + "type": "string", + "enum": [ + "bool" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "int8" + }, + "type": { + "type": "string", + "enum": [ + "i8" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "u8" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "int16" + }, + "type": { + "type": "string", + "enum": [ + "i16" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "u16" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "int32" + }, + "type": { + "type": "string", + "enum": [ + "i32" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "u32" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "int64" + }, + "type": { + "type": "string", + "enum": [ + "i64" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "type": { + "type": "string", + "enum": [ + "u64" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "number", + "format": "float" + }, + "type": { + "type": "string", + "enum": [ + "f32" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "number", + "format": "double" + }, + "type": { + "type": "string", + "enum": [ + "f64" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "string" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "type": "array", + "items": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "type": { + "type": "string", + "enum": [ + "bytes" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Cumulativeint64" + }, + "type": { + "type": "string", + "enum": [ + "cumulative_i64" + ] + } + }, + "required": [ + "datum", + "type" ] }, - "priority": { - "nullable": true, - "description": "The priority ordering for preserving old zone bundles.", + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Cumulativeuint64" + }, + "type": { + "type": "string", + "enum": [ + "cumulative_u64" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Cumulativefloat" + }, + "type": { + "type": "string", + "enum": [ + "cumulative_f32" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Cumulativedouble" + }, + "type": { + "type": "string", + "enum": [ + "cumulative_f64" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramint8" + }, + "type": { + "type": "string", + "enum": [ + "histogram_i8" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramuint8" + }, + "type": { + "type": "string", + "enum": [ + "histogram_u8" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramint16" + }, + "type": { + "type": "string", + "enum": [ + "histogram_i16" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramuint16" + }, + "type": { + "type": "string", + "enum": [ + "histogram_u16" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramint32" + }, + "type": { + "type": "string", + "enum": [ + "histogram_i32" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramuint32" + }, + "type": { + "type": "string", + "enum": [ + "histogram_u32" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramint64" + }, + "type": { + "type": "string", + "enum": [ + "histogram_i64" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramuint64" + }, + "type": { + "type": "string", + "enum": [ + "histogram_u64" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramfloat" + }, + "type": { + "type": "string", + "enum": [ + "histogram_f32" + ] + } + }, + "required": [ + "datum", + "type" + ] + }, + { + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Histogramdouble" + }, + "type": { + "type": "string", + "enum": [ + "histogram_f64" + ] + } + }, + "required": [ + "datum", + "type" + ] + } + ] + }, + "DeleteVirtualNetworkInterfaceHost": { + "description": "The data needed to identify a virtual IP for which a sled maintains an OPTE virtual-to-physical mapping such that that mapping can be deleted.", + "type": "object", + "properties": { + "virtual_ip": { + "description": "The virtual IP whose mapping should be deleted.", + "type": "string", + "format": "ip" + }, + "vni": { + "description": "The VNI for the network containing the virtual IP whose mapping should be deleted.", "allOf": [ { - "$ref": "#/components/schemas/PriorityOrder" + "$ref": "#/components/schemas/Vni" } ] + } + }, + "required": [ + "virtual_ip", + "vni" + ] + }, + "DhcpConfig": { + "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.", + "type": "object", + "properties": { + "dns_servers": { + "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)", + "type": "array", + "items": { + "type": "string", + "format": "ip" + } }, - "storage_limit": { + "host_domain": { "nullable": true, - "description": "The new limit on the underlying dataset quota allowed for bundles.", - "type": "integer", - "format": "uint8", - "minimum": 0 + "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)", + "type": "string" + }, + "search_domains": { + "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)", + "type": "array", + "items": { + "type": "string" + } } - } + }, + "required": [ + "dns_servers", + "search_domains" + ] }, - "CleanupCount": { - "description": "The count of bundles / bytes removed during a cleanup operation.", + "DiskEnsureBody": { + "description": "Sent from to a sled agent to establish the runtime state of a Disk", "type": "object", "properties": { - "bundles": { - "description": "The number of bundles removed.", - "type": "integer", - "format": "uint64", - "minimum": 0 + "initial_runtime": { + "description": "Last runtime state of the Disk known to Nexus (used if the agent has never seen this Disk before).", + "allOf": [ + { + "$ref": "#/components/schemas/DiskRuntimeState" + } + ] }, - "bytes": { - "description": "The number of bytes removed.", - "type": "integer", - "format": "uint64", - "minimum": 0 + "target": { + "description": "requested runtime state of the Disk", + "allOf": [ + { + "$ref": "#/components/schemas/DiskStateRequested" + } + ] } }, "required": [ - "bundles", - "bytes" - ] - }, - "CleanupPeriod": { - "description": "A period on which bundles are automatically cleaned up.", - "allOf": [ - { - "$ref": "#/components/schemas/Duration" - } + "initial_runtime", + "target" ] }, - "CrucibleOpts": { + "DiskRequest": { "type": "object", "properties": { - "cert_pem": { - "nullable": true, - "type": "string" - }, - "control": { - "nullable": true, - "type": "string" - }, - "flush_timeout": { - "nullable": true, - "type": "number", - "format": "float" - }, - "id": { - "type": "string", - "format": "uuid" - }, - "key": { - "nullable": true, + "device": { "type": "string" }, - "key_pem": { - "nullable": true, + "name": { "type": "string" }, - "lossy": { - "type": "boolean" - }, "read_only": { "type": "boolean" }, - "root_cert_pem": { - "nullable": true, - "type": "string" + "slot": { + "$ref": "#/components/schemas/Slot" }, - "target": { - "type": "array", - "items": { - "type": "string" - } + "volume_construction_request": { + "$ref": "#/components/schemas/VolumeConstructionRequest" } }, "required": [ - "id", - "lossy", + "device", + "name", "read_only", - "target" + "slot", + "volume_construction_request" ] }, - "DatasetKind": { - "description": "The type of a dataset, and an auxiliary information necessary to successfully launch a zone managing the associated data.", + "DiskRuntimeState": { + "description": "Runtime state of the Disk, which includes its attach state and some minimal metadata", + "type": "object", + "properties": { + "disk_state": { + "description": "runtime state of the Disk", + "allOf": [ + { + "$ref": "#/components/schemas/DiskState" + } + ] + }, + "gen": { + "description": "generation number for this state", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "time_updated": { + "description": "timestamp for this information", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "disk_state", + "gen", + "time_updated" + ] + }, + "DiskState": { + "description": "State of a Disk", "oneOf": [ { + "description": "Disk is being initialized", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "creating" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is ready but detached from any Instance", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "detached" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is ready to receive blocks from an external source", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "import_ready" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is importing blocks from a URL", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "importing_from_url" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is importing blocks from bulk writes", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "importing_from_bulk_writes" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is being finalized to state Detached", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "finalizing" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is undergoing maintenance", + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "maintenance" + ] + } + }, + "required": [ + "state" + ] + }, + { + "description": "Disk is being attached to the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attaching" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk is attached to the given Instance", + "type": "object", + "properties": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { + "type": "string", + "enum": [ + "attached" + ] + } + }, + "required": [ + "instance", + "state" + ] + }, + { + "description": "Disk is being detached from the given Instance", "type": "object", "properties": { - "type": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { "type": "string", "enum": [ - "cockroach_db" + "detaching" ] } }, "required": [ - "type" + "instance", + "state" ] }, { + "description": "Disk has been destroyed", "type": "object", "properties": { - "type": { + "state": { "type": "string", "enum": [ - "crucible" + "destroyed" ] } }, "required": [ - "type" + "state" ] }, { + "description": "Disk is unavailable", "type": "object", "properties": { - "type": { + "state": { "type": "string", "enum": [ - "clickhouse" + "faulted" ] } }, "required": [ - "type" + "state" + ] + } + ] + }, + "DiskStateRequested": { + "description": "Used to request a Disk state change", + "oneOf": [ + { + "type": "object", + "properties": { + "state": { + "type": "string", + "enum": [ + "detached" + ] + } + }, + "required": [ + "state" ] }, { "type": "object", "properties": { - "type": { + "instance": { + "type": "string", + "format": "uuid" + }, + "state": { "type": "string", "enum": [ - "clickhouse_keeper" + "attached" ] } }, "required": [ - "type" + "instance", + "state" ] }, { "type": "object", "properties": { - "type": { + "state": { "type": "string", "enum": [ - "external_dns" + "destroyed" ] } }, "required": [ - "type" + "state" ] }, { "type": "object", "properties": { - "type": { + "state": { "type": "string", "enum": [ - "internal_dns" + "faulted" ] } }, "required": [ - "type" + "state" ] } ] }, - "DatasetName": { + "DiskType": { + "type": "string", + "enum": [ + "U2", + "M2" + ] + }, + "Duration": { "type": "object", "properties": { - "kind": { - "$ref": "#/components/schemas/DatasetKind" + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0 }, - "pool_name": { - "$ref": "#/components/schemas/ZpoolName" + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0 } }, "required": [ - "kind", - "pool_name" + "nanos", + "secs" ] }, - "DatasetRequest": { - "description": "Describes a request to provision a specific dataset", + "EarlyNetworkConfig": { + "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.", "type": "object", "properties": { - "id": { - "type": "string", - "format": "uuid" + "body": { + "$ref": "#/components/schemas/EarlyNetworkConfigBody" }, - "name": { - "$ref": "#/components/schemas/DatasetName" + "generation": { + "type": "integer", + "format": "uint64", + "minimum": 0 }, - "service_address": { - "type": "string" + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ - "id", - "name", - "service_address" + "body", + "generation", + "schema_version" ] }, - "DeleteVirtualNetworkInterfaceHost": { - "description": "The data needed to identify a virtual IP for which a sled maintains an OPTE virtual-to-physical mapping such that that mapping can be deleted.", + "EarlyNetworkConfigBody": { + "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", "type": "object", "properties": { - "virtual_ip": { - "description": "The virtual IP whose mapping should be deleted.", - "type": "string", - "format": "ip" + "ntp_servers": { + "description": "The external NTP server addresses.", + "type": "array", + "items": { + "type": "string" + } }, - "vni": { - "description": "The VNI for the network containing the virtual IP whose mapping should be deleted.", + "rack_network_config": { + "nullable": true, "allOf": [ { - "$ref": "#/components/schemas/Vni" + "$ref": "#/components/schemas/RackNetworkConfigV1" } ] } }, "required": [ - "virtual_ip", - "vni" + "ntp_servers" ] }, - "DhcpConfig": { - "description": "DHCP configuration for a port\n\nNot present here: Hostname (DHCPv4 option 12; used in DHCPv6 option 39); we use `InstanceRuntimeState::hostname` for this value.", + "Error": { + "description": "Error information from a response.", "type": "object", "properties": { - "dns_servers": { - "description": "DNS servers to send to the instance\n\n(DHCPv4 option 6; DHCPv6 option 23)", - "type": "array", - "items": { - "type": "string", - "format": "ip" - } - }, - "host_domain": { - "nullable": true, - "description": "DNS zone this instance's hostname belongs to (e.g. the `project.example` part of `instance1.project.example`)\n\n(DHCPv4 option 15; used in DHCPv6 option 39)", + "error_code": { "type": "string" }, - "search_domains": { - "description": "DNS search domains\n\n(DHCPv4 option 119; DHCPv6 option 24)", - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "dns_servers", - "search_domains" - ] - }, - "DiskEnsureBody": { - "description": "Sent from to a sled agent to establish the runtime state of a Disk", - "type": "object", - "properties": { - "initial_runtime": { - "description": "Last runtime state of the Disk known to Nexus (used if the agent has never seen this Disk before).", - "allOf": [ - { - "$ref": "#/components/schemas/DiskRuntimeState" - } - ] + "message": { + "type": "string" }, - "target": { - "description": "requested runtime state of the Disk", - "allOf": [ - { - "$ref": "#/components/schemas/DiskStateRequested" - } - ] + "request_id": { + "type": "string" } }, "required": [ - "initial_runtime", - "target" + "message", + "request_id" ] }, - "DiskRequest": { + "Field": { + "description": "A `Field` is a named aspect of a target or metric.", "type": "object", "properties": { - "device": { - "type": "string" - }, "name": { "type": "string" }, - "read_only": { - "type": "boolean" - }, - "slot": { - "$ref": "#/components/schemas/Slot" - }, - "volume_construction_request": { - "$ref": "#/components/schemas/VolumeConstructionRequest" + "value": { + "$ref": "#/components/schemas/FieldValue" } }, "required": [ - "device", "name", - "read_only", - "slot", - "volume_construction_request" + "value" ] }, - "DiskRuntimeState": { - "description": "Runtime state of the Disk, which includes its attach state and some minimal metadata", + "FieldSet": { "type": "object", "properties": { - "disk_state": { - "description": "runtime state of the Disk", - "allOf": [ - { - "$ref": "#/components/schemas/DiskState" - } - ] - }, - "gen": { - "description": "generation number for this state", - "allOf": [ - { - "$ref": "#/components/schemas/Generation" - } - ] + "fields": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/Field" + } }, - "time_updated": { - "description": "timestamp for this information", - "type": "string", - "format": "date-time" + "name": { + "type": "string" } }, "required": [ - "disk_state", - "gen", - "time_updated" + "fields", + "name" ] }, - "DiskState": { - "description": "State of a Disk", + "FieldValue": { + "description": "The `FieldValue` contains the value of a target or metric field.", "oneOf": [ { - "description": "Disk is being initialized", "type": "object", "properties": { - "state": { + "type": { + "type": "string", + "enum": [ + "string" + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { "type": "string", "enum": [ - "creating" + "i8" ] + }, + "value": { + "type": "integer", + "format": "int8" } }, "required": [ - "state" + "type", + "value" ] }, { - "description": "Disk is ready but detached from any Instance", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "detached" + "u8" ] + }, + "value": { + "type": "integer", + "format": "uint8", + "minimum": 0 } }, "required": [ - "state" + "type", + "value" ] }, { - "description": "Disk is ready to receive blocks from an external source", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "import_ready" + "i16" ] + }, + "value": { + "type": "integer", + "format": "int16" } }, "required": [ - "state" + "type", + "value" ] }, { - "description": "Disk is importing blocks from a URL", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "importing_from_url" + "u16" ] + }, + "value": { + "type": "integer", + "format": "uint16", + "minimum": 0 } }, "required": [ - "state" + "type", + "value" ] }, { - "description": "Disk is importing blocks from bulk writes", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "importing_from_bulk_writes" + "i32" ] + }, + "value": { + "type": "integer", + "format": "int32" } }, "required": [ - "state" + "type", + "value" ] }, { - "description": "Disk is being finalized to state Detached", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "finalizing" + "u32" ] + }, + "value": { + "type": "integer", + "format": "uint32", + "minimum": 0 } }, "required": [ - "state" + "type", + "value" ] }, { - "description": "Disk is undergoing maintenance", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "maintenance" + "i64" ] + }, + "value": { + "type": "integer", + "format": "int64" } }, "required": [ - "state" + "type", + "value" ] }, { - "description": "Disk is being attached to the given Instance", "type": "object", "properties": { - "instance": { - "type": "string", - "format": "uuid" - }, - "state": { + "type": { "type": "string", "enum": [ - "attaching" + "u64" ] + }, + "value": { + "type": "integer", + "format": "uint64", + "minimum": 0 } }, "required": [ - "instance", - "state" + "type", + "value" ] }, { - "description": "Disk is attached to the given Instance", "type": "object", "properties": { - "instance": { - "type": "string", - "format": "uuid" - }, - "state": { + "type": { "type": "string", "enum": [ - "attached" + "ip_addr" ] + }, + "value": { + "type": "string", + "format": "ip" } }, "required": [ - "instance", - "state" + "type", + "value" ] }, { - "description": "Disk is being detached from the given Instance", "type": "object", "properties": { - "instance": { + "type": { "type": "string", - "format": "uuid" + "enum": [ + "uuid" + ] }, - "state": { + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { "type": "string", "enum": [ - "detaching" + "bool" ] + }, + "value": { + "type": "boolean" } }, "required": [ - "instance", - "state" + "type", + "value" ] - }, + } + ] + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "HistogramError": { + "description": "Errors related to constructing histograms or adding samples into them.", + "oneOf": [ { - "description": "Disk has been destroyed", + "description": "An attempt to construct a histogram with an empty set of bins.", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "destroyed" + "empty_bins" ] } }, "required": [ - "state" + "type" ] }, { - "description": "Disk is unavailable", + "description": "An attempt to construct a histogram with non-monotonic bins.", "type": "object", "properties": { - "state": { + "type": { "type": "string", "enum": [ - "faulted" + "nonmonotonic_bins" ] } }, "required": [ - "state" + "type" ] - } - ] - }, - "DiskStateRequested": { - "description": "Used to request a Disk state change", - "oneOf": [ + }, { + "description": "A non-finite was encountered, either as a bin edge or a sample.", "type": "object", "properties": { - "state": { + "content": { + "type": "string" + }, + "type": { "type": "string", "enum": [ - "detached" + "non_finite_value" ] } }, "required": [ - "state" + "content", + "type" ] }, { + "description": "Error returned when two neighboring bins are not adjoining (there's space between them)", "type": "object", "properties": { - "instance": { - "type": "string", - "format": "uuid" + "content": { + "type": "object", + "properties": { + "left": { + "type": "string" + }, + "right": { + "type": "string" + } + }, + "required": [ + "left", + "right" + ] }, - "state": { + "type": { "type": "string", "enum": [ - "attached" + "non_adjoining_bins" ] } }, "required": [ - "instance", - "state" + "content", + "type" ] }, { + "description": "Bin and count arrays are of different sizes.", "type": "object", "properties": { - "state": { + "content": { + "type": "object", + "properties": { + "n_bins": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "n_counts": { + "type": "integer", + "format": "uint", + "minimum": 0 + } + }, + "required": [ + "n_bins", + "n_counts" + ] + }, + "type": { "type": "string", "enum": [ - "destroyed" + "array_size_mismatch" ] } }, "required": [ - "state" + "content", + "type" ] }, { "type": "object", "properties": { - "state": { + "content": { + "$ref": "#/components/schemas/QuantizationError" + }, + "type": { "type": "string", "enum": [ - "faulted" + "quantization" ] } }, "required": [ - "state" + "content", + "type" ] } ] }, - "DiskType": { - "type": "string", - "enum": [ - "U2", - "M2" + "Histogramdouble": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Bindouble" + } + }, + "n_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start_time": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "bins", + "n_samples", + "start_time" + ] + }, + "Histogramfloat": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binfloat" + } + }, + "n_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start_time": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "bins", + "n_samples", + "start_time" + ] + }, + "Histogramint16": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binint16" + } + }, + "n_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start_time": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "bins", + "n_samples", + "start_time" ] }, - "Duration": { + "Histogramint32": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { - "nanos": { + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binint32" + } + }, + "n_samples": { "type": "integer", - "format": "uint32", + "format": "uint64", "minimum": 0 }, - "secs": { + "start_time": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "bins", + "n_samples", + "start_time" + ] + }, + "Histogramint64": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binint64" + } + }, + "n_samples": { "type": "integer", "format": "uint64", "minimum": 0 + }, + "start_time": { + "type": "string", + "format": "date-time" } }, "required": [ - "nanos", - "secs" + "bins", + "n_samples", + "start_time" ] }, - "EarlyNetworkConfig": { - "description": "Network configuration required to bring up the control plane\n\nThe fields in this structure are those from [`super::params::RackInitializeRequest`] necessary for use beyond RSS. This is just for the initial rack configuration and cold boot purposes. Updates come from Nexus.", + "Histogramint8": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { - "body": { - "$ref": "#/components/schemas/EarlyNetworkConfigBody" + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binint8" + } }, - "generation": { + "n_samples": { "type": "integer", "format": "uint64", "minimum": 0 }, - "schema_version": { + "start_time": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "bins", + "n_samples", + "start_time" + ] + }, + "Histogramuint16": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binuint16" + } + }, + "n_samples": { "type": "integer", - "format": "uint32", + "format": "uint64", "minimum": 0 + }, + "start_time": { + "type": "string", + "format": "date-time" } }, "required": [ - "body", - "generation", - "schema_version" + "bins", + "n_samples", + "start_time" ] }, - "EarlyNetworkConfigBody": { - "description": "This is the actual configuration of EarlyNetworking.\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "Histogramuint32": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { - "ntp_servers": { - "description": "The external NTP server addresses.", + "bins": { "type": "array", "items": { - "type": "string" + "$ref": "#/components/schemas/Binuint32" } }, - "rack_network_config": { - "nullable": true, - "allOf": [ - { - "$ref": "#/components/schemas/RackNetworkConfigV1" - } - ] + "n_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start_time": { + "type": "string", + "format": "date-time" } }, "required": [ - "ntp_servers" + "bins", + "n_samples", + "start_time" ] }, - "Error": { - "description": "Error information from a response.", + "Histogramuint64": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { - "error_code": { - "type": "string" + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binuint64" + } }, - "message": { - "type": "string" + "n_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 }, - "request_id": { - "type": "string" + "start_time": { + "type": "string", + "format": "date-time" } }, "required": [ - "message", - "request_id" + "bins", + "n_samples", + "start_time" ] }, - "Generation": { - "description": "Generation numbers stored in the database, used for optimistic concurrency control", - "type": "integer", - "format": "uint64", - "minimum": 0 + "Histogramuint8": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Binuint8" + } + }, + "n_samples": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "start_time": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "bins", + "n_samples", + "start_time" + ] }, "HostIdentifier": { "description": "A `HostIdentifier` represents either an IP host or network (v4 or v6), or an entire VPC (identified by its VNI). It is used in firewall rule host filters.", @@ -2343,61 +4722,210 @@ } ] }, - "Ipv4Net": { - "example": "192.168.1.0/24", - "title": "An IPv4 subnet", - "description": "An IPv4 subnet, including prefix and subnet mask", - "type": "string", - "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$" - }, - "Ipv4Network": { - "type": "string", - "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" - }, - "Ipv6Net": { - "example": "fd12:3456::/64", - "title": "An IPv6 subnet", - "description": "An IPv6 subnet, including prefix and subnet mask", - "type": "string", - "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" - }, - "Ipv6Network": { - "type": "string", - "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" - }, - "KnownArtifactKind": { - "description": "Kinds of update artifacts, as used by Nexus to determine what updates are available and by sled-agent to determine how to apply an update when asked.", - "type": "string", - "enum": [ - "gimlet_sp", - "gimlet_rot", - "host", - "trampoline", - "control_plane", - "psc_sp", - "psc_rot", - "switch_sp", - "switch_rot" - ] - }, - "L4PortRange": { - "example": "22", - "title": "A range of IP ports", - "description": "An inclusive-inclusive range of IP ports. The second port may be omitted to represent a single port", - "type": "string", - "pattern": "^[0-9]{1,5}(-[0-9]{1,5})?$", - "minLength": 1, - "maxLength": 11 - }, - "MacAddr": { - "example": "ff:ff:ff:ff:ff:ff", - "title": "A MAC address", - "description": "A Media Access Control address, in EUI-48 format", - "type": "string", - "pattern": "^([0-9a-fA-F]{0,2}:){5}[0-9a-fA-F]{0,2}$", - "minLength": 5, - "maxLength": 17 - }, + "Ipv4Net": { + "example": "192.168.1.0/24", + "title": "An IPv4 subnet", + "description": "An IPv4 subnet, including prefix and subnet mask", + "type": "string", + "pattern": "^(([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])/([0-9]|1[0-9]|2[0-9]|3[0-2])$" + }, + "Ipv4Network": { + "type": "string", + "pattern": "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\/(3[0-2]|[0-2]?[0-9])$" + }, + "Ipv6Net": { + "example": "fd12:3456::/64", + "title": "An IPv6 subnet", + "description": "An IPv6 subnet, including prefix and subnet mask", + "type": "string", + "pattern": "^([fF][dD])[0-9a-fA-F]{2}:(([0-9a-fA-F]{1,4}:){6}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,6}:)([0-9a-fA-F]{1,4})?\\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])$" + }, + "Ipv6Network": { + "type": "string", + "pattern": "^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\")[/](12[0-8]|1[0-1][0-9]|[0-9]?[0-9])$" + }, + "Ipv6Subnet": { + "description": "Wraps an [`Ipv6Network`] with a compile-time prefix length.", + "type": "object", + "properties": { + "net": { + "$ref": "#/components/schemas/Ipv6Net" + } + }, + "required": [ + "net" + ] + }, + "KnownArtifactKind": { + "description": "Kinds of update artifacts, as used by Nexus to determine what updates are available and by sled-agent to determine how to apply an update when asked.", + "type": "string", + "enum": [ + "gimlet_sp", + "gimlet_rot", + "host", + "trampoline", + "control_plane", + "psc_sp", + "psc_rot", + "switch_sp", + "switch_rot" + ] + }, + "L4PortRange": { + "example": "22", + "title": "A range of IP ports", + "description": "An inclusive-inclusive range of IP ports. The second port may be omitted to represent a single port", + "type": "string", + "pattern": "^[0-9]{1,5}(-[0-9]{1,5})?$", + "minLength": 1, + "maxLength": 11 + }, + "MacAddr": { + "example": "ff:ff:ff:ff:ff:ff", + "title": "A MAC address", + "description": "A Media Access Control address, in EUI-48 format", + "type": "string", + "pattern": "^([0-9a-fA-F]{0,2}:){5}[0-9a-fA-F]{0,2}$", + "minLength": 5, + "maxLength": 17 + }, + "Measurement": { + "description": "A `Measurement` is a timestamped datum from a single metric", + "type": "object", + "properties": { + "datum": { + "$ref": "#/components/schemas/Datum" + }, + "timestamp": { + "type": "string", + "format": "date-time" + } + }, + "required": [ + "datum", + "timestamp" + ] + }, + "MetricsError": { + "description": "Errors related to the generation or collection of metrics.", + "oneOf": [ + { + "description": "An error related to generating metric data points", + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "datum_error" + ] + } + }, + "required": [ + "content", + "type" + ] + }, + { + "description": "An error running an `Oximeter` server", + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "oximeter_server" + ] + } + }, + "required": [ + "content", + "type" + ] + }, + { + "description": "An error related to creating or sampling a [`histogram::Histogram`] metric.", + "type": "object", + "properties": { + "content": { + "$ref": "#/components/schemas/HistogramError" + }, + "type": { + "type": "string", + "enum": [ + "histogram_error" + ] + } + }, + "required": [ + "content", + "type" + ] + }, + { + "description": "An error parsing a field or measurement from a string.", + "type": "object", + "properties": { + "content": { + "type": "object", + "properties": { + "src": { + "type": "string" + }, + "typ": { + "type": "string" + } + }, + "required": [ + "src", + "typ" + ] + }, + "type": { + "type": "string", + "enum": [ + "parse_error" + ] + } + }, + "required": [ + "content", + "type" + ] + }, + { + "description": "A field name is duplicated between the target and metric.", + "type": "object", + "properties": { + "content": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "type": { + "type": "string", + "enum": [ + "duplicate_field_name" + ] + } + }, + "required": [ + "content", + "type" + ] + } + ] + }, "Name": { "title": "A name unique within the parent collection", "description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.", @@ -2614,6 +5142,138 @@ "minItems": 2, "maxItems": 2 }, + "ProducerResultsItem": { + "oneOf": [ + { + "type": "object", + "properties": { + "info": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Sample" + } + }, + "status": { + "type": "string", + "enum": [ + "ok" + ] + } + }, + "required": [ + "info", + "status" + ] + }, + { + "type": "object", + "properties": { + "info": { + "$ref": "#/components/schemas/MetricsError" + }, + "status": { + "type": "string", + "enum": [ + "err" + ] + } + }, + "required": [ + "info", + "status" + ] + } + ] + }, + "QuantizationError": { + "description": "Errors occurring during quantizated bin generation.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "overflow" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "precision" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "invalid_base" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "invalid_steps" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "uneven_steps_for_base" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "powers_out_of_order" + ] + } + }, + "required": [ + "type" + ] + } + ] + }, "RackNetworkConfigV1": { "description": "Initial network configuration", "type": "object", @@ -2676,6 +5336,36 @@ "nexthop" ] }, + "Sample": { + "description": "A concrete type representing a single, timestamped measurement from a timeseries.", + "type": "object", + "properties": { + "measurement": { + "description": "The measured value of the metric at this sample", + "allOf": [ + { + "$ref": "#/components/schemas/Measurement" + } + ] + }, + "metric": { + "$ref": "#/components/schemas/FieldSet" + }, + "target": { + "$ref": "#/components/schemas/FieldSet" + }, + "timeseries_name": { + "description": "The name of the timeseries this sample belongs to", + "type": "string" + } + }, + "required": [ + "measurement", + "metric", + "target", + "timeseries_name" + ] + }, "SemverVersion": { "type": "string", "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" @@ -3184,6 +5874,70 @@ "last_port" ] }, + "StartSledAgentRequest": { + "description": "Configuration information for launching a Sled Agent.", + "type": "object", + "properties": { + "body": { + "$ref": "#/components/schemas/StartSledAgentRequestBody" + }, + "generation": { + "description": "The current generation number of data as stored in CRDB.\n\nThe initial generation is set during RSS time and then only mutated by Nexus. For now, we don't actually anticipate mutating this data, but we leave open the possiblity.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "body", + "generation", + "schema_version" + ] + }, + "StartSledAgentRequestBody": { + "description": "This is the actual app level data of `StartSledAgentRequest`\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "properties": { + "id": { + "description": "Uuid of the Sled Agent to be created.", + "type": "string", + "format": "uuid" + }, + "is_lrtq_learner": { + "description": "Is this node an LRTQ learner node?\n\nWe only put the node into learner mode if `use_trust_quorum` is also true.", + "type": "boolean" + }, + "rack_id": { + "description": "Uuid of the rack to which this sled agent belongs.", + "type": "string", + "format": "uuid" + }, + "subnet": { + "description": "Portion of the IP space to be managed by the Sled Agent.", + "allOf": [ + { + "$ref": "#/components/schemas/Ipv6Subnet" + } + ] + }, + "use_trust_quorum": { + "description": "Use trust quorum for key generation", + "type": "boolean" + } + }, + "required": [ + "id", + "is_lrtq_learner", + "rack_id", + "subnet", + "use_trust_quorum" + ] + }, "StorageLimit": { "description": "The limit on space allowed for zone bundles, as a percentage of the overall dataset's quota.", "type": "integer", @@ -3714,6 +6468,18 @@ "type": "string", "pattern": "^ox[ip]_[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/openapi/wicketd.json b/openapi/wicketd.json index a75c965ad89..60ad9a42dff 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -131,44 +131,31 @@ } } }, - "/clear-update-state/{type}/{slot}": { + "/clear-update-state": { "post": { "summary": "Resets update state for a sled.", "description": "Use this to clear update state after a failed update.", "operationId": "post_clear_update_state", - "parameters": [ - { - "in": "path", - "name": "slot", - "required": true, - "schema": { - "type": "integer", - "format": "uint32", - "minimum": 0 - } - }, - { - "in": "path", - "name": "type", - "required": true, - "schema": { - "$ref": "#/components/schemas/SpType" - } - } - ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ClearUpdateStateOptions" + "$ref": "#/components/schemas/ClearUpdateStateParams" } } }, "required": true }, "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClearUpdateStateResponse" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -690,18 +677,6 @@ } }, "components": { - "responses": { - "Error": { - "description": "Error", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Error" - } - } - } - } - }, "schemas": { "AbortUpdateOptions": { "type": "object", @@ -874,6 +849,41 @@ "format": "uint32", "minimum": 0 }, + "connect_retry": { + "nullable": true, + "description": "The interval in seconds between peer connection retry attempts.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "delay_open": { + "nullable": true, + "description": "How long to delay sending open messages to a peer. In seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "hold_time": { + "nullable": true, + "description": "How long to keep a session alive without a keepalive in seconds. Defaults to 6.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "idle_hold_time": { + "nullable": true, + "description": "How long to keep a peer in idle after a state machine reset in seconds.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "keepalive": { + "nullable": true, + "description": "The interval to send keepalive messages at.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, "port": { "description": "Switch port the peer is reachable on.", "type": "string" @@ -1014,6 +1024,56 @@ } } }, + "ClearUpdateStateParams": { + "type": "object", + "properties": { + "options": { + "description": "Options for clearing update state", + "allOf": [ + { + "$ref": "#/components/schemas/ClearUpdateStateOptions" + } + ] + }, + "targets": { + "description": "The SP identifiers to clear the update state for. Must be non-empty.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SpIdentifier" + }, + "uniqueItems": true + } + }, + "required": [ + "options", + "targets" + ] + }, + "ClearUpdateStateResponse": { + "type": "object", + "properties": { + "cleared": { + "description": "The SPs for which update data was cleared.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SpIdentifier" + }, + "uniqueItems": true + }, + "no_update_data": { + "description": "The SPs that had no update state to clear.", + "type": "array", + "items": { + "$ref": "#/components/schemas/SpIdentifier" + }, + "uniqueItems": true + } + }, + "required": [ + "cleared", + "no_update_data" + ] + }, "CurrentRssUserConfig": { "type": "object", "properties": { @@ -4634,6 +4694,18 @@ "power_reset" ] } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } } } } \ No newline at end of file diff --git a/oximeter/collector/Cargo.toml b/oximeter/collector/Cargo.toml index 470d9db312b..92c91ca101b 100644 --- a/oximeter/collector/Cargo.toml +++ b/oximeter/collector/Cargo.toml @@ -7,6 +7,8 @@ license = "MPL-2.0" [dependencies] anyhow.workspace = true +camino.workspace = true +chrono.workspace = true clap.workspace = true dropshot.workspace = true futures.workspace = true @@ -25,6 +27,7 @@ slog.workspace = true slog-async.workspace = true slog-dtrace.workspace = true slog-term.workspace = true +strum.workspace = true thiserror.workspace = true tokio.workspace = true toml.workspace = true @@ -33,6 +36,7 @@ omicron-workspace-hack.workspace = true [dev-dependencies] expectorate.workspace = true +hyper.workspace = true omicron-test-utils.workspace = true openapi-lint.workspace = true openapiv3.workspace = true diff --git a/oximeter/collector/src/agent.rs b/oximeter/collector/src/agent.rs new file mode 100644 index 00000000000..23ff32ed668 --- /dev/null +++ b/oximeter/collector/src/agent.rs @@ -0,0 +1,889 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The oximeter agent handles collection tasks for each producer. + +// Copyright 2023 Oxide Computer Company + +use crate::self_stats; +use crate::DbConfig; +use crate::Error; +use crate::ProducerEndpoint; +use anyhow::anyhow; +use internal_dns::resolver::Resolver; +use internal_dns::ServiceName; +use omicron_common::address::CLICKHOUSE_PORT; +use oximeter::types::ProducerResults; +use oximeter::types::ProducerResultsItem; +use oximeter_db::Client; +use oximeter_db::DbWrite; +use slog::debug; +use slog::error; +use slog::info; +use slog::o; +use slog::trace; +use slog::warn; +use slog::Logger; +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::ops::Bound; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use tokio::time::interval; +use uuid::Uuid; + +type CollectionToken = oneshot::Sender<()>; + +// Messages for controlling a collection task +#[derive(Debug)] +enum CollectionMessage { + // Explicit request that the task collect data from its producer + // + // Also sends a oneshot that is signalled once the task scrapes + // data from the Producer, and places it in the Clickhouse server. + Collect(CollectionToken), + // Request that the task update its interval and the socket address on which it collects data + // from its producer. + Update(ProducerEndpoint), + // Request that the task exit + Shutdown, + // Return the current statistics from a single task. + #[cfg(test)] + Statistics { + reply_tx: oneshot::Sender, + }, +} + +async fn perform_collection( + log: &Logger, + self_target: &mut self_stats::CollectionTaskStats, + client: &reqwest::Client, + producer: &ProducerEndpoint, + outbox: &mpsc::Sender<(Option, ProducerResults)>, + token: Option, +) { + debug!(log, "collecting from producer"); + let res = client + .get(format!( + "http://{}{}", + producer.address, + producer.collection_route() + )) + .send() + .await; + match res { + Ok(res) => { + if res.status().is_success() { + match res.json::().await { + Ok(results) => { + debug!( + log, + "collected results from producer"; + "n_results" => results.len() + ); + self_target.collections.datum.increment(); + outbox.send((token, results)).await.unwrap(); + } + Err(e) => { + warn!( + log, + "failed to collect results from producer"; + "error" => ?e, + ); + self_target + .failures_for_reason( + self_stats::FailureReason::Deserialization, + ) + .datum + .increment() + } + } + } else { + warn!( + log, + "failed to receive metric results from producer"; + "status_code" => res.status().as_u16(), + ); + self_target + .failures_for_reason(self_stats::FailureReason::Other( + res.status(), + )) + .datum + .increment() + } + } + Err(e) => { + error!( + log, + "failed to send collection request to producer"; + "error" => ?e + ); + self_target + .failures_for_reason(self_stats::FailureReason::Unreachable) + .datum + .increment() + } + } +} + +// Background task used to collect metrics from one producer on an interval. +// +// This function is started by the `OximeterAgent`, when a producer is registered. The task loops +// endlessly, and collects metrics from the assigned producer on a timeout. The assigned agent can +// also send a `CollectionMessage`, for example to update the collection interval. This is not +// currently used, but will likely be exposed via control plane interfaces in the future. +async fn collection_task( + log: Logger, + collector: self_stats::OximeterCollector, + mut producer: ProducerEndpoint, + mut inbox: mpsc::Receiver, + outbox: mpsc::Sender<(Option, ProducerResults)>, +) { + let client = reqwest::Client::new(); + let mut collection_timer = interval(producer.interval); + collection_timer.tick().await; // completes immediately + debug!( + log, + "starting oximeter collection task"; + "interval" => ?producer.interval, + ); + + // Set up the collection of self statistics about this collection task. + let mut stats = self_stats::CollectionTaskStats::new(collector, &producer); + let mut self_collection_timer = interval(self_stats::COLLECTION_INTERVAL); + self_collection_timer.tick().await; + + loop { + tokio::select! { + message = inbox.recv() => { + match message { + None => { + debug!(log, "collection task inbox closed, shutting down"); + return; + } + Some(CollectionMessage::Shutdown) => { + debug!(log, "collection task received shutdown request"); + return; + }, + Some(CollectionMessage::Collect(token)) => { + debug!(log, "collection task received explicit request to collect"); + perform_collection(&log, &mut stats, &client, &producer, &outbox, Some(token)).await; + }, + Some(CollectionMessage::Update(new_info)) => { + producer = new_info; + debug!( + log, + "collection task received request to update its producer information"; + "interval" => ?producer.interval, + "address" => producer.address, + ); + collection_timer = interval(producer.interval); + collection_timer.tick().await; // completes immediately + } + #[cfg(test)] + Some(CollectionMessage::Statistics { reply_tx }) => { + debug!( + log, + "received request for current task statistics" + ); + reply_tx.send(stats.clone()).expect("failed to send statistics"); + } + } + } + _ = self_collection_timer.tick() => { + debug!(log, "reporting oximeter self-collection statistics"); + outbox.send((None, stats.sample())).await.unwrap(); + } + _ = collection_timer.tick() => { + perform_collection(&log, &mut stats, &client, &producer, &outbox, None).await; + } + } + } +} + +// Struct representing a task for collecting metric data from a single producer +#[derive(Debug)] +struct CollectionTask { + // Channel used to send messages from the agent to the actual task. The task owns the other + // side. + pub inbox: mpsc::Sender, + // Handle to the actual tokio task running the collection loop. + #[allow(dead_code)] + pub task: JoinHandle<()>, +} + +// A task run by `oximeter` in standalone mode, which simply prints results as +// they're received. +async fn results_printer( + log: Logger, + mut rx: mpsc::Receiver<(Option, ProducerResults)>, +) { + loop { + match rx.recv().await { + Some((_, results)) => { + for res in results.into_iter() { + match res { + ProducerResultsItem::Ok(samples) => { + for sample in samples.into_iter() { + info!( + log, + ""; + "sample" => ?sample, + ); + } + } + ProducerResultsItem::Err(e) => { + error!( + log, + "received error from a producer"; + "err" => ?e, + ); + } + } + } + } + None => { + debug!(log, "result queue closed, exiting"); + return; + } + } + } +} + +// Aggregation point for all results, from all collection tasks. +async fn results_sink( + log: Logger, + client: Client, + batch_size: usize, + batch_interval: Duration, + mut rx: mpsc::Receiver<(Option, ProducerResults)>, +) { + let mut timer = interval(batch_interval); + timer.tick().await; // completes immediately + let mut batch = Vec::with_capacity(batch_size); + loop { + let mut collection_token = None; + let insert = tokio::select! { + _ = timer.tick() => { + if batch.is_empty() { + trace!(log, "batch interval expired, but no samples to insert"); + false + } else { + true + } + } + results = rx.recv() => { + match results { + Some((token, results)) => { + let flattened_results = { + let mut flattened = Vec::with_capacity(results.len()); + for inner_batch in results.into_iter() { + match inner_batch { + ProducerResultsItem::Ok(samples) => flattened.extend(samples.into_iter()), + ProducerResultsItem::Err(e) => { + debug!( + log, + "received error (not samples) from a producer: {}", + e.to_string() + ); + } + } + } + flattened + }; + batch.extend(flattened_results); + + collection_token = token; + if collection_token.is_some() { + true + } else { + batch.len() >= batch_size + } + } + None => { + warn!(log, "result queue closed, exiting"); + return; + } + } + } + }; + + if insert { + debug!(log, "inserting {} samples into database", batch.len()); + match client.insert_samples(&batch).await { + Ok(()) => trace!(log, "successfully inserted samples"), + Err(e) => { + warn!( + log, + "failed to insert some results into metric DB: {}", + e.to_string() + ); + } + } + // TODO-correctness The `insert_samples` call above may fail. The method itself needs + // better handling of partially-inserted results in that case, but we may need to retry + // or otherwise handle an error here as well. + // + // See https://github.com/oxidecomputer/omicron/issues/740 for a + // disucssion. + batch.clear(); + } + + if let Some(token) = collection_token { + let _ = token.send(()); + } + } +} + +/// The internal agent the oximeter server uses to collect metrics from producers. +#[derive(Debug)] +pub struct OximeterAgent { + /// The collector ID for this agent + pub id: Uuid, + log: Logger, + // Oximeter target used by this agent to produce metrics about itself. + collection_target: self_stats::OximeterCollector, + // Handle to the TX-side of a channel for collecting results from the collection tasks + result_sender: mpsc::Sender<(Option, ProducerResults)>, + // The actual tokio tasks running the collection on a timer. + collection_tasks: + Arc>>, +} + +impl OximeterAgent { + /// Construct a new agent with the given ID and logger. + pub async fn with_id( + id: Uuid, + address: SocketAddrV6, + db_config: DbConfig, + resolver: &Resolver, + log: &Logger, + ) -> Result { + let (result_sender, result_receiver) = mpsc::channel(8); + let log = log.new(o!( + "component" => "oximeter-agent", + "collector_id" => id.to_string(), + )); + let insertion_log = log.new(o!("component" => "results-sink")); + + // Construct the ClickHouse client first, propagate an error if we can't reach the + // database. + let db_address = if let Some(address) = db_config.address { + address + } else { + SocketAddr::new( + resolver.lookup_ip(ServiceName::Clickhouse).await?, + CLICKHOUSE_PORT, + ) + }; + + // Determine the version of the database. + // + // There are three cases + // + // - The database exists and is at the expected version. Continue in + // this case. + // + // - The database exists and is at a lower-than-expected version. We + // fail back to the caller here, which will retry indefinitely until the + // DB has been updated. + // + // - The DB doesn't exist at all. This reports a version number of 0. We + // need to create the DB here, at the latest version. This is used in + // fresh installations and tests. + let client = Client::new(db_address, &log); + match client.check_db_is_at_expected_version().await { + Ok(_) => {} + Err(oximeter_db::Error::DatabaseVersionMismatch { + found: 0, + .. + }) => { + debug!(log, "oximeter database does not exist, creating"); + let replicated = client.is_oximeter_cluster().await?; + client + .initialize_db_with_version( + replicated, + oximeter_db::OXIMETER_VERSION, + ) + .await?; + } + Err(e) => return Err(Error::from(e)), + } + + // Set up tracking of statistics about ourselves. + let collection_target = self_stats::OximeterCollector { + collector_id: id, + collector_ip: (*address.ip()).into(), + collector_port: address.port(), + }; + + // Spawn the task for aggregating and inserting all metrics + tokio::spawn(async move { + results_sink( + insertion_log, + client, + db_config.batch_size, + Duration::from_secs(db_config.batch_interval), + result_receiver, + ) + .await + }); + Ok(Self { + id, + log, + collection_target, + result_sender, + collection_tasks: Arc::new(Mutex::new(BTreeMap::new())), + }) + } + + /// Construct a new standalone `oximeter` collector. + /// + /// In this mode, `oximeter` can be used to test the collection of metrics + /// from producers, without requiring all the normal machinery of the + /// control plane. The collector is run as usual, but additionally starts a + /// API server to stand-in for Nexus. The registrations of the producers and + /// collectors occurs through the normal code path, but uses this mock Nexus + /// instead of the real thing. + pub async fn new_standalone( + id: Uuid, + address: SocketAddrV6, + db_config: Option, + log: &Logger, + ) -> Result { + let (result_sender, result_receiver) = mpsc::channel(8); + let log = log.new(o!( + "component" => "oximeter-standalone", + "collector_id" => id.to_string(), + )); + + // If we have configuration for ClickHouse, we'll spawn the results + // sink task as usual. If not, we'll spawn a dummy task that simply + // prints the results as they're received. + let insertion_log = log.new(o!("component" => "results-sink")); + if let Some(db_config) = db_config { + let Some(address) = db_config.address else { + return Err(Error::Standalone(anyhow!( + "Must provide explicit IP address in standalone mode" + ))); + }; + let client = Client::new(address, &log); + let replicated = client.is_oximeter_cluster().await?; + if !replicated { + client.init_single_node_db().await?; + } else { + client.init_replicated_db().await?; + } + + // Spawn the task for aggregating and inserting all metrics + tokio::spawn(async move { + results_sink( + insertion_log, + client, + db_config.batch_size, + Duration::from_secs(db_config.batch_interval), + result_receiver, + ) + .await + }); + } else { + tokio::spawn(results_printer(insertion_log, result_receiver)); + } + + // Set up tracking of statistics about ourselves. + let collection_target = self_stats::OximeterCollector { + collector_id: id, + collector_ip: (*address.ip()).into(), + collector_port: address.port(), + }; + Ok(Self { + id, + log, + collection_target, + result_sender, + collection_tasks: Arc::new(Mutex::new(BTreeMap::new())), + }) + } + + /// Register a new producer with this oximeter instance. + pub async fn register_producer( + &self, + info: ProducerEndpoint, + ) -> Result<(), Error> { + let id = info.id; + match self.collection_tasks.lock().await.entry(id) { + Entry::Vacant(value) => { + debug!( + self.log, + "registered new metric producer"; + "producer_id" => id.to_string(), + "address" => info.address, + ); + + // Build channel to control the task and receive results. + let (tx, rx) = mpsc::channel(4); + let q = self.result_sender.clone(); + let log = self.log.new(o!("component" => "collection-task", "producer_id" => id.to_string())); + let info_clone = info.clone(); + let target = self.collection_target; + let task = tokio::spawn(async move { + collection_task(log, target, info_clone, rx, q).await; + }); + value.insert((info, CollectionTask { inbox: tx, task })); + } + Entry::Occupied(mut value) => { + debug!( + self.log, + "received request to register existing metric \ + producer, updating collection information"; + "producer_id" => id.to_string(), + "interval" => ?info.interval, + "address" => info.address, + ); + value.get_mut().0 = info.clone(); + value + .get() + .1 + .inbox + .send(CollectionMessage::Update(info)) + .await + .unwrap(); + } + } + Ok(()) + } + + /// Forces a collection from all producers. + /// + /// Returns once all those values have been inserted into Clickhouse, + /// or an error occurs trying to perform the collection. + pub async fn force_collection(&self) { + let mut collection_oneshots = vec![]; + let collection_tasks = self.collection_tasks.lock().await; + for (_id, (_endpoint, task)) in collection_tasks.iter() { + let (tx, rx) = oneshot::channel(); + // Scrape from each producer, into oximeter... + task.inbox.send(CollectionMessage::Collect(tx)).await.unwrap(); + // ... and keep track of the token that indicates once the metric + // has made it into Clickhouse. + collection_oneshots.push(rx); + } + drop(collection_tasks); + + // Only return once all producers finish processing the token we + // provided. + // + // NOTE: This can either mean that the collection completed + // successfully, or an error occurred in the collection pathway. + futures::future::join_all(collection_oneshots).await; + } + + /// List existing producers. + pub async fn list_producers( + &self, + start_id: Option, + limit: usize, + ) -> Vec { + let start = if let Some(id) = start_id { + Bound::Excluded(id) + } else { + Bound::Unbounded + }; + self.collection_tasks + .lock() + .await + .range((start, Bound::Unbounded)) + .take(limit) + .map(|(_id, (info, _t))| info.clone()) + .collect() + } + + /// Delete a producer by ID, stopping its collection task. + pub async fn delete_producer(&self, id: Uuid) -> Result<(), Error> { + let (_info, task) = self + .collection_tasks + .lock() + .await + .remove(&id) + .ok_or_else(|| Error::NoSuchProducer(id))?; + debug!( + self.log, + "removed collection task from set"; + "producer_id" => %id, + ); + match task.inbox.send(CollectionMessage::Shutdown).await { + Ok(_) => debug!( + self.log, + "shut down collection task"; + "producer_id" => %id, + ), + Err(e) => error!( + self.log, + "failed to shut down collection task"; + "producer_id" => %id, + "error" => ?e, + ), + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::CollectionMessage; + use super::OximeterAgent; + use super::ProducerEndpoint; + use crate::self_stats::FailureReason; + use hyper::service::make_service_fn; + use hyper::service::service_fn; + use hyper::Body; + use hyper::Request; + use hyper::Response; + use hyper::Server; + use hyper::StatusCode; + use omicron_test_utils::dev::test_setup_log; + use std::convert::Infallible; + use std::net::Ipv6Addr; + use std::net::SocketAddr; + use std::net::SocketAddrV6; + use std::time::Duration; + use tokio::sync::oneshot; + use tokio::time::Instant; + use uuid::Uuid; + + // Test that we count successful collections from a target correctly. + #[tokio::test] + async fn test_self_stat_collection_count() { + let logctx = test_setup_log("test_self_stat_collection_count"); + let log = &logctx.log; + + // Spawn an oximeter collector ... + let collector = OximeterAgent::new_standalone( + Uuid::new_v4(), + SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0), + None, + log, + ) + .await + .unwrap(); + + // And a dummy server that will always report empty statistics. There + // will be no actual data here, but the sample counter will increment. + let addr = + SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0)); + async fn handler( + _: Request, + ) -> Result, Infallible> { + Ok(Response::new(Body::from("[]"))) + } + let make_svc = make_service_fn(|_conn| async { + Ok::<_, Infallible>(service_fn(handler)) + }); + let server = Server::bind(&addr).serve(make_svc); + let address = server.local_addr(); + let _task = tokio::task::spawn(server); + + // Register the dummy producer. + let interval = Duration::from_secs(1); + let endpoint = ProducerEndpoint { + id: Uuid::new_v4(), + address, + base_route: String::from("/"), + interval, + }; + collector + .register_producer(endpoint) + .await + .expect("failed to register dummy producer"); + + // Step time until there has been exactly `N_COLLECTIONS` collections. + tokio::time::pause(); + let now = Instant::now(); + const N_COLLECTIONS: usize = 5; + let wait_for = interval * N_COLLECTIONS as u32 + interval / 2; + while now.elapsed() < wait_for { + tokio::time::advance(interval / 10).await; + } + + // Request the statistics from the task itself. + let (reply_tx, rx) = oneshot::channel(); + collector + .collection_tasks + .lock() + .await + .values() + .next() + .unwrap() + .1 + .inbox + .send(CollectionMessage::Statistics { reply_tx }) + .await + .expect("failed to request statistics from task"); + let stats = rx.await.expect("failed to receive statistics from task"); + assert_eq!(stats.collections.datum.value(), N_COLLECTIONS as u64); + assert!(stats.failed_collections.is_empty()); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_self_stat_unreachable_counter() { + let logctx = test_setup_log("test_self_stat_unreachable_counter"); + let log = &logctx.log; + + // Spawn an oximeter collector ... + let collector = OximeterAgent::new_standalone( + Uuid::new_v4(), + SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0), + None, + log, + ) + .await + .unwrap(); + + // Register a bogus producer, which is equivalent to a producer that is + // unreachable. + let interval = Duration::from_secs(1); + let endpoint = ProducerEndpoint { + id: Uuid::new_v4(), + address: SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + )), + base_route: String::from("/"), + interval, + }; + collector + .register_producer(endpoint) + .await + .expect("failed to register bogus producer"); + + // Step time until there has been exactly `N_COLLECTIONS` collections. + tokio::time::pause(); + let now = Instant::now(); + const N_COLLECTIONS: usize = 5; + let wait_for = interval * N_COLLECTIONS as u32 + interval / 2; + while now.elapsed() < wait_for { + tokio::time::advance(interval / 10).await; + } + + // Request the statistics from the task itself. + let (reply_tx, rx) = oneshot::channel(); + collector + .collection_tasks + .lock() + .await + .values() + .next() + .unwrap() + .1 + .inbox + .send(CollectionMessage::Statistics { reply_tx }) + .await + .expect("failed to request statistics from task"); + let stats = rx.await.expect("failed to receive statistics from task"); + assert_eq!(stats.collections.datum.value(), 0); + assert_eq!( + stats + .failed_collections + .get(&FailureReason::Unreachable) + .unwrap() + .datum + .value(), + N_COLLECTIONS as u64 + ); + assert_eq!(stats.failed_collections.len(), 1); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_self_stat_error_counter() { + let logctx = test_setup_log("test_self_stat_error_counter"); + let log = &logctx.log; + + // Spawn an oximeter collector ... + let collector = OximeterAgent::new_standalone( + Uuid::new_v4(), + SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0), + None, + log, + ) + .await + .unwrap(); + + // And a dummy server that will always fail with a 500. + let addr = + SocketAddr::V6(SocketAddrV6::new(Ipv6Addr::LOCALHOST, 0, 0, 0)); + async fn handler( + _: Request, + ) -> Result, Infallible> { + let mut res = Response::new(Body::from("im ded")); + *res.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; + Ok(res) + } + let make_svc = make_service_fn(|_conn| async { + Ok::<_, Infallible>(service_fn(handler)) + }); + let server = Server::bind(&addr).serve(make_svc); + let address = server.local_addr(); + let _task = tokio::task::spawn(server); + + // Register the rather flaky producer. + let interval = Duration::from_secs(1); + let endpoint = ProducerEndpoint { + id: Uuid::new_v4(), + address, + base_route: String::from("/"), + interval, + }; + collector + .register_producer(endpoint) + .await + .expect("failed to register flaky producer"); + + // Step time until there has been exactly `N_COLLECTIONS` collections. + tokio::time::pause(); + let now = Instant::now(); + const N_COLLECTIONS: usize = 5; + let wait_for = interval * N_COLLECTIONS as u32 + interval / 2; + while now.elapsed() < wait_for { + tokio::time::advance(interval / 10).await; + } + + // Request the statistics from the task itself. + let (reply_tx, rx) = oneshot::channel(); + collector + .collection_tasks + .lock() + .await + .values() + .next() + .unwrap() + .1 + .inbox + .send(CollectionMessage::Statistics { reply_tx }) + .await + .expect("failed to request statistics from task"); + let stats = rx.await.expect("failed to receive statistics from task"); + assert_eq!(stats.collections.datum.value(), 0); + assert_eq!( + stats + .failed_collections + .get(&FailureReason::Other(StatusCode::INTERNAL_SERVER_ERROR)) + .unwrap() + .datum + .value(), + N_COLLECTIONS as u64 + ); + assert_eq!(stats.failed_collections.len(), 1); + logctx.cleanup_successful(); + } +} diff --git a/oximeter/collector/src/bin/clickhouse-schema-updater.rs b/oximeter/collector/src/bin/clickhouse-schema-updater.rs new file mode 100644 index 00000000000..20780c37e07 --- /dev/null +++ b/oximeter/collector/src/bin/clickhouse-schema-updater.rs @@ -0,0 +1,126 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! CLI tool to apply offline updates to ClickHouse schema. + +// Copyright 2023 Oxide Computer Company + +use anyhow::anyhow; +use anyhow::Context; +use camino::Utf8PathBuf; +use clap::Parser; +use clap::Subcommand; +use omicron_common::address::CLICKHOUSE_PORT; +use oximeter_db::model::OXIMETER_VERSION; +use oximeter_db::Client; +use slog::Drain; +use slog::Level; +use slog::LevelFilter; +use slog::Logger; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use std::net::SocketAddrV6; + +const DEFAULT_HOST: SocketAddr = SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + CLICKHOUSE_PORT, + 0, + 0, +)); + +fn parse_log_level(s: &str) -> anyhow::Result { + s.parse().map_err(|_| anyhow!("Invalid log level")) +} + +/// Tool to apply offline updates to ClickHouse schema. +#[derive(Clone, Debug, Parser)] +struct Args { + /// IP address and port at which to access ClickHouse. + #[arg(long, default_value_t = DEFAULT_HOST, env = "CLICKHOUSE_HOST")] + host: SocketAddr, + + /// Directory from which to read schema files for each version. + #[arg( + short = 's', + long, + default_value_t = Utf8PathBuf::from("/opt/oxide/oximeter/schema") + )] + schema_directory: Utf8PathBuf, + + /// The log level while running the command. + #[arg( + short, + long, + value_parser = parse_log_level, + default_value_t = Level::Warning + )] + log_level: Level, + + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Clone, Debug, Subcommand)] +enum Cmd { + /// List all schema in the directory available for an upgrade + #[clap(visible_alias = "ls")] + List, + /// Apply an upgrade to a specific version + #[clap(visible_aliases = ["up", "apply"])] + Upgrade { + /// The version to which to upgrade. + #[arg(default_value_t = OXIMETER_VERSION)] + version: u64, + }, +} + +fn build_logger(level: Level) -> Logger { + let decorator = slog_term::TermDecorator::new().build(); + let drain = slog_term::FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + let drain = LevelFilter::new(drain, level).fuse(); + Logger::root(drain, slog::o!("unit" => "clickhouse_schema_updater")) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let args = Args::parse(); + let log = build_logger(args.log_level); + let client = Client::new(args.host, &log); + let is_replicated = client.is_oximeter_cluster().await?; + match args.cmd { + Cmd::List => { + let latest = client + .read_latest_version() + .await + .context("Failed to read latest version")?; + let available_versions = Client::read_available_schema_versions( + &log, + is_replicated, + &args.schema_directory, + ) + .await?; + println!("Latest version: {latest}"); + println!("Available versions:"); + for ver in available_versions { + print!(" {ver}"); + if ver == latest { + print!(" (reported by database)"); + } + if ver == OXIMETER_VERSION { + print!(" (expected by oximeter)"); + } + println!(); + } + } + Cmd::Upgrade { version } => { + client + .ensure_schema(is_replicated, version, args.schema_directory) + .await + .context("Failed to upgrade schema")?; + println!("Upgrade to oximeter database version {version} complete"); + } + } + Ok(()) +} diff --git a/oximeter/collector/src/bin/oximeter.rs b/oximeter/collector/src/bin/oximeter.rs index 8c4bf0e27cf..d97ae5e72e2 100644 --- a/oximeter/collector/src/bin/oximeter.rs +++ b/oximeter/collector/src/bin/oximeter.rs @@ -6,6 +6,7 @@ // Copyright 2023 Oxide Computer Company +use anyhow::{anyhow, Context}; use clap::Parser; use omicron_common::cmd::fatal; use omicron_common::cmd::CmdError; @@ -132,7 +133,9 @@ async fn main() { async fn do_run() -> Result<(), CmdError> { let args = Args::parse(); match args { - Args::Openapi => run_openapi().map_err(CmdError::Failure), + Args::Openapi => { + run_openapi().map_err(|err| CmdError::Failure(anyhow!(err))) + } Args::Run { config_file, id, address } => { let config = Config::from_file(config_file).unwrap(); let args = OximeterArguments { id, address }; @@ -141,13 +144,15 @@ async fn do_run() -> Result<(), CmdError> { .unwrap() .serve_forever() .await - .map_err(|e| CmdError::Failure(e.to_string())) + .context("Failed to create oximeter") + .map_err(CmdError::Failure) } Args::Standalone { id, address, nexus, clickhouse, log_level } => { // Start the standalone Nexus server, for registration of both the // collector and producers. let nexus_server = StandaloneNexus::new(nexus.into(), log_level) - .map_err(|e| CmdError::Failure(e.to_string()))?; + .context("Failed to create nexus") + .map_err(CmdError::Failure)?; let args = OximeterArguments { id, address }; Oximeter::new_standalone( nexus_server.log(), @@ -159,10 +164,10 @@ async fn do_run() -> Result<(), CmdError> { .unwrap() .serve_forever() .await - .map_err(|e| CmdError::Failure(e.to_string())) - } - Args::StandaloneOpenapi => { - run_standalone_openapi().map_err(CmdError::Failure) + .context("Failed to create standalone oximeter") + .map_err(CmdError::Failure) } + Args::StandaloneOpenapi => run_standalone_openapi() + .map_err(|err| CmdError::Failure(anyhow!(err))), } } diff --git a/oximeter/collector/src/http_entrypoints.rs b/oximeter/collector/src/http_entrypoints.rs new file mode 100644 index 00000000000..493083a40de --- /dev/null +++ b/oximeter/collector/src/http_entrypoints.rs @@ -0,0 +1,133 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Oximeter collector server HTTP API + +// Copyright 2023 Oxide Computer Company + +use crate::OximeterAgent; +use dropshot::endpoint; +use dropshot::ApiDescription; +use dropshot::EmptyScanParams; +use dropshot::HttpError; +use dropshot::HttpResponseDeleted; +use dropshot::HttpResponseOk; +use dropshot::HttpResponseUpdatedNoContent; +use dropshot::PaginationParams; +use dropshot::Query; +use dropshot::RequestContext; +use dropshot::ResultsPage; +use dropshot::TypedBody; +use dropshot::WhichPage; +use omicron_common::api::internal::nexus::ProducerEndpoint; +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; +use std::sync::Arc; +use uuid::Uuid; + +// Build the HTTP API internal to the control plane +pub fn oximeter_api() -> ApiDescription> { + let mut api = ApiDescription::new(); + api.register(producers_post) + .expect("Could not register producers_post API handler"); + api.register(producers_list) + .expect("Could not register producers_list API handler"); + api.register(producer_delete) + .expect("Could not register producers_delete API handler"); + api.register(collector_info) + .expect("Could not register collector_info API handler"); + api +} + +// Handle a request from Nexus to register a new producer with this collector. +#[endpoint { + method = POST, + path = "/producers", +}] +async fn producers_post( + request_context: RequestContext>, + body: TypedBody, +) -> Result { + let agent = request_context.context(); + let producer_info = body.into_inner(); + agent + .register_producer(producer_info) + .await + .map_err(HttpError::from) + .map(|_| HttpResponseUpdatedNoContent()) +} + +// Parameters for paginating the list of producers. +#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, Serialize)] +struct ProducerPage { + id: Uuid, +} + +// List all producers +#[endpoint { + method = GET, + path = "/producers", +}] +async fn producers_list( + request_context: RequestContext>, + query: Query>, +) -> Result>, HttpError> { + let agent = request_context.context(); + let pagination = query.into_inner(); + let limit = request_context.page_limit(&pagination)?.get() as usize; + let start = match &pagination.page { + WhichPage::First(..) => None, + WhichPage::Next(ProducerPage { id }) => Some(*id), + }; + let producers = agent.list_producers(start, limit).await; + ResultsPage::new( + producers, + &EmptyScanParams {}, + |info: &ProducerEndpoint, _| ProducerPage { id: info.id }, + ) + .map(HttpResponseOk) +} + +#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, Serialize)] +struct ProducerIdPathParams { + producer_id: Uuid, +} + +// Delete a producer by ID. +#[endpoint { + method = DELETE, + path = "/producers/{producer_id}", +}] +async fn producer_delete( + request_context: RequestContext>, + path: dropshot::Path, +) -> Result { + let agent = request_context.context(); + let producer_id = path.into_inner().producer_id; + agent + .delete_producer(producer_id) + .await + .map_err(HttpError::from) + .map(|_| HttpResponseDeleted()) +} + +#[derive(Clone, Copy, Debug, Deserialize, JsonSchema, Serialize)] +pub struct CollectorInfo { + /// The collector's UUID. + pub id: Uuid, +} + +// Return identifying information about this collector +#[endpoint { + method = GET, + path = "/info", +}] +async fn collector_info( + request_context: RequestContext>, +) -> Result, HttpError> { + let agent = request_context.context(); + let info = CollectorInfo { id: agent.id }; + Ok(HttpResponseOk(info)) +} diff --git a/oximeter/collector/src/lib.rs b/oximeter/collector/src/lib.rs index b7a14cec453..f3c793d5c23 100644 --- a/oximeter/collector/src/lib.rs +++ b/oximeter/collector/src/lib.rs @@ -6,65 +6,41 @@ // Copyright 2023 Oxide Computer Company -use anyhow::anyhow; -use anyhow::Context; -use dropshot::endpoint; -use dropshot::ApiDescription; use dropshot::ConfigDropshot; use dropshot::ConfigLogging; -use dropshot::EmptyScanParams; use dropshot::HttpError; -use dropshot::HttpResponseDeleted; -use dropshot::HttpResponseOk; -use dropshot::HttpResponseUpdatedNoContent; use dropshot::HttpServer; use dropshot::HttpServerStarter; -use dropshot::PaginationParams; -use dropshot::Query; -use dropshot::RequestContext; -use dropshot::ResultsPage; -use dropshot::TypedBody; -use dropshot::WhichPage; use internal_dns::resolver::ResolveError; use internal_dns::resolver::Resolver; use internal_dns::ServiceName; -use omicron_common::address::CLICKHOUSE_PORT; use omicron_common::address::NEXUS_INTERNAL_PORT; use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_common::backoff; use omicron_common::FileKv; -use oximeter::types::ProducerResults; -use oximeter::types::ProducerResultsItem; -use oximeter_db::model::OXIMETER_VERSION; -use oximeter_db::Client; -use oximeter_db::DbWrite; use serde::Deserialize; use serde::Serialize; use slog::debug; use slog::error; use slog::info; use slog::o; -use slog::trace; use slog::warn; use slog::Drain; use slog::Logger; -use std::collections::btree_map::Entry; -use std::collections::BTreeMap; use std::net::SocketAddr; use std::net::SocketAddrV6; -use std::ops::Bound; use std::path::Path; use std::sync::Arc; -use std::time::Duration; use thiserror::Error; -use tokio::sync::mpsc; -use tokio::sync::oneshot; -use tokio::sync::Mutex; -use tokio::task::JoinHandle; -use tokio::time::interval; use uuid::Uuid; +mod agent; +mod http_entrypoints; +mod self_stats; mod standalone; + +pub use agent::OximeterAgent; +pub use http_entrypoints::oximeter_api; pub use standalone::standalone_nexus_api; pub use standalone::Server as StandaloneNexus; @@ -102,289 +78,6 @@ impl From for HttpError { } } -/// A simple representation of a producer, used mostly for standalone mode. -/// -/// These are usually specified as a structured string, formatted like: -/// `"@
"`. -#[derive(Copy, Clone, Debug)] -pub struct ProducerInfo { - /// The ID of the producer. - pub id: Uuid, - /// The address on which the producer listens. - pub address: SocketAddr, -} - -impl std::str::FromStr for ProducerInfo { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let (id, addr) = s - .split_once('@') - .context("Producer info should written as @
")?; - let id = id.parse().context("Invalid UUID")?; - let address = addr.parse().context("Invalid address")?; - Ok(Self { id, address }) - } -} - -type CollectionToken = oneshot::Sender<()>; - -// Messages for controlling a collection task -#[derive(Debug)] -enum CollectionMessage { - // Explicit request that the task collect data from its producer - // - // Also sends a oneshot that is signalled once the task scrapes - // data from the Producer, and places it in the Clickhouse server. - Collect(CollectionToken), - // Request that the task update its interval and the socket address on which it collects data - // from its producer. - Update(ProducerEndpoint), - // Request that the task exit - Shutdown, -} - -async fn perform_collection( - log: &Logger, - client: &reqwest::Client, - producer: &ProducerEndpoint, - outbox: &mpsc::Sender<(Option, ProducerResults)>, - token: Option, -) { - debug!(log, "collecting from producer"); - let res = client - .get(format!( - "http://{}{}", - producer.address, - producer.collection_route() - )) - .send() - .await; - match res { - Ok(res) => { - if res.status().is_success() { - match res.json::().await { - Ok(results) => { - debug!( - log, - "collected {} total results", - results.len(); - ); - outbox.send((token, results)).await.unwrap(); - } - Err(e) => { - warn!( - log, - "failed to collect results from producer: {}", - e.to_string(); - ); - } - } - } else { - warn!( - log, - "failed to receive metric results from producer"; - "status_code" => res.status().as_u16(), - ); - } - } - Err(e) => { - warn!( - log, - "failed to send collection request to producer: {}", - e.to_string(); - ); - } - } -} - -// Background task used to collect metrics from one producer on an interval. -// -// This function is started by the `OximeterAgent`, when a producer is registered. The task loops -// endlessly, and collects metrics from the assigned producer on a timeout. The assigned agent can -// also send a `CollectionMessage`, for example to update the collection interval. This is not -// currently used, but will likely be exposed via control plane interfaces in the future. -async fn collection_task( - log: Logger, - mut producer: ProducerEndpoint, - mut inbox: mpsc::Receiver, - outbox: mpsc::Sender<(Option, ProducerResults)>, -) { - let client = reqwest::Client::new(); - let mut collection_timer = interval(producer.interval); - collection_timer.tick().await; // completes immediately - debug!( - log, - "starting oximeter collection task"; - "interval" => ?producer.interval, - ); - - loop { - tokio::select! { - message = inbox.recv() => { - match message { - None => { - debug!(log, "collection task inbox closed, shutting down"); - return; - } - Some(CollectionMessage::Shutdown) => { - debug!(log, "collection task received shutdown request"); - return; - }, - Some(CollectionMessage::Collect(token)) => { - debug!(log, "collection task received explicit request to collect"); - perform_collection(&log, &client, &producer, &outbox, Some(token)).await; - }, - Some(CollectionMessage::Update(new_info)) => { - producer = new_info; - debug!( - log, - "collection task received request to update its producer information"; - "interval" => ?producer.interval, - "address" => producer.address, - ); - collection_timer = interval(producer.interval); - collection_timer.tick().await; // completes immediately - } - } - } - _ = collection_timer.tick() => { - perform_collection(&log, &client, &producer, &outbox, None).await; - } - } - } -} - -// Struct representing a task for collecting metric data from a single producer -#[derive(Debug)] -struct CollectionTask { - // Channel used to send messages from the agent to the actual task. The task owns the other - // side. - pub inbox: mpsc::Sender, - // Handle to the actual tokio task running the collection loop. - #[allow(dead_code)] - pub task: JoinHandle<()>, -} - -// A task run by `oximeter` in standalone mode, which simply prints results as -// they're received. -async fn results_printer( - log: Logger, - mut rx: mpsc::Receiver<(Option, ProducerResults)>, -) { - loop { - match rx.recv().await { - Some((_, results)) => { - for res in results.into_iter() { - match res { - ProducerResultsItem::Ok(samples) => { - for sample in samples.into_iter() { - info!( - log, - ""; - "sample" => ?sample, - ); - } - } - ProducerResultsItem::Err(e) => { - error!( - log, - "received error from a producer"; - "err" => ?e, - ); - } - } - } - } - None => { - debug!(log, "result queue closed, exiting"); - return; - } - } - } -} - -// Aggregation point for all results, from all collection tasks. -async fn results_sink( - log: Logger, - client: Client, - batch_size: usize, - batch_interval: Duration, - mut rx: mpsc::Receiver<(Option, ProducerResults)>, -) { - let mut timer = interval(batch_interval); - timer.tick().await; // completes immediately - let mut batch = Vec::with_capacity(batch_size); - loop { - let mut collection_token = None; - let insert = tokio::select! { - _ = timer.tick() => { - if batch.is_empty() { - trace!(log, "batch interval expired, but no samples to insert"); - false - } else { - true - } - } - results = rx.recv() => { - match results { - Some((token, results)) => { - let flattened_results = { - let mut flattened = Vec::with_capacity(results.len()); - for inner_batch in results.into_iter() { - match inner_batch { - ProducerResultsItem::Ok(samples) => flattened.extend(samples.into_iter()), - ProducerResultsItem::Err(e) => { - debug!( - log, - "received error (not samples) from a producer: {}", - e.to_string() - ); - } - } - } - flattened - }; - batch.extend(flattened_results); - - collection_token = token; - if collection_token.is_some() { - true - } else { - batch.len() >= batch_size - } - } - None => { - warn!(log, "result queue closed, exiting"); - return; - } - } - } - }; - - if insert { - debug!(log, "inserting {} samples into database", batch.len()); - match client.insert_samples(&batch).await { - Ok(()) => trace!(log, "successfully inserted samples"), - Err(e) => { - warn!( - log, - "failed to insert some results into metric DB: {}", - e.to_string() - ); - } - } - // TODO-correctness The `insert_samples` call above may fail. The method itself needs - // better handling of partially-inserted results in that case, but we may need to retry - // or otherwise handle an error here as well. - batch.clear(); - } - - if let Some(token) = collection_token { - let _ = token.send(()); - } - } -} - /// Configuration for interacting with the metric database. #[derive(Debug, Clone, Copy, Deserialize, Serialize)] pub struct DbConfig { @@ -403,7 +96,12 @@ pub struct DbConfig { } impl DbConfig { + /// Default number of samples to wait for before inserting a batch into + /// ClickHouse. pub const DEFAULT_BATCH_SIZE: usize = 1000; + + /// Default number of seconds to wait before inserting a batch into + /// ClickHouse. pub const DEFAULT_BATCH_INTERVAL: u64 = 5; // Construct config with an address, using the defaults for other fields @@ -416,244 +114,6 @@ impl DbConfig { } } -/// The internal agent the oximeter server uses to collect metrics from producers. -#[derive(Debug)] -pub struct OximeterAgent { - /// The collector ID for this agent - pub id: Uuid, - log: Logger, - // Handle to the TX-side of a channel for collecting results from the collection tasks - result_sender: mpsc::Sender<(Option, ProducerResults)>, - // The actual tokio tasks running the collection on a timer. - collection_tasks: - Arc>>, -} - -impl OximeterAgent { - /// Construct a new agent with the given ID and logger. - pub async fn with_id( - id: Uuid, - db_config: DbConfig, - resolver: &Resolver, - log: &Logger, - ) -> Result { - let (result_sender, result_receiver) = mpsc::channel(8); - let log = log.new(o!( - "component" => "oximeter-agent", - "collector_id" => id.to_string(), - )); - let insertion_log = log.new(o!("component" => "results-sink")); - - // Construct the ClickHouse client first, propagate an error if we can't reach the - // database. - let db_address = if let Some(address) = db_config.address { - address - } else { - SocketAddr::new( - resolver.lookup_ip(ServiceName::Clickhouse).await?, - CLICKHOUSE_PORT, - ) - }; - let client = Client::new(db_address, &log); - let replicated = client.is_oximeter_cluster().await?; - client.initialize_db_with_version(replicated, OXIMETER_VERSION).await?; - - // Spawn the task for aggregating and inserting all metrics - tokio::spawn(async move { - results_sink( - insertion_log, - client, - db_config.batch_size, - Duration::from_secs(db_config.batch_interval), - result_receiver, - ) - .await - }); - Ok(Self { - id, - log, - result_sender, - collection_tasks: Arc::new(Mutex::new(BTreeMap::new())), - }) - } - - /// Construct a new standalone `oximeter` collector. - pub async fn new_standalone( - id: Uuid, - db_config: Option, - log: &Logger, - ) -> Result { - let (result_sender, result_receiver) = mpsc::channel(8); - let log = log.new(o!( - "component" => "oximeter-standalone", - "collector_id" => id.to_string(), - )); - - // If we have configuration for ClickHouse, we'll spawn the results - // sink task as usual. If not, we'll spawn a dummy task that simply - // prints the results as they're received. - let insertion_log = log.new(o!("component" => "results-sink")); - if let Some(db_config) = db_config { - let Some(address) = db_config.address else { - return Err(Error::Standalone(anyhow!( - "Must provide explicit IP address in standalone mode" - ))); - }; - let client = Client::new(address, &log); - let replicated = client.is_oximeter_cluster().await?; - if !replicated { - client.init_single_node_db().await?; - } else { - client.init_replicated_db().await?; - } - - // Spawn the task for aggregating and inserting all metrics - tokio::spawn(async move { - results_sink( - insertion_log, - client, - db_config.batch_size, - Duration::from_secs(db_config.batch_interval), - result_receiver, - ) - .await - }); - } else { - tokio::spawn(results_printer(insertion_log, result_receiver)); - } - - // Construct the ClickHouse client first, propagate an error if we can't reach the - // database. - Ok(Self { - id, - log, - result_sender, - collection_tasks: Arc::new(Mutex::new(BTreeMap::new())), - }) - } - - /// Register a new producer with this oximeter instance. - pub async fn register_producer( - &self, - info: ProducerEndpoint, - ) -> Result<(), Error> { - let id = info.id; - match self.collection_tasks.lock().await.entry(id) { - Entry::Vacant(value) => { - debug!( - self.log, - "registered new metric producer"; - "producer_id" => id.to_string(), - "address" => info.address, - ); - - // Build channel to control the task and receive results. - let (tx, rx) = mpsc::channel(4); - let q = self.result_sender.clone(); - let log = self.log.new(o!("component" => "collection-task", "producer_id" => id.to_string())); - let info_clone = info.clone(); - let task = tokio::spawn(async move { - collection_task(log, info_clone, rx, q).await; - }); - value.insert((info, CollectionTask { inbox: tx, task })); - } - Entry::Occupied(mut value) => { - debug!( - self.log, - "received request to register existing metric \ - producer, updating collection information"; - "producer_id" => id.to_string(), - "interval" => ?info.interval, - "address" => info.address, - ); - value.get_mut().0 = info.clone(); - value - .get() - .1 - .inbox - .send(CollectionMessage::Update(info)) - .await - .unwrap(); - } - } - Ok(()) - } - - /// Forces a collection from all producers. - /// - /// Returns once all those values have been inserted into Clickhouse, - /// or an error occurs trying to perform the collection. - pub async fn force_collection(&self) { - let mut collection_oneshots = vec![]; - let collection_tasks = self.collection_tasks.lock().await; - for (_id, (_endpoint, task)) in collection_tasks.iter() { - let (tx, rx) = oneshot::channel(); - // Scrape from each producer, into oximeter... - task.inbox.send(CollectionMessage::Collect(tx)).await.unwrap(); - // ... and keep track of the token that indicates once the metric - // has made it into Clickhouse. - collection_oneshots.push(rx); - } - drop(collection_tasks); - - // Only return once all producers finish processing the token we - // provided. - // - // NOTE: This can either mean that the collection completed - // successfully, or an error occurred in the collection pathway. - futures::future::join_all(collection_oneshots).await; - } - - /// List existing producers. - pub async fn list_producers( - &self, - start_id: Option, - limit: usize, - ) -> Vec { - let start = if let Some(id) = start_id { - Bound::Excluded(id) - } else { - Bound::Unbounded - }; - self.collection_tasks - .lock() - .await - .range((start, Bound::Unbounded)) - .take(limit) - .map(|(_id, (info, _t))| info.clone()) - .collect() - } - - /// Delete a producer by ID, stopping its collection task. - pub async fn delete_producer(&self, id: Uuid) -> Result<(), Error> { - let (_info, task) = self - .collection_tasks - .lock() - .await - .remove(&id) - .ok_or_else(|| Error::NoSuchProducer(id))?; - debug!( - self.log, - "removed collection task from set"; - "producer_id" => %id, - ); - match task.inbox.send(CollectionMessage::Shutdown).await { - Ok(_) => debug!( - self.log, - "shut down collection task"; - "producer_id" => %id, - ), - Err(e) => error!( - self.log, - "failed to shut down collection task"; - "producer_id" => %id, - "error" => ?e, - ), - } - Ok(()) - } -} - /// Configuration used to initialize an oximeter server #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Config { @@ -712,6 +172,9 @@ impl Oximeter { /// /// This can be used to override / ignore the logging configuration in /// `config`, using `log` instead. + /// + /// Note that this blocks until the ClickHouse database is available **and + /// at the expected version**. pub async fn with_logger( config: &Config, args: &OximeterArguments, @@ -736,14 +199,21 @@ impl Oximeter { let make_agent = || async { debug!(log, "creating ClickHouse client"); Ok(Arc::new( - OximeterAgent::with_id(args.id, config.db, &resolver, &log) - .await?, + OximeterAgent::with_id( + args.id, + args.address, + config.db, + &resolver, + &log, + ) + .await?, )) }; let log_client_failure = |error, delay| { warn!( log, - "failed to initialize ClickHouse database, will retry in {:?}", delay; + "failed to create ClickHouse client"; + "retry_after" => ?delay, "error" => ?error, ); }; @@ -825,7 +295,13 @@ impl Oximeter { ) -> Result { let db_config = clickhouse.map(DbConfig::with_address); let agent = Arc::new( - OximeterAgent::new_standalone(args.id, db_config, &log).await?, + OximeterAgent::new_standalone( + args.id, + args.address, + db_config, + &log, + ) + .await?, ); let dropshot_log = log.new(o!("component" => "dropshot")); @@ -908,108 +384,3 @@ impl Oximeter { self.agent.delete_producer(id).await } } - -// Build the HTTP API internal to the control plane -pub fn oximeter_api() -> ApiDescription> { - let mut api = ApiDescription::new(); - api.register(producers_post) - .expect("Could not register producers_post API handler"); - api.register(producers_list) - .expect("Could not register producers_list API handler"); - api.register(producer_delete) - .expect("Could not register producers_delete API handler"); - api.register(collector_info) - .expect("Could not register collector_info API handler"); - api -} - -// Handle a request from Nexus to register a new producer with this collector. -#[endpoint { - method = POST, - path = "/producers", -}] -async fn producers_post( - request_context: RequestContext>, - body: TypedBody, -) -> Result { - let agent = request_context.context(); - let producer_info = body.into_inner(); - agent - .register_producer(producer_info) - .await - .map_err(HttpError::from) - .map(|_| HttpResponseUpdatedNoContent()) -} - -// Parameters for paginating the list of producers. -#[derive(Clone, Copy, Debug, Deserialize, schemars::JsonSchema, Serialize)] -struct ProducerPage { - id: Uuid, -} - -// List all producers -#[endpoint { - method = GET, - path = "/producers", -}] -async fn producers_list( - request_context: RequestContext>, - query: Query>, -) -> Result>, HttpError> { - let agent = request_context.context(); - let pagination = query.into_inner(); - let limit = request_context.page_limit(&pagination)?.get() as usize; - let start = match &pagination.page { - WhichPage::First(..) => None, - WhichPage::Next(ProducerPage { id }) => Some(*id), - }; - let producers = agent.list_producers(start, limit).await; - ResultsPage::new( - producers, - &EmptyScanParams {}, - |info: &ProducerEndpoint, _| ProducerPage { id: info.id }, - ) - .map(HttpResponseOk) -} - -#[derive(Clone, Copy, Debug, Deserialize, schemars::JsonSchema, Serialize)] -struct ProducerIdPathParams { - producer_id: Uuid, -} - -// Delete a producer by ID. -#[endpoint { - method = DELETE, - path = "/producers/{producer_id}", -}] -async fn producer_delete( - request_context: RequestContext>, - path: dropshot::Path, -) -> Result { - let agent = request_context.context(); - let producer_id = path.into_inner().producer_id; - agent - .delete_producer(producer_id) - .await - .map_err(HttpError::from) - .map(|_| HttpResponseDeleted()) -} - -#[derive(Clone, Copy, Debug, Deserialize, schemars::JsonSchema, Serialize)] -pub struct CollectorInfo { - /// The collector's UUID. - pub id: Uuid, -} - -// Return identifying information about this collector -#[endpoint { - method = GET, - path = "/info", -}] -async fn collector_info( - request_context: RequestContext>, -) -> Result, HttpError> { - let agent = request_context.context(); - let info = CollectorInfo { id: agent.id }; - Ok(HttpResponseOk(info)) -} diff --git a/oximeter/collector/src/self_stats.rs b/oximeter/collector/src/self_stats.rs new file mode 100644 index 00000000000..dd1701203e1 --- /dev/null +++ b/oximeter/collector/src/self_stats.rs @@ -0,0 +1,171 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Metrics oximeter reports about itself + +// Copyright 2023 Oxide Computer Company + +use crate::ProducerEndpoint; +use oximeter::types::Cumulative; +use oximeter::types::ProducerResultsItem; +use oximeter::Metric; +use oximeter::MetricsError; +use oximeter::Sample; +use oximeter::Target; +use reqwest::StatusCode; +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::time::Duration; +use uuid::Uuid; + +/// The interval on which we report self statistics +pub const COLLECTION_INTERVAL: Duration = Duration::from_secs(60); + +/// A target representing a single oximeter collector. +#[derive(Clone, Copy, Debug, Target)] +pub struct OximeterCollector { + /// The collector's ID. + pub collector_id: Uuid, + /// The collector server's IP address. + pub collector_ip: IpAddr, + /// The collector server's port. + pub collector_port: u16, +} + +/// The number of successful collections from a single producer. +#[derive(Clone, Debug, Metric)] +pub struct Collections { + /// The producer's ID. + pub producer_id: Uuid, + /// The producer's IP address. + pub producer_ip: IpAddr, + /// The producer's port. + pub producer_port: u16, + /// The base route in the producer server used to collect metrics. + /// + /// The full route is `{base_route}/{producer_id}`. + pub base_route: String, + pub datum: Cumulative, +} + +/// Small enum to help understand why oximeter failed to collect from a +/// producer. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[non_exhaustive] +pub enum FailureReason { + /// The producer could not be reached. + Unreachable, + /// Error during deserialization. + Deserialization, + /// Some other reason, which includes the status code. + Other(StatusCode), +} + +impl std::fmt::Display for FailureReason { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Unreachable => write!(f, "unreachable"), + Self::Deserialization => write!(f, "deserialization"), + Self::Other(c) => write!(f, "{}", c.as_u16()), + } + } +} + +/// The number of failed collections from a single producer. +#[derive(Clone, Debug, Metric)] +pub struct FailedCollections { + /// The producer's ID. + pub producer_id: Uuid, + /// The producer's IP address. + pub producer_ip: IpAddr, + /// The producer's port. + pub producer_port: u16, + /// The base route in the producer server used to collect metrics. + /// + /// The full route is `{base_route}/{producer_id}`. + pub base_route: String, + /// The reason we could not collect. + // + // NOTE: This should always be generated through a `FailureReason`. + pub reason: String, + pub datum: Cumulative, +} + +/// Oximeter collection statistics maintained by each collection task. +#[derive(Clone, Debug)] +pub struct CollectionTaskStats { + pub collector: OximeterCollector, + pub collections: Collections, + pub failed_collections: BTreeMap, +} + +impl CollectionTaskStats { + pub fn new( + collector: OximeterCollector, + producer: &ProducerEndpoint, + ) -> Self { + Self { + collector, + collections: Collections { + producer_id: producer.id, + producer_ip: producer.address.ip(), + producer_port: producer.address.port(), + base_route: producer.base_route.clone(), + datum: Cumulative::new(0), + }, + failed_collections: BTreeMap::new(), + } + } + + pub fn failures_for_reason( + &mut self, + reason: FailureReason, + ) -> &mut FailedCollections { + self.failed_collections.entry(reason).or_insert_with(|| { + FailedCollections { + producer_id: self.collections.producer_id, + producer_ip: self.collections.producer_ip, + producer_port: self.collections.producer_port, + base_route: self.collections.base_route.clone(), + reason: reason.to_string(), + datum: Cumulative::new(0), + } + }) + } + + pub fn sample(&self) -> Vec { + fn to_item(res: Result) -> ProducerResultsItem { + match res { + Ok(s) => ProducerResultsItem::Ok(vec![s]), + Err(s) => ProducerResultsItem::Err(s), + } + } + let mut samples = Vec::with_capacity(1 + self.failed_collections.len()); + samples.push(to_item(Sample::new(&self.collector, &self.collections))); + samples.extend( + self.failed_collections + .values() + .map(|metric| to_item(Sample::new(&self.collector, metric))), + ); + samples + } +} + +#[cfg(test)] +mod tests { + use super::FailureReason; + use super::StatusCode; + + #[test] + fn test_failure_reason_serialization() { + let data = &[ + (FailureReason::Deserialization, "deserialization"), + (FailureReason::Unreachable, "unreachable"), + (FailureReason::Other(StatusCode::INTERNAL_SERVER_ERROR), "500"), + ]; + for (variant, as_str) in data.iter() { + assert_eq!(variant.to_string(), *as_str); + } + } +} diff --git a/oximeter/db/Cargo.toml b/oximeter/db/Cargo.toml index d37c57ccced..4d53869d0dc 100644 --- a/oximeter/db/Cargo.toml +++ b/oximeter/db/Cargo.toml @@ -10,10 +10,12 @@ anyhow.workspace = true async-trait.workspace = true bcs.workspace = true bytes = { workspace = true, features = [ "serde" ] } +camino.workspace = true chrono.workspace = true clap.workspace = true dropshot.workspace = true highway.workspace = true +omicron-common.workspace = true oximeter.workspace = true regex.workspace = true reqwest = { workspace = true, features = [ "json" ] } @@ -35,6 +37,7 @@ itertools.workspace = true omicron-test-utils.workspace = true slog-dtrace.workspace = true strum.workspace = true +tempfile.workspace = true [[bin]] name = "oxdb" diff --git a/oximeter/db/schema/README.md b/oximeter/db/schema/README.md new file mode 100644 index 00000000000..2f1633138d2 --- /dev/null +++ b/oximeter/db/schema/README.md @@ -0,0 +1,39 @@ +# ClickHouse schema files + +This directory contains the SQL files for different versions of the ClickHouse +timeseries database used by `oximeter`. In general, schema are expected to be +applied while the database is online, but no other clients exist. This is +similar to the current situation for _offline upgrade_ we use when updating the +main control plane database in CockroachDB. + +## Constraints, or why ClickHouse is weird + +While this tool is modeled after the mechanism for applying updates in +CockroachDB, ClickHouse is a significantly different DBMS. There are no +transactions; no unique primary keys; a single DB server can house both +replicated and single-node tables. This means we need to be pretty careful when +updating the schema. Changes must be idempotent, as with the CRDB schema, but at +this point we do not support inserting or modifying data at all. + +Similar to the CRDB offline update tool, we assume no non-update modifications +of the database are running concurrently. However, given ClickHouse's lack of +transactions, we actually require that there are no writes of any kind. In +practice, this means `oximeter` **must not** be running when this is called. +Similarly, there must be only a single instance of this program at a time. + +To run this program: + +- Ensure the ClickHouse server is running, and grab its IP address; + ```bash + $ pfexec zlogin oxz_clickhouse_e449eb80-3371-40a6-a316-d6e64b039357 'ipadm show-addr -o addrobj,addr | grep omicron6' + oxControlService20/omicron6 fd00:1122:3344:101::e/64 + ``` +- Log into the `oximeter` zone, `zlogin oxz_oximeter_` +- Run this tool, pointing it at the desired schema directory, e.g.: + +```bash +# /opt/oxide/oximeter/bin/clickhouse-schema-updater \ + --host \ + --schema-dir /opt/oxide/oximeter/sql + up VERSION +``` diff --git a/oximeter/db/schema/replicated/2/up.sql b/oximeter/db/schema/replicated/2/up.sql new file mode 100644 index 00000000000..eb01b8c1a3f --- /dev/null +++ b/oximeter/db/schema/replicated/2/up.sql @@ -0,0 +1,819 @@ +CREATE DATABASE IF NOT EXISTS oximeter ON CLUSTER oximeter_cluster; + +/* The version table contains metadata about the `oximeter` database */ +CREATE TABLE IF NOT EXISTS oximeter.version ON CLUSTER oximeter_cluster +( + value UInt64, + timestamp DateTime64(9, 'UTC') +) +ENGINE = ReplicatedMergeTree() +ORDER BY (value, timestamp); + +/* The measurement tables contain all individual samples from each timeseries. + * + * Each table stores a single datum type, and otherwise contains nearly the same + * structure. The primary sorting key is on the timeseries name, key, and then + * timestamp, so that all timeseries from the same schema are grouped, followed + * by all samples from the same timeseries. + * + * This reflects that one usually looks up the _key_ in one or more field table, + * and then uses that to index quickly into the measurements tables. + */ +CREATE TABLE IF NOT EXISTS oximeter.measurements_bool_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt8 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_bool_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_bool ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt8 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_bool_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i8_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int8 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i8_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int8 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i8_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u8_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt8 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u8_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt8 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u8_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i16_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int16 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i16_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int16 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i16_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u16_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt16 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u16_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt16 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u16_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int32 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int32 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt32 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt32 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int64 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int64 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt64 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt64 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float32 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_f32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float32 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_f32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float32 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_f32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float64 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_f32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float64 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_f64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float64 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_f64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_string_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum String +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_string_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_string ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum String +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_string_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Array(UInt8) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_bytes_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Array(UInt8) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_bytes_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Int64 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativei64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Int64 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativei64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum UInt64 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativeu64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum UInt64 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativeu64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Float32 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativef32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Float32 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativef32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Float64 +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativef64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Float64 +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativef64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int8), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami8_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int8), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami8_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt8), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu8_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt8), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu8_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int16), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami16_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int16), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami16_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt16), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu16_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt16), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu16_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int32), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int32), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt32), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt32), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int64), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int64), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt64), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt64), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Float32), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramf32_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Float32), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramf32_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64_local ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Float64), + counts Array(UInt64) +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramf64_local', '{replica}') +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Float64), + counts Array(UInt64) +) +ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramf64_local', xxHash64(splitByChar(':', timeseries_name)[1])); + +/* The field tables store named dimensions of each timeseries. + * + * As with the measurement tables, there is one field table for each field data + * type. Fields are deduplicated by using the "replacing merge tree", though + * this behavior **must not** be relied upon for query correctness. + * + * The index for the fields differs from the measurements, however. Rows are + * sorted by timeseries name, then field name, field value, and finally + * timeseries key. This reflects the most common pattern for looking them up: + * by field name and possibly value, within a timeseries. The resulting keys are + * usually then used to look up measurements. + * + * NOTE: We may want to consider a secondary index on these tables, sorting by + * timeseries name and then key, since it would improve lookups where one + * already has the key. Realistically though, these tables are quite small and + * so performance benefits will be low in absolute terms. + */ +CREATE TABLE IF NOT EXISTS oximeter.fields_bool ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt8 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i8 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int8 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u8 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt8 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i16 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int16 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u16 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt16 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int32 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u32 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt32 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int64 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u64 ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt64 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_ipaddr ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value IPv6 +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_string ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value String +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_uuid ON CLUSTER oximeter_cluster +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UUID +) +ENGINE = ReplicatedReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +/* The timeseries schema table stores the extracted schema for the samples + * oximeter collects. + */ +CREATE TABLE IF NOT EXISTS oximeter.timeseries_schema ON CLUSTER oximeter_cluster +( + timeseries_name String, + fields Nested( + name String, + type Enum( + 'Bool' = 1, + 'I64' = 2, + 'IpAddr' = 3, + 'String' = 4, + 'Uuid' = 6 + ), + source Enum( + 'Target' = 1, + 'Metric' = 2 + ) + ), + datum_type Enum( + 'Bool' = 1, + 'I64' = 2, + 'F64' = 3, + 'String' = 4, + 'Bytes' = 5, + 'CumulativeI64' = 6, + 'CumulativeF64' = 7, + 'HistogramI64' = 8, + 'HistogramF64' = 9, + 'I8' = 10, + 'U8' = 11, + 'I16' = 12, + 'U16' = 13, + 'I32' = 14, + 'U32' = 15, + 'U64' = 16, + 'F32' = 17, + 'CumulativeU64' = 18, + 'CumulativeF32' = 19, + 'HistogramI8' = 20, + 'HistogramU8' = 21, + 'HistogramI16' = 22, + 'HistogramU16' = 23, + 'HistogramI32' = 24, + 'HistogramU32' = 25, + 'HistogramU64' = 26, + 'HistogramF32' = 27 + ), + created DateTime64(9, 'UTC') +) +ENGINE = ReplicatedMergeTree() +ORDER BY (timeseries_name, fields.name); diff --git a/oximeter/db/schema/replicated/3/up.sql b/oximeter/db/schema/replicated/3/up.sql new file mode 100644 index 00000000000..073d643564f --- /dev/null +++ b/oximeter/db/schema/replicated/3/up.sql @@ -0,0 +1,22 @@ +/* This adds missing field types to the timeseries schema table field.type + * column, by augmentin the enum to capture new values. Note that the existing + * variants can't be moved or changed, so the new ones are added at the end. The + * client never sees this discriminant, only the string, so it should not + * matter. + */ +ALTER TABLE oximeter.timeseries_schema + MODIFY COLUMN IF EXISTS fields.type + Array(Enum( + 'Bool' = 1, + 'I64' = 2, + 'IpAddr' = 3, + 'String' = 4, + 'Uuid' = 6, + 'I8' = 7, + 'U8' = 8, + 'I16' = 9, + 'U16' = 10, + 'I32' = 11, + 'U32' = 12, + 'U64' = 13 + )); diff --git a/oximeter/db/src/db-replicated-init.sql b/oximeter/db/schema/replicated/db-init.sql similarity index 93% rename from oximeter/db/src/db-replicated-init.sql rename to oximeter/db/schema/replicated/db-init.sql index ec11854e446..4429f41364b 100644 --- a/oximeter/db/src/db-replicated-init.sql +++ b/oximeter/db/schema/replicated/db-init.sql @@ -1,5 +1,6 @@ CREATE DATABASE IF NOT EXISTS oximeter ON CLUSTER oximeter_cluster; --- + +/* The version table contains metadata about the `oximeter` database */ CREATE TABLE IF NOT EXISTS oximeter.version ON CLUSTER oximeter_cluster ( value UInt64, @@ -7,7 +8,17 @@ CREATE TABLE IF NOT EXISTS oximeter.version ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedMergeTree() ORDER BY (value, timestamp); --- + +/* The measurement tables contain all individual samples from each timeseries. + * + * Each table stores a single datum type, and otherwise contains nearly the same + * structure. The primary sorting key is on the timeseries name, key, and then + * timestamp, so that all timeseries from the same schema are grouped, followed + * by all samples from the same timeseries. + * + * This reflects that one usually looks up the _key_ in one or more field table, + * and then uses that to index quickly into the measurements tables. + */ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -18,7 +29,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool_local ON CLUSTER oximeter_ ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_bool_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_bool ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -27,7 +38,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool ON CLUSTER oximeter_cluste datum UInt8 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_bool_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i8_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -38,7 +49,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i8_local ON CLUSTER oximeter_cl ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i8_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -47,7 +58,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 ON CLUSTER oximeter_cluster datum Int8 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i8_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u8_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -58,7 +69,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u8_local ON CLUSTER oximeter_cl ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u8_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -67,7 +78,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 ON CLUSTER oximeter_cluster datum UInt8 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u8_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i16_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -78,7 +89,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i16_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i16_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -87,7 +98,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 ON CLUSTER oximeter_cluster datum Int16 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i16_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u16_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -98,7 +109,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u16_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u16_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -107,7 +118,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 ON CLUSTER oximeter_cluster datum UInt16 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u16_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i32_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -118,7 +129,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i32_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -127,7 +138,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 ON CLUSTER oximeter_cluster datum Int32 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i32_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u32_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -138,7 +149,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u32_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -147,7 +158,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 ON CLUSTER oximeter_cluster datum UInt32 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u32_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -158,7 +169,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i64_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_i64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -167,7 +178,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 ON CLUSTER oximeter_cluster datum Int64 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_i64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -178,7 +189,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u64_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_u64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -187,7 +198,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 ON CLUSTER oximeter_cluster datum UInt64 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_u64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_f32_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -198,7 +209,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f32_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_f32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -207,7 +218,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 ON CLUSTER oximeter_cluster datum Float32 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_f32_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_f64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -218,7 +229,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f64_local ON CLUSTER oximeter_c ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_f64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -227,7 +238,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 ON CLUSTER oximeter_cluster datum Float64 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_f64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_string_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -238,7 +249,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_string_local ON CLUSTER oximete ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_string_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_string ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -247,7 +258,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_string ON CLUSTER oximeter_clus datum String ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_string_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -258,7 +269,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes_local ON CLUSTER oximeter ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_bytes_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -267,7 +278,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes ON CLUSTER oximeter_clust datum Array(UInt8) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_bytes_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -279,7 +290,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64_local ON CLUSTER ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativei64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -289,7 +300,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 ON CLUSTER oximet datum Int64 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativei64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -301,7 +312,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64_local ON CLUSTER ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativeu64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -311,7 +322,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 ON CLUSTER oximet datum UInt64 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativeu64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -323,7 +334,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32_local ON CLUSTER ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativef32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -333,7 +344,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 ON CLUSTER oximet datum Float32 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativef32_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -345,7 +356,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64_local ON CLUSTER ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_cumulativef64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -355,7 +366,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 ON CLUSTER oximet datum Float64 ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_cumulativef64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -368,7 +379,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8_local ON CLUSTER ox ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami8_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -379,7 +390,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8 ON CLUSTER oximeter counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami8_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -392,7 +403,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8_local ON CLUSTER ox ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu8_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -403,7 +414,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8 ON CLUSTER oximeter counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu8_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -416,7 +427,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami16_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -427,7 +438,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami16_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -440,7 +451,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu16_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -451,7 +462,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu16_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -464,7 +475,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -475,7 +486,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami32_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -488,7 +499,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -499,7 +510,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu32_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -512,7 +523,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogrami64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -523,7 +534,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogrami64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -536,7 +547,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramu64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -547,7 +558,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramu64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -560,7 +571,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramf32_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -571,7 +582,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramf32_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64_local ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -584,7 +595,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64_local ON CLUSTER o ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/measurements_histogramf64_local', '{replica}') ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -595,7 +606,24 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64 ON CLUSTER oximete counts Array(UInt64) ) ENGINE = Distributed('oximeter_cluster', 'oximeter', 'measurements_histogramf64_local', xxHash64(splitByChar(':', timeseries_name)[1])); --- + +/* The field tables store named dimensions of each timeseries. + * + * As with the measurement tables, there is one field table for each field data + * type. Fields are deduplicated by using the "replacing merge tree", though + * this behavior **must not** be relied upon for query correctness. + * + * The index for the fields differs from the measurements, however. Rows are + * sorted by timeseries name, then field name, field value, and finally + * timeseries key. This reflects the most common pattern for looking them up: + * by field name and possibly value, within a timeseries. The resulting keys are + * usually then used to look up measurements. + * + * NOTE: We may want to consider a secondary index on these tables, sorting by + * timeseries name and then key, since it would improve lookups where one + * already has the key. Realistically though, these tables are quite small and + * so performance benefits will be low in absolute terms. + */ CREATE TABLE IF NOT EXISTS oximeter.fields_bool ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -605,7 +633,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_bool ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i8 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -615,7 +643,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i8 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u8 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -625,7 +653,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u8 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i16 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -635,7 +663,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i16 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u16 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -645,7 +673,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u16 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -655,7 +683,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i32 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u32 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -665,7 +693,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u32 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -675,7 +703,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i64 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u64 ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -685,7 +713,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u64 ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_ipaddr ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -695,7 +723,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_ipaddr ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_string ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -705,7 +733,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_string ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_uuid ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -715,7 +743,10 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_uuid ON CLUSTER oximeter_cluster ) ENGINE = ReplicatedReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + +/* The timeseries schema table stores the extracted schema for the samples + * oximeter collects. + */ CREATE TABLE IF NOT EXISTS oximeter.timeseries_schema ON CLUSTER oximeter_cluster ( timeseries_name String, @@ -726,7 +757,14 @@ CREATE TABLE IF NOT EXISTS oximeter.timeseries_schema ON CLUSTER oximeter_cluste 'I64' = 2, 'IpAddr' = 3, 'String' = 4, - 'Uuid' = 6 + 'Uuid' = 6, + 'I8' = 7, + 'U8' = 8, + 'I16' = 9, + 'U16' = 10, + 'I32' = 11, + 'U32' = 12, + 'U64' = 13 ), source Enum( 'Target' = 1, diff --git a/oximeter/db/src/db-wipe-replicated.sql b/oximeter/db/schema/replicated/db-wipe.sql similarity index 100% rename from oximeter/db/src/db-wipe-replicated.sql rename to oximeter/db/schema/replicated/db-wipe.sql diff --git a/oximeter/db/src/db-single-node-init.sql b/oximeter/db/schema/single-node/2/up.sql similarity index 88% rename from oximeter/db/src/db-single-node-init.sql rename to oximeter/db/schema/single-node/2/up.sql index 2fb5c363977..4756e2897d0 100644 --- a/oximeter/db/src/db-single-node-init.sql +++ b/oximeter/db/schema/single-node/2/up.sql @@ -1,5 +1,6 @@ CREATE DATABASE IF NOT EXISTS oximeter; --- + +/* The version table contains metadata about the `oximeter` database */ CREATE TABLE IF NOT EXISTS oximeter.version ( value UInt64, @@ -7,7 +8,17 @@ CREATE TABLE IF NOT EXISTS oximeter.version ) ENGINE = MergeTree() ORDER BY (value, timestamp); --- + +/* The measurement tables contain all individual samples from each timeseries. + * + * Each table stores a single datum type, and otherwise contains nearly the same + * structure. The primary sorting key is on the timeseries name, key, and then + * timestamp, so that all timeseries from the same schema are grouped, followed + * by all samples from the same timeseries. + * + * This reflects that one usually looks up the _key_ in one or more field table, + * and then uses that to index quickly into the measurements tables. + */ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool ( timeseries_name String, @@ -18,7 +29,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bool ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 ( timeseries_name String, @@ -29,7 +40,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 ( timeseries_name String, @@ -40,7 +51,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 ( timeseries_name String, @@ -51,7 +62,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 ( timeseries_name String, @@ -62,7 +73,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 ( timeseries_name String, @@ -73,7 +84,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 ( timeseries_name String, @@ -84,7 +95,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 ( timeseries_name String, @@ -95,7 +106,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 ( timeseries_name String, @@ -106,7 +117,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 ( timeseries_name String, @@ -117,7 +128,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 ( timeseries_name String, @@ -128,7 +139,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_string ( timeseries_name String, @@ -139,7 +150,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_string ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes ( timeseries_name String, @@ -150,7 +161,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 ( timeseries_name String, @@ -162,7 +173,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 ( timeseries_name String, @@ -174,7 +185,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 ( timeseries_name String, @@ -186,8 +197,8 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- --- + + CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 ( timeseries_name String, @@ -199,7 +210,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8 ( timeseries_name String, @@ -212,7 +223,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8 ( timeseries_name String, @@ -225,7 +236,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16 ( timeseries_name String, @@ -238,7 +249,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16 ( timeseries_name String, @@ -251,7 +262,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32 ( timeseries_name String, @@ -264,7 +275,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32 ( timeseries_name String, @@ -277,7 +288,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64 ( timeseries_name String, @@ -290,7 +301,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64 ( timeseries_name String, @@ -303,7 +314,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32 ( timeseries_name String, @@ -316,7 +327,7 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64 ( timeseries_name String, @@ -329,7 +340,24 @@ CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64 ENGINE = MergeTree() ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) TTL toDateTime(timestamp) + INTERVAL 30 DAY; --- + +/* The field tables store named dimensions of each timeseries. + * + * As with the measurement tables, there is one field table for each field data + * type. Fields are deduplicated by using the "replacing merge tree", though + * this behavior **must not** be relied upon for query correctness. + * + * The index for the fields differs from the measurements, however. Rows are + * sorted by timeseries name, then field name, field value, and finally + * timeseries key. This reflects the most common pattern for looking them up: + * by field name and possibly value, within a timeseries. The resulting keys are + * usually then used to look up measurements. + * + * NOTE: We may want to consider a secondary index on these tables, sorting by + * timeseries name and then key, since it would improve lookups where one + * already has the key. Realistically though, these tables are quite small and + * so performance benefits will be low in absolute terms. + */ CREATE TABLE IF NOT EXISTS oximeter.fields_bool ( timeseries_name String, @@ -339,7 +367,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_bool ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i8 ( timeseries_name String, @@ -349,7 +377,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i8 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u8 ( timeseries_name String, @@ -359,7 +387,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u8 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i16 ( timeseries_name String, @@ -369,7 +397,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i16 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u16 ( timeseries_name String, @@ -379,7 +407,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u16 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i32 ( timeseries_name String, @@ -389,7 +417,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i32 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u32 ( timeseries_name String, @@ -399,7 +427,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u32 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_i64 ( timeseries_name String, @@ -409,7 +437,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_i64 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_u64 ( timeseries_name String, @@ -419,7 +447,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_u64 ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_ipaddr ( timeseries_name String, @@ -429,7 +457,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_ipaddr ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_string ( timeseries_name String, @@ -439,7 +467,7 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_string ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + CREATE TABLE IF NOT EXISTS oximeter.fields_uuid ( timeseries_name String, @@ -449,7 +477,10 @@ CREATE TABLE IF NOT EXISTS oximeter.fields_uuid ) ENGINE = ReplacingMergeTree() ORDER BY (timeseries_name, field_name, field_value, timeseries_key); --- + +/* The timeseries schema table stores the extracted schema for the samples + * oximeter collects. + */ CREATE TABLE IF NOT EXISTS oximeter.timeseries_schema ( timeseries_name String, diff --git a/oximeter/db/schema/single-node/3/up.sql b/oximeter/db/schema/single-node/3/up.sql new file mode 100644 index 00000000000..073d643564f --- /dev/null +++ b/oximeter/db/schema/single-node/3/up.sql @@ -0,0 +1,22 @@ +/* This adds missing field types to the timeseries schema table field.type + * column, by augmentin the enum to capture new values. Note that the existing + * variants can't be moved or changed, so the new ones are added at the end. The + * client never sees this discriminant, only the string, so it should not + * matter. + */ +ALTER TABLE oximeter.timeseries_schema + MODIFY COLUMN IF EXISTS fields.type + Array(Enum( + 'Bool' = 1, + 'I64' = 2, + 'IpAddr' = 3, + 'String' = 4, + 'Uuid' = 6, + 'I8' = 7, + 'U8' = 8, + 'I16' = 9, + 'U16' = 10, + 'I32' = 11, + 'U32' = 12, + 'U64' = 13 + )); diff --git a/oximeter/db/schema/single-node/db-init.sql b/oximeter/db/schema/single-node/db-init.sql new file mode 100644 index 00000000000..ee5e91c4b7e --- /dev/null +++ b/oximeter/db/schema/single-node/db-init.sql @@ -0,0 +1,540 @@ +CREATE DATABASE IF NOT EXISTS oximeter; + +/* The version table contains metadata about the `oximeter` database */ +CREATE TABLE IF NOT EXISTS oximeter.version +( + value UInt64, + timestamp DateTime64(9, 'UTC') +) +ENGINE = MergeTree() +ORDER BY (value, timestamp); + +/* The measurement tables contain all individual samples from each timeseries. + * + * Each table stores a single datum type, and otherwise contains nearly the same + * structure. The primary sorting key is on the timeseries name, key, and then + * timestamp, so that all timeseries from the same schema are grouped, followed + * by all samples from the same timeseries. + * + * This reflects that one usually looks up the _key_ in one or more field table, + * and then uses that to index quickly into the measurements tables. + */ +CREATE TABLE IF NOT EXISTS oximeter.measurements_bool +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt8 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i8 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int8 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u8 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt8 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i16 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int16 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u16 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt16 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i32 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int32 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u32 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt32 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_i64 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Int64 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_u64 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum UInt64 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f32 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float32 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_f64 +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Float64 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_string +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum String +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_bytes +( + timeseries_name String, + timeseries_key UInt64, + timestamp DateTime64(9, 'UTC'), + datum Array(UInt8) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativei64 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Int64 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativeu64 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum UInt64 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef32 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Float32 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + + +CREATE TABLE IF NOT EXISTS oximeter.measurements_cumulativef64 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + datum Float64 +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami8 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int8), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu8 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt8), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami16 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int16), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu16 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt16), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami32 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int32), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu32 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt32), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogrami64 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Int64), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramu64 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(UInt64), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf32 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Float32), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS oximeter.measurements_histogramf64 +( + timeseries_name String, + timeseries_key UInt64, + start_time DateTime64(9, 'UTC'), + timestamp DateTime64(9, 'UTC'), + bins Array(Float64), + counts Array(UInt64) +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, timeseries_key, start_time, timestamp) +TTL toDateTime(timestamp) + INTERVAL 30 DAY; + +/* The field tables store named dimensions of each timeseries. + * + * As with the measurement tables, there is one field table for each field data + * type. Fields are deduplicated by using the "replacing merge tree", though + * this behavior **must not** be relied upon for query correctness. + * + * The index for the fields differs from the measurements, however. Rows are + * sorted by timeseries name, then field name, field value, and finally + * timeseries key. This reflects the most common pattern for looking them up: + * by field name and possibly value, within a timeseries. The resulting keys are + * usually then used to look up measurements. + * + * NOTE: We may want to consider a secondary index on these tables, sorting by + * timeseries name and then key, since it would improve lookups where one + * already has the key. Realistically though, these tables are quite small and + * so performance benefits will be low in absolute terms. + */ +CREATE TABLE IF NOT EXISTS oximeter.fields_bool +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt8 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i8 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int8 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u8 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt8 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i16 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int16 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u16 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt16 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i32 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int32 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u32 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt32 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_i64 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value Int64 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_u64 +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UInt64 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_ipaddr +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value IPv6 +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_string +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value String +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +CREATE TABLE IF NOT EXISTS oximeter.fields_uuid +( + timeseries_name String, + timeseries_key UInt64, + field_name String, + field_value UUID +) +ENGINE = ReplacingMergeTree() +ORDER BY (timeseries_name, field_name, field_value, timeseries_key); + +/* The timeseries schema table stores the extracted schema for the samples + * oximeter collects. + */ +CREATE TABLE IF NOT EXISTS oximeter.timeseries_schema +( + timeseries_name String, + fields Nested( + name String, + type Enum( + 'Bool' = 1, + 'I64' = 2, + 'IpAddr' = 3, + 'String' = 4, + 'Uuid' = 6, + 'I8' = 7, + 'U8' = 8, + 'I16' = 9, + 'U16' = 10, + 'I32' = 11, + 'U32' = 12, + 'U64' = 13 + ), + source Enum( + 'Target' = 1, + 'Metric' = 2 + ) + ), + datum_type Enum( + 'Bool' = 1, + 'I64' = 2, + 'F64' = 3, + 'String' = 4, + 'Bytes' = 5, + 'CumulativeI64' = 6, + 'CumulativeF64' = 7, + 'HistogramI64' = 8, + 'HistogramF64' = 9, + 'I8' = 10, + 'U8' = 11, + 'I16' = 12, + 'U16' = 13, + 'I32' = 14, + 'U32' = 15, + 'U64' = 16, + 'F32' = 17, + 'CumulativeU64' = 18, + 'CumulativeF32' = 19, + 'HistogramI8' = 20, + 'HistogramU8' = 21, + 'HistogramI16' = 22, + 'HistogramU16' = 23, + 'HistogramI32' = 24, + 'HistogramU32' = 25, + 'HistogramU64' = 26, + 'HistogramF32' = 27 + ), + created DateTime64(9, 'UTC') +) +ENGINE = MergeTree() +ORDER BY (timeseries_name, fields.name); diff --git a/oximeter/db/src/db-wipe-single-node.sql b/oximeter/db/schema/single-node/db-wipe.sql similarity index 100% rename from oximeter/db/src/db-wipe-single-node.sql rename to oximeter/db/schema/single-node/db-wipe.sql diff --git a/oximeter/db/src/client.rs b/oximeter/db/src/client.rs index 69e91f888af..e1ed06554ca 100644 --- a/oximeter/db/src/client.rs +++ b/oximeter/db/src/client.rs @@ -23,6 +23,8 @@ use dropshot::PaginationOrder; use dropshot::ResultsPage; use dropshot::WhichPage; use oximeter::types::Sample; +use regex::Regex; +use regex::RegexBuilder; use slog::debug; use slog::error; use slog::info; @@ -35,6 +37,11 @@ use std::collections::BTreeSet; use std::convert::TryFrom; use std::net::SocketAddr; use std::num::NonZeroU32; +use std::ops::Bound; +use std::path::Path; +use std::path::PathBuf; +use std::sync::OnceLock; +use tokio::fs; use tokio::sync::Mutex; use uuid::Uuid; @@ -267,14 +274,319 @@ impl Client { .map_err(|e| Error::Database(e.to_string())) } + /// Read the available schema versions in the provided directory. + pub async fn read_available_schema_versions( + log: &Logger, + is_replicated: bool, + schema_dir: impl AsRef, + ) -> Result, Error> { + let dir = schema_dir.as_ref().join(if is_replicated { + "replicated" + } else { + "single-node" + }); + let mut rd = + fs::read_dir(&dir).await.map_err(|err| Error::ReadSchemaDir { + context: format!( + "Failed to read schema directory '{}'", + dir.display() + ), + err, + })?; + let mut versions = BTreeSet::new(); + debug!(log, "reading entries from schema dir"; "dir" => dir.display()); + while let Some(entry) = + rd.next_entry().await.map_err(|err| Error::ReadSchemaDir { + context: String::from("Failed to read directory entry"), + err, + })? + { + let name = entry + .file_name() + .into_string() + .map_err(|bad| Error::NonUtf8SchemaDirEntry(bad.to_owned()))?; + let md = + entry.metadata().await.map_err(|err| Error::ReadSchemaDir { + context: String::from("Failed to fetch entry metatdata"), + err, + })?; + if !md.is_dir() { + debug!(log, "skipping non-directory"; "name" => &name); + continue; + } + match name.parse() { + Ok(ver) => { + debug!(log, "valid version dir"; "ver" => ver); + assert!(versions.insert(ver), "Versions should be unique"); + } + Err(e) => warn!( + log, + "found directory with non-u64 name, skipping"; + "name" => name, + "error" => ?e, + ), + } + } + Ok(versions) + } + + /// Ensure that the database is upgraded to the desired version of the + /// schema. + /// + /// NOTE: This function is not safe for concurrent usage! + pub async fn ensure_schema( + &self, + replicated: bool, + desired_version: u64, + schema_dir: impl AsRef, + ) -> Result<(), Error> { + let schema_dir = schema_dir.as_ref(); + let latest = self.read_latest_version().await?; + if latest == desired_version { + debug!( + self.log, + "database already at desired version"; + "version" => latest, + ); + return Ok(()); + } + debug!( + self.log, + "starting upgrade to desired version {}", desired_version + ); + let available = Self::read_available_schema_versions( + &self.log, replicated, schema_dir, + ) + .await?; + // We explicitly ignore version 0, which implies the database doesn't + // exist at all. + if latest > 0 && !available.contains(&latest) { + return Err(Error::MissingSchemaVersion(latest)); + } + if !available.contains(&desired_version) { + return Err(Error::MissingSchemaVersion(desired_version)); + } + + // Check we have no gaps in version numbers, starting with the latest + // version and walking through all available ones strictly greater. This + // is to check that the _next_ version is also 1 greater than the + // latest. + let range = (Bound::Excluded(latest), Bound::Included(desired_version)); + if available + .range(latest..) + .zip(available.range(range)) + .any(|(current, next)| next - current != 1) + { + return Err(Error::NonSequentialSchemaVersions); + } + + // Walk through all changes between current version (exclusive) and + // the desired version (inclusive). + let versions_to_apply = available.range(range); + let mut current = latest; + for version in versions_to_apply { + if let Err(e) = self + .apply_one_schema_upgrade(replicated, *version, schema_dir) + .await + { + error!( + self.log, + "failed to apply schema upgrade"; + "current_version" => current, + "next_version" => *version, + "replicated" => replicated, + "schema_dir" => schema_dir.display(), + "error" => ?e, + ); + return Err(e); + } + current = *version; + self.insert_version(current).await?; + } + Ok(()) + } + + fn verify_schema_upgrades( + files: &BTreeMap, + ) -> Result<(), Error> { + let re = schema_validation_regex(); + for (path, sql) in files.values() { + if re.is_match(&sql) { + return Err(Error::SchemaUpdateModifiesData { + path: path.clone(), + statement: sql.clone(), + }); + } + if sql.matches(';').count() > 1 { + return Err(Error::MultipleSqlStatementsInSchemaUpdate { + path: path.clone(), + }); + } + } + Ok(()) + } + + async fn apply_one_schema_upgrade( + &self, + replicated: bool, + next_version: u64, + schema_dir: impl AsRef, + ) -> Result<(), Error> { + let schema_dir = schema_dir.as_ref(); + let upgrade_file_contents = Self::read_schema_upgrade_sql_files( + &self.log, + replicated, + next_version, + schema_dir, + ) + .await?; + + // We need to be pretty careful at this point with any data-modifying + // statements. There should be no INSERT queries, for example, which we + // check here. ClickHouse doesn't support much in the way of data + // modification, which makes this pretty easy. + Self::verify_schema_upgrades(&upgrade_file_contents)?; + + // Apply each file in sequence in the upgrade directory. + for (name, (path, sql)) in upgrade_file_contents.into_iter() { + debug!( + self.log, + "apply schema upgrade file"; + "version" => next_version, + "path" => path.display(), + "filename" => &name, + ); + match self.execute(sql).await { + Ok(_) => debug!( + self.log, + "successfully applied schema upgrade file"; + "version" => next_version, + "path" => path.display(), + "name" => name, + ), + Err(e) => { + return Err(e); + } + } + } + Ok(()) + } + + fn full_upgrade_path( + replicated: bool, + version: u64, + schema_dir: impl AsRef, + ) -> PathBuf { + schema_dir + .as_ref() + .join(if replicated { "replicated" } else { "single-node" }) + .join(version.to_string()) + } + + // Read all SQL files, in order, in the schema directory for the provided + // version. + async fn read_schema_upgrade_sql_files( + log: &Logger, + replicated: bool, + version: u64, + schema_dir: impl AsRef, + ) -> Result, Error> { + let version_schema_dir = + Self::full_upgrade_path(replicated, version, schema_dir.as_ref()); + let mut rd = + fs::read_dir(&version_schema_dir).await.map_err(|err| { + Error::ReadSchemaDir { + context: format!( + "Failed to read schema directory '{}'", + version_schema_dir.display() + ), + err, + } + })?; + + let mut upgrade_files = BTreeMap::new(); + debug!(log, "reading SQL files from schema dir"; "dir" => version_schema_dir.display()); + while let Some(entry) = + rd.next_entry().await.map_err(|err| Error::ReadSchemaDir { + context: String::from("Failed to read directory entry"), + err, + })? + { + let path = entry.path(); + let Some(ext) = path.extension() else { + warn!( + log, + "skipping schema dir entry without an extension"; + "dir" => version_schema_dir.display(), + "path" => path.display(), + ); + continue; + }; + let Some(ext) = ext.to_str() else { + warn!( + log, + "skipping schema dir entry with non-UTF8 extension"; + "dir" => version_schema_dir.display(), + "path" => path.display(), + ); + continue; + }; + if ext.eq_ignore_ascii_case("sql") { + let Some(stem) = path.file_stem() else { + warn!( + log, + "skipping schema SQL file with no name"; + "dir" => version_schema_dir.display(), + "path" => path.display(), + ); + continue; + }; + let Some(name) = stem.to_str() else { + warn!( + log, + "skipping schema SQL file with non-UTF8 name"; + "dir" => version_schema_dir.display(), + "path" => path.display(), + ); + continue; + }; + let contents = + fs::read_to_string(&path).await.map_err(|err| { + Error::ReadSqlFile { + context: format!( + "Reading SQL file '{}' for upgrade", + path.display(), + ), + err, + } + })?; + upgrade_files + .insert(name.to_string(), (path.to_owned(), contents)); + } else { + warn!( + log, + "skipping non-SQL schema dir entry"; + "dir" => version_schema_dir.display(), + "path" => path.display(), + ); + continue; + } + } + Ok(upgrade_files) + } + /// Validates that the schema used by the DB matches the version used by /// the executable using it. /// - /// This function will wipe metrics data if the version stored within + /// This function will **wipe** metrics data if the version stored within /// the DB is less than the schema version of Oximeter. /// If the version in the DB is newer than what is known to Oximeter, an /// error is returned. /// + /// If you would like to non-destructively upgrade the database, then either + /// the included binary `clickhouse-schema-updater` or the method + /// [`Client::ensure_schema()`] should be used instead. + /// /// NOTE: This function is not safe for concurrent usage! pub async fn initialize_db_with_version( &self, @@ -304,11 +616,10 @@ impl Client { } else if version > expected_version { // If the on-storage version is greater than the constant embedded // into this binary, we may have downgraded. - return Err(Error::Database( - format!( - "Expected version {expected_version}, saw {version}. Downgrading is not supported.", - ) - )); + return Err(Error::DatabaseVersionMismatch { + expected: crate::model::OXIMETER_VERSION, + found: version, + }); } else { // If the version matches, we don't need to update the DB return Ok(()); @@ -319,7 +630,8 @@ impl Client { Ok(()) } - async fn read_latest_version(&self) -> Result { + /// Read the latest version applied in the database. + pub async fn read_latest_version(&self) -> Result { let sql = format!( "SELECT MAX(value) FROM {db_name}.version;", db_name = crate::DATABASE_NAME, @@ -354,6 +666,20 @@ impl Client { Ok(version) } + /// Return Ok if the DB is at exactly the version compatible with this + /// client. + pub async fn check_db_is_at_expected_version(&self) -> Result<(), Error> { + let ver = self.read_latest_version().await?; + if ver == crate::model::OXIMETER_VERSION { + Ok(()) + } else { + Err(Error::DatabaseVersionMismatch { + expected: crate::model::OXIMETER_VERSION, + found: ver, + }) + } + } + async fn insert_version(&self, version: u64) -> Result<(), Error> { let sql = format!( "INSERT INTO {db_name}.version (*) VALUES ({version}, now());", @@ -365,7 +691,7 @@ impl Client { /// Verifies if instance is part of oximeter_cluster pub async fn is_oximeter_cluster(&self) -> Result { - let sql = String::from("SHOW CLUSTERS FORMAT JSONEachRow;"); + let sql = "SHOW CLUSTERS FORMAT JSONEachRow;"; let res = self.execute_with_body(sql).await?; Ok(res.contains("oximeter_cluster")) } @@ -501,7 +827,11 @@ impl Client { S: AsRef, { let sql = sql.as_ref().to_string(); - trace!(self.log, "executing SQL query: {}", sql); + trace!( + self.log, + "executing SQL query"; + "sql" => &sql, + ); let id = usdt::UniqueId::new(); probes::query__start!(|| (&id, &sql)); let response = handle_db_response( @@ -720,6 +1050,42 @@ impl Client { // many as one per sample. It's not clear how to structure this in a way that's useful. Ok(()) } + + // Run one or more SQL statements. + // + // This is intended to be used for the methods which run SQL from one of the + // SQL files in the crate, e.g., the DB initialization or update files. + async fn run_many_sql_statements( + &self, + sql: impl AsRef, + ) -> Result<(), Error> { + for stmt in sql.as_ref().split(';').filter(|s| !s.trim().is_empty()) { + self.execute(stmt).await?; + } + Ok(()) + } +} + +// A regex used to validate supported schema updates. +static SCHEMA_VALIDATION_REGEX: OnceLock = OnceLock::new(); +fn schema_validation_regex() -> &'static Regex { + SCHEMA_VALIDATION_REGEX.get_or_init(|| { + RegexBuilder::new(concat!( + // Cannot insert rows + r#"(INSERT INTO)|"#, + // Cannot delete rows in a table + r#"(ALTER TABLE .* DELETE)|"#, + // Cannot update values in a table + r#"(ALTER TABLE .* UPDATE)|"#, + // Cannot drop column values + r#"(ALTER TABLE .* CLEAR COLUMN)|"#, + // Or issue lightweight deletes + r#"(DELETE FROM)"#, + )) + .case_insensitive(true) + .build() + .expect("Invalid regex") + }) } #[derive(Debug)] @@ -767,40 +1133,38 @@ impl DbWrite for Client { /// Initialize the replicated telemetry database, creating tables as needed. async fn init_replicated_db(&self) -> Result<(), Error> { - // The HTTP client doesn't support multiple statements per query, so we break them out here - // manually. debug!(self.log, "initializing ClickHouse database"); - let sql = include_str!("./db-replicated-init.sql"); - for query in sql.split("\n--\n") { - self.execute(query.to_string()).await?; - } - Ok(()) + self.run_many_sql_statements(include_str!( + "../schema/replicated/db-init.sql" + )) + .await + } + + /// Wipe the ClickHouse database entirely from a replicated set up. + async fn wipe_replicated_db(&self) -> Result<(), Error> { + debug!(self.log, "wiping ClickHouse database"); + self.run_many_sql_statements(include_str!( + "../schema/replicated/db-wipe.sql" + )) + .await } /// Initialize a single node telemetry database, creating tables as needed. async fn init_single_node_db(&self) -> Result<(), Error> { - // The HTTP client doesn't support multiple statements per query, so we break them out here - // manually. debug!(self.log, "initializing ClickHouse database"); - let sql = include_str!("./db-single-node-init.sql"); - for query in sql.split("\n--\n") { - self.execute(query.to_string()).await?; - } - Ok(()) + self.run_many_sql_statements(include_str!( + "../schema/single-node/db-init.sql" + )) + .await } /// Wipe the ClickHouse database entirely from a single node set up. async fn wipe_single_node_db(&self) -> Result<(), Error> { debug!(self.log, "wiping ClickHouse database"); - let sql = include_str!("./db-wipe-single-node.sql").to_string(); - self.execute(sql).await - } - - /// Wipe the ClickHouse database entirely from a replicated set up. - async fn wipe_replicated_db(&self) -> Result<(), Error> { - debug!(self.log, "wiping ClickHouse database"); - let sql = include_str!("./db-wipe-replicated.sql").to_string(); - self.execute(sql).await + self.run_many_sql_statements(include_str!( + "../schema/single-node/db-wipe.sql" + )) + .await } } @@ -839,7 +1203,9 @@ mod tests { use oximeter::Metric; use oximeter::Target; use std::net::Ipv6Addr; + use std::path::PathBuf; use std::time::Duration; + use tempfile::TempDir; use tokio::time::sleep; use uuid::Uuid; @@ -1062,11 +1428,20 @@ mod tests { db.cleanup().await.expect("Failed to cleanup ClickHouse server"); } + async fn create_cluster() -> ClickHouseCluster { + let cur_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let replica_config = + cur_dir.as_path().join("src/configs/replica_config.xml"); + let keeper_config = + cur_dir.as_path().join("src/configs/keeper_config.xml"); + ClickHouseCluster::new(replica_config, keeper_config) + .await + .expect("Failed to initialise ClickHouse Cluster") + } + #[tokio::test] async fn test_replicated() { - let mut cluster = ClickHouseCluster::new() - .await - .expect("Failed to initialise ClickHouse Cluster"); + let mut cluster = create_cluster().await; // Tests that the expected error is returned on a wrong address bad_db_connection_test().await.unwrap(); @@ -3157,7 +3532,7 @@ mod tests { ) -> Result<(), Error> { use strum::IntoEnumIterator; usdt::register_probes().unwrap(); - let logctx = test_setup_log("test_update_schema_cache_on_new_sample"); + let logctx = test_setup_log("test_select_all_datum_types"); let log = &logctx.log; let client = Client::new(address, &log); @@ -3338,4 +3713,521 @@ mod tests { ); } } + + async fn create_test_upgrade_schema_directory( + replicated: bool, + versions: &[u64], + ) -> (TempDir, Vec) { + assert!(!versions.is_empty()); + let schema_dir = TempDir::new().expect("failed to create tempdir"); + let mut paths = Vec::with_capacity(versions.len()); + for version in versions.iter() { + let version_dir = Client::full_upgrade_path( + replicated, + *version, + schema_dir.as_ref(), + ); + fs::create_dir_all(&version_dir) + .await + .expect("failed to make version directory"); + paths.push(version_dir); + } + (schema_dir, paths) + } + + #[tokio::test] + async fn test_read_schema_upgrade_sql_files() { + let logctx = test_setup_log("test_read_schema_upgrade_sql_files"); + let log = &logctx.log; + const REPLICATED: bool = false; + const VERSION: u64 = 1; + let (schema_dir, version_dirs) = + create_test_upgrade_schema_directory(REPLICATED, &[VERSION]).await; + let version_dir = &version_dirs[0]; + + // Create a few SQL files in there. + const SQL: &str = "SELECT NOW();"; + let filenames: Vec<_> = (0..3).map(|i| format!("up-{i}.sql")).collect(); + for name in filenames.iter() { + let full_path = version_dir.join(name); + fs::write(full_path, SQL).await.expect("Failed to write dummy SQL"); + } + + let upgrade_files = Client::read_schema_upgrade_sql_files( + log, + REPLICATED, + VERSION, + schema_dir.path(), + ) + .await + .expect("Failed to read schema upgrade files"); + for filename in filenames.iter() { + let stem = filename.split_once('.').unwrap().0; + assert_eq!( + upgrade_files.get(stem).unwrap().1, + SQL, + "upgrade SQL file contents are not correct" + ); + } + logctx.cleanup_successful(); + } + + async fn test_apply_one_schema_upgrade_impl( + log: &Logger, + address: SocketAddr, + replicated: bool, + ) { + let test_name = format!( + "test_apply_one_schema_upgrade_{}", + if replicated { "replicated" } else { "single_node" } + ); + let client = Client::new(address, &log); + + // We'll test moving from version 1, which just creates a database and + // table, to version 2, which adds two columns to that table in + // different SQL files. + client.execute(format!("CREATE DATABASE {test_name};")).await.unwrap(); + client + .execute(format!( + "\ + CREATE TABLE {test_name}.tbl (\ + `col0` UInt8 \ + )\ + ENGINE = MergeTree() + ORDER BY `col0`;\ + " + )) + .await + .unwrap(); + + // Write out the upgrading SQL files. + // + // Note that all of these statements are going in the version 2 schema + // directory. + let (schema_dir, version_dirs) = + create_test_upgrade_schema_directory(replicated, &[NEXT_VERSION]) + .await; + const NEXT_VERSION: u64 = 2; + let first_sql = + format!("ALTER TABLE {test_name}.tbl ADD COLUMN `col1` UInt16;"); + let second_sql = + format!("ALTER TABLE {test_name}.tbl ADD COLUMN `col2` String;"); + let all_sql = [first_sql, second_sql]; + let version_dir = &version_dirs[0]; + for (i, sql) in all_sql.iter().enumerate() { + let path = version_dir.join(format!("up-{i}.sql")); + fs::write(path, sql) + .await + .expect("failed to write out upgrade SQL file"); + } + + // Apply the upgrade itself. + client + .apply_one_schema_upgrade( + replicated, + NEXT_VERSION, + schema_dir.path(), + ) + .await + .expect("Failed to apply one schema upgrade"); + + // Check that it actually worked! + let body = client + .execute_with_body(format!( + "\ + SELECT name, type FROM system.columns \ + WHERE database = '{test_name}' AND table = 'tbl' \ + ORDER BY name \ + FORMAT CSV;\ + " + )) + .await + .unwrap(); + let mut lines = body.lines(); + assert_eq!(lines.next().unwrap(), "\"col0\",\"UInt8\""); + assert_eq!(lines.next().unwrap(), "\"col1\",\"UInt16\""); + assert_eq!(lines.next().unwrap(), "\"col2\",\"String\""); + assert!(lines.next().is_none()); + } + + #[tokio::test] + async fn test_apply_one_schema_upgrade_replicated() { + const TEST_NAME: &str = "test_apply_one_schema_upgrade_replicated"; + let logctx = test_setup_log(TEST_NAME); + let log = &logctx.log; + let mut cluster = create_cluster().await; + let address = cluster.replica_1.address; + test_apply_one_schema_upgrade_impl(log, address, true).await; + + // TODO-cleanup: These should be arrays. + // See https://github.com/oxidecomputer/omicron/issues/4460. + cluster + .keeper_1 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 1"); + cluster + .keeper_2 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 2"); + cluster + .keeper_3 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 3"); + cluster + .replica_1 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse server 1"); + cluster + .replica_2 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse server 2"); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_apply_one_schema_upgrade_single_node() { + const TEST_NAME: &str = "test_apply_one_schema_upgrade_single_node"; + let logctx = test_setup_log(TEST_NAME); + let log = &logctx.log; + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new(Ipv6Addr::LOCALHOST.into(), db.port()); + test_apply_one_schema_upgrade_impl(log, address, false).await; + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_ensure_schema_with_version_gaps_fails() { + let logctx = + test_setup_log("test_ensure_schema_with_version_gaps_fails"); + let log = &logctx.log; + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new(Ipv6Addr::LOCALHOST.into(), db.port()); + let client = Client::new(address, &log); + const REPLICATED: bool = false; + client + .initialize_db_with_version( + REPLICATED, + crate::model::OXIMETER_VERSION, + ) + .await + .expect("failed to initialize DB"); + + const BOGUS_VERSION: u64 = u64::MAX; + let (schema_dir, _) = create_test_upgrade_schema_directory( + REPLICATED, + &[crate::model::OXIMETER_VERSION, BOGUS_VERSION], + ) + .await; + + let err = client + .ensure_schema(REPLICATED, BOGUS_VERSION, schema_dir.path()) + .await + .expect_err( + "Should have received an error when ensuring \ + non-sequential version numbers", + ); + let Error::NonSequentialSchemaVersions = err else { + panic!( + "Expected an Error::NonSequentialSchemaVersions, found {err:?}" + ); + }; + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_ensure_schema_with_missing_desired_schema_version_fails() { + let logctx = test_setup_log( + "test_ensure_schema_with_missing_desired_schema_version_fails", + ); + let log = &logctx.log; + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new(Ipv6Addr::LOCALHOST.into(), db.port()); + let client = Client::new(address, &log); + const REPLICATED: bool = false; + client + .initialize_db_with_version( + REPLICATED, + crate::model::OXIMETER_VERSION, + ) + .await + .expect("failed to initialize DB"); + + let (schema_dir, _) = create_test_upgrade_schema_directory( + REPLICATED, + &[crate::model::OXIMETER_VERSION], + ) + .await; + + const BOGUS_VERSION: u64 = u64::MAX; + let err = client.ensure_schema( + REPLICATED, + BOGUS_VERSION, + schema_dir.path(), + ).await + .expect_err("Should have received an error when ensuring a non-existing version"); + let Error::MissingSchemaVersion(missing) = err else { + panic!("Expected an Error::MissingSchemaVersion, found {err:?}"); + }; + assert_eq!(missing, BOGUS_VERSION); + + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } + + async fn test_ensure_schema_walks_through_multiple_steps_impl( + log: &Logger, + address: SocketAddr, + replicated: bool, + ) { + let test_name = format!( + "test_ensure_schema_walks_through_multiple_steps_{}", + if replicated { "replicated" } else { "single_node" } + ); + let client = Client::new(address, &log); + + // We need to actually have the oximeter DB here, and the version table, + // since `ensure_schema()` writes out versions to the DB as they're + // applied. + client.initialize_db_with_version(replicated, 1).await.unwrap(); + + // We'll test moving from version 1, which just creates a database and + // table, to version 3, stopping off at version 2. This is similar to + // the `test_apply_one_schema_upgrade` test, but we split the two + // modifications over two versions, rather than as multiple schema + // upgrades in one version bump. + client.execute(format!("CREATE DATABASE {test_name};")).await.unwrap(); + client + .execute(format!( + "\ + CREATE TABLE {test_name}.tbl (\ + `col0` UInt8 \ + )\ + ENGINE = MergeTree() + ORDER BY `col0`;\ + " + )) + .await + .unwrap(); + + // Write out the upgrading SQL files. + // + // Note that each statement goes into a different version. + const VERSIONS: [u64; 3] = [1, 2, 3]; + let (schema_dir, version_dirs) = + create_test_upgrade_schema_directory(replicated, &VERSIONS).await; + let first_sql = String::new(); + let second_sql = + format!("ALTER TABLE {test_name}.tbl ADD COLUMN `col1` UInt16;"); + let third_sql = + format!("ALTER TABLE {test_name}.tbl ADD COLUMN `col2` String;"); + let all_sql = [first_sql, second_sql, third_sql]; + for (version_dir, sql) in version_dirs.iter().zip(all_sql) { + let path = version_dir.join("up.sql"); + fs::write(path, sql) + .await + .expect("failed to write out upgrade SQL file"); + } + + // Apply the sequence of upgrades. + client + .ensure_schema( + replicated, + *VERSIONS.last().unwrap(), + schema_dir.path(), + ) + .await + .expect("Failed to apply one schema upgrade"); + + // Check that it actually worked! + let body = client + .execute_with_body(format!( + "\ + SELECT name, type FROM system.columns \ + WHERE database = '{test_name}' AND table = 'tbl' \ + ORDER BY name \ + FORMAT CSV;\ + " + )) + .await + .unwrap(); + let mut lines = body.lines(); + assert_eq!(lines.next().unwrap(), "\"col0\",\"UInt8\""); + assert_eq!(lines.next().unwrap(), "\"col1\",\"UInt16\""); + assert_eq!(lines.next().unwrap(), "\"col2\",\"String\""); + assert!(lines.next().is_none()); + + let latest_version = client.read_latest_version().await.unwrap(); + assert_eq!( + latest_version, + *VERSIONS.last().unwrap(), + "Updated version not written to the database" + ); + } + + #[tokio::test] + async fn test_ensure_schema_walks_through_multiple_steps_single_node() { + const TEST_NAME: &str = + "test_ensure_schema_walks_through_multiple_steps_single_node"; + let logctx = test_setup_log(TEST_NAME); + let log = &logctx.log; + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new(Ipv6Addr::LOCALHOST.into(), db.port()); + test_ensure_schema_walks_through_multiple_steps_impl( + log, address, false, + ) + .await; + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_ensure_schema_walks_through_multiple_steps_replicated() { + const TEST_NAME: &str = + "test_ensure_schema_walks_through_multiple_steps_replicated"; + let logctx = test_setup_log(TEST_NAME); + let log = &logctx.log; + let mut cluster = create_cluster().await; + let address = cluster.replica_1.address; + test_ensure_schema_walks_through_multiple_steps_impl( + log, address, true, + ) + .await; + cluster + .keeper_1 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 1"); + cluster + .keeper_2 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 2"); + cluster + .keeper_3 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse keeper 3"); + cluster + .replica_1 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse server 1"); + cluster + .replica_2 + .cleanup() + .await + .expect("Failed to cleanup ClickHouse server 2"); + logctx.cleanup_successful(); + } + + #[test] + fn test_verify_schema_upgrades() { + let mut map = BTreeMap::new(); + + // Check that we fail if the upgrade tries to insert data. + map.insert( + "up".into(), + ( + PathBuf::from("/foo/bar/up.sql"), + String::from( + "INSERT INTO oximeter.version (*) VALUES (100, now());", + ), + ), + ); + assert!(Client::verify_schema_upgrades(&map).is_err()); + + // Sanity check for the normal case. + map.clear(); + map.insert( + "up".into(), + ( + PathBuf::from("/foo/bar/up.sql"), + String::from("ALTER TABLE oximeter.measurements_bool ADD COLUMN foo UInt64;") + ), + ); + assert!(Client::verify_schema_upgrades(&map).is_ok()); + + // Check that we fail if the upgrade ties to delete any data. + map.clear(); + map.insert( + "up".into(), + ( + PathBuf::from("/foo/bar/up.sql"), + String::from("ALTER TABLE oximeter.measurements_bool DELETE WHERE timestamp < NOW();") + ), + ); + assert!(Client::verify_schema_upgrades(&map).is_err()); + + // Check that we fail if the upgrade contains multiple SQL statements. + map.clear(); + map.insert( + "up".into(), + ( + PathBuf::from("/foo/bar/up.sql"), + String::from( + "\ + ALTER TABLE oximeter.measurements_bool \ + ADD COLUMN foo UInt8; \ + ALTER TABLE oximeter.measurements_bool \ + ADD COLUMN bar UInt8; \ + ", + ), + ), + ); + assert!(Client::verify_schema_upgrades(&map).is_err()); + } + + // Regression test for https://github.com/oxidecomputer/omicron/issues/4369. + // + // This tests that we can successfully query all extant field types from the + // schema table. There may be no such values, but the query itself should + // succeed. + #[tokio::test] + async fn test_select_all_field_types() { + use strum::IntoEnumIterator; + usdt::register_probes().unwrap(); + let logctx = test_setup_log("test_select_all_field_types"); + let log = &logctx.log; + + let mut db = ClickHouseInstance::new_single_node(0) + .await + .expect("Failed to start ClickHouse"); + let address = SocketAddr::new(Ipv6Addr::LOCALHOST.into(), db.port()); + let client = Client::new(address, &log); + client + .init_single_node_db() + .await + .expect("Failed to initialize timeseries database"); + + // Attempt to select all schema with each field type. + for ty in oximeter::FieldType::iter() { + let sql = format!( + "SELECT COUNT() \ + FROM {}.timeseries_schema \ + WHERE arrayFirstIndex(x -> x = '{:?}', fields.type) > 0;", + crate::DATABASE_NAME, + crate::model::DbFieldType::from(ty), + ); + let res = client.execute_with_body(sql).await.unwrap(); + let count = res.trim().parse::().unwrap(); + assert_eq!(count, 0); + } + db.cleanup().await.expect("Failed to cleanup ClickHouse server"); + logctx.cleanup_successful(); + } } diff --git a/oximeter/db/src/lib.rs b/oximeter/db/src/lib.rs index 11ecbeddc87..425c5189eec 100644 --- a/oximeter/db/src/lib.rs +++ b/oximeter/db/src/lib.rs @@ -15,7 +15,9 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::convert::TryFrom; +use std::io; use std::num::NonZeroU32; +use std::path::PathBuf; use thiserror::Error; mod client; @@ -23,7 +25,9 @@ pub mod model; pub mod query; pub use client::{Client, DbWrite}; -#[derive(Clone, Debug, Error)] +pub use model::OXIMETER_VERSION; + +#[derive(Debug, Error)] pub enum Error { #[error("Oximeter core error: {0}")] Oximeter(#[from] oximeter::MetricsError), @@ -79,6 +83,38 @@ pub enum Error { #[error("Query must resolve to a single timeseries if limit is specified")] InvalidLimitQuery, + + #[error("Database is not at expected version")] + DatabaseVersionMismatch { expected: u64, found: u64 }, + + #[error("Could not read schema directory")] + ReadSchemaDir { + context: String, + #[source] + err: io::Error, + }, + + #[error("Could not read SQL file from path")] + ReadSqlFile { + context: String, + #[source] + err: io::Error, + }, + + #[error("Non-UTF8 schema directory entry")] + NonUtf8SchemaDirEntry(std::ffi::OsString), + + #[error("Missing desired schema version: {0}")] + MissingSchemaVersion(u64), + + #[error("Data-modifying operations are not supported in schema updates")] + SchemaUpdateModifiesData { path: PathBuf, statement: String }, + + #[error("Schema update SQL files should contain at most 1 statement")] + MultipleSqlStatementsInSchemaUpdate { path: PathBuf }, + + #[error("Schema update versions must be sequential without gaps")] + NonSequentialSchemaVersions, } /// A timeseries name. diff --git a/oximeter/db/src/model.rs b/oximeter/db/src/model.rs index 41c7ab9d249..715e025a04d 100644 --- a/oximeter/db/src/model.rs +++ b/oximeter/db/src/model.rs @@ -38,11 +38,12 @@ use uuid::Uuid; /// Describes the version of the Oximeter database. /// -/// See: [crate::Client::initialize_db_with_version] for usage. +/// For usage and details see: /// -/// TODO(#4271): The current implementation of versioning will wipe the metrics -/// database if this number is incremented. -pub const OXIMETER_VERSION: u64 = 2; +/// - [`crate::Client::initialize_db_with_version`] +/// - [`crate::Client::ensure_schema`] +/// - The `clickhouse-schema-updater` binary in this crate +pub const OXIMETER_VERSION: u64 = 3; // Wrapper type to represent a boolean in the database. // diff --git a/oximeter/instruments/Cargo.toml b/oximeter/instruments/Cargo.toml index 3653ab80112..8372b7c5602 100644 --- a/oximeter/instruments/Cargo.toml +++ b/oximeter/instruments/Cargo.toml @@ -5,15 +5,27 @@ edition = "2021" license = "MPL-2.0" [dependencies] +cfg-if.workspace = true chrono.workspace = true dropshot.workspace = true futures.workspace = true +http = { workspace = true, optional = true } oximeter.workspace = true +slog.workspace = true tokio.workspace = true -http = { workspace = true, optional = true } +thiserror.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true [features] -default = ["http-instruments"] +default = ["http-instruments", "kstat"] http-instruments = ["http"] +kstat = ["kstat-rs"] + +[dev-dependencies] +rand.workspace = true +slog-async.workspace = true +slog-term.workspace = true + +[target.'cfg(target_os = "illumos")'.dependencies] +kstat-rs = { workspace = true, optional = true } diff --git a/oximeter/instruments/src/kstat/link.rs b/oximeter/instruments/src/kstat/link.rs new file mode 100644 index 00000000000..d22ac60378c --- /dev/null +++ b/oximeter/instruments/src/kstat/link.rs @@ -0,0 +1,653 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Report metrics about Ethernet data links on the host system + +use crate::kstat::hrtime_to_utc; +use crate::kstat::ConvertNamedData; +use crate::kstat::Error; +use crate::kstat::KstatList; +use crate::kstat::KstatTarget; +use chrono::DateTime; +use chrono::Utc; +use kstat_rs::Data; +use kstat_rs::Kstat; +use kstat_rs::Named; +use oximeter::types::Cumulative; +use oximeter::Metric; +use oximeter::Sample; +use oximeter::Target; +use uuid::Uuid; + +/// Information about a single physical Ethernet link on a host. +#[derive(Clone, Debug, Target)] +pub struct PhysicalDataLink { + /// The ID of the rack (cluster) containing this host. + pub rack_id: Uuid, + /// The ID of the sled itself. + pub sled_id: Uuid, + /// The serial number of the hosting sled. + pub serial: String, + /// The name of the host. + pub hostname: String, + /// The name of the link. + pub link_name: String, +} + +/// Information about a virtual Ethernet link on a host. +/// +/// Note that this is specifically for a VNIC in on the host system, not a guest +/// data link. +#[derive(Clone, Debug, Target)] +pub struct VirtualDataLink { + /// The ID of the rack (cluster) containing this host. + pub rack_id: Uuid, + /// The ID of the sled itself. + pub sled_id: Uuid, + /// The serial number of the hosting sled. + pub serial: String, + /// The name of the host, or the zone name for links in a zone. + pub hostname: String, + /// The name of the link. + pub link_name: String, +} + +/// Information about a guest virtual Ethernet link. +#[derive(Clone, Debug, Target)] +pub struct GuestDataLink { + /// The ID of the rack (cluster) containing this host. + pub rack_id: Uuid, + /// The ID of the sled itself. + pub sled_id: Uuid, + /// The serial number of the hosting sled. + pub serial: String, + /// The name of the host, or the zone name for links in a zone. + pub hostname: String, + /// The ID of the project containing the instance. + pub project_id: Uuid, + /// The ID of the instance. + pub instance_id: Uuid, + /// The name of the link. + pub link_name: String, +} + +/// The number of packets received on the link. +#[derive(Clone, Copy, Metric)] +pub struct PacketsReceived { + pub datum: Cumulative, +} + +/// The number of packets sent on the link. +#[derive(Clone, Copy, Metric)] +pub struct PacketsSent { + pub datum: Cumulative, +} + +/// The number of bytes sent on the link. +#[derive(Clone, Copy, Metric)] +pub struct BytesSent { + pub datum: Cumulative, +} + +/// The number of bytes received on the link. +#[derive(Clone, Copy, Metric)] +pub struct BytesReceived { + pub datum: Cumulative, +} + +/// The number of errors received on the link. +#[derive(Clone, Copy, Metric)] +pub struct ErrorsReceived { + pub datum: Cumulative, +} + +/// The number of errors sent on the link. +#[derive(Clone, Copy, Metric)] +pub struct ErrorsSent { + pub datum: Cumulative, +} + +// Helper function to extract the same kstat metrics from all link targets. +fn extract_link_kstats( + target: &T, + named_data: &Named, + creation_time: DateTime, + snapshot_time: DateTime, +) -> Option> +where + T: KstatTarget, +{ + let Named { name, value } = named_data; + if *name == "rbytes64" { + Some(value.as_u64().and_then(|x| { + let metric = BytesReceived { + datum: Cumulative::with_start_time(creation_time, x), + }; + Sample::new_with_timestamp(snapshot_time, target, &metric) + .map_err(Error::Sample) + })) + } else if *name == "obytes64" { + Some(value.as_u64().and_then(|x| { + let metric = BytesSent { + datum: Cumulative::with_start_time(creation_time, x), + }; + Sample::new_with_timestamp(snapshot_time, target, &metric) + .map_err(Error::Sample) + })) + } else if *name == "ipackets64" { + Some(value.as_u64().and_then(|x| { + let metric = PacketsReceived { + datum: Cumulative::with_start_time(creation_time, x), + }; + Sample::new_with_timestamp(snapshot_time, target, &metric) + .map_err(Error::Sample) + })) + } else if *name == "opackets64" { + Some(value.as_u64().and_then(|x| { + let metric = PacketsSent { + datum: Cumulative::with_start_time(creation_time, x), + }; + Sample::new_with_timestamp(snapshot_time, target, &metric) + .map_err(Error::Sample) + })) + } else if *name == "ierrors" { + Some(value.as_u32().and_then(|x| { + let metric = ErrorsReceived { + datum: Cumulative::with_start_time(creation_time, x.into()), + }; + Sample::new_with_timestamp(snapshot_time, target, &metric) + .map_err(Error::Sample) + })) + } else if *name == "oerrors" { + Some(value.as_u32().and_then(|x| { + let metric = ErrorsSent { + datum: Cumulative::with_start_time(creation_time, x.into()), + }; + Sample::new_with_timestamp(snapshot_time, target, &metric) + .map_err(Error::Sample) + })) + } else { + None + } +} + +// Helper trait for defining `KstatTarget` for all the link-based stats. +trait LinkKstatTarget: KstatTarget { + fn link_name(&self) -> &str; +} + +impl LinkKstatTarget for PhysicalDataLink { + fn link_name(&self) -> &str { + &self.link_name + } +} + +impl LinkKstatTarget for VirtualDataLink { + fn link_name(&self) -> &str { + &self.link_name + } +} + +impl LinkKstatTarget for GuestDataLink { + fn link_name(&self) -> &str { + &self.link_name + } +} + +impl KstatTarget for T +where + T: LinkKstatTarget, +{ + fn interested(&self, kstat: &Kstat<'_>) -> bool { + kstat.ks_module == "link" + && kstat.ks_instance == 0 + && kstat.ks_name == self.link_name() + } + + fn to_samples( + &self, + kstats: KstatList<'_, '_>, + ) -> Result, Error> { + let Some((creation_time, kstat, data)) = kstats.first() else { + return Ok(vec![]); + }; + let snapshot_time = hrtime_to_utc(kstat.ks_snaptime)?; + let Data::Named(named) = data else { + return Err(Error::ExpectedNamedKstat); + }; + named + .iter() + .filter_map(|nd| { + extract_link_kstats(self, nd, *creation_time, snapshot_time) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::kstat::sampler::KstatPath; + use crate::kstat::sampler::CREATION_TIME_PRUNE_INTERVAL; + use crate::kstat::CollectionDetails; + use crate::kstat::KstatSampler; + use crate::kstat::TargetStatus; + use kstat_rs::Ctl; + use oximeter::Producer; + use rand::distributions::Uniform; + use rand::Rng; + use slog::info; + use slog::Drain; + use slog::Logger; + use std::time::Duration; + use tokio::time::Instant; + use uuid::uuid; + use uuid::Uuid; + + fn test_logger() -> Logger { + let dec = + slog_term::PlainSyncDecorator::new(slog_term::TestStdoutWriter); + let drain = slog_term::FullFormat::new(dec).build().fuse(); + let log = + Logger::root(drain, slog::o!("component" => "fake-cleanup-task")); + log + } + + const RACK_ID: Uuid = uuid!("de784702-cafb-41a9-b3e5-93af189def29"); + const SLED_ID: Uuid = uuid!("88240343-5262-45f4-86f1-3c82fe383f2a"); + + // An etherstub we can use for testing. + // + // This is not meant to produce real data. It is simply a data link that + // shows up with the `link:::` kstat scheme, and which doesn't require us to + // decide which physical link over which to create something like a VNIC. + #[derive(Debug)] + struct TestEtherstub { + name: String, + } + + impl TestEtherstub { + const PFEXEC: &str = "/usr/bin/pfexec"; + const DLADM: &str = "/usr/sbin/dladm"; + fn new() -> Self { + let name = format!( + "kstest{}0", + rand::thread_rng() + .sample_iter(Uniform::new('a', 'z')) + .take(5) + .map(char::from) + .collect::(), + ); + Self::create(&name); + Self { name } + } + + fn create(name: &str) { + let output = std::process::Command::new(Self::PFEXEC) + .env_clear() + .arg(Self::DLADM) + .arg("create-etherstub") + .arg("-t") + .arg(name) + .output() + .expect("failed to spawn dladm"); + assert!( + output.status.success(), + "failed to create test etherstub:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + } + } + + impl Drop for TestEtherstub { + fn drop(&mut self) { + let output = std::process::Command::new(Self::PFEXEC) + .env_clear() + .arg(Self::DLADM) + .arg("delete-etherstub") + .arg(&self.name) + .output() + .expect("failed to spawn dladm"); + if !output.status.success() { + eprintln!( + "Failed to delete etherstub '{}'.\n\ + Delete manually with `dladm delete-etherstub {}`:\n{}", + &self.name, + &self.name, + String::from_utf8_lossy(&output.stderr), + ); + } + } + } + + #[test] + fn test_physical_datalink() { + let link = TestEtherstub::new(); + let sn = String::from("BRM000001"); + let dl = PhysicalDataLink { + rack_id: RACK_ID, + sled_id: SLED_ID, + serial: sn.clone(), + hostname: sn, + link_name: link.name.to_string(), + }; + let ctl = Ctl::new().unwrap(); + let ctl = ctl.update().unwrap(); + let mut kstat = ctl + .filter(Some("link"), Some(0), Some(dl.link_name.as_str())) + .next() + .unwrap(); + let creation_time = hrtime_to_utc(kstat.ks_crtime).unwrap(); + let data = ctl.read(&mut kstat).unwrap(); + let samples = dl.to_samples(&[(creation_time, kstat, data)]).unwrap(); + println!("{samples:#?}"); + } + + #[tokio::test] + async fn test_kstat_sampler() { + let mut sampler = KstatSampler::new(&test_logger()).unwrap(); + let sn = String::from("BRM000001"); + let link = TestEtherstub::new(); + let dl = PhysicalDataLink { + rack_id: RACK_ID, + sled_id: SLED_ID, + serial: sn.clone(), + hostname: sn, + link_name: link.name.to_string(), + }; + let details = CollectionDetails::never(Duration::from_secs(1)); + let id = sampler.add_target(dl, details).await.unwrap(); + let samples: Vec<_> = sampler.produce().unwrap().collect(); + assert!(samples.is_empty()); + + // Pause time, and advance until we're notified of new samples. + tokio::time::pause(); + const MAX_DURATION: Duration = Duration::from_secs(3); + const STEP_DURATION: Duration = Duration::from_secs(1); + let now = Instant::now(); + let expected_counts = loop { + tokio::time::advance(STEP_DURATION).await; + if now.elapsed() > MAX_DURATION { + panic!("Waited too long for samples"); + } + if let Some(counts) = sampler.sample_counts() { + break counts; + } + }; + let samples: Vec<_> = sampler.produce().unwrap().collect(); + println!("{samples:#?}"); + assert_eq!(samples.len(), expected_counts.total); + assert_eq!(expected_counts.overflow, 0); + + // Test status and remove behavior. + tokio::time::resume(); + assert!(matches!( + sampler.target_status(id).await.unwrap(), + TargetStatus::Ok { .. }, + )); + sampler.remove_target(id).await.unwrap(); + assert!(sampler.target_status(id).await.is_err()); + } + + #[tokio::test] + async fn test_kstat_sampler_with_overflow() { + let limit = 2; + let mut sampler = + KstatSampler::with_sample_limit(&test_logger(), limit).unwrap(); + let sn = String::from("BRM000001"); + let link = TestEtherstub::new(); + let dl = PhysicalDataLink { + rack_id: RACK_ID, + sled_id: SLED_ID, + serial: sn.clone(), + hostname: sn, + link_name: link.name.to_string(), + }; + let details = CollectionDetails::never(Duration::from_secs(1)); + sampler.add_target(dl, details).await.unwrap(); + let samples: Vec<_> = sampler.produce().unwrap().collect(); + assert!(samples.is_empty()); + + // Pause time, and advance until we're notified of new samples. + tokio::time::pause(); + const MAX_DURATION: Duration = Duration::from_secs(3); + const STEP_DURATION: Duration = Duration::from_secs(1); + let now = Instant::now(); + let expected_counts = loop { + tokio::time::advance(STEP_DURATION).await; + if now.elapsed() > MAX_DURATION { + panic!("Waited too long for samples"); + } + if let Some(counts) = sampler.sample_counts() { + break counts; + } + }; + + // We should have produced 2 samples from the actual target, plus one + // from the counter indicating we've dropped some samples! + let samples: Vec<_> = sampler.produce().unwrap().collect(); + let (link_samples, dropped_samples): (Vec<_>, Vec<_>) = samples + .iter() + .partition(|s| s.timeseries_name.contains("physical_data_link")); + println!("{link_samples:#?}"); + assert_eq!(link_samples.len(), limit); + + // The total number of samples less overflow should be match the number + // of samples for the link we've produced. + assert_eq!( + link_samples.len(), + expected_counts.total - expected_counts.overflow + ); + + // The worker must have produced one sample representing the number of + // overflows. + println!("{dropped_samples:#?}"); + assert_eq!(dropped_samples.len(), 1); + + // Verify that we actually counted the correct number of dropped + // samples. + let oximeter::Datum::CumulativeU64(overflow) = + dropped_samples[0].measurement.datum() + else { + unreachable!(); + }; + assert_eq!(overflow.value(), expected_counts.overflow as u64); + } + + #[tokio::test] + async fn test_kstat_with_expiration() { + // Create a VNIC, which we'll start tracking from, then delete it and + // make sure we expire after the expected period. + let log = test_logger(); + let mut sampler = KstatSampler::new(&log).unwrap(); + let sn = String::from("BRM000001"); + let link = TestEtherstub::new(); + info!(log, "created test etherstub"; "name" => &link.name); + let dl = PhysicalDataLink { + rack_id: RACK_ID, + sled_id: SLED_ID, + serial: sn.clone(), + hostname: sn, + link_name: link.name.to_string(), + }; + let collection_interval = Duration::from_secs(1); + let expiry = Duration::from_secs(1); + let details = CollectionDetails::duration(collection_interval, expiry); + let id = sampler.add_target(dl, details).await.unwrap(); + info!(log, "target added"; "id" => ?id); + assert!(matches!( + sampler.target_status(id).await.unwrap(), + TargetStatus::Ok { .. }, + )); + + // Delete the link right away. + drop(link); + info!(log, "dropped test etherstub"); + + // Pause time, and advance until we should have expired the target. + tokio::time::pause(); + const MAX_DURATION: Duration = Duration::from_secs(3); + let now = Instant::now(); + let is_expired = loop { + tokio::time::advance(expiry).await; + if now.elapsed() > MAX_DURATION { + panic!("Waited too long for samples"); + } + if let TargetStatus::Expired { .. } = + sampler.target_status(id).await.unwrap() + { + break true; + } + }; + assert!(is_expired, "Target should have expired by now"); + + // We should have some self-stat expiration samples now. + let samples = sampler.produce().unwrap(); + let expiration_samples: Vec<_> = samples + .filter(|sample| { + sample.timeseries_name == "kstat_sampler:expired_targets" + }) + .collect(); + assert_eq!(expiration_samples.len(), 1); + } + + // A sanity check that a cumulative start time does not change over time, + // since we've fixed the time reference at the time it was added. + #[tokio::test] + async fn test_kstat_start_time_is_equal() { + let log = test_logger(); + let mut sampler = KstatSampler::new(&log).unwrap(); + let sn = String::from("BRM000001"); + let link = TestEtherstub::new(); + info!(log, "created test etherstub"; "name" => &link.name); + let dl = PhysicalDataLink { + rack_id: RACK_ID, + sled_id: SLED_ID, + serial: sn.clone(), + hostname: sn, + link_name: link.name.to_string(), + }; + let collection_interval = Duration::from_secs(1); + let expiry = Duration::from_secs(1); + let details = CollectionDetails::duration(collection_interval, expiry); + let id = sampler.add_target(dl, details).await.unwrap(); + info!(log, "target added"; "id" => ?id); + assert!(matches!( + sampler.target_status(id).await.unwrap(), + TargetStatus::Ok { .. }, + )); + tokio::time::pause(); + let now = Instant::now(); + while now.elapsed() < (expiry * 10) { + tokio::time::advance(expiry).await; + } + let samples = sampler.produce().unwrap(); + let mut start_times = samples + .filter(|sample| { + sample.timeseries_name.as_str().starts_with("physical") + }) + .map(|sample| sample.measurement.start_time().unwrap()); + let first = start_times.next().unwrap(); + println!("{first}"); + assert!(start_times.all(|t| { + println!("{t}"); + t == first + })); + } + + #[tokio::test] + async fn test_prune_creation_times_when_kstat_is_gone() { + // Create a VNIC, which we'll start tracking from, then delete it and + // make sure the creation times are pruned. + let log = test_logger(); + let sampler = KstatSampler::new(&log).unwrap(); + let sn = String::from("BRM000001"); + let link = TestEtherstub::new(); + let path = KstatPath { + module: "link".to_string(), + instance: 0, + name: link.name.clone(), + }; + info!(log, "created test etherstub"; "name" => &link.name); + let dl = PhysicalDataLink { + rack_id: RACK_ID, + sled_id: SLED_ID, + serial: sn.clone(), + hostname: sn, + link_name: link.name.to_string(), + }; + let collection_interval = Duration::from_secs(1); + let expiry = Duration::from_secs(1); + let details = CollectionDetails::duration(collection_interval, expiry); + let id = sampler.add_target(dl, details).await.unwrap(); + info!(log, "target added"; "id" => ?id); + assert!(matches!( + sampler.target_status(id).await.unwrap(), + TargetStatus::Ok { .. }, + )); + + // Delete the link right away. + drop(link); + info!(log, "dropped test etherstub"); + + // Advance time through the prune interval. + tokio::time::pause(); + let now = Instant::now(); + while now.elapsed() < CREATION_TIME_PRUNE_INTERVAL + expiry { + tokio::time::advance(expiry).await; + } + + // Now check that the creation times are pruned. + let times = sampler.creation_times().await; + assert!(!times.contains_key(&path)); + } + + #[tokio::test] + async fn test_prune_creation_times_when_target_is_removed() { + // Create a VNIC, which we'll start tracking from, then delete it and + // make sure the creation times are pruned. + let log = test_logger(); + let sampler = KstatSampler::new(&log).unwrap(); + let sn = String::from("BRM000001"); + let link = TestEtherstub::new(); + let path = KstatPath { + module: "link".to_string(), + instance: 0, + name: link.name.clone(), + }; + info!(log, "created test etherstub"; "name" => &link.name); + let dl = PhysicalDataLink { + rack_id: RACK_ID, + sled_id: SLED_ID, + serial: sn.clone(), + hostname: sn, + link_name: link.name.to_string(), + }; + let collection_interval = Duration::from_secs(1); + let expiry = Duration::from_secs(1); + let details = CollectionDetails::duration(collection_interval, expiry); + let id = sampler.add_target(dl, details).await.unwrap(); + info!(log, "target added"; "id" => ?id); + assert!(matches!( + sampler.target_status(id).await.unwrap(), + TargetStatus::Ok { .. }, + )); + + // Remove the target, but do not drop the link. This will mean that the + // underlying kstat is still around, even though there's no target + // that's interested in it. We should keep it, in this case. + sampler.remove_target(id).await.unwrap(); + + // Advance time through the prune interval. + tokio::time::pause(); + let now = Instant::now(); + while now.elapsed() < CREATION_TIME_PRUNE_INTERVAL + expiry { + tokio::time::advance(expiry).await; + } + + // Now check that the creation time is still around. + let times = sampler.creation_times().await; + assert!(times.contains_key(&path)); + } +} diff --git a/oximeter/instruments/src/kstat/mod.rs b/oximeter/instruments/src/kstat/mod.rs new file mode 100644 index 00000000000..90f34acae85 --- /dev/null +++ b/oximeter/instruments/src/kstat/mod.rs @@ -0,0 +1,267 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +//! Types for publishing kernel statistics via oximeter. +//! +//! # illumos kernel statistics +//! +//! illumos defines a generic framework for tracking statistics throughout the +//! OS, called `kstat`. These statistics are generally defined by the kernel or +//! device drivers, and made available to userspace for reading through +//! `libkstat`. +//! +//! Kernel statistics are defined by a 4-tuple of names, canonically separated +//! with a ":". For example, the `cpu:0:vm:pgin` kstat tracks the number of +//! memory pages paged in for CPU 0. In this case, the kstat is tracked by a +//! u64, though several data types are supported. +//! +//! This module uses the `kstat-rs` crate, which is a fairly low-level wrapper +//! around `libkstat`. For the purposes of this module, most folks will be +//! interested in the types [`kstat_rs::Kstat`], [`kstat_rs::Data`], and +//! [`kstat_rs::NamedData`]. +//! +//! # Oximeter +//! +//! Oximeter is the Oxide control plane component which collects telemetry from +//! other parts of the system, called _producers_. Statistics are defined using +//! a _target_, which is an object about which statistics are collected, and +//! _metrics_, which are the actual measurements about a target. As an example, +//! a target might be an NVMe drive on the rack, and a metric might be its +//! temperature, or the estimated remaining drive lifetime. +//! +//! Targets and metrics are encapsulated by traits of the same name. Using +//! these, producers can generate timestamped [`Sample`]s, which the `oximeter` +//! collector program pulls at regular intervals for storage in the timeseries +//! database. +//! +//! # What does this mod do? +//! +//! This module is intended to connect illumos kstats with oximeter. Developers +//! can use this to define a mapping betweeen one or more kstats, and an +//! oximeter `Target` and `Metric`. This mapping is encapsulated in the +//! [`KstatTarget`] trait, which extends the `Target` trait itself. +//! +//! To implement the trait, developers register their interest in a particular +//! [`Kstat`] through the [`KstatTarget::interested()`] method. They then +//! describe how to generate any number of [`Sample`]s from that set of kstats, +//! through the [`KstatTarget::to_samples()`] method. +//! +//! # The [`KstatSampler`] +//! +//! Most folks will instantiate a [`KstatSampler`], which manages any number of +//! tracked `KstatTarget`s. Users can register their implementation of +//! `KstatTarget` with the sampler, and it will periodically generate samples +//! from it, converting the "interesting" kstats into `Sample`s. +//! +//! # Intervals and expiration +//! +//! When users register a target for sampling, they are required to include +//! details about how often their target should be sampled, and what to do if we +//! cannot produce samples due to an error, or if there are _no kstats_ that the +//! target is interested in. These details are captured in the +//! [`CollectionDetails`] type. +//! +//! After a configurable period of errors (expressed in either consecutive error +//! counts or a duration of concecutive errors), a target is _expired_, and will +//! no longer be collected from. A target's status may be queried with the +//! [`KstatSampler::target_status()`] method, which will inform the caller if +//! the target has expired. In this case, users can re-register a target, which +//! will replace the expired one, generating new samples (assuming the error +//! condition has been resolved). + +use chrono::DateTime; +use chrono::Utc; +use kstat_rs::Data; +use kstat_rs::Error as KstatError; +use kstat_rs::Kstat; +use kstat_rs::NamedData; +use kstat_rs::NamedType; +use oximeter::FieldValue; +use oximeter::MetricsError; +use oximeter::Sample; +use oximeter::Target; +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::time::Duration; + +pub mod link; +mod sampler; + +pub use sampler::CollectionDetails; +pub use sampler::ExpirationBehavior; +pub use sampler::KstatSampler; +pub use sampler::TargetId; +pub use sampler::TargetStatus; + +/// The reason a kstat target was expired and removed from a sampler. +#[derive(Clone, Copy, Debug)] +pub enum ExpirationReason { + /// Expired after too many failed attempts. + Attempts(usize), + /// Expired after a defined interval of consistent failures. + Duration(Duration), +} + +/// An error describing why a kstat target was expired. +#[derive(Debug)] +pub struct Expiration { + /// The reason for expiration. + pub reason: ExpirationReason, + /// The last error before expiration. + pub error: Box, + /// The time at which the expiration occurred. + #[cfg(test)] + pub expired_at: tokio::time::Instant, + #[cfg(not(test))] + pub expired_at: DateTime, +} + +/// Errors resulting from reporting kernel statistics. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Could not find kstat with the expected name")] + NoSuchKstat, + + #[error("Kstat does not have the expected data type")] + UnexpectedDataType { expected: NamedType, found: NamedType }, + + #[error("Expected a named kstat")] + ExpectedNamedKstat, + + #[error("Duplicate target instance")] + DuplicateTarget { + target_name: String, + fields: BTreeMap, + }, + + #[error("No such kstat target exists")] + NoSuchTarget, + + #[error("Error generating sample")] + Sample(#[from] MetricsError), + + #[error("Kstat library error")] + Kstat(#[from] KstatError), + + #[error("Kstat control handle is not available")] + NoKstatCtl, + + #[error("Overflow computing kstat creation or snapshot time")] + TimestampOverflow, + + #[error("Failed to send message to background sampling task")] + SendError, + + #[error("Failed to receive message from background sampling task")] + RecvError, + + #[error("Expired following too many unsuccessful collection attempts")] + Expired(Expiration), + + #[error("Expired after unsucessfull collections for {duration:?}")] + ExpiredAfterDuration { duration: Duration, error: Box }, +} + +/// Type alias for a list of kstats. +/// +/// This includes the kstat's creation time, the kstat itself, and its data. +pub type KstatList<'a, 'k> = &'a [(DateTime, Kstat<'k>, Data<'k>)]; + +/// A trait for generating oximeter samples from a kstat. +/// +/// This trait is used to describe the kernel statistics that are relevant for +/// an `oximeter::Target`, and how to generate samples from them. +pub trait KstatTarget: + Target + Send + Sync + 'static + std::fmt::Debug +{ + /// Return true for any kstat you're interested in. + fn interested(&self, kstat: &Kstat<'_>) -> bool; + + /// Convert from a kstat and its data to a list of samples. + fn to_samples( + &self, + kstats: KstatList<'_, '_>, + ) -> Result, Error>; +} + +/// Convert from a high-res timestamp into UTC, if possible. +pub fn hrtime_to_utc(hrtime: i64) -> Result, Error> { + let utc_now = Utc::now(); + let hrtime_now = unsafe { gethrtime() }; + match hrtime_now.cmp(&hrtime) { + Ordering::Equal => Ok(utc_now), + Ordering::Less => { + let offset = u64::try_from(hrtime - hrtime_now) + .map_err(|_| Error::TimestampOverflow)?; + Ok(utc_now + Duration::from_nanos(offset)) + } + Ordering::Greater => { + let offset = u64::try_from(hrtime_now - hrtime) + .map_err(|_| Error::TimestampOverflow)?; + Ok(utc_now - Duration::from_nanos(offset)) + } + } +} + +// Helper trait for converting a `NamedData` item into a specific contained data +// type, if possible. +pub(crate) trait ConvertNamedData { + fn as_i32(&self) -> Result; + fn as_u32(&self) -> Result; + fn as_i64(&self) -> Result; + fn as_u64(&self) -> Result; +} + +impl<'a> ConvertNamedData for NamedData<'a> { + fn as_i32(&self) -> Result { + if let NamedData::Int32(x) = self { + Ok(*x) + } else { + Err(Error::UnexpectedDataType { + expected: NamedType::Int32, + found: self.data_type(), + }) + } + } + + fn as_u32(&self) -> Result { + if let NamedData::UInt32(x) = self { + Ok(*x) + } else { + Err(Error::UnexpectedDataType { + expected: NamedType::UInt32, + found: self.data_type(), + }) + } + } + + fn as_i64(&self) -> Result { + if let NamedData::Int64(x) = self { + Ok(*x) + } else { + Err(Error::UnexpectedDataType { + expected: NamedType::Int64, + found: self.data_type(), + }) + } + } + + fn as_u64(&self) -> Result { + if let NamedData::UInt64(x) = self { + Ok(*x) + } else { + Err(Error::UnexpectedDataType { + expected: NamedType::UInt64, + found: self.data_type(), + }) + } + } +} + +#[link(name = "c")] +extern "C" { + fn gethrtime() -> i64; +} diff --git a/oximeter/instruments/src/kstat/sampler.rs b/oximeter/instruments/src/kstat/sampler.rs new file mode 100644 index 00000000000..bab8ad0ba51 --- /dev/null +++ b/oximeter/instruments/src/kstat/sampler.rs @@ -0,0 +1,1225 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Generate oximeter samples from kernel statistics. + +use crate::kstat::hrtime_to_utc; +use crate::kstat::Error; +use crate::kstat::Expiration; +use crate::kstat::ExpirationReason; +use crate::kstat::KstatTarget; +use chrono::DateTime; +use chrono::Utc; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use kstat_rs::Ctl; +use kstat_rs::Kstat; +use oximeter::types::Cumulative; +use oximeter::Metric; +use oximeter::MetricsError; +use oximeter::Sample; +use slog::debug; +use slog::error; +use slog::o; +use slog::trace; +use slog::warn; +use slog::Logger; +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::fmt; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::Mutex; +use std::task::Context; +use std::task::Poll; +use std::time::Duration; +use tokio::sync::mpsc; +use tokio::sync::oneshot; +use tokio::time::interval; +use tokio::time::sleep; +use tokio::time::Sleep; + +#[cfg(test)] +use tokio::time::Instant; + +// The `KstatSampler` generates some statistics about its own operation, mostly +// for surfacing failures to collect and dropped samples. +mod self_stats { + use super::BTreeMap; + use super::Cumulative; + use super::TargetId; + + /// Information identifying this kstat sampler. + #[derive(Debug, oximeter::Target)] + pub struct KstatSampler { + /// The hostname (or zonename) of the host machine. + pub hostname: String, + } + + /// The total number of samples dropped for a single target. + #[derive(Debug, oximeter::Metric)] + pub struct SamplesDropped { + /// The ID of the target being tracked. + pub target_id: u64, + /// The name of the target being tracked. + pub target_name: String, + pub datum: Cumulative, + } + + /// The cumulative number of expired targets. + #[derive(Debug, oximeter::Metric)] + pub struct ExpiredTargets { + pub datum: Cumulative, + } + + #[derive(Debug)] + pub struct SelfStats { + pub target: KstatSampler, + // We'll store it this way for quick lookups, and build the type as we + // need it when publishing the samples, from the key and value. + pub drops: BTreeMap<(TargetId, String), Cumulative>, + pub expired: ExpiredTargets, + } + + impl SelfStats { + pub fn new(hostname: String) -> Self { + Self { + target: KstatSampler { hostname }, + drops: BTreeMap::new(), + expired: ExpiredTargets { datum: Cumulative::new(0) }, + } + } + } +} + +/// An identifier for a single tracked kstat target. +/// +/// This opaque identifier can be used to unregister targets from the sampler. +/// If not removed, data from the targets will be produced according to the +/// [`ExpirationBehavior`] configured for the target. +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct TargetId(u64); + +impl fmt::Debug for TargetId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl fmt::Display for TargetId { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// When to expire kstat which can no longer be collected from. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum ExpirationBehavior { + /// Never stop attempting to produce data from this target. + Never, + /// Expire after a number of sequential failed collections. + /// + /// If the payload is 0, expire after the first failure. + Attempts(usize), + /// Expire after a specified period of failing to collect. + Duration(Duration), +} + +/// Details about the collection and expiration intervals for a target. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct CollectionDetails { + /// The interval on which data from the target is collected. + pub interval: Duration, + /// The expiration behavior, specifying how to handle situations in which we + /// cannot collect kstat samples. + /// + /// Note that this includes both errors during an attempt to collect, and + /// situations in which the target doesn't signal interest in any kstats. + /// The latter can occur if the physical resource (such as a datalink) that + /// underlies the kstat has disappeared, for example. + pub expiration: ExpirationBehavior, +} + +impl CollectionDetails { + /// Return collection details with no expiration. + pub fn never(interval: Duration) -> Self { + Self { interval, expiration: ExpirationBehavior::Never } + } + + /// Return collection details that expires after a number of attempts. + pub fn attempts(interval: Duration, count: usize) -> Self { + Self { interval, expiration: ExpirationBehavior::Attempts(count) } + } + + /// Return collection details that expires after a duration has elapsed. + pub fn duration(interval: Duration, duration: Duration) -> Self { + Self { interval, expiration: ExpirationBehavior::Duration(duration) } + } +} + +/// The status of a sampled kstat-based target. +#[derive(Clone, Debug)] +pub enum TargetStatus { + /// The target is currently being collected from normally. + /// + /// The timestamp of the last collection is included. + Ok { + #[cfg(test)] + last_collection: Option, + #[cfg(not(test))] + last_collection: Option>, + }, + /// The target has been expired. + /// + /// The details about the expiration are included. + Expired { + reason: ExpirationReason, + // NOTE: The error is a string, because it's not cloneable. + error: String, + #[cfg(test)] + expired_at: Instant, + #[cfg(not(test))] + expired_at: DateTime, + }, +} + +/// A request sent from `KstatSampler` to the worker task. +// NOTE: The docstrings here are public for ease of consumption by IDEs and +// other tooling. +#[derive(Debug)] +enum Request { + /// Add a new target for sampling. + AddTarget { + target: Box, + details: CollectionDetails, + reply_tx: oneshot::Sender>, + }, + /// Request the status for a target + TargetStatus { + id: TargetId, + reply_tx: oneshot::Sender>, + }, + /// Remove a target. + RemoveTarget { id: TargetId, reply_tx: oneshot::Sender> }, + /// Return the creation times of all tracked / extant kstats. + #[cfg(test)] + CreationTimes { + reply_tx: oneshot::Sender>>, + }, +} + +/// Data about a single kstat target. +#[derive(Debug)] +struct SampledKstat { + /// The target from which to collect. + target: Box, + /// The details around collection and expiration behavior. + details: CollectionDetails, + /// The time at which we _added_ this target to the sampler. + #[cfg(test)] + time_added: Instant, + #[cfg(not(test))] + time_added: DateTime, + /// The last time we successfully collected from the target. + #[cfg(test)] + time_of_last_collection: Option, + #[cfg(not(test))] + time_of_last_collection: Option>, + /// Attempts since we last successfully collected from the target. + attempts_since_last_collection: usize, +} + +/// Represents the current state of a registered target. +/// +/// We use this to report the status of a kstat target, such as reporting if its +/// been expired. +#[derive(Debug)] +enum SampledObject { + Kstat(SampledKstat), + Expired(Expiration), +} + +/// Helper to hash a target, used for creating unique IDs for them. +fn hash_target(t: &dyn KstatTarget) -> TargetId { + use std::hash::Hash; + use std::hash::Hasher; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + t.name().hash(&mut hasher); + for f in t.fields() { + f.hash(&mut hasher); + } + TargetId(hasher.finish()) +} + +/// Small future that yields an ID for a target to be sampled after its +/// predefined interval expires. +struct YieldIdAfter { + /// Future to which we delegate to awake us after our interval. + sleep: Pin>, + /// The next interval to yield when we complete. + interval: Duration, + /// The ID of the target to yield when we complete. + id: TargetId, +} + +impl std::fmt::Debug for YieldIdAfter { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("YieldIdAfter") + .field("sleep", &"_") + .field("interval", &self.interval) + .field("id", &self.id) + .finish() + } +} + +impl YieldIdAfter { + fn new(id: TargetId, interval: Duration) -> Self { + Self { sleep: Box::pin(sleep(interval)), interval, id } + } +} + +impl core::future::Future for YieldIdAfter { + type Output = (TargetId, Duration); + + fn poll( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll { + match self.sleep.as_mut().poll(cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(_) => Poll::Ready((self.id, self.interval)), + } + } +} + +/// An owned type used to keep track of the creation time for each kstat in +/// which interest has been signaled. +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct KstatPath { + pub module: String, + pub instance: i32, + pub name: String, +} + +impl<'a> From> for KstatPath { + fn from(k: Kstat<'a>) -> Self { + Self { + module: k.ks_module.to_string(), + instance: k.ks_instance, + name: k.ks_name.to_string(), + } + } +} + +/// The interval on which we prune creation times. +/// +/// As targets are added and kstats are tracked, we store their creation times +/// in the `KstatSamplerWorker::creation_times` mapping. This lets us keep a +/// consistent start time, which is important for cumulative metrics. In order +/// to keep these consistent, we don't necessarily prune them as a target is +/// removed or interest in the kstat changes, since the target may be added +/// again. Instead, we prune the creation times only when the _kstat itself_ is +/// removed from the kstat chain. +pub(crate) const CREATION_TIME_PRUNE_INTERVAL: Duration = + Duration::from_secs(60); + +/// Type which owns the `kstat` chain and samples each target on an interval. +/// +/// This type runs in a separate tokio task. As targets are added, it schedules +/// futures which expire after the target's sampling interval, yielding the +/// target's ID. (This is the `YieldIdAfter` type.) When those futures complete, +/// this type then samples the kstats they indicate, and push those onto a +/// per-target queue of samples. +#[derive(Debug)] +struct KstatSamplerWorker { + log: Logger, + + /// The kstat chain. + ctl: Option, + + /// The set of registered targets to collect kstats from, ordered by their + /// IDs. + targets: BTreeMap, + + /// The set of creation times for all tracked kstats. + /// + /// As interest in kstats is noted, we add a creation time for those kstats + /// here. It is removed only when the kstat itself no longer appears on the + /// kstat chain, even if a caller removes the target or no longer signals + /// interest in it. + creation_times: BTreeMap>, + + /// The per-target queue of samples, pulled by the main `KstatSampler` type + /// when producing metrics. + samples: Arc>>>, + + /// Inbox channel on which the `KstatSampler` sends messages. + inbox: mpsc::Receiver, + + /// The maximum number of samples we allow in each per-target buffer, to + /// avoid huge allocations when we produce data faster than it's collected. + sample_limit: usize, + + /// Outbound queue on which to publish self statistics, which are expected to + /// be low-volume. + self_stat_queue: mpsc::Sender, + + /// The statistics we maintain about ourselves. + /// + /// This is an option, since it's possible we fail to extract the hostname + /// at construction time. In that case, we'll try again the next time we + /// need it. + self_stats: Option, +} + +fn hostname() -> Option { + let out = + std::process::Command::new("hostname").env_clear().output().ok()?; + if !out.status.success() { + return None; + } + Some(String::from_utf8_lossy(&out.stdout).trim().to_string()) +} + +/// Stores the number of samples taken, used for testing. +#[cfg(test)] +pub(crate) struct SampleCounts { + pub total: usize, + pub overflow: usize, +} + +impl KstatSamplerWorker { + /// Create a new sampler worker. + fn new( + log: Logger, + inbox: mpsc::Receiver, + self_stat_queue: mpsc::Sender, + samples: Arc>>>, + sample_limit: usize, + ) -> Result { + let ctl = Some(Ctl::new().map_err(Error::Kstat)?); + let self_stats = hostname().map(self_stats::SelfStats::new); + Ok(Self { + ctl, + log, + targets: BTreeMap::new(), + creation_times: BTreeMap::new(), + samples, + inbox, + sample_limit, + self_stat_queue, + self_stats, + }) + } + + /// Consume self and run its main polling loop. + /// + /// This will accept messages on its inbox, and also sample the registered + /// kstats at their intervals. Samples will be pushed onto the queue. + async fn run( + mut self, + #[cfg(test)] sample_count_tx: mpsc::UnboundedSender, + ) { + let mut sample_timeouts = FuturesUnordered::new(); + let mut creation_prune_interval = + interval(CREATION_TIME_PRUNE_INTERVAL); + creation_prune_interval.tick().await; // Completes immediately. + loop { + tokio::select! { + _ = creation_prune_interval.tick() => { + if let Err(e) = self.prune_creation_times() { + error!( + self.log, + "failed to prune creation times"; + "error" => ?e, + ); + } + } + maybe_id = sample_timeouts.next(), if !sample_timeouts.is_empty() => { + let Some((id, interval)) = maybe_id else { + unreachable!(); + }; + match self.sample_one(id) { + Ok(Some(samples)) => { + if samples.is_empty() { + debug!( + self.log, + "no new samples from target, requeueing"; + "id" => ?id, + ); + sample_timeouts.push(YieldIdAfter::new(id, interval)); + continue; + } + let n_samples = samples.len(); + debug!( + self.log, + "pulled samples from target"; + "id" => ?id, + "n_samples" => n_samples, + ); + + // Append any samples to the per-target queues. + // + // This returns None if there was no queue, and logs + // the error internally. We'll just go round the + // loop again in that case. + let Some(n_overflow_samples) = + self.append_per_target_samples(id, samples) else { + continue; + }; + + // Safety: We only get here if the `sample_one()` + // method works, which means we have a record of the + // target, and it's not expired. + if n_overflow_samples > 0 { + let SampledObject::Kstat(ks) = self.targets.get(&id).unwrap() else { + unreachable!(); + }; + self.increment_dropped_sample_counter( + id, + ks.target.name().to_string(), + n_overflow_samples, + ).await; + } + + // Send the total number of samples we've actually + // taken and the number we've appended over to any + // testing code which might be listening. + #[cfg(test)] + sample_count_tx.send(SampleCounts { + total: n_samples, + overflow: n_overflow_samples, + }).unwrap(); + + trace!( + self.log, + "re-queueing target for sampling"; + "id" => ?id, + "interval" => ?interval, + ); + sample_timeouts.push(YieldIdAfter::new(id, interval)); + } + Ok(None) => { + debug!( + self.log, + "sample timeout triggered for non-existent target"; + "id" => ?id, + ); + } + Err(Error::Expired(expiration)) => { + error!( + self.log, + "expiring kstat after too many failures"; + "id" => ?id, + "reason" => ?expiration.reason, + "error" => ?expiration.error, + ); + let _ = self.targets.insert(id, SampledObject::Expired(expiration)); + self.increment_expired_target_counter().await; + } + Err(e) => { + error!( + self.log, + "failed to sample kstat target, requeueing"; + "id" => ?id, + "error" => ?e, + ); + sample_timeouts.push(YieldIdAfter::new(id, interval)); + } + } + } + maybe_request = self.inbox.recv() => { + let Some(request) = maybe_request else { + debug!(self.log, "inbox returned None, exiting"); + return; + }; + trace!( + self.log, + "received request on inbox"; + "request" => ?request, + ); + match request { + Request::AddTarget { + target, + details, + reply_tx, + } => { + match self.add_target(target, details) { + Ok(id) => { + let timeout = YieldIdAfter::new(id, details.interval); + sample_timeouts.push(timeout); + trace!( + self.log, + "added target with timeout"; + "id" => ?id, + "details" => ?details, + ); + match reply_tx.send(Ok(id)) { + Ok(_) => trace!(self.log, "sent reply"), + Err(e) => error!( + self.log, + "failed to send reply"; + "id" => ?id, + "error" => ?e, + ) + } + } + Err(e) => { + error!( + self.log, + "failed to add target"; + "error" => ?e, + ); + match reply_tx.send(Err(e)) { + Ok(_) => trace!(self.log, "sent reply"), + Err(e) => error!( + self.log, + "failed to send reply"; + "error" => ?e, + ) + } + } + } + } + Request::RemoveTarget { id, reply_tx } => { + self.targets.remove(&id); + if let Some(remaining_samples) = self.samples.lock().unwrap().remove(&id) { + if !remaining_samples.is_empty() { + warn!( + self.log, + "target removed with queued samples"; + "id" => ?id, + "n_samples" => remaining_samples.len(), + ); + } + } + match reply_tx.send(Ok(())) { + Ok(_) => trace!(self.log, "sent reply"), + Err(e) => error!( + self.log, + "failed to send reply"; + "error" => ?e, + ) + } + } + Request::TargetStatus { id, reply_tx } => { + trace!( + self.log, + "request for target status"; + "id" => ?id, + ); + let response = match self.targets.get(&id) { + None => Err(Error::NoSuchTarget), + Some(SampledObject::Kstat(k)) => { + Ok(TargetStatus::Ok { + last_collection: k.time_of_last_collection, + }) + } + Some(SampledObject::Expired(e)) => { + Ok(TargetStatus::Expired { + reason: e.reason, + error: e.error.to_string(), + expired_at: e.expired_at, + }) + } + }; + match reply_tx.send(response) { + Ok(_) => trace!(self.log, "sent reply"), + Err(e) => error!( + self.log, + "failed to send reply"; + "id" => ?id, + "error" => ?e, + ), + } + } + #[cfg(test)] + Request::CreationTimes { reply_tx } => { + debug!(self.log, "request for creation times"); + reply_tx.send(self.creation_times.clone()).unwrap(); + debug!(self.log, "sent reply for creation times"); + } + } + } + } + } + } + + fn append_per_target_samples( + &self, + id: TargetId, + mut samples: Vec, + ) -> Option { + // Limit the number of samples we actually contain + // in the sample queue. This is to avoid huge + // allocations when we produce lots of data, but + // we're not polled quickly enough by oximeter. + // + // Note that this is a _per-target_ queue. + let mut all_samples = self.samples.lock().unwrap(); + let Some(current_samples) = all_samples.get_mut(&id) else { + error!( + self.log, + "per-target sample queue not found!"; + "id" => ?id, + ); + return None; + }; + let n_new_samples = samples.len(); + let n_current_samples = current_samples.len(); + let n_total_samples = n_new_samples + n_current_samples; + let n_overflow_samples = + n_total_samples.checked_sub(self.sample_limit).unwrap_or(0); + if n_overflow_samples > 0 { + warn!( + self.log, + "sample queue is too full, dropping oldest samples"; + "n_new_samples" => n_new_samples, + "n_current_samples" => n_current_samples, + "n_overflow_samples" => n_overflow_samples, + ); + // It's possible that the number of new samples + // is big enough to overflow the current + // capacity, and also require removing new + // samples. + if n_overflow_samples < n_current_samples { + let _ = current_samples.drain(..n_overflow_samples); + } else { + // Clear all the current samples, and some + // of the new ones. The subtraction below + // cannot panic, because + // `n_overflow_samples` is computed above by + // adding `n_current_samples`. + current_samples.clear(); + let _ = + samples.drain(..(n_overflow_samples - n_current_samples)); + } + } + current_samples.extend(samples); + Some(n_overflow_samples) + } + + /// Add samples for one target to the internal queue. + /// + /// Note that this updates the kstat chain. + fn sample_one( + &mut self, + id: TargetId, + ) -> Result>, Error> { + self.update_chain()?; + let ctl = self.ctl.as_ref().unwrap(); + let Some(maybe_kstat) = self.targets.get_mut(&id) else { + return Ok(None); + }; + let SampledObject::Kstat(sampled_kstat) = maybe_kstat else { + panic!("Should not be sampling an expired kstat"); + }; + + // Fetch each interested kstat, and include the data and creation times + // for each of them. + let kstats = ctl + .iter() + .filter(|kstat| sampled_kstat.target.interested(kstat)) + .map(|mut kstat| { + let data = ctl.read(&mut kstat)?; + let creation_time = Self::ensure_kstat_creation_time( + &self.log, + kstat, + &mut self.creation_times, + )?; + Ok((creation_time, kstat, data)) + }) + .collect::, _>>(); + match kstats { + Ok(k) if !k.is_empty() => { + cfg_if::cfg_if! { + if #[cfg(test)] { + sampled_kstat.time_of_last_collection = Some(Instant::now()); + } else { + sampled_kstat.time_of_last_collection = Some(Utc::now()); + } + } + sampled_kstat.attempts_since_last_collection = 0; + sampled_kstat.target.to_samples(&k).map(Option::Some) + } + other => { + // Convert a list of zero interested kstats into an error. + let e = match other { + Err(e) => e, + Ok(k) if k.is_empty() => { + trace!( + self.log, + "no matching samples for target, converting \ + to sampling error"; + "id" => ?id, + ); + Error::NoSuchKstat + } + _ => unreachable!(), + }; + sampled_kstat.attempts_since_last_collection += 1; + + // Check if this kstat should be expired, based on the attempts + // we've previously made and the expiration policy. + match sampled_kstat.details.expiration { + ExpirationBehavior::Never => {} + ExpirationBehavior::Attempts(n_attempts) => { + if sampled_kstat.attempts_since_last_collection + >= n_attempts + { + cfg_if::cfg_if! { + if #[cfg(test)] { + let expired_at = Instant::now(); + } else { + let expired_at = Utc::now(); + } + } + return Err(Error::Expired(Expiration { + reason: ExpirationReason::Attempts(n_attempts), + error: Box::new(e), + expired_at, + })); + } + } + ExpirationBehavior::Duration(duration) => { + // Use the time of the last collection, if one exists, + // or the time we added the kstat if not. + let start = sampled_kstat + .time_of_last_collection + .unwrap_or_else(|| sampled_kstat.time_added); + let expire_at = start + duration; + cfg_if::cfg_if! { + if #[cfg(test)] { + let now = Instant::now(); + } else { + let now = Utc::now(); + } + } + if now >= expire_at { + return Err(Error::Expired(Expiration { + reason: ExpirationReason::Duration(duration), + error: Box::new(e), + expired_at: now, + })); + } + } + } + + // Do not expire the kstat, simply fail this collection. + Err(e) + } + } + } + + async fn increment_dropped_sample_counter( + &mut self, + target_id: TargetId, + target_name: String, + n_overflow_samples: usize, + ) { + assert!(n_overflow_samples > 0); + if let Some(stats) = self.get_or_try_build_self_stats() { + // Get the entry for this target, or build a counter starting a 0. + // We'll always add the number of overflow samples afterwards. + let drops = stats + .drops + .entry((target_id, target_name.clone())) + .or_default(); + *drops += n_overflow_samples as u64; + let metric = self_stats::SamplesDropped { + target_id: target_id.0, + target_name, + datum: *drops, + }; + let sample = match Sample::new(&stats.target, &metric) { + Ok(s) => s, + Err(e) => { + error!( + self.log, + "could not generate sample for dropped sample counter"; + "error" => ?e, + ); + return; + } + }; + match self.self_stat_queue.send(sample).await { + Ok(_) => trace!(self.log, "sent dropped sample counter stat"), + Err(e) => error!( + self.log, + "failed to send dropped sample counter to self stat queue"; + "error" => ?e, + ), + } + } else { + warn!( + self.log, + "cannot record dropped sample statistic, failed to get hostname" + ); + } + } + + async fn increment_expired_target_counter(&mut self) { + if let Some(stats) = self.get_or_try_build_self_stats() { + stats.expired.datum_mut().increment(); + let sample = match Sample::new(&stats.target, &stats.expired) { + Ok(s) => s, + Err(e) => { + error!( + self.log, + "could not generate sample for expired target counter"; + "error" => ?e, + ); + return; + } + }; + match self.self_stat_queue.send(sample).await { + Ok(_) => trace!(self.log, "sent expired target counter stat"), + Err(e) => error!( + self.log, + "failed to send target counter to self stat queue"; + "error" => ?e, + ), + } + } else { + warn!( + self.log, + "cannot record expiration statistic, failed to get hostname" + ); + } + } + + /// If we have an actual `SelfStats` struct, return it, or try to create one. + /// We'll still return `None` in that latter case, and we fail to make one. + fn get_or_try_build_self_stats( + &mut self, + ) -> Option<&mut self_stats::SelfStats> { + if self.self_stats.is_none() { + self.self_stats = hostname().map(self_stats::SelfStats::new); + } + self.self_stats.as_mut() + } + + /// Ensure that we have recorded the creation time for all interested kstats + /// for a new target. + fn ensure_creation_times_for_target( + &mut self, + target: &dyn KstatTarget, + ) -> Result<(), Error> { + self.update_chain()?; + let ctl = self.ctl.as_ref().unwrap(); + for kstat in ctl.iter().filter(|k| target.interested(k)) { + Self::ensure_kstat_creation_time( + &self.log, + kstat, + &mut self.creation_times, + )?; + } + Ok(()) + } + + /// Ensure that we store the creation time for the provided kstat. + fn ensure_kstat_creation_time( + log: &Logger, + kstat: Kstat, + creation_times: &mut BTreeMap>, + ) -> Result, Error> { + let path = KstatPath::from(kstat); + match creation_times.entry(path.clone()) { + Entry::Occupied(entry) => { + trace!( + log, + "creation time already exists for tracked target"; + "path" => ?path, + ); + Ok(*entry.get()) + } + Entry::Vacant(entry) => { + let creation_time = hrtime_to_utc(kstat.ks_crtime)?; + debug!( + log, + "storing new creation time for tracked target"; + "path" => ?path, + "creation_time" => ?creation_time, + ); + entry.insert(creation_time); + Ok(creation_time) + } + } + } + + /// Prune the stored creation times, removing any that no longer have + /// corresponding kstats on the chain. + fn prune_creation_times(&mut self) -> Result<(), Error> { + if self.creation_times.is_empty() { + trace!(self.log, "no creation times to prune"); + return Ok(()); + } + // We'll create a list of all the creation times to prune, by + // progressively removing any _extant_ kstats from the set of keys we + // currently have. If something is _not_ on the chain, it'll remain in + // this map at the end of the loop below, and thus we know we need to + // remove it. + let mut to_remove: BTreeSet<_> = + self.creation_times.keys().cloned().collect(); + + // Iterate the chain, and remove any current keys that do _not_ appear + // on the chain. + self.update_chain()?; + let ctl = self.ctl.as_ref().unwrap(); + for kstat in ctl.iter() { + let path = KstatPath::from(kstat); + let _ = to_remove.remove(&path); + } + + if to_remove.is_empty() { + trace!(self.log, "kstat creation times is already pruned"); + } else { + debug!( + self.log, + "pruning creation times for kstats that are gone"; + "to_remove" => ?to_remove, + "n_to_remove" => to_remove.len(), + ); + self.creation_times.retain(|key, _value| !to_remove.contains(key)); + } + Ok(()) + } + + /// Start tracking a single KstatTarget object. + fn add_target( + &mut self, + target: Box, + details: CollectionDetails, + ) -> Result { + let id = hash_target(&*target); + match self.targets.get(&id) { + Some(SampledObject::Kstat(_)) => { + return Err(Error::DuplicateTarget { + target_name: target.name().to_string(), + fields: target + .field_names() + .iter() + .map(ToString::to_string) + .zip(target.field_values()) + .collect(), + }); + } + Some(SampledObject::Expired(e)) => { + warn!( + self.log, + "replacing expired kstat target"; + "id" => ?id, + "expiration_reason" => ?e.reason, + "error" => ?e.error, + "expired_at" => ?e.expired_at, + ); + } + None => {} + } + self.ensure_creation_times_for_target(&*target)?; + + cfg_if::cfg_if! { + if #[cfg(test)] { + let time_added = Instant::now(); + } else { + let time_added = Utc::now(); + } + } + let item = SampledKstat { + target, + details, + time_added, + time_of_last_collection: None, + attempts_since_last_collection: 0, + }; + let _ = self.targets.insert(id, SampledObject::Kstat(item)); + + // Add to the per-target queues, making sure to keep any samples that + // were already there previously. This would be a bit odd, since it + // means that the target expired, but we hadn't been polled by oximeter. + // Nonetheless keep these samples anyway. + let n_samples = + self.samples.lock().unwrap().entry(id).or_default().len(); + match n_samples { + 0 => debug!( + self.log, + "inserted empty per-target sample queue"; + "id" => ?id, + ), + n => debug!( + self.log, + "per-target queue appears to have old samples"; + "id" => ?id, + "n_samples" => n, + ), + } + Ok(id) + } + + fn update_chain(&mut self) -> Result<(), Error> { + let new_ctl = match self.ctl.take() { + None => Ctl::new(), + Some(old) => old.update(), + } + .map_err(Error::Kstat)?; + let _ = self.ctl.insert(new_ctl); + Ok(()) + } +} + +/// A type for reporting kernel statistics as oximeter samples. +#[derive(Clone, Debug)] +pub struct KstatSampler { + samples: Arc>>>, + outbox: mpsc::Sender, + self_stat_rx: Arc>>, + _worker_task: Arc>, + #[cfg(test)] + sample_count_rx: Arc>>, +} + +impl KstatSampler { + /// The maximum number of samples allowed in the internal buffer, before + /// oldest samples are dropped. + /// + /// This is to avoid unbounded allocations in situations where data is + /// produced faster than it is collected. + /// + /// Note that this is a _per-target_ sample limit! + pub const DEFAULT_SAMPLE_LIMIT: usize = 500; + + /// Create a new sampler. + pub fn new(log: &Logger) -> Result { + Self::with_sample_limit(log, Self::DEFAULT_SAMPLE_LIMIT) + } + + /// Create a new sampler with a sample limit. + pub fn with_sample_limit( + log: &Logger, + limit: usize, + ) -> Result { + let samples = Arc::new(Mutex::new(BTreeMap::new())); + let (self_stat_tx, self_stat_rx) = mpsc::channel(4096); + let (outbox, inbox) = mpsc::channel(1); + let worker = KstatSamplerWorker::new( + log.new(o!("component" => "kstat-sampler-worker")), + inbox, + self_stat_tx, + samples.clone(), + limit, + )?; + #[cfg(test)] + let (sample_count_rx, _worker_task) = { + let (sample_count_tx, sample_count_rx) = mpsc::unbounded_channel(); + ( + Arc::new(Mutex::new(sample_count_rx)), + Arc::new(tokio::task::spawn(worker.run(sample_count_tx))), + ) + }; + #[cfg(not(test))] + let _worker_task = Arc::new(tokio::task::spawn(worker.run())); + Ok(Self { + samples, + outbox, + self_stat_rx: Arc::new(Mutex::new(self_stat_rx)), + _worker_task, + #[cfg(test)] + sample_count_rx, + }) + } + + /// Add a target, which can be used to produce zero or more samples. + /// + /// Note that adding a target which has previously expired is _not_ an + /// error, and instead replaces the expired target. + pub async fn add_target( + &self, + target: impl KstatTarget, + details: CollectionDetails, + ) -> Result { + let (reply_tx, reply_rx) = oneshot::channel(); + let request = + Request::AddTarget { target: Box::new(target), details, reply_tx }; + self.outbox.send(request).await.map_err(|_| Error::SendError)?; + reply_rx.await.map_err(|_| Error::RecvError)? + } + + /// Remove a tracked target. + pub async fn remove_target(&self, id: TargetId) -> Result<(), Error> { + let (reply_tx, reply_rx) = oneshot::channel(); + let request = Request::RemoveTarget { id, reply_tx }; + self.outbox.send(request).await.map_err(|_| Error::SendError)?; + reply_rx.await.map_err(|_| Error::RecvError)? + } + + /// Fetch the status for a target. + /// + /// If the target is being collected normally, then `TargetStatus::Ok` is + /// returned, which contains the time of the last collection, if any. + /// + /// If the target exists, but has been expired, then the details about the + /// expiration are returned in `TargetStatus::Expired`. + /// + /// If the target doesn't exist at all, then an error is returned. + pub async fn target_status( + &self, + id: TargetId, + ) -> Result { + let (reply_tx, reply_rx) = oneshot::channel(); + let request = Request::TargetStatus { id, reply_tx }; + self.outbox.send(request).await.map_err(|_| Error::SendError)?; + reply_rx.await.map_err(|_| Error::RecvError)? + } + + /// Return the number of samples pushed by the sampling task, if any. + #[cfg(test)] + pub(crate) fn sample_counts(&self) -> Option { + match self.sample_count_rx.lock().unwrap().try_recv() { + Ok(c) => Some(c), + Err(mpsc::error::TryRecvError::Empty) => None, + _ => panic!("sample_tx disconnected"), + } + } + + /// Return the creation times for all tracked kstats. + #[cfg(test)] + pub(crate) async fn creation_times( + &self, + ) -> BTreeMap> { + let (reply_tx, reply_rx) = oneshot::channel(); + let request = Request::CreationTimes { reply_tx }; + self.outbox.send(request).await.map_err(|_| Error::SendError).unwrap(); + reply_rx.await.map_err(|_| Error::RecvError).unwrap() + } +} + +impl oximeter::Producer for KstatSampler { + fn produce( + &mut self, + ) -> Result)>, MetricsError> { + // Swap the _entries_ of all the existing per-target sample queues, but + // we need to leave empty queues in their place. I.e., we can't remove + // keys. + let mut samples = Vec::new(); + for (_id, queue) in self.samples.lock().unwrap().iter_mut() { + samples.append(queue); + } + + // Append any self-stat samples as well. + let mut rx = self.self_stat_rx.lock().unwrap(); + loop { + match rx.try_recv() { + Ok(sample) => samples.push(sample), + Err(mpsc::error::TryRecvError::Empty) => break, + Err(mpsc::error::TryRecvError::Disconnected) => { + panic!("kstat stampler self-stat queue tx disconnected"); + } + } + } + drop(rx); + + Ok(Box::new(samples.into_iter())) + } +} diff --git a/oximeter/instruments/src/lib.rs b/oximeter/instruments/src/lib.rs index d5c53fd05c0..d003e717395 100644 --- a/oximeter/instruments/src/lib.rs +++ b/oximeter/instruments/src/lib.rs @@ -4,7 +4,10 @@ //! General-purpose types for instrumenting code to producer oximeter metrics. -// Copyright 2021 Oxide Computer Company +// Copyright 2023 Oxide Computer Company #[cfg(feature = "http-instruments")] pub mod http; + +#[cfg(all(feature = "kstat", target_os = "illumos"))] +pub mod kstat; diff --git a/oximeter/oximeter/src/types.rs b/oximeter/oximeter/src/types.rs index 0cc3299ec40..325974781e2 100644 --- a/oximeter/oximeter/src/types.rs +++ b/oximeter/oximeter/src/types.rs @@ -42,6 +42,7 @@ use uuid::Uuid; JsonSchema, Serialize, Deserialize, + strum::EnumIter, )] #[serde(rename_all = "snake_case")] pub enum FieldType { diff --git a/oximeter/producer/Cargo.toml b/oximeter/producer/Cargo.toml index ef2f16c8adb..79f6c754f7e 100644 --- a/oximeter/producer/Cargo.toml +++ b/oximeter/producer/Cargo.toml @@ -11,7 +11,6 @@ dropshot.workspace = true nexus-client.workspace = true omicron-common.workspace = true oximeter.workspace = true -reqwest = { workspace = true, features = [ "json" ] } schemars = { workspace = true, features = [ "uuid1", "bytes", "chrono" ] } serde.workspace = true slog.workspace = true diff --git a/package-manifest.toml b/package-manifest.toml index b8ffb2756aa..61c90a3e750 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -107,9 +107,12 @@ setup_hint = """ service_name = "oximeter" only_for_targets.image = "standard" source.type = "local" -source.rust.binary_names = ["oximeter"] +source.rust.binary_names = ["oximeter", "clickhouse-schema-updater"] source.rust.release = true -source.paths = [ { from = "smf/oximeter", to = "/var/svc/manifest/site/oximeter" } ] +source.paths = [ + { from = "smf/oximeter", to = "/var/svc/manifest/site/oximeter" }, + { from = "oximeter/db/schema", to = "/opt/oxide/oximeter/schema" }, +] output.type = "zone" [package.clickhouse] @@ -422,7 +425,7 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" +source.commit = "aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt source.sha256 = "d871406ed926571efebdab248de08d4f1ca6c31d4f9a691ce47b186474165c57" @@ -438,7 +441,7 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" +source.commit = "aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt source.sha256 = "85ec05a8726989b5cb0a567de6b0855f6f84b6f3409ac99ccaf372be5821e45d" @@ -453,10 +456,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" +source.commit = "aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "452dfb3491e1b6d4df6be1cb689921f59623aed082e47606a78c0f44d918f66a" +source.sha256 = "aa7241cd35976f28f25aaf3ce2ce2af14dae1da9d67585c7de3b724dbcc55e60" output.type = "zone" output.intermediate_only = true @@ -473,8 +476,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" -source.sha256 = "0808f331741e02d55e199847579dfd01f3658b21c7122cef8c3f9279f43dbab0" +source.commit = "147b03901aa8305b5271e0133a09f628b8140949" +source.sha256 = "14fe7f904f963b50188d6e060106b63df6d061ca64238f7b21623c432b5944e3" output.type = "zone" output.intermediate_only = true @@ -498,8 +501,8 @@ only_for_targets.image = "standard" # 2. Copy the output zone image from dendrite/out to omicron/out source.type = "prebuilt" source.repo = "dendrite" -source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" -source.sha256 = "c359de1be5073a484d86d4c58e8656a36002ce1dc38506f97b730e21615ccae1" +source.commit = "147b03901aa8305b5271e0133a09f628b8140949" +source.sha256 = "f3aa685e4096f8f6e2ea6c169f391dbb88707abcbf1d2bde29163d81736e8ec6" output.type = "zone" output.intermediate_only = true @@ -516,8 +519,8 @@ only_for_targets.image = "standard" # 2. Copy dendrite.tar.gz from dendrite/out to omicron/out/dendrite-softnpu.tar.gz source.type = "prebuilt" source.repo = "dendrite" -source.commit = "343e3a572cc02efe3f8b68f9affd008623a33966" -source.sha256 = "110bfbfb2cf3d3471f3e3a64d26129c7a02f6c5857f9623ebb99690728c3b2ff" +source.commit = "147b03901aa8305b5271e0133a09f628b8140949" +source.sha256 = "dece729ce4127216fba48e9cfed90ec2e5a57ee4ca6c4afc5fa770de6ea636bf" output.type = "zone" output.intermediate_only = true diff --git a/package/Cargo.toml b/package/Cargo.toml index b840938db08..6cc0e343dba 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -12,7 +12,6 @@ futures.workspace = true hex.workspace = true illumos-utils.workspace = true indicatif.workspace = true -omicron-common.workspace = true omicron-zone-package.workspace = true petgraph.workspace = true rayon.workspace = true @@ -20,15 +19,14 @@ reqwest = { workspace = true, features = [ "rustls-tls" ] } ring.workspace = true semver.workspace = true serde.workspace = true -serde_derive.workspace = true sled-hardware.workspace = true slog.workspace = true slog-async.workspace = true slog-term.workspace = true smf.workspace = true strum.workspace = true +swrite.workspace = true tar.workspace = true -tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = [ "full" ] } toml.workspace = true diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index bc07b612348..357a217fe5f 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -29,6 +29,7 @@ use std::io::Write; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; +use swrite::{swrite, SWrite}; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::process::Command; @@ -153,12 +154,11 @@ async fn do_for_all_rust_packages( }) .partition(|(_, release)| *release); - let features = config - .target - .0 - .iter() - .map(|(name, value)| format!("{}-{} ", name, value)) - .collect::(); + let features = + config.target.0.iter().fold(String::new(), |mut acc, (name, value)| { + swrite!(acc, "{}-{} ", name, value); + acc + }); // Execute all the release / debug packages at the same time. if !release_pkgs.is_empty() { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index eacfef491be..804ff08cce8 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -4,5 +4,5 @@ # # We choose a specific toolchain (rather than "stable") for repeatability. The # intent is to keep this up-to-date with recently-released stable Rust. -channel = "1.72.1" +channel = "1.73.0" profile = "default" diff --git a/schema/crdb/10.0.0/README.md b/schema/crdb/10.0.0/README.md new file mode 100644 index 00000000000..9a98141e37d --- /dev/null +++ b/schema/crdb/10.0.0/README.md @@ -0,0 +1,12 @@ +# Why? + +This migration is part of a PR that adds a check to the schema tests ensuring that the order of enum members is the same when starting from scratch with `dbinit.sql` as it is when building up from existing deployments by running the migrations. The problem: there were already two enums, `dataset_kind` and `service_kind`, where the order did not match, so we have to fix that by putting the enums in the "right" order even on an existing deployment where the order is wrong. To do that, for each of those enums, we: + +1. add `clickhouse_keeper2` member +1. change existing uses of `clickhouse_keeper` to `clickhouse_keeper2` +1. drop `clickhouse_keeper` member +1. add `clickhouse_keeper` back in the right order using `AFTER 'clickhouse'` +1. change uses of `clickhouse_keeper2` back to `clickhouse_keeper` +1. drop `clickhouse_keeper2` + +As there are 6 steps here and two different enums to do them for, there are 12 `up*.sql` files. diff --git a/schema/crdb/10.0.0/up01.sql b/schema/crdb/10.0.0/up01.sql new file mode 100644 index 00000000000..6b92700215d --- /dev/null +++ b/schema/crdb/10.0.0/up01.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.dataset_kind ADD VALUE IF NOT EXISTS 'clickhouse_keeper2' AFTER 'clickhouse'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up02.sql b/schema/crdb/10.0.0/up02.sql new file mode 100644 index 00000000000..d7c15e1959d --- /dev/null +++ b/schema/crdb/10.0.0/up02.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.service_kind ADD VALUE IF NOT EXISTS 'clickhouse_keeper2' AFTER 'clickhouse'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up03.sql b/schema/crdb/10.0.0/up03.sql new file mode 100644 index 00000000000..52a32f9f8af --- /dev/null +++ b/schema/crdb/10.0.0/up03.sql @@ -0,0 +1,5 @@ +set local disallow_full_table_scans = off; + +UPDATE omicron.public.dataset +SET kind = 'clickhouse_keeper2' +WHERE kind = 'clickhouse_keeper'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up04.sql b/schema/crdb/10.0.0/up04.sql new file mode 100644 index 00000000000..c9e193f1c3d --- /dev/null +++ b/schema/crdb/10.0.0/up04.sql @@ -0,0 +1,5 @@ +set local disallow_full_table_scans = off; + +UPDATE omicron.public.service +SET kind = 'clickhouse_keeper2' +WHERE kind = 'clickhouse_keeper'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up05.sql b/schema/crdb/10.0.0/up05.sql new file mode 100644 index 00000000000..4e64de94255 --- /dev/null +++ b/schema/crdb/10.0.0/up05.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.dataset_kind DROP VALUE 'clickhouse_keeper'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up06.sql b/schema/crdb/10.0.0/up06.sql new file mode 100644 index 00000000000..4be0ddf6165 --- /dev/null +++ b/schema/crdb/10.0.0/up06.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.service_kind DROP VALUE 'clickhouse_keeper'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up07.sql b/schema/crdb/10.0.0/up07.sql new file mode 100644 index 00000000000..8971f7cf474 --- /dev/null +++ b/schema/crdb/10.0.0/up07.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.dataset_kind ADD VALUE 'clickhouse_keeper' AFTER 'clickhouse'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up08.sql b/schema/crdb/10.0.0/up08.sql new file mode 100644 index 00000000000..4a53d9b8123 --- /dev/null +++ b/schema/crdb/10.0.0/up08.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.service_kind ADD VALUE 'clickhouse_keeper' AFTER 'clickhouse'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up09.sql b/schema/crdb/10.0.0/up09.sql new file mode 100644 index 00000000000..60f2bbbb278 --- /dev/null +++ b/schema/crdb/10.0.0/up09.sql @@ -0,0 +1,5 @@ +set local disallow_full_table_scans = off; + +UPDATE omicron.public.dataset +SET kind = 'clickhouse_keeper' +WHERE kind = 'clickhouse_keeper2'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up10.sql b/schema/crdb/10.0.0/up10.sql new file mode 100644 index 00000000000..ad8801709d5 --- /dev/null +++ b/schema/crdb/10.0.0/up10.sql @@ -0,0 +1,5 @@ +set local disallow_full_table_scans = off; + +UPDATE omicron.public.service +SET kind = 'clickhouse_keeper' +WHERE kind = 'clickhouse_keeper2'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up11.sql b/schema/crdb/10.0.0/up11.sql new file mode 100644 index 00000000000..2c50c0064e9 --- /dev/null +++ b/schema/crdb/10.0.0/up11.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.dataset_kind DROP VALUE 'clickhouse_keeper2'; \ No newline at end of file diff --git a/schema/crdb/10.0.0/up12.sql b/schema/crdb/10.0.0/up12.sql new file mode 100644 index 00000000000..376c25bfcda --- /dev/null +++ b/schema/crdb/10.0.0/up12.sql @@ -0,0 +1 @@ +ALTER TYPE omicron.public.service_kind DROP VALUE 'clickhouse_keeper2'; \ No newline at end of file diff --git a/schema/crdb/README.adoc b/schema/crdb/README.adoc index f92748f1018..fba36ed73b6 100644 --- a/schema/crdb/README.adoc +++ b/schema/crdb/README.adoc @@ -7,25 +7,31 @@ This directory describes the schema(s) used by CockroachDB. We use the following conventions: -* `schema/crdb/VERSION/up.sql`: The necessary idempotent migrations to transition from the - previous version of CockroachDB to this version. These migrations will always be placed - within one transaction per file. -** If more than one change is needed per version, any number of files starting with `up` - and ending with `.sql` may be used. These files will be sorted in lexicographic order - before being executed, and each will be executed in a separate transaction. -** CockroachDB documentation recommends the following: "Execute schema changes... in a single - explicit transaction consisting of the single schema change statement". - Practically this means: If you want to change multiple tables, columns, - types, indices, or constraints, do so in separate files. -** More information can be found here: https://www.cockroachlabs.com/docs/stable/online-schema-changes -* `schema/crdb/dbinit.sql`: The necessary operations to create the latest version - of the schema. Should be equivalent to running all `up.sql` migrations, in-order. -* `schema/crdb/dbwipe.sql`: The necessary operations to delete the latest version - of the schema. - -Note that to upgrade from version N to version N+2, we always need to apply the -N+1 upgrade first, before applying the N+2 upgrade. This simplifies our model -of DB schema changes as an incremental linear history. +* `schema/crdb/VERSION/up*.sql`: Files containing the necessary idempotent + statements to transition from the previous version of the database schema to + this version. All of the statements in a given file will be executed + together in one transaction; however, usually only one statement should + appear in each file. More on this below. +** If there's only one statement required, we put it into `up.sql`. +** If more than one change is needed, any number of files starting with `up` + and ending with `.sql` may be used. These files will be sorted in + lexicographic order before being executed. Each will be executed in a + separate transaction. +** CockroachDB documentation recommends the following: "Execute schema + changes ... in an explicit transaction consisting of the single schema + change statement.". Practically this means: If you want to change multiple + tables, columns, types, indices, or constraints, do so in separate files. + See https://www.cockroachlabs.com/docs/stable/online-schema-changes for + more. +* `schema/crdb/dbinit.sql`: The necessary operations to create the latest + version of the schema. Should be equivalent to running all `up.sql` + migrations, in-order. +* `schema/crdb/dbwipe.sql`: The necessary operations to delete the latest + version of the schema. + +Note that to upgrade from version N to version N+2, we always apply the N+1 +upgrade first, before applying the N+2 upgrade. This simplifies our model of DB +schema changes by ensuring an incremental, linear history. == Offline Upgrade @@ -34,9 +40,9 @@ This means we're operating with the following constraints: * We assume that downtime is acceptable to perform an update. * We assume that while an update is occuring, all Nexus services -are running the same version of software. + are running the same version of software. * We assume that no (non-upgrade) concurrent database requests will happen for -the duration of the migration. + the duration of the migration. This is not an acceptable long-term solution - we must be able to update without downtime - but it is an interim solution, and one which provides a @@ -44,7 +50,7 @@ fall-back pathway for performing upgrades. See RFD 319 for more discussion of the online upgrade plans. -=== How to change the schema +== How to change the schema Assumptions: @@ -53,11 +59,22 @@ Assumptions: Process: -* Choose a `NEW_VERSION` number. This should almost certainly be a major version bump over `OLD_VERSION`. -* Add a file to `schema/crdb/NEW_VERSION/up.sql` with your changes to the schema. -** This file should validate the expected current version transactionally. -** This file should only issue a single schema-modifying statement per transaction. -** This file should not issue any data-modifying operations within the schema-modifying transactions. +* Choose a `NEW_VERSION` number. This should almost certainly be a major + version bump over `OLD_VERSION`. +* Create directory `schema/crdb/NEW_VERSION`. +* If only one SQL statement is necessary to get from `OLD_VERSION` to + `NEW_VERSION`, put that statement into `schema/crdb/NEW_VERSION/up.sql`. If + multiple statements are required, put each one into a separate file, naming + these `schema/crdb/NEW_VERSION/upN.sql` for as many `N` as you need. +** Each file should contain _either_ one schema-modifying statement _or_ some + number of data-modifying statements. You can combine multiple data-modifying + statements. But you should not mix schema-modifying statements and + data-modifying statements in one file. And you should not include multiple + schema-modifying statements in one file. +** Beware that the entire file will be run in one transaction. Expensive data- + modifying operations leading to long-running transactions are generally + to-be-avoided; however, there's no better way to do this today if you really + do need to update thousands of rows as part of the update. * Update `schema/crdb/dbinit.sql` to match what the database should look like after your update is applied. Don't forget to update the version field of `db_metadata` at the bottom of the file! @@ -68,35 +85,23 @@ Process: SQL Validation, via Automated Tests: * The `SCHEMA_VERSION` matches the version used in `dbinit.sql` -* The combination of all `up.sql` files results in the same schema as `dbinit.sql` +* The combination of all `up.sql` files results in the same schema as + `dbinit.sql` * All `up.sql` files can be applied twice without error -==== Handling common schema changes +=== General notes -Although CockroachDB's schema includes some opaque internally-generated fields -that are order dependent - such as the names of anonymous CHECK constraints - -our schema comparison tools intentionally ignore these values. As a result, -when performing schema changes, the order of new tables and constraints should -generally not be important. +CockroachDB's representation of the schema includes some opaque +internally-generated fields that are order dependent, like the names of +anonymous CHECK constraints. Our schema comparison tools intentionally ignore +these values. As a result, when performing schema changes, the order of new +tables and constraints should generally not be important. -As convention, however, we recommend keeping the `db_metadata` file at the end of -`dbinit.sql`, so that the database does not contain a version until it is fully -populated. +As convention, however, we recommend keeping the `db_metadata` file at the end +of `dbinit.sql`, so that the database does not contain a version until it is +fully populated. -==== Adding new source tables to an existing view - -An upgrade can add a new table and then use a `CREATE OR REPLACE VIEW` statement -to make an existing view depend on that table. To do this in `dbinit.sql` while -maintaining table and view ordering, use `CREATE VIEW` to create a "placeholder" -view in the correct position, then add the table to the bottom of `dbinit.sql` -and use `CREATE OR REPLACE VIEW` to "fill out" the placeholder definition to -refer to the new table. (You may need to do the `CREATE OR REPLACE VIEW` in a -separate transaction from the original `CREATE VIEW`.) - -Note that `CREATE OR REPLACE VIEW` requires that the new view maintain all of -the columns of the old view with the same type and same order (though the query -used to populate them can change. See -https://www.postgresql.org/docs/15/sql-createview.html. +=== Scenario-specific gotchas ==== Renaming columns diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index da842cbfebd..875877ee967 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -2838,7 +2838,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '9.0.0', NULL) + ( TRUE, NOW(), NOW(), '10.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 39a9a68acc7..2ce8ae3bdcd 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -132,6 +132,51 @@ "format": "uint32", "minimum": 0.0 }, + "connect_retry": { + "description": "The interval in seconds between peer connection retry attempts.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "delay_open": { + "description": "How long to delay sending open messages to a peer. In seconds.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "hold_time": { + "description": "How long to keep a session alive without a keepalive in seconds. Defaults to 6.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "idle_hold_time": { + "description": "How long to keep a peer in idle after a state machine reset in seconds.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "keepalive": { + "description": "The interval to send keepalive messages at.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, "port": { "description": "Switch port the peer is reachable on.", "type": "string" @@ -588,33 +633,46 @@ "description": "Configuration information for launching a Sled Agent.", "type": "object", "required": [ - "dns_servers", + "body", + "generation", + "schema_version" + ], + "properties": { + "body": { + "$ref": "#/definitions/StartSledAgentRequestBody" + }, + "generation": { + "description": "The current generation number of data as stored in CRDB.\n\nThe initial generation is set during RSS time and then only mutated by Nexus. For now, we don't actually anticipate mutating this data, but we leave open the possiblity.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "StartSledAgentRequestBody": { + "description": "This is the actual app level data of `StartSledAgentRequest`\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", + "type": "object", + "required": [ "id", - "ntp_servers", + "is_lrtq_learner", "rack_id", "subnet", "use_trust_quorum" ], "properties": { - "dns_servers": { - "description": "The external DNS servers to use", - "type": "array", - "items": { - "type": "string", - "format": "ip" - } - }, "id": { "description": "Uuid of the Sled Agent to be created.", "type": "string", "format": "uuid" }, - "ntp_servers": { - "description": "The external NTP servers to use", - "type": "array", - "items": { - "type": "string" - } + "is_lrtq_learner": { + "description": "Is this node an LRTQ learner node?\n\nWe only put the node into learner mode if `use_trust_quorum` is also true.", + "type": "boolean" }, "rack_id": { "description": "Uuid of the rack to which this sled agent belongs.", diff --git a/schema/persistent-sled-agent-request.json b/schema/start-sled-agent-request.json similarity index 54% rename from schema/persistent-sled-agent-request.json rename to schema/start-sled-agent-request.json index 5679165c323..b03058d106f 100644 --- a/schema/persistent-sled-agent-request.json +++ b/schema/start-sled-agent-request.json @@ -1,13 +1,27 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "PersistentSledAgentRequest", + "title": "StartSledAgentRequest", + "description": "Configuration information for launching a Sled Agent.", "type": "object", "required": [ - "request" + "body", + "generation", + "schema_version" ], "properties": { - "request": { - "$ref": "#/definitions/StartSledAgentRequest" + "body": { + "$ref": "#/definitions/StartSledAgentRequestBody" + }, + "generation": { + "description": "The current generation number of data as stored in CRDB.\n\nThe initial generation is set during RSS time and then only mutated by Nexus. For now, we don't actually anticipate mutating this data, but we leave open the possiblity.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "schema_version": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 } }, "definitions": { @@ -32,37 +46,25 @@ } } }, - "StartSledAgentRequest": { - "description": "Configuration information for launching a Sled Agent.", + "StartSledAgentRequestBody": { + "description": "This is the actual app level data of `StartSledAgentRequest`\n\nWe nest it below the \"header\" of `generation` and `schema_version` so that we can perform partial deserialization of `EarlyNetworkConfig` to only read the header and defer deserialization of the body once we know the schema version. This is possible via the use of [`serde_json::value::RawValue`] in future (post-v1) deserialization paths.", "type": "object", "required": [ - "dns_servers", "id", - "ntp_servers", + "is_lrtq_learner", "rack_id", "subnet", "use_trust_quorum" ], "properties": { - "dns_servers": { - "description": "The external DNS servers to use", - "type": "array", - "items": { - "type": "string", - "format": "ip" - } - }, "id": { "description": "Uuid of the Sled Agent to be created.", "type": "string", "format": "uuid" }, - "ntp_servers": { - "description": "The external NTP servers to use", - "type": "array", - "items": { - "type": "string" - } + "is_lrtq_learner": { + "description": "Is this node an LRTQ learner node?\n\nWe only put the node into learner mode if `use_trust_quorum` is also true.", + "type": "boolean" }, "rack_id": { "description": "Uuid of the rack to which this sled agent belongs.", diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index ff9644773a6..61e61709e16 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -9,7 +9,6 @@ license = "MPL-2.0" anyhow.workspace = true async-trait.workspace = true base64.workspace = true -bincode.workspace = true bootstore.workspace = true bootstrap-agent-client.workspace = true bytes.workspace = true @@ -20,7 +19,6 @@ cfg-if.workspace = true chrono.workspace = true clap.workspace = true # Only used by the simulated sled agent. -crucible-client-types.workspace = true crucible-agent-client.workspace = true ddm-admin-client.workspace = true derive_more.workspace = true @@ -41,15 +39,15 @@ itertools.workspace = true key-manager.workspace = true libc.workspace = true macaddr.workspace = true +mg-admin-client.workspace = true nexus-client.workspace = true omicron-common.workspace = true once_cell.workspace = true oximeter.workspace = true +oximeter-instruments.workspace = true oximeter-producer.workspace = true -percent-encoding.workspace = true -progenitor.workspace = true -propolis-client = { workspace = true, features = [ "generated-migration" ] } -propolis-server.workspace = true # Only used by the simulated sled agent +propolis-client.workspace = true +propolis-mock-server.workspace = true # Only used by the simulated sled agent rand = { workspace = true, features = ["getrandom"] } reqwest = { workspace = true, features = ["rustls-tls", "stream"] } schemars = { workspace = true, features = [ "chrono", "uuid1" ] } @@ -59,6 +57,7 @@ serde_json = {workspace = true, features = ["raw_value"]} sha3.workspace = true sled-agent-client.workspace = true sled-hardware.workspace = true +sled-storage.workspace = true slog.workspace = true slog-async.workspace = true slog-dtrace.workspace = true @@ -68,7 +67,6 @@ tar.workspace = true thiserror.workspace = true tofino.workspace = true tokio = { workspace = true, features = [ "full" ] } -tokio-tungstenite.workspace = true toml.workspace = true usdt.workspace = true uuid.workspace = true @@ -96,7 +94,8 @@ slog-async.workspace = true slog-term.workspace = true tempfile.workspace = true -illumos-utils = { workspace = true, features = ["testing"] } +illumos-utils = { workspace = true, features = ["testing", "tmp_keypath"] } +sled-storage = { workspace = true, features = ["testing"] } # # Disable doc builds by default for our binaries to work around issue @@ -119,5 +118,6 @@ machine-non-gimlet = [] switch-asic = [] switch-stub = [] switch-softnpu = [] +switch-hypersoftnpu = [] rack-topology-single-sled = [] rack-topology-multi-sled = [] diff --git a/sled-agent/src/backing_fs.rs b/sled-agent/src/backing_fs.rs index 5014ac59991..2e9ea4c8d91 100644 --- a/sled-agent/src/backing_fs.rs +++ b/sled-agent/src/backing_fs.rs @@ -25,6 +25,7 @@ use camino::Utf8PathBuf; use illumos_utils::zfs::{ EnsureFilesystemError, GetValueError, Mountpoint, SizeDetails, Zfs, }; +use std::io; #[derive(Debug, thiserror::Error)] pub enum BackingFsError { @@ -36,9 +37,12 @@ pub enum BackingFsError { #[error("Error initializing dataset: {0}")] Mount(#[from] EnsureFilesystemError), + + #[error("Failed to ensure subdirectory {0}")] + EnsureSubdir(#[from] io::Error), } -struct BackingFs { +struct BackingFs<'a> { // Dataset name name: &'static str, // Mountpoint @@ -49,9 +53,11 @@ struct BackingFs { compression: Option<&'static str>, // Linked service service: Option<&'static str>, + // Subdirectories to ensure + subdirs: Option<&'a [&'static str]>, } -impl BackingFs { +impl<'a> BackingFs<'a> { const fn new(name: &'static str) -> Self { Self { name, @@ -59,6 +65,7 @@ impl BackingFs { quota: None, compression: None, service: None, + subdirs: None, } } @@ -81,10 +88,16 @@ impl BackingFs { self.service = Some(service); self } + + const fn subdirs(mut self, subdirs: &'a [&'static str]) -> Self { + self.subdirs = Some(subdirs); + self + } } const BACKING_FMD_DATASET: &'static str = "fmd"; const BACKING_FMD_MOUNTPOINT: &'static str = "/var/fm/fmd"; +const BACKING_FMD_SUBDIRS: [&'static str; 3] = ["rsrc", "ckpt", "xprt"]; const BACKING_FMD_SERVICE: &'static str = "svc:/system/fmd:default"; const BACKING_FMD_QUOTA: usize = 500 * (1 << 20); // 500 MiB @@ -94,6 +107,7 @@ const BACKINGFS_COUNT: usize = 1; static BACKINGFS: [BackingFs; BACKINGFS_COUNT] = [BackingFs::new(BACKING_FMD_DATASET) .mountpoint(BACKING_FMD_MOUNTPOINT) + .subdirs(&BACKING_FMD_SUBDIRS) .quota(BACKING_FMD_QUOTA) .compression(BACKING_COMPRESSION) .service(BACKING_FMD_SERVICE)]; @@ -114,7 +128,7 @@ pub(crate) fn ensure_backing_fs( let dataset = format!( "{}/{}/{}", boot_zpool_name, - sled_hardware::disk::M2_BACKING_DATASET, + sled_storage::dataset::M2_BACKING_DATASET, bfs.name ); let mountpoint = Mountpoint::Path(Utf8PathBuf::from(bfs.mountpoint)); @@ -165,6 +179,15 @@ pub(crate) fn ensure_backing_fs( Zfs::mount_overlay_dataset(&dataset, &mountpoint)?; + if let Some(subdirs) = bfs.subdirs { + for dir in subdirs { + let subdir = format!("{}/{}", mountpoint, dir); + + info!(log, "Ensuring directory {}", subdir); + std::fs::create_dir_all(subdir)?; + } + } + if let Some(service) = bfs.service { info!(log, "Starting service {}", service); smf::Adm::new() diff --git a/sled-agent/src/bin/sled-agent-sim.rs b/sled-agent/src/bin/sled-agent-sim.rs index adba8eab6bb..ee0ebda71e6 100644 --- a/sled-agent/src/bin/sled-agent-sim.rs +++ b/sled-agent/src/bin/sled-agent-sim.rs @@ -6,7 +6,7 @@ // TODO see the TODO for nexus. -use anyhow::Context; +use anyhow::{anyhow, Context}; use camino::Utf8PathBuf; use clap::Parser; use dropshot::ConfigDropshot; @@ -96,7 +96,7 @@ async fn do_run() -> Result<(), CmdError> { let args = Args::parse(); let tmp = camino_tempfile::tempdir() - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(|e| CmdError::Failure(anyhow!(e)))?; let config = Config { id: args.uuid, sim_mode: args.sim_mode, @@ -125,10 +125,10 @@ async fn do_run() -> Result<(), CmdError> { (Some(cert_path), Some(key_path)) => { let cert_bytes = std::fs::read_to_string(&cert_path) .with_context(|| format!("read {:?}", &cert_path)) - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(CmdError::Failure)?; let key_bytes = std::fs::read_to_string(&key_path) .with_context(|| format!("read {:?}", &key_path)) - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(CmdError::Failure)?; Some(NexusTypes::Certificate { cert: cert_bytes, key: key_bytes }) } _ => { @@ -145,7 +145,5 @@ async fn do_run() -> Result<(), CmdError> { tls_certificate, }; - run_standalone_server(&config, &rss_args) - .await - .map_err(|e| CmdError::Failure(e.to_string())) + run_standalone_server(&config, &rss_args).await.map_err(CmdError::Failure) } diff --git a/sled-agent/src/bin/sled-agent.rs b/sled-agent/src/bin/sled-agent.rs index 5764bf8309f..b8b5abf07fd 100644 --- a/sled-agent/src/bin/sled-agent.rs +++ b/sled-agent/src/bin/sled-agent.rs @@ -4,6 +4,7 @@ //! Executable program to run the sled agent +use anyhow::anyhow; use camino::Utf8PathBuf; use clap::{Parser, Subcommand}; use omicron_common::cmd::fatal; @@ -51,16 +52,14 @@ async fn do_run() -> Result<(), CmdError> { match args { Args::Openapi(flavor) => match flavor { - OpenapiFlavor::Sled => { - sled_server::run_openapi().map_err(CmdError::Failure) - } - OpenapiFlavor::Bootstrap => { - bootstrap_server::run_openapi().map_err(CmdError::Failure) - } + OpenapiFlavor::Sled => sled_server::run_openapi() + .map_err(|err| CmdError::Failure(anyhow!(err))), + OpenapiFlavor::Bootstrap => bootstrap_server::run_openapi() + .map_err(|err| CmdError::Failure(anyhow!(err))), }, Args::Run { config_path } => { let config = SledConfig::from_file(&config_path) - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(|e| CmdError::Failure(anyhow!(e)))?; // - Sled agent starts with the normal config file - typically // called "config.toml". @@ -83,7 +82,7 @@ async fn do_run() -> Result<(), CmdError> { let rss_config = if rss_config_path.exists() { Some( RssConfig::from_file(rss_config_path) - .map_err(|e| CmdError::Failure(e.to_string()))?, + .map_err(|e| CmdError::Failure(anyhow!(e)))?, ) } else { None @@ -91,7 +90,7 @@ async fn do_run() -> Result<(), CmdError> { let server = bootstrap_server::Server::start(config) .await - .map_err(|err| CmdError::Failure(format!("{err:#}")))?; + .map_err(|err| CmdError::Failure(anyhow!(err)))?; // If requested, automatically supply the RSS configuration. // @@ -103,12 +102,15 @@ async fn do_run() -> Result<(), CmdError> { // abandon the server. Ok(_) | Err(RssAccessError::AlreadyInitialized) => {} Err(e) => { - return Err(CmdError::Failure(e.to_string())); + return Err(CmdError::Failure(anyhow!(e))); } } } - server.wait_for_finish().await.map_err(CmdError::Failure)?; + server + .wait_for_finish() + .await + .map_err(|err| CmdError::Failure(anyhow!(err)))?; Ok(()) } diff --git a/sled-agent/src/bootstrap/bootstore.rs b/sled-agent/src/bootstrap/bootstore_setup.rs similarity index 55% rename from sled-agent/src/bootstrap/bootstore.rs rename to sled-agent/src/bootstrap/bootstore_setup.rs index 17267bef55a..9eb0a87c033 100644 --- a/sled-agent/src/bootstrap/bootstore.rs +++ b/sled-agent/src/bootstrap/bootstore_setup.rs @@ -5,124 +5,78 @@ //! Helpers for configuring and starting the bootstore during bootstrap agent //! startup. +#![allow(clippy::result_large_err)] + use super::config::BOOTSTORE_PORT; use super::server::StartError; -use crate::storage_manager::StorageResources; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use ddm_admin_client::Client as DdmAdminClient; use sled_hardware::underlay::BootstrapInterface; use sled_hardware::Baseboard; +use sled_storage::dataset::CLUSTER_DATASET; +use sled_storage::resources::StorageResources; use slog::Logger; use std::collections::BTreeSet; use std::net::Ipv6Addr; use std::net::SocketAddrV6; use std::time::Duration; -use tokio::task::JoinHandle; const BOOTSTORE_FSM_STATE_FILE: &str = "bootstore-fsm-state.json"; const BOOTSTORE_NETWORK_CONFIG_FILE: &str = "bootstore-network-config.json"; -pub(super) struct BootstoreHandles { - pub(super) node_handle: bootstore::NodeHandle, - - // These two are never used; we keep them to show ownership of the spawned - // tasks. - _node_task_handle: JoinHandle<()>, - _peer_update_task_handle: JoinHandle<()>, -} - -impl BootstoreHandles { - pub(super) async fn spawn( - storage_resources: &StorageResources, - ddm_admin_client: DdmAdminClient, - baseboard: Baseboard, - global_zone_bootstrap_ip: Ipv6Addr, - base_log: &Logger, - ) -> Result { - let config = bootstore::Config { - id: baseboard, - addr: SocketAddrV6::new( - global_zone_bootstrap_ip, - BOOTSTORE_PORT, - 0, - 0, - ), - time_per_tick: Duration::from_millis(250), - learn_timeout: Duration::from_secs(5), - rack_init_timeout: Duration::from_secs(300), - rack_secret_request_timeout: Duration::from_secs(5), - fsm_state_ledger_paths: bootstore_fsm_state_paths( - &storage_resources, - ) - .await?, - network_config_ledger_paths: bootstore_network_config_paths( - &storage_resources, - ) - .await?, - }; - - let (mut node, node_handle) = - bootstore::Node::new(config, base_log).await; - - let join_handle = tokio::spawn(async move { node.run().await }); - - // Spawn a task for polling DDMD and updating bootstore - let peer_update_handle = - tokio::spawn(poll_ddmd_for_bootstore_peer_update( - base_log.new(o!("component" => "bootstore_ddmd_poller")), - node_handle.clone(), - ddm_admin_client, - )); - - Ok(Self { - node_handle, - _node_task_handle: join_handle, - _peer_update_task_handle: peer_update_handle, - }) - } +pub fn new_bootstore_config( + storage_resources: &StorageResources, + baseboard: Baseboard, + global_zone_bootstrap_ip: Ipv6Addr, +) -> Result { + Ok(bootstore::Config { + id: baseboard, + addr: SocketAddrV6::new(global_zone_bootstrap_ip, BOOTSTORE_PORT, 0, 0), + time_per_tick: Duration::from_millis(250), + learn_timeout: Duration::from_secs(5), + rack_init_timeout: Duration::from_secs(300), + rack_secret_request_timeout: Duration::from_secs(5), + fsm_state_ledger_paths: bootstore_fsm_state_paths(&storage_resources)?, + network_config_ledger_paths: bootstore_network_config_paths( + &storage_resources, + )?, + }) } -async fn bootstore_fsm_state_paths( +fn bootstore_fsm_state_paths( storage: &StorageResources, ) -> Result, StartError> { let paths: Vec<_> = storage - .all_m2_mountpoints(sled_hardware::disk::CLUSTER_DATASET) - .await + .all_m2_mountpoints(CLUSTER_DATASET) .into_iter() .map(|p| p.join(BOOTSTORE_FSM_STATE_FILE)) .collect(); if paths.is_empty() { - return Err(StartError::MissingM2Paths( - sled_hardware::disk::CLUSTER_DATASET, - )); + return Err(StartError::MissingM2Paths(CLUSTER_DATASET)); } Ok(paths) } -async fn bootstore_network_config_paths( +fn bootstore_network_config_paths( storage: &StorageResources, ) -> Result, StartError> { let paths: Vec<_> = storage - .all_m2_mountpoints(sled_hardware::disk::CLUSTER_DATASET) - .await + .all_m2_mountpoints(CLUSTER_DATASET) .into_iter() .map(|p| p.join(BOOTSTORE_NETWORK_CONFIG_FILE)) .collect(); if paths.is_empty() { - return Err(StartError::MissingM2Paths( - sled_hardware::disk::CLUSTER_DATASET, - )); + return Err(StartError::MissingM2Paths(CLUSTER_DATASET)); } Ok(paths) } -async fn poll_ddmd_for_bootstore_peer_update( +pub async fn poll_ddmd_for_bootstore_peer_update( log: Logger, bootstore_node_handle: bootstore::NodeHandle, - ddmd_client: DdmAdminClient, ) { let mut current_peers: BTreeSet = BTreeSet::new(); // We're talking to a service's admin interface on localhost and @@ -132,7 +86,7 @@ async fn poll_ddmd_for_bootstore_peer_update( // We also use this timeout in the case of spurious ddmd failures // that require a reconnection from the ddmd_client. const RETRY: tokio::time::Duration = tokio::time::Duration::from_secs(5); - + let ddmd_client = DdmAdminClient::localhost(&log).unwrap(); loop { match ddmd_client .derive_bootstrap_addrs_from_prefixes(&[ @@ -154,7 +108,7 @@ async fn poll_ddmd_for_bootstore_peer_update( log, concat!( "Bootstore comms error: {}. ", - "bootstore::Node task must have paniced", + "bootstore::Node task must have panicked", ), e ); diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index 9adfa47d9bc..bec309dc279 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -17,7 +17,9 @@ use gateway_client::Client as MgsClient; use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; use internal_dns::ServiceName; use ipnetwork::{IpNetwork, Ipv6Network}; -use omicron_common::address::{Ipv6Subnet, MGS_PORT}; +use mg_admin_client::types::{ApplyRequest, BgpPeerConfig, Prefix4}; +use mg_admin_client::Client as MgdClient; +use omicron_common::address::{Ipv6Subnet, MGD_PORT, MGS_PORT}; use omicron_common::address::{DDMD_PORT, DENDRITE_PORT}; use omicron_common::api::internal::shared::{ PortConfigV1, PortFec, PortSpeed, RackNetworkConfig, RackNetworkConfigV1, @@ -27,6 +29,7 @@ use omicron_common::backoff::{ retry_notify, retry_policy_local, BackoffError, ExponentialBackoff, ExponentialBackoffBuilder, }; +use omicron_common::OMICRON_DPD_TAG; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use slog::Logger; @@ -36,6 +39,7 @@ use std::time::{Duration, Instant}; use thiserror::Error; static BOUNDARY_SERVICES_ADDR: &str = "fd00:99::1"; +const BGP_SESSION_RESOLUTION: u64 = 100; /// Errors that can occur during early network setup #[derive(Error, Debug)] @@ -54,6 +58,12 @@ pub enum EarlyNetworkSetupError { #[error("Error during DNS lookup: {0}")] DnsResolver(#[from] ResolveError), + + #[error("BGP configuration error: {0}")] + BgpConfigurationError(String), + + #[error("MGD error: {0}")] + MgdError(String), } enum LookupSwitchZoneAddrsResult { @@ -403,7 +413,7 @@ impl<'a> EarlyNetworkSetup<'a> { let dpd = DpdClient::new( &format!("http://[{}]:{}", switch_zone_underlay_ip, DENDRITE_PORT), dpd_client::ClientState { - tag: "early_networking".to_string(), + tag: OMICRON_DPD_TAG.into(), log: self.log.new(o!("component" => "DpdClient")), }, ); @@ -432,13 +442,17 @@ impl<'a> EarlyNetworkSetup<'a> { "Configuring default uplink on switch"; "config" => #?dpd_port_settings ); - dpd.port_settings_apply(&port_id, &dpd_port_settings) - .await - .map_err(|e| { - EarlyNetworkSetupError::Dendrite(format!( - "unable to apply uplink port configuration: {e}" - )) - })?; + dpd.port_settings_apply( + &port_id, + Some(OMICRON_DPD_TAG), + &dpd_port_settings, + ) + .await + .map_err(|e| { + EarlyNetworkSetupError::Dendrite(format!( + "unable to apply uplink port configuration: {e}" + )) + })?; info!(self.log, "advertising boundary services loopback address"); @@ -448,6 +462,67 @@ impl<'a> EarlyNetworkSetup<'a> { ddmd_client.advertise_prefix(Ipv6Subnet::new(ipv6_entry.addr)); } + let mgd = MgdClient::new( + &self.log, + SocketAddrV6::new(switch_zone_underlay_ip, MGD_PORT, 0, 0).into(), + ) + .map_err(|e| { + EarlyNetworkSetupError::MgdError(format!( + "initialize mgd client: {e}" + )) + })?; + + // Iterate through ports and apply BGP config. + for port in &our_ports { + let mut bgp_peer_configs = Vec::new(); + for peer in &port.bgp_peers { + let config = rack_network_config + .bgp + .iter() + .find(|x| x.asn == peer.asn) + .ok_or(EarlyNetworkSetupError::BgpConfigurationError( + format!( + "asn {} referenced by peer undefined", + peer.asn + ), + ))?; + + let bpc = BgpPeerConfig { + asn: peer.asn, + name: format!("{}", peer.addr), + host: format!("{}:179", peer.addr), + hold_time: peer.hold_time.unwrap_or(6), + idle_hold_time: peer.idle_hold_time.unwrap_or(3), + delay_open: peer.delay_open.unwrap_or(0), + connect_retry: peer.connect_retry.unwrap_or(3), + keepalive: peer.keepalive.unwrap_or(2), + resolution: BGP_SESSION_RESOLUTION, + originate: config + .originate + .iter() + .map(|x| Prefix4 { length: x.prefix(), value: x.ip() }) + .collect(), + }; + bgp_peer_configs.push(bpc); + } + + if bgp_peer_configs.is_empty() { + continue; + } + + mgd.inner + .bgp_apply(&ApplyRequest { + peer_group: port.port.clone(), + peers: bgp_peer_configs, + }) + .await + .map_err(|e| { + EarlyNetworkSetupError::BgpConfigurationError(format!( + "BGP peer configuration failed: {e}", + )) + })?; + } + Ok(our_ports) } @@ -462,10 +537,9 @@ impl<'a> EarlyNetworkSetup<'a> { "failed to parse `BOUNDARY_SERVICES_ADDR` as `Ipv6Addr`: {e}" )) })?, - tag: "rss".into(), + tag: OMICRON_DPD_TAG.into(), }; let mut dpd_port_settings = PortSettings { - tag: "rss".into(), links: HashMap::new(), v4_routes: HashMap::new(), v6_routes: HashMap::new(), @@ -488,6 +562,7 @@ impl<'a> EarlyNetworkSetup<'a> { kr: false, fec: convert_fec(&port_config.uplink_port_fec), speed: convert_speed(&port_config.uplink_port_speed), + lane: Some(LinkId(0)), }, //addrs: vec![addr], addrs, diff --git a/sled-agent/src/bootstrap/http_entrypoints.rs b/sled-agent/src/bootstrap/http_entrypoints.rs index c69bdeb0ceb..7c32bf48a5f 100644 --- a/sled-agent/src/bootstrap/http_entrypoints.rs +++ b/sled-agent/src/bootstrap/http_entrypoints.rs @@ -12,7 +12,6 @@ use super::BootstrapError; use super::RssAccessError; use crate::bootstrap::params::RackInitializeRequest; use crate::bootstrap::rack_ops::{RackInitId, RackResetId}; -use crate::storage_manager::StorageResources; use crate::updates::ConfigUpdates; use crate::updates::{Component, UpdateManager}; use bootstore::schemes::v0 as bootstore; @@ -25,6 +24,7 @@ use omicron_common::api::external::Error; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_hardware::Baseboard; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::net::Ipv6Addr; use tokio::sync::mpsc::error::TrySendError; @@ -33,7 +33,7 @@ use tokio::sync::{mpsc, oneshot}; pub(crate) struct BootstrapServerContext { pub(crate) base_log: Logger, pub(crate) global_zone_bootstrap_ip: Ipv6Addr, - pub(crate) storage_resources: StorageResources, + pub(crate) storage_manager: StorageHandle, pub(crate) bootstore_node_handle: bootstore::NodeHandle, pub(crate) baseboard: Baseboard, pub(crate) rss_access: RssAccess, @@ -50,7 +50,7 @@ impl BootstrapServerContext { self.rss_access.start_initializing( &self.base_log, self.global_zone_bootstrap_ip, - &self.storage_resources, + &self.storage_manager, &self.bootstore_node_handle, request, ) diff --git a/sled-agent/src/bootstrap/mod.rs b/sled-agent/src/bootstrap/mod.rs index 96e674acf31..590e13c8919 100644 --- a/sled-agent/src/bootstrap/mod.rs +++ b/sled-agent/src/bootstrap/mod.rs @@ -4,7 +4,7 @@ //! Bootstrap-related utilities -mod bootstore; +pub(crate) mod bootstore_setup; pub mod client; pub mod config; pub mod early_networking; @@ -14,7 +14,7 @@ pub(crate) mod params; mod pre_server; mod rack_ops; pub(crate) mod rss_handle; -mod secret_retriever; +pub mod secret_retriever; pub mod server; mod sprockets_server; mod views; diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs index 49833834703..ab85915dc16 100644 --- a/sled-agent/src/bootstrap/params.rs +++ b/sled-agent/src/bootstrap/params.rs @@ -5,8 +5,10 @@ //! Request types for the bootstrap agent use anyhow::{bail, Result}; +use async_trait::async_trait; use omicron_common::address::{self, Ipv6Subnet, SLED_PREFIX}; use omicron_common::api::internal::shared::RackNetworkConfig; +use omicron_common::ledger::Ledgerable; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sha3::{Digest, Sha3_256}; @@ -172,9 +174,23 @@ impl TryFrom for RackInitializeRequest { pub type Certificate = nexus_client::types::Certificate; pub type RecoverySiloConfig = nexus_client::types::RecoverySiloConfig; -/// Configuration information for launching a Sled Agent. +/// A request to Add a given sled after rack initialization has occurred #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] -pub struct StartSledAgentRequest { +pub struct AddSledRequest { + pub sled_id: Baseboard, + pub start_request: StartSledAgentRequest, +} + +// A wrapper around StartSledAgentRequestV0 that was used +// for the ledger format. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] +struct PersistentSledAgentRequest { + request: StartSledAgentRequestV0, +} + +/// The version of `StartSledAgentRequest` we originally shipped with. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] +pub struct StartSledAgentRequestV0 { /// Uuid of the Sled Agent to be created. pub id: Uuid, @@ -197,13 +213,62 @@ pub struct StartSledAgentRequest { pub subnet: Ipv6Subnet, } +/// Configuration information for launching a Sled Agent. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] +pub struct StartSledAgentRequest { + /// The current generation number of data as stored in CRDB. + /// + /// The initial generation is set during RSS time and then only mutated + /// by Nexus. For now, we don't actually anticipate mutating this data, + /// but we leave open the possiblity. + pub generation: u64, + + // Which version of the data structure do we have. This is to help with + // deserialization and conversion in future updates. + pub schema_version: u32, + + // The actual configuration details + pub body: StartSledAgentRequestBody, +} + +/// This is the actual app level data of `StartSledAgentRequest` +/// +/// We nest it below the "header" of `generation` and `schema_version` so that +/// we can perform partial deserialization of `EarlyNetworkConfig` to only read +/// the header and defer deserialization of the body once we know the schema +/// version. This is possible via the use of [`serde_json::value::RawValue`] in +/// future (post-v1) deserialization paths. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, JsonSchema)] +pub struct StartSledAgentRequestBody { + /// Uuid of the Sled Agent to be created. + pub id: Uuid, + + /// Uuid of the rack to which this sled agent belongs. + pub rack_id: Uuid, + + /// Use trust quorum for key generation + pub use_trust_quorum: bool, + + /// Is this node an LRTQ learner node? + /// + /// We only put the node into learner mode if `use_trust_quorum` is also + /// true. + pub is_lrtq_learner: bool, + + // Note: The order of these fields is load bearing, because we serialize + // `SledAgentRequest`s as toml. `subnet` serializes as a TOML table, so it + // must come after non-table fields. + /// Portion of the IP space to be managed by the Sled Agent. + pub subnet: Ipv6Subnet, +} + impl StartSledAgentRequest { pub fn sled_address(&self) -> SocketAddrV6 { - address::get_sled_address(self.subnet) + address::get_sled_address(self.body.subnet) } pub fn switch_zone_ip(&self) -> Ipv6Addr { - address::get_switch_zone_address(self.subnet) + address::get_switch_zone_address(self.body.subnet) } /// Compute the sha3_256 digest of `self.rack_id` to use as a `salt` @@ -212,7 +277,57 @@ impl StartSledAgentRequest { /// between sleds. pub fn hash_rack_id(&self) -> [u8; 32] { // We know the unwrap succeeds as a Sha3_256 digest is 32 bytes - Sha3_256::digest(self.rack_id.as_bytes()).as_slice().try_into().unwrap() + Sha3_256::digest(self.body.rack_id.as_bytes()) + .as_slice() + .try_into() + .unwrap() + } +} + +impl From for StartSledAgentRequest { + fn from(v0: StartSledAgentRequestV0) -> Self { + StartSledAgentRequest { + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { + id: v0.id, + rack_id: v0.rack_id, + use_trust_quorum: v0.use_trust_quorum, + is_lrtq_learner: false, + subnet: v0.subnet, + }, + } + } +} + +#[async_trait] +impl Ledgerable for StartSledAgentRequest { + fn is_newer_than(&self, other: &Self) -> bool { + self.generation > other.generation + } + + fn generation_bump(&mut self) { + // DO NOTHING! + // + // Generation bumps must only ever come from nexus and will be encoded + // in the struct itself + } + + // Attempt to deserialize the v1 or v0 version and return + // the v1 version. + fn deserialize( + s: &str, + ) -> Result { + // Try to deserialize the latest version of the data structure (v1). If + // that succeeds we are done. + if let Ok(val) = serde_json::from_str::(s) { + return Ok(val); + } + + // We don't have the latest version. Try to deserialize v0 and then + // convert it to the latest version. + let v0 = serde_json::from_str::(s)?.request; + Ok(v0.into()) } } @@ -291,12 +406,15 @@ mod tests { version: 1, request: Request::StartSledAgentRequest(Cow::Owned( StartSledAgentRequest { - id: Uuid::new_v4(), - rack_id: Uuid::new_v4(), - ntp_servers: vec![String::from("test.pool.example.com")], - dns_servers: vec!["1.1.1.1".parse().unwrap()], - use_trust_quorum: false, - subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { + id: Uuid::new_v4(), + rack_id: Uuid::new_v4(), + use_trust_quorum: false, + is_lrtq_learner: false, + subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + }, }, )), }; @@ -308,6 +426,36 @@ mod tests { assert!(envelope == deserialized, "serialization round trip failed"); } + #[test] + fn serialize_start_sled_agent_v0_deserialize_v1() { + let v0 = PersistentSledAgentRequest { + request: StartSledAgentRequestV0 { + id: Uuid::new_v4(), + rack_id: Uuid::new_v4(), + ntp_servers: vec![String::from("test.pool.example.com")], + dns_servers: vec!["1.1.1.1".parse().unwrap()], + use_trust_quorum: false, + subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), + }, + }; + let serialized = serde_json::to_string(&v0).unwrap(); + let expected = StartSledAgentRequest { + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { + id: v0.request.id, + rack_id: v0.request.rack_id, + use_trust_quorum: v0.request.use_trust_quorum, + is_lrtq_learner: false, + subnet: v0.request.subnet, + }, + }; + + let actual: StartSledAgentRequest = + Ledgerable::deserialize(&serialized).unwrap(); + assert_eq!(expected, actual); + } + #[test] fn validate_external_dns_ips_must_be_in_internal_services_ip_pools() { // Conjure up a config; we'll tweak the internal services pools and diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs index 0c19c30865d..02710ff583a 100644 --- a/sled-agent/src/bootstrap/pre_server.rs +++ b/sled-agent/src/bootstrap/pre_server.rs @@ -11,12 +11,15 @@ #![allow(clippy::result_large_err)] use super::maghemite; -use super::secret_retriever::LrtqOrHardcodedSecretRetriever; use super::server::StartError; use crate::config::Config; +use crate::config::SidecarRevision; +use crate::long_running_tasks::{ + spawn_all_longrunning_tasks, LongRunningTaskHandles, +}; use crate::services::ServiceManager; use crate::sled_agent::SledAgent; -use crate::storage_manager::StorageManager; +use crate::storage_monitor::UnderlayAccess; use camino::Utf8PathBuf; use cancel_safe_futures::TryStreamExt; use ddm_admin_client::Client as DdmAdminClient; @@ -29,115 +32,16 @@ use illumos_utils::zfs; use illumos_utils::zfs::Zfs; use illumos_utils::zone; use illumos_utils::zone::Zones; -use key_manager::KeyManager; -use key_manager::StorageKeyRequester; use omicron_common::address::Ipv6Subnet; use omicron_common::FileKv; use sled_hardware::underlay; use sled_hardware::DendriteAsic; -use sled_hardware::HardwareManager; -use sled_hardware::HardwareUpdate; use sled_hardware::SledMode; use slog::Drain; use slog::Logger; use std::net::IpAddr; use std::net::Ipv6Addr; -use tokio::sync::broadcast; -use tokio::task::JoinHandle; - -pub(super) struct BootstrapManagers { - pub(super) hardware: HardwareManager, - pub(super) storage: StorageManager, - pub(super) service: ServiceManager, -} - -impl BootstrapManagers { - pub(super) async fn handle_hardware_update( - &self, - update: Result, - sled_agent: Option<&SledAgent>, - log: &Logger, - ) { - match update { - Ok(update) => match update { - HardwareUpdate::TofinoLoaded => { - let baseboard = self.hardware.baseboard(); - if let Err(e) = self - .service - .activate_switch( - sled_agent.map(|sa| sa.switch_zone_underlay_info()), - baseboard, - ) - .await - { - warn!(log, "Failed to activate switch: {e}"); - } - } - HardwareUpdate::TofinoUnloaded => { - if let Err(e) = self.service.deactivate_switch().await { - warn!(log, "Failed to deactivate switch: {e}"); - } - } - HardwareUpdate::TofinoDeviceChange => { - if let Some(sled_agent) = sled_agent { - sled_agent.notify_nexus_about_self(log); - } - } - HardwareUpdate::DiskAdded(disk) => { - self.storage.upsert_disk(disk).await; - } - HardwareUpdate::DiskRemoved(disk) => { - self.storage.delete_disk(disk).await; - } - }, - Err(broadcast::error::RecvError::Lagged(count)) => { - warn!(log, "Hardware monitor missed {count} messages"); - self.check_latest_hardware_snapshot(sled_agent, log).await; - } - Err(broadcast::error::RecvError::Closed) => { - // The `HardwareManager` monitoring task is an infinite loop - - // the only way for us to get `Closed` here is if it panicked, - // so we will propagate such a panic. - panic!("Hardware manager monitor task panicked"); - } - } - } - - // Observe the current hardware state manually. - // - // We use this when we're monitoring hardware for the first - // time, and if we miss notifications. - pub(super) async fn check_latest_hardware_snapshot( - &self, - sled_agent: Option<&SledAgent>, - log: &Logger, - ) { - let underlay_network = sled_agent.map(|sled_agent| { - sled_agent.notify_nexus_about_self(log); - sled_agent.switch_zone_underlay_info() - }); - info!( - log, "Checking current full hardware snapshot"; - "underlay_network_info" => ?underlay_network, - ); - if self.hardware.is_scrimlet_driver_loaded() { - let baseboard = self.hardware.baseboard(); - if let Err(e) = - self.service.activate_switch(underlay_network, baseboard).await - { - warn!(log, "Failed to activate switch: {e}"); - } - } else { - if let Err(e) = self.service.deactivate_switch().await { - warn!(log, "Failed to deactivate switch: {e}"); - } - } - - self.storage - .ensure_using_exactly_these_disks(self.hardware.disks()) - .await; - } -} +use tokio::sync::oneshot; pub(super) struct BootstrapAgentStartup { pub(super) config: Config, @@ -145,8 +49,10 @@ pub(super) struct BootstrapAgentStartup { pub(super) ddm_admin_localhost_client: DdmAdminClient, pub(super) base_log: Logger, pub(super) startup_log: Logger, - pub(super) managers: BootstrapManagers, - pub(super) key_manager_handle: JoinHandle<()>, + pub(super) service_manager: ServiceManager, + pub(super) long_running_task_handles: LongRunningTaskHandles, + pub(super) sled_agent_started_tx: oneshot::Sender, + pub(super) underlay_available_tx: oneshot::Sender, } impl BootstrapAgentStartup { @@ -200,36 +106,23 @@ impl BootstrapAgentStartup { // This should be a no-op if already enabled. BootstrapNetworking::enable_ipv6_forwarding().await?; - // Spawn the `KeyManager` which is needed by the the StorageManager to - // retrieve encryption keys. - let (storage_key_requester, key_manager_handle) = - spawn_key_manager_task(&base_log); - + // Are we a gimlet or scrimlet? let sled_mode = sled_mode_from_config(&config)?; - // Start monitoring hardware. This is blocking so we use - // `spawn_blocking`; similar to above, we move some things in and (on - // success) it gives them back. - let (base_log, log, hardware_manager) = { - tokio::task::spawn_blocking(move || { - info!( - log, "Starting hardware monitor"; - "sled_mode" => ?sled_mode, - ); - let hardware_manager = - HardwareManager::new(&base_log, sled_mode) - .map_err(StartError::StartHardwareManager)?; - Ok::<_, StartError>((base_log, log, hardware_manager)) - }) - .await - .unwrap()? - }; - - // Create a `StorageManager` and (possibly) synthetic disks. - let storage_manager = - StorageManager::new(&base_log, storage_key_requester).await; - upsert_synthetic_zpools_if_needed(&log, &storage_manager, &config) - .await; + // Spawn all important long running tasks that live for the lifetime of + // the process and are used by both the bootstrap agent and sled agent + let ( + long_running_task_handles, + sled_agent_started_tx, + service_manager_ready_tx, + underlay_available_tx, + ) = spawn_all_longrunning_tasks( + &base_log, + sled_mode, + startup_networking.global_zone_bootstrap_ip, + &config, + ) + .await; let global_zone_bootstrap_ip = startup_networking.global_zone_bootstrap_ip; @@ -242,22 +135,27 @@ impl BootstrapAgentStartup { config.skip_timesync, config.sidecar_revision.clone(), config.switch_zone_maghemite_links.clone(), - storage_manager.resources().clone(), - storage_manager.zone_bundler().clone(), + long_running_task_handles.storage_manager.clone(), + long_running_task_handles.zone_bundler.clone(), ); + // Inform the hardware monitor that the service manager is ready + // This is a onetime operation, and so we use a oneshot channel + service_manager_ready_tx + .send(service_manager.clone()) + .map_err(|_| ()) + .expect("Failed to send to StorageMonitor"); + Ok(Self { config, global_zone_bootstrap_ip, ddm_admin_localhost_client, base_log, startup_log: log, - managers: BootstrapManagers { - hardware: hardware_manager, - storage: storage_manager, - service: service_manager, - }, - key_manager_handle, + service_manager, + long_running_task_handles, + sled_agent_started_tx, + underlay_available_tx, }) } } @@ -339,6 +237,7 @@ async fn cleanup_all_old_global_state(log: &Logger) -> Result<(), StartError> { } fn enable_mg_ddm(config: &Config, log: &Logger) -> Result<(), StartError> { + info!(log, "finding links {:?}", config.data_links); let mg_addr_objs = underlay::find_nics(&config.data_links) .map_err(StartError::FindMaghemiteAddrObjs)?; if mg_addr_objs.is_empty() { @@ -357,13 +256,10 @@ fn ensure_zfs_key_directory_exists(log: &Logger) -> Result<(), StartError> { // to create and mount encrypted datasets. info!( log, "Ensuring zfs key directory exists"; - "path" => sled_hardware::disk::KEYPATH_ROOT, + "path" => zfs::KEYPATH_ROOT, ); - std::fs::create_dir_all(sled_hardware::disk::KEYPATH_ROOT).map_err(|err| { - StartError::CreateZfsKeyDirectory { - dir: sled_hardware::disk::KEYPATH_ROOT, - err, - } + std::fs::create_dir_all(zfs::KEYPATH_ROOT).map_err(|err| { + StartError::CreateZfsKeyDirectory { dir: zfs::KEYPATH_ROOT, err } }) } @@ -386,23 +282,6 @@ fn ensure_zfs_ramdisk_dataset() -> Result<(), StartError> { .map_err(StartError::EnsureZfsRamdiskDataset) } -async fn upsert_synthetic_zpools_if_needed( - log: &Logger, - storage_manager: &StorageManager, - config: &Config, -) { - if let Some(pools) = &config.zpools { - for pool in pools { - info!( - log, - "Upserting synthetic zpool to Storage Manager: {}", - pool.to_string() - ); - storage_manager.upsert_synthetic_disk(pool.clone()).await; - } - } -} - // Combine the `sled_mode` config with the build-time switch type to determine // the actual sled mode. fn sled_mode_from_config(config: &Config) -> Result { @@ -423,7 +302,16 @@ fn sled_mode_from_config(config: &Config) -> Result { } else if cfg!(feature = "switch-stub") { DendriteAsic::TofinoStub } else if cfg!(feature = "switch-softnpu") { - DendriteAsic::SoftNpu + match config.sidecar_revision { + SidecarRevision::SoftZone(_) => DendriteAsic::SoftNpuZone, + SidecarRevision::SoftPropolis(_) => { + DendriteAsic::SoftNpuPropolisDevice + } + _ => return Err(StartError::IncorrectBuildPackaging( + "sled-agent configured to run on softnpu zone but dosen't \ + have a softnpu sidecar revision", + )), + } } else { return Err(StartError::IncorrectBuildPackaging( "sled-agent configured to run on scrimlet but wasn't \ @@ -436,19 +324,6 @@ fn sled_mode_from_config(config: &Config) -> Result { Ok(sled_mode) } -fn spawn_key_manager_task( - log: &Logger, -) -> (StorageKeyRequester, JoinHandle<()>) { - let secret_retriever = LrtqOrHardcodedSecretRetriever::new(); - let (mut key_manager, storage_key_requester) = - KeyManager::new(log, secret_retriever); - - let key_manager_handle = - tokio::spawn(async move { key_manager.run().await }); - - (storage_key_requester, key_manager_handle) -} - #[derive(Debug, Clone)] pub(crate) struct BootstrapNetworking { pub(crate) bootstrap_etherstub: dladm::Etherstub, diff --git a/sled-agent/src/bootstrap/rack_ops.rs b/sled-agent/src/bootstrap/rack_ops.rs index b8721f83329..5cfd0b074a0 100644 --- a/sled-agent/src/bootstrap/rack_ops.rs +++ b/sled-agent/src/bootstrap/rack_ops.rs @@ -8,11 +8,11 @@ use crate::bootstrap::http_entrypoints::RackOperationStatus; use crate::bootstrap::params::RackInitializeRequest; use crate::bootstrap::rss_handle::RssHandle; use crate::rack_setup::service::SetupServiceError; -use crate::storage_manager::StorageResources; use bootstore::schemes::v0 as bootstore; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::mem; use std::net::Ipv6Addr; @@ -171,7 +171,7 @@ impl RssAccess { &self, parent_log: &Logger, global_zone_bootstrap_ip: Ipv6Addr, - storage_resources: &StorageResources, + storage_manager: &StorageHandle, bootstore_node_handle: &bootstore::NodeHandle, request: RackInitializeRequest, ) -> Result { @@ -207,14 +207,14 @@ impl RssAccess { mem::drop(status); let parent_log = parent_log.clone(); - let storage_resources = storage_resources.clone(); + let storage_manager = storage_manager.clone(); let bootstore_node_handle = bootstore_node_handle.clone(); let status = Arc::clone(&self.status); tokio::spawn(async move { let result = rack_initialize( &parent_log, global_zone_bootstrap_ip, - storage_resources, + storage_manager, bootstore_node_handle, request, ) @@ -342,7 +342,7 @@ enum RssStatus { async fn rack_initialize( parent_log: &Logger, global_zone_bootstrap_ip: Ipv6Addr, - storage_resources: StorageResources, + storage_manager: StorageHandle, bootstore_node_handle: bootstore::NodeHandle, request: RackInitializeRequest, ) -> Result<(), SetupServiceError> { @@ -350,7 +350,7 @@ async fn rack_initialize( parent_log, request, global_zone_bootstrap_ip, - storage_resources, + storage_manager, bootstore_node_handle, ) .await diff --git a/sled-agent/src/bootstrap/rss_handle.rs b/sled-agent/src/bootstrap/rss_handle.rs index c82873d91da..5d9c01e7f27 100644 --- a/sled-agent/src/bootstrap/rss_handle.rs +++ b/sled-agent/src/bootstrap/rss_handle.rs @@ -9,7 +9,6 @@ use super::params::StartSledAgentRequest; use crate::rack_setup::config::SetupServiceConfig; use crate::rack_setup::service::RackSetupService; use crate::rack_setup::service::SetupServiceError; -use crate::storage_manager::StorageResources; use ::bootstrap_agent_client::Client as BootstrapAgentClient; use bootstore::schemes::v0 as bootstore; use futures::stream::FuturesUnordered; @@ -17,6 +16,7 @@ use futures::StreamExt; use omicron_common::backoff::retry_notify; use omicron_common::backoff::retry_policy_local; use omicron_common::backoff::BackoffError; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::net::Ipv6Addr; use std::net::SocketAddrV6; @@ -46,7 +46,7 @@ impl RssHandle { log: &Logger, config: SetupServiceConfig, our_bootstrap_address: Ipv6Addr, - storage_resources: StorageResources, + storage_manager: StorageHandle, bootstore: bootstore::NodeHandle, ) -> Result<(), SetupServiceError> { let (tx, rx) = rss_channel(our_bootstrap_address); @@ -54,7 +54,7 @@ impl RssHandle { let rss = RackSetupService::new( log.new(o!("component" => "RSS")), config, - storage_resources, + storage_manager, tx, bootstore, ); diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 9ed3ad582d8..f4948de83bd 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -8,12 +8,10 @@ use super::config::BOOTSTRAP_AGENT_HTTP_PORT; use super::http_entrypoints; use super::params::RackInitializeRequest; use super::params::StartSledAgentRequest; -use super::pre_server::BootstrapManagers; use super::rack_ops::RackInitId; use super::views::SledAgentResponse; use super::BootstrapError; use super::RssAccessError; -use crate::bootstrap::bootstore::BootstoreHandles; use crate::bootstrap::config::BOOTSTRAP_AGENT_RACK_INIT_PORT; use crate::bootstrap::http_entrypoints::api as http_api; use crate::bootstrap::http_entrypoints::BootstrapServerContext; @@ -24,16 +22,17 @@ use crate::bootstrap::secret_retriever::LrtqOrHardcodedSecretRetriever; use crate::bootstrap::sprockets_server::SprocketsServer; use crate::config::Config as SledConfig; use crate::config::ConfigError; +use crate::long_running_tasks::LongRunningTaskHandles; use crate::server::Server as SledAgentServer; +use crate::services::ServiceManager; use crate::sled_agent::SledAgent; -use crate::storage_manager::StorageResources; +use crate::storage_monitor::UnderlayAccess; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use cancel_safe_futures::TryStreamExt; use ddm_admin_client::Client as DdmAdminClient; use ddm_admin_client::DdmError; use dropshot::HttpServer; -use futures::Future; use futures::StreamExt; use illumos_utils::dladm; use illumos_utils::zfs; @@ -41,18 +40,13 @@ use illumos_utils::zone; use illumos_utils::zone::Zones; use omicron_common::ledger; use omicron_common::ledger::Ledger; -use omicron_common::ledger::Ledgerable; -use schemars::JsonSchema; -use serde::Deserialize; -use serde::Serialize; use sled_hardware::underlay; -use sled_hardware::HardwareUpdate; +use sled_storage::dataset::CONFIG_DATASET; +use sled_storage::manager::StorageHandle; use slog::Logger; -use std::borrow::Cow; use std::io; use std::net::SocketAddr; use std::net::SocketAddrV6; -use tokio::sync::broadcast; use tokio::sync::mpsc; use tokio::sync::oneshot; use tokio::task::JoinHandle; @@ -153,6 +147,9 @@ pub enum StartError { #[error("Failed to bind sprocket server")] BindSprocketsServer(#[source] io::Error), + + #[error("Failed to initialize lrtq node as learner: {0}")] + FailedLearnerInit(bootstore::NodeRequestError), } /// Server for the bootstrap agent. @@ -177,68 +174,18 @@ impl Server { ddm_admin_localhost_client, base_log, startup_log, - managers, - key_manager_handle, + service_manager, + long_running_task_handles, + sled_agent_started_tx, + underlay_available_tx, } = BootstrapAgentStartup::run(config).await?; - // From this point on we will listen for hardware notifications and - // potentially start the switch zone and be notified of new disks; we - // are responsible for responding to updates from this point on. - let mut hardware_monitor = managers.hardware.monitor(); - let storage_resources = managers.storage.resources(); - - // Check the latest hardware snapshot; we could have missed events - // between the creation of the hardware manager and our subscription of - // its monitor. - managers.check_latest_hardware_snapshot(None, &startup_log).await; - - // Wait for our boot M.2 to show up. - wait_while_handling_hardware_updates( - wait_for_boot_m2(storage_resources, &startup_log), - &mut hardware_monitor, - &managers, - None, // No underlay network yet - &startup_log, - "waiting for boot M.2", - ) - .await; - - // Wait for the bootstore to start. - let bootstore_handles = wait_while_handling_hardware_updates( - BootstoreHandles::spawn( - storage_resources, - ddm_admin_localhost_client.clone(), - managers.hardware.baseboard(), - global_zone_bootstrap_ip, - &base_log, - ), - &mut hardware_monitor, - &managers, - None, // No underlay network yet - &startup_log, - "initializing bootstore", - ) - .await?; - // Do we have a StartSledAgentRequest stored in the ledger? - let maybe_ledger = wait_while_handling_hardware_updates( - async { - let paths = sled_config_paths(storage_resources).await?; - let maybe_ledger = - Ledger::>::new( - &startup_log, - paths, - ) - .await; - Ok::<_, StartError>(maybe_ledger) - }, - &mut hardware_monitor, - &managers, - None, // No underlay network yet - &startup_log, - "loading sled-agent request from ledger", - ) - .await?; + let paths = + sled_config_paths(&long_running_task_handles.storage_manager) + .await?; + let maybe_ledger = + Ledger::::new(&startup_log, paths).await; // We don't yet _act_ on the `StartSledAgentRequest` if we have one, but // if we have one we init our `RssAccess` noting that we're already @@ -255,9 +202,9 @@ impl Server { let bootstrap_context = BootstrapServerContext { base_log: base_log.clone(), global_zone_bootstrap_ip, - storage_resources: storage_resources.clone(), - bootstore_node_handle: bootstore_handles.node_handle.clone(), - baseboard: managers.hardware.baseboard(), + storage_manager: long_running_task_handles.storage_manager.clone(), + bootstore_node_handle: long_running_task_handles.bootstore.clone(), + baseboard: long_running_task_handles.hardware_manager.baseboard(), rss_access, updates: config.updates.clone(), sled_reset_tx, @@ -288,56 +235,37 @@ impl Server { // Do we have a persistent sled-agent request that we need to restore? let state = if let Some(ledger) = maybe_ledger { - let sled_request = ledger.data(); - let sled_agent_server = wait_while_handling_hardware_updates( - start_sled_agent( - &config, - &sled_request.request, - &bootstore_handles.node_handle, - &managers, - &ddm_admin_localhost_client, - &base_log, - &startup_log, - ), - &mut hardware_monitor, - &managers, - None, // No underlay network yet + let start_sled_agent_request = ledger.into_inner(); + let sled_agent_server = start_sled_agent( + &config, + start_sled_agent_request, + long_running_task_handles.clone(), + underlay_available_tx, + service_manager.clone(), + &ddm_admin_localhost_client, + &base_log, &startup_log, - "restoring sled-agent (cold boot)", ) .await?; + // Give the HardwareMonitory access to the `SledAgent` let sled_agent = sled_agent_server.sled_agent(); - - // We've created sled-agent; we need to (possibly) reconfigure the - // switch zone, if we're a scrimlet, to give it our underlay network - // information. - let underlay_network_info = sled_agent.switch_zone_underlay_info(); - info!( - startup_log, "Sled Agent started; rescanning hardware"; - "underlay_network_info" => ?underlay_network_info, - ); - managers - .check_latest_hardware_snapshot(Some(&sled_agent), &startup_log) - .await; + sled_agent_started_tx + .send(sled_agent.clone()) + .map_err(|_| ()) + .expect("Failed to send to StorageMonitor"); // For cold boot specifically, we now need to load the services // we're responsible for, while continuing to handle hardware // notifications. This cannot fail: we retry indefinitely until // we're done loading services. - wait_while_handling_hardware_updates( - sled_agent.cold_boot_load_services(), - &mut hardware_monitor, - &managers, - Some(&sled_agent), - &startup_log, - "restoring sled-agent services (cold boot)", - ) - .await; - + sled_agent.cold_boot_load_services().await; SledAgentState::ServerStarted(sled_agent_server) } else { - SledAgentState::Bootstrapping + SledAgentState::Bootstrapping( + Some(sled_agent_started_tx), + Some(underlay_available_tx), + ) }; // Spawn our inner task that handles any future hardware updates and any @@ -345,15 +273,13 @@ impl Server { // agent state. let inner = Inner { config, - hardware_monitor, state, sled_init_rx, sled_reset_rx, - managers, ddm_admin_localhost_client, - bootstore_handles, + long_running_task_handles, + service_manager, _sprockets_server_handle: sprockets_server_handle, - _key_manager_handle: key_manager_handle, base_log, }; let inner_task = tokio::spawn(inner.run()); @@ -382,20 +308,14 @@ impl Server { // bootstrap server). enum SledAgentState { // We're still in the bootstrapping phase, waiting for a sled-agent request. - Bootstrapping, + Bootstrapping( + Option>, + Option>, + ), // ... or the sled agent server is running. ServerStarted(SledAgentServer), } -impl SledAgentState { - fn sled_agent(&self) -> Option<&SledAgent> { - match self { - SledAgentState::Bootstrapping => None, - SledAgentState::ServerStarted(server) => Some(server.sled_agent()), - } - } -} - #[derive(thiserror::Error, Debug)] pub enum SledAgentServerStartError { #[error("Failed to start sled-agent server: {0}")] @@ -406,6 +326,9 @@ pub enum SledAgentServerStartError { #[error("Failed to commit sled agent request to ledger")] CommitToLedger(#[from] ledger::Error), + + #[error("Failed to initialize this lrtq node as a learner: {0}")] + FailedLearnerInit(#[from] bootstore::NodeRequestError), } impl From for StartError { @@ -420,15 +343,20 @@ impl From for StartError { SledAgentServerStartError::CommitToLedger(err) => { Self::CommitToLedger(err) } + SledAgentServerStartError::FailedLearnerInit(err) => { + Self::FailedLearnerInit(err) + } } } } +#[allow(clippy::too_many_arguments)] async fn start_sled_agent( config: &SledConfig, - request: &StartSledAgentRequest, - bootstore: &bootstore::NodeHandle, - managers: &BootstrapManagers, + request: StartSledAgentRequest, + long_running_task_handles: LongRunningTaskHandles, + underlay_available_tx: oneshot::Sender, + service_manager: ServiceManager, ddmd_client: &DdmAdminClient, base_log: &Logger, log: &Logger, @@ -440,17 +368,33 @@ async fn start_sled_agent( // storage manager about keys, advertising prefixes, ...). // Initialize the secret retriever used by the `KeyManager` - if request.use_trust_quorum { + if request.body.use_trust_quorum { info!(log, "KeyManager: using lrtq secret retriever"); let salt = request.hash_rack_id(); - LrtqOrHardcodedSecretRetriever::init_lrtq(salt, bootstore.clone()) + LrtqOrHardcodedSecretRetriever::init_lrtq( + salt, + long_running_task_handles.bootstore.clone(), + ) } else { info!(log, "KeyManager: using hardcoded secret retriever"); LrtqOrHardcodedSecretRetriever::init_hardcoded(); } + if request.body.use_trust_quorum && request.body.is_lrtq_learner { + info!(log, "Initializing sled as learner"); + match long_running_task_handles.bootstore.init_learner().await { + Err(bootstore::NodeRequestError::Fsm( + bootstore::ApiError::AlreadyInitialized, + )) => { + // This is a cold boot. Let's ignore this error and continue. + } + Err(e) => return Err(e.into()), + Ok(()) => (), + } + } + // Inform the storage service that the key manager is available - managers.storage.key_manager_ready().await; + long_running_task_handles.storage_manager.key_manager_ready().await; // Start trying to notify ddmd of our sled prefix so it can // advertise it to other sleds. @@ -463,16 +407,16 @@ async fn start_sled_agent( // we'll need to do something different here for underlay vs bootstrap // addrs (either talk to a differently-configured ddmd, or include info // indicating which kind of address we're advertising). - ddmd_client.advertise_prefix(request.subnet); + ddmd_client.advertise_prefix(request.body.subnet); // Server does not exist, initialize it. let server = SledAgentServer::start( config, base_log.clone(), request.clone(), - managers.service.clone(), - managers.storage.clone(), - bootstore.clone(), + long_running_task_handles.clone(), + service_manager, + underlay_available_tx, ) .await .map_err(SledAgentServerStartError::FailedStartingServer)?; @@ -481,13 +425,10 @@ async fn start_sled_agent( // Record this request so the sled agent can be automatically // initialized on the next boot. - let paths = sled_config_paths(managers.storage.resources()).await?; + let paths = + sled_config_paths(&long_running_task_handles.storage_manager).await?; - let mut ledger = Ledger::new_with( - &log, - paths, - PersistentSledAgentRequest { request: Cow::Borrowed(request) }, - ); + let mut ledger = Ledger::new_with(&log, paths, request); ledger.commit().await?; Ok(server) @@ -524,28 +465,6 @@ fn start_dropshot_server( Ok(http_server) } -/// Wait for at least the M.2 we booted from to show up. -/// -/// TODO-correctness Subsequent steps may assume all M.2s that will ever be -/// present are present once we return from this function; see -/// . -async fn wait_for_boot_m2(storage_resources: &StorageResources, log: &Logger) { - // Wait for at least the M.2 we booted from to show up. - loop { - match storage_resources.boot_disk().await { - Some(disk) => { - info!(log, "Found boot disk M.2: {disk:?}"); - break; - } - None => { - info!(log, "Waiting for boot disk M.2..."); - tokio::time::sleep(core::time::Duration::from_millis(250)) - .await; - } - } - } -} - struct MissingM2Paths(&'static str); impl From for StartError { @@ -561,68 +480,21 @@ impl From for SledAgentServerStartError { } async fn sled_config_paths( - storage: &StorageResources, + storage: &StorageHandle, ) -> Result, MissingM2Paths> { - let paths: Vec<_> = storage - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) - .await + let resources = storage.get_latest_resources().await; + let paths: Vec<_> = resources + .all_m2_mountpoints(CONFIG_DATASET) .into_iter() .map(|p| p.join(SLED_AGENT_REQUEST_FILE)) .collect(); if paths.is_empty() { - return Err(MissingM2Paths(sled_hardware::disk::CONFIG_DATASET)); + return Err(MissingM2Paths(CONFIG_DATASET)); } Ok(paths) } -// Helper function to wait for `fut` while handling any updates about hardware. -async fn wait_while_handling_hardware_updates, T>( - fut: F, - hardware_monitor: &mut broadcast::Receiver, - managers: &BootstrapManagers, - sled_agent: Option<&SledAgent>, - log: &Logger, - log_phase: &str, -) -> T { - tokio::pin!(fut); - loop { - tokio::select! { - // Cancel-safe per the docs on `broadcast::Receiver::recv()`. - hardware_update = hardware_monitor.recv() => { - info!( - log, - "Handling hardware update message"; - "phase" => log_phase, - "update" => ?hardware_update, - ); - - managers.handle_hardware_update( - hardware_update, - sled_agent, - log, - ).await; - } - - // Cancel-safe: we're using a `&mut Future`; dropping the - // reference does not cancel the underlying future. - result = &mut fut => return result, - } - } -} - -#[derive(Clone, Serialize, Deserialize, PartialEq, JsonSchema)] -struct PersistentSledAgentRequest<'a> { - request: Cow<'a, StartSledAgentRequest>, -} - -impl<'a> Ledgerable for PersistentSledAgentRequest<'a> { - fn is_newer_than(&self, _other: &Self) -> bool { - true - } - fn generation_bump(&mut self) {} -} - /// Runs the OpenAPI generator, emitting the spec to stdout. pub fn run_openapi() -> Result<(), String> { http_api() @@ -636,18 +508,16 @@ pub fn run_openapi() -> Result<(), String> { struct Inner { config: SledConfig, - hardware_monitor: broadcast::Receiver, state: SledAgentState, sled_init_rx: mpsc::Receiver<( StartSledAgentRequest, oneshot::Sender>, )>, sled_reset_rx: mpsc::Receiver>>, - managers: BootstrapManagers, ddm_admin_localhost_client: DdmAdminClient, - bootstore_handles: BootstoreHandles, + long_running_task_handles: LongRunningTaskHandles, + service_manager: ServiceManager, _sprockets_server_handle: JoinHandle<()>, - _key_manager_handle: JoinHandle<()>, base_log: Logger, } @@ -655,14 +525,7 @@ impl Inner { async fn run(mut self) { let log = self.base_log.new(o!("component" => "SledAgentMain")); loop { - // TODO-correctness We pause handling hardware update messages while - // we handle sled init/reset requests - is that okay? tokio::select! { - // Cancel-safe per the docs on `broadcast::Receiver::recv()`. - hardware_update = self.hardware_monitor.recv() => { - self.handle_hardware_update(hardware_update, &log).await; - } - // Cancel-safe per the docs on `mpsc::Receiver::recv()`. Some((request, response_tx)) = self.sled_init_rx.recv() => { self.handle_start_sled_agent_request( @@ -690,40 +553,36 @@ impl Inner { } } - async fn handle_hardware_update( - &self, - hardware_update: Result, - log: &Logger, - ) { - info!( - log, - "Handling hardware update message"; - "phase" => "bootstore-steady-state", - "update" => ?hardware_update, - ); - - self.managers - .handle_hardware_update( - hardware_update, - self.state.sled_agent(), - &log, - ) - .await; - } - async fn handle_start_sled_agent_request( &mut self, request: StartSledAgentRequest, response_tx: oneshot::Sender>, log: &Logger, ) { - match &self.state { - SledAgentState::Bootstrapping => { + match &mut self.state { + SledAgentState::Bootstrapping( + sled_agent_started_tx, + underlay_available_tx, + ) => { + let request_id = request.body.id; + + // Extract from options to satisfy the borrow checker. + // It is not possible for `start_sled_agent` to be cancelled + // or fail in a safe, restartable manner. Therefore, for now, + // we explicitly unwrap here, and panic on error below. + // + // See https://github.com/oxidecomputer/omicron/issues/4494 + let sled_agent_started_tx = + sled_agent_started_tx.take().unwrap(); + let underlay_available_tx = + underlay_available_tx.take().unwrap(); + let response = match start_sled_agent( &self.config, - &request, - &self.bootstore_handles.node_handle, - &self.managers, + request, + self.long_running_task_handles.clone(), + underlay_available_tx, + self.service_manager.clone(), &self.ddm_admin_localhost_client, &self.base_log, &log, @@ -734,37 +593,31 @@ impl Inner { // We've created sled-agent; we need to (possibly) // reconfigure the switch zone, if we're a scrimlet, to // give it our underlay network information. - self.managers - .check_latest_hardware_snapshot( - Some(server.sled_agent()), - log, - ) - .await; - + sled_agent_started_tx + .send(server.sled_agent().clone()) + .map_err(|_| ()) + .expect("Failed to send to StorageMonitor"); self.state = SledAgentState::ServerStarted(server); - Ok(SledAgentResponse { id: request.id }) + Ok(SledAgentResponse { id: request_id }) + } + Err(err) => { + // This error is unrecoverable, and if returned we'd + // end up in maintenance mode anyway. + error!(log, "Failed to start sled agent: {err:#}"); + panic!("Failed to start sled agent"); } - Err(err) => Err(format!("{err:#}")), }; _ = response_tx.send(response); } SledAgentState::ServerStarted(server) => { info!(log, "Sled Agent already loaded"); - - let sled_address = request.sled_address(); - let response = if server.id() != request.id { + let initial = server.sled_agent().start_request(); + let response = if initial != &request { Err(format!( - "Sled Agent already running with UUID {}, \ - but {} was requested", - server.id(), - request.id, - )) - } else if &server.address().ip() != sled_address.ip() { - Err(format!( - "Sled Agent already running on address {}, \ - but {} was requested", - server.address().ip(), - sled_address.ip(), + "Sled Agent already running: + initital request = {:?}, + current request: {:?}", + initial, request )) } else { Ok(SledAgentResponse { id: server.id() }) @@ -796,11 +649,11 @@ impl Inner { async fn uninstall_sled_local_config(&self) -> Result<(), BootstrapError> { let config_dirs = self - .managers - .storage - .resources() - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + .long_running_task_handles + .storage_manager + .get_latest_resources() .await + .all_m2_mountpoints(CONFIG_DATASET) .into_iter(); for dir in config_dirs { @@ -873,6 +726,8 @@ impl Inner { #[cfg(test)] mod tests { + use crate::bootstrap::params::StartSledAgentRequestBody; + use super::*; use omicron_common::address::Ipv6Subnet; use omicron_test_utils::dev::test_setup_log; @@ -880,20 +735,21 @@ mod tests { use uuid::Uuid; #[tokio::test] - async fn persistent_sled_agent_request_serialization() { + async fn start_sled_agent_request_serialization() { let logctx = test_setup_log("persistent_sled_agent_request_serialization"); let log = &logctx.log; - let request = PersistentSledAgentRequest { - request: Cow::Owned(StartSledAgentRequest { + let request = StartSledAgentRequest { + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { id: Uuid::new_v4(), rack_id: Uuid::new_v4(), - ntp_servers: vec![String::from("test.pool.example.com")], - dns_servers: vec!["1.1.1.1".parse().unwrap()], use_trust_quorum: false, + is_lrtq_learner: false, subnet: Ipv6Subnet::new(Ipv6Addr::LOCALHOST), - }), + }, }; let tempdir = camino_tempfile::Utf8TempDir::new().unwrap(); @@ -902,7 +758,7 @@ mod tests { let mut ledger = Ledger::new_with(log, paths.clone(), request.clone()); ledger.commit().await.expect("Failed to write to ledger"); - let ledger = Ledger::::new(log, paths) + let ledger = Ledger::::new(log, paths) .await .expect("Failed to read request"); @@ -912,9 +768,9 @@ mod tests { #[test] fn test_persistent_sled_agent_request_schema() { - let schema = schemars::schema_for!(PersistentSledAgentRequest<'_>); + let schema = schemars::schema_for!(StartSledAgentRequest); expectorate::assert_contents( - "../schema/persistent-sled-agent-request.json", + "../schema/start-sled-agent-request.json", &serde_json::to_string_pretty(&schema).unwrap(), ); } diff --git a/sled-agent/src/common/disk.rs b/sled-agent/src/common/disk.rs index 18160950d3f..57868937d07 100644 --- a/sled-agent/src/common/disk.rs +++ b/sled-agent/src/common/disk.rs @@ -9,7 +9,7 @@ use chrono::Utc; use omicron_common::api::external::DiskState; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::DiskRuntimeState; -use propolis_client::api::DiskAttachmentState as PropolisDiskState; +use propolis_client::types::DiskAttachmentState as PropolisDiskState; use uuid::Uuid; /// Action to be taken on behalf of state transition. diff --git a/sled-agent/src/common/instance.rs b/sled-agent/src/common/instance.rs index 9e285840e02..d7ee8982e08 100644 --- a/sled-agent/src/common/instance.rs +++ b/sled-agent/src/common/instance.rs @@ -10,8 +10,9 @@ use omicron_common::api::external::InstanceState as ApiInstanceState; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, SledInstanceState, VmmRuntimeState, }; -use propolis_client::api::{ +use propolis_client::types::{ InstanceState as PropolisApiState, InstanceStateMonitorResponse, + MigrationState, }; use uuid::Uuid; @@ -36,7 +37,7 @@ impl From for PropolisInstanceState { impl From for ApiInstanceState { fn from(value: PropolisInstanceState) -> Self { - use propolis_client::api::InstanceState as State; + use propolis_client::types::InstanceState as State; match value.0 { // Nexus uses the VMM state as the externally-visible instance state // when an instance has an active VMM. A Propolis that is "creating" @@ -119,7 +120,6 @@ impl ObservedPropolisState { (Some(this_id), Some(propolis_migration)) if this_id == propolis_migration.migration_id => { - use propolis_client::api::MigrationState; match propolis_migration.state { MigrationState::Finish => { ObservedMigrationStatus::Succeeded @@ -510,7 +510,7 @@ mod test { use chrono::Utc; use omicron_common::api::external::Generation; use omicron_common::api::internal::nexus::InstanceRuntimeState; - use propolis_client::api::InstanceState as Observed; + use propolis_client::types::InstanceState as Observed; use uuid::Uuid; fn make_instance() -> InstanceStates { diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 2473c14566a..a596cf83dbb 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -28,7 +28,8 @@ pub enum SledMode { #[serde(rename_all = "snake_case")] pub enum SidecarRevision { Physical(String), - Soft(SoftPortConfig), + SoftZone(SoftPortConfig), + SoftPropolis(SoftPortConfig), } #[derive(Debug, Clone, Deserialize)] @@ -51,6 +52,9 @@ pub struct Config { pub sidecar_revision: SidecarRevision, /// Optional percentage of DRAM to reserve for guest memory pub vmm_reservoir_percentage: Option, + /// Optional DRAM to reserve for guest memory in MiB (mutually exclusive + /// option with vmm_reservoir_percentage). + pub vmm_reservoir_size_mb: Option, /// Optional swap device size in GiB pub swap_device_size_gb: Option, /// Optional VLAN ID to be used for tagging guest VNICs. diff --git a/sled-agent/src/storage/dump_setup.rs b/sled-agent/src/dump_setup.rs similarity index 93% rename from sled-agent/src/storage/dump_setup.rs rename to sled-agent/src/dump_setup.rs index 9b5edc0a7e5..e675e6e12d9 100644 --- a/sled-agent/src/storage/dump_setup.rs +++ b/sled-agent/src/dump_setup.rs @@ -1,4 +1,3 @@ -use crate::storage_manager::DiskWrapper; use camino::Utf8PathBuf; use derive_more::{AsRef, Deref, From}; use illumos_utils::dumpadm::DumpAdmError; @@ -6,13 +5,15 @@ use illumos_utils::zone::{AdmError, Zones}; use illumos_utils::zpool::{ZpoolHealth, ZpoolName}; use omicron_common::disk::DiskIdentity; use sled_hardware::DiskVariant; +use sled_storage::dataset::{CRASH_DATASET, DUMP_DATASET}; +use sled_storage::disk::Disk; +use sled_storage::pool::Pool; use slog::Logger; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::sync::{Arc, Weak}; use std::time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH}; -use tokio::sync::MutexGuard; pub struct DumpSetup { worker: Arc>, @@ -70,11 +71,11 @@ trait GetMountpoint: std::ops::Deref { } impl GetMountpoint for DebugZpool { type NewType = DebugDataset; - const MOUNTPOINT: &'static str = sled_hardware::disk::DUMP_DATASET; + const MOUNTPOINT: &'static str = DUMP_DATASET; } impl GetMountpoint for CoreZpool { type NewType = CoreDataset; - const MOUNTPOINT: &'static str = sled_hardware::disk::CRASH_DATASET; + const MOUNTPOINT: &'static str = CRASH_DATASET; } struct DumpSetupWorker { @@ -99,50 +100,51 @@ const ARCHIVAL_INTERVAL: Duration = Duration::from_secs(300); impl DumpSetup { pub(crate) async fn update_dumpdev_setup( &self, - disks: &mut MutexGuard<'_, HashMap>, + disks: &BTreeMap, ) { let log = &self.log; let mut m2_dump_slices = Vec::new(); let mut u2_debug_datasets = Vec::new(); let mut m2_core_datasets = Vec::new(); - for (_id, disk_wrapper) in disks.iter() { - match disk_wrapper { - DiskWrapper::Real { disk, .. } => match disk.variant() { - DiskVariant::M2 => { - match disk.dump_device_devfs_path(false) { - Ok(path) => { - m2_dump_slices.push(DumpSlicePath(path)) - } - Err(err) => { - warn!(log, "Error getting dump device devfs path: {err:?}"); - } + for (_id, (disk, _)) in disks.iter() { + if disk.is_synthetic() { + // We only setup dump devices on real disks + continue; + } + match disk.variant() { + DiskVariant::M2 => { + match disk.dump_device_devfs_path(false) { + Ok(path) => m2_dump_slices.push(DumpSlicePath(path)), + Err(err) => { + warn!( + log, + "Error getting dump device devfs path: {err:?}" + ); } - let name = disk.zpool_name(); - if let Ok(info) = illumos_utils::zpool::Zpool::get_info( - &name.to_string(), - ) { - if info.health() == ZpoolHealth::Online { - m2_core_datasets.push(CoreZpool(name.clone())); - } else { - warn!(log, "Zpool {name:?} not online, won't attempt to save process core dumps there"); - } + } + let name = disk.zpool_name(); + if let Ok(info) = + illumos_utils::zpool::Zpool::get_info(&name.to_string()) + { + if info.health() == ZpoolHealth::Online { + m2_core_datasets.push(CoreZpool(name.clone())); + } else { + warn!(log, "Zpool {name:?} not online, won't attempt to save process core dumps there"); } } - DiskVariant::U2 => { - let name = disk.zpool_name(); - if let Ok(info) = illumos_utils::zpool::Zpool::get_info( - &name.to_string(), - ) { - if info.health() == ZpoolHealth::Online { - u2_debug_datasets - .push(DebugZpool(name.clone())); - } else { - warn!(log, "Zpool {name:?} not online, won't attempt to save kernel core dumps there"); - } + } + DiskVariant::U2 => { + let name = disk.zpool_name(); + if let Ok(info) = + illumos_utils::zpool::Zpool::get_info(&name.to_string()) + { + if info.health() == ZpoolHealth::Online { + u2_debug_datasets.push(DebugZpool(name.clone())); + } else { + warn!(log, "Zpool {name:?} not online, won't attempt to save kernel core dumps there"); } } - }, - DiskWrapper::Synthetic { .. } => {} + } } } diff --git a/sled-agent/src/hardware_monitor.rs b/sled-agent/src/hardware_monitor.rs new file mode 100644 index 00000000000..698d2d4608a --- /dev/null +++ b/sled-agent/src/hardware_monitor.rs @@ -0,0 +1,257 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! A task that listens for hardware events from the +//! [`sled_hardware::HardwareManager`] and dispatches them to other parts +//! of the bootstrap agent and sled-agent code. + +use crate::services::ServiceManager; +use crate::sled_agent::SledAgent; +use sled_hardware::{Baseboard, HardwareManager, HardwareUpdate}; +use sled_storage::disk::RawDisk; +use sled_storage::manager::StorageHandle; +use slog::Logger; +use tokio::sync::broadcast::error::RecvError; +use tokio::sync::{broadcast, oneshot}; + +// A thin wrapper around the the [`ServiceManager`] that caches the state +// whether or not the tofino is loaded if the [`ServiceManager`] doesn't exist +// yet. +enum TofinoManager { + Ready(ServiceManager), + NotReady { tofino_loaded: bool }, +} + +impl TofinoManager { + pub fn new() -> TofinoManager { + TofinoManager::NotReady { tofino_loaded: false } + } + + // Must only be called once on the transition from `NotReady` to `Ready`. + // Panics otherwise. + // + // Returns whether the tofino was loaded or not + pub fn become_ready(&mut self, service_manager: ServiceManager) -> bool { + let tofino_loaded = match self { + Self::Ready(_) => panic!("ServiceManager is already available"), + Self::NotReady { tofino_loaded } => *tofino_loaded, + }; + *self = Self::Ready(service_manager); + tofino_loaded + } + + pub fn is_ready(&self) -> bool { + match self { + TofinoManager::Ready(_) => true, + _ => false, + } + } +} + +// A monitor for hardware events +pub struct HardwareMonitor { + log: Logger, + + baseboard: Baseboard, + + // Receive a onetime notification that the SledAgent has started + sled_agent_started_rx: oneshot::Receiver, + + // Receive a onetime notification that the ServiceManager is ready + service_manager_ready_rx: oneshot::Receiver, + + // Receive messages from the [`HardwareManager`] + hardware_rx: broadcast::Receiver, + + // A reference to the hardware manager + hardware_manager: HardwareManager, + + // A handle to [`sled_hardware::manager::StorageManger`] + storage_manager: StorageHandle, + + // A handle to the sled-agent + // + // This will go away once Nexus updates are polled: + // See: + // * https://github.com/oxidecomputer/omicron/issues/1917 + // * https://rfd.shared.oxide.computer/rfd/0433 + sled_agent: Option, + + // The [`ServiceManager`] is instantiated after we start the [`HardwareMonitor`] + // task. However, it is only used to load and unload the switch zone when thes + // state of the tofino changes. We keep track of the tofino state so that we + // can properly load the tofino when the [`ServiceManager`] becomes available + // available. + tofino_manager: TofinoManager, +} + +impl HardwareMonitor { + pub fn new( + log: &Logger, + hardware_manager: &HardwareManager, + storage_manager: &StorageHandle, + ) -> ( + HardwareMonitor, + oneshot::Sender, + oneshot::Sender, + ) { + let (sled_agent_started_tx, sled_agent_started_rx) = oneshot::channel(); + let (service_manager_ready_tx, service_manager_ready_rx) = + oneshot::channel(); + let baseboard = hardware_manager.baseboard(); + let hardware_rx = hardware_manager.monitor(); + let log = log.new(o!("component" => "HardwareMonitor")); + let tofino_manager = TofinoManager::new(); + ( + HardwareMonitor { + log, + baseboard, + sled_agent_started_rx, + service_manager_ready_rx, + hardware_rx, + hardware_manager: hardware_manager.clone(), + storage_manager: storage_manager.clone(), + sled_agent: None, + tofino_manager, + }, + sled_agent_started_tx, + service_manager_ready_tx, + ) + } + + /// Run the main receive loop of the `HardwareMonitor` + /// + /// This should be spawned into a tokio task + pub async fn run(&mut self) { + // Check the latest hardware snapshot; we could have missed events + // between the creation of the hardware manager and our subscription of + // its monitor. + self.check_latest_hardware_snapshot().await; + + loop { + tokio::select! { + Ok(sled_agent) = &mut self.sled_agent_started_rx, + if self.sled_agent.is_none() => + { + info!(self.log, "Sled Agent Started"); + self.sled_agent = Some(sled_agent); + self.check_latest_hardware_snapshot().await; + } + Ok(service_manager) = &mut self.service_manager_ready_rx, + if !self.tofino_manager.is_ready() => + { + let tofino_loaded = + self.tofino_manager.become_ready(service_manager); + if tofino_loaded { + self.activate_switch().await; + } + } + update = self.hardware_rx.recv() => { + info!( + self.log, + "Received hardware update message"; + "update" => ?update, + ); + self.handle_hardware_update(update).await; + } + } + } + } + + // Handle an update from the [`HardwareMonitor`] + async fn handle_hardware_update( + &mut self, + update: Result, + ) { + match update { + Ok(update) => match update { + HardwareUpdate::TofinoLoaded => self.activate_switch().await, + HardwareUpdate::TofinoUnloaded => { + self.deactivate_switch().await + } + HardwareUpdate::TofinoDeviceChange => { + if let Some(sled_agent) = &mut self.sled_agent { + sled_agent.notify_nexus_about_self(&self.log); + } + } + HardwareUpdate::DiskAdded(disk) => { + self.storage_manager.upsert_disk(disk.into()).await; + } + HardwareUpdate::DiskRemoved(disk) => { + self.storage_manager.delete_disk(disk.into()).await; + } + }, + Err(broadcast::error::RecvError::Lagged(count)) => { + warn!(self.log, "Hardware monitor missed {count} messages"); + self.check_latest_hardware_snapshot().await; + } + Err(broadcast::error::RecvError::Closed) => { + // The `HardwareManager` monitoring task is an infinite loop - + // the only way for us to get `Closed` here is if it panicked, + // so we will propagate such a panic. + panic!("Hardware manager monitor task panicked"); + } + } + } + + async fn activate_switch(&mut self) { + match &mut self.tofino_manager { + TofinoManager::Ready(service_manager) => { + if let Err(e) = service_manager + .activate_switch( + self.sled_agent + .as_ref() + .map(|sa| sa.switch_zone_underlay_info()), + self.baseboard.clone(), + ) + .await + { + warn!(self.log, "Failed to activate switch: {e}"); + } + } + TofinoManager::NotReady { tofino_loaded } => { + *tofino_loaded = true; + } + } + } + + async fn deactivate_switch(&mut self) { + match &mut self.tofino_manager { + TofinoManager::Ready(service_manager) => { + if let Err(e) = service_manager.deactivate_switch().await { + warn!(self.log, "Failed to deactivate switch: {e}"); + } + } + TofinoManager::NotReady { tofino_loaded } => { + *tofino_loaded = false; + } + } + } + + // Observe the current hardware state manually. + // + // We use this when we're monitoring hardware for the first + // time, and if we miss notifications. + async fn check_latest_hardware_snapshot(&mut self) { + let underlay_network = self.sled_agent.as_ref().map(|sled_agent| { + sled_agent.notify_nexus_about_self(&self.log); + sled_agent.switch_zone_underlay_info() + }); + info!( + self.log, "Checking current full hardware snapshot"; + "underlay_network_info" => ?underlay_network, + ); + if self.hardware_manager.is_scrimlet_driver_loaded() { + self.activate_switch().await; + } else { + self.deactivate_switch().await; + } + + self.storage_manager + .ensure_using_exactly_these_disks( + self.hardware_manager.disks().into_iter().map(RawDisk::from), + ) + .await; + } +} diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 68330d0c0e7..2d0e2c40010 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -6,6 +6,7 @@ use super::sled_agent::SledAgent; use crate::bootstrap::early_networking::EarlyNetworkConfig; +use crate::bootstrap::params::AddSledRequest; use crate::params::{ CleanupContextUpdate, DiskEnsureBody, InstanceEnsureBody, InstancePutMigrationIdsBody, InstancePutStateBody, @@ -30,6 +31,9 @@ use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledInstanceState, UpdateArtifactId, }; use omicron_common::api::internal::shared::SwitchPorts; +use oximeter::types::ProducerResults; +use oximeter_producer::collect; +use oximeter_producer::ProducerIdPathParams; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; @@ -68,6 +72,8 @@ pub fn api() -> SledApiDescription { api.register(uplink_ensure)?; api.register(read_network_bootstore_config_cache)?; api.register(write_network_bootstore_config)?; + api.register(add_sled_to_initialized_rack)?; + api.register(metrics_collect)?; Ok(()) } @@ -358,7 +364,7 @@ async fn zpools_get( rqctx: RequestContext, ) -> Result>, HttpError> { let sa = rqctx.context(); - Ok(HttpResponseOk(sa.zpools_get().await.map_err(|e| Error::from(e))?)) + Ok(HttpResponseOk(sa.zpools_get().await)) } #[endpoint { @@ -706,3 +712,58 @@ async fn write_network_bootstore_config( Ok(HttpResponseUpdatedNoContent()) } + +/// Add a sled to a rack that was already initialized via RSS +#[endpoint { + method = PUT, + path = "/sleds" +}] +async fn add_sled_to_initialized_rack( + rqctx: RequestContext, + body: TypedBody, +) -> Result { + let sa = rqctx.context(); + let request = body.into_inner(); + + // Perform some minimal validation + if request.start_request.body.use_trust_quorum + && !request.start_request.body.is_lrtq_learner + { + return Err(HttpError::for_bad_request( + None, + "New sleds must be LRTQ learners if trust quorum is in use" + .to_string(), + )); + } + + crate::sled_agent::add_sled_to_initialized_rack( + sa.logger().clone(), + request.sled_id, + request.start_request, + ) + .await + .map_err(|e| { + let message = format!("Failed to add sled to rack cluster: {e}"); + HttpError { + status_code: http::StatusCode::INTERNAL_SERVER_ERROR, + error_code: None, + external_message: message.clone(), + internal_message: message, + } + })?; + Ok(HttpResponseUpdatedNoContent()) +} + +/// Collect oximeter samples from the sled agent. +#[endpoint { + method = GET, + path = "/metrics/collect/{producer_id}", +}] +async fn metrics_collect( + request_context: RequestContext, + path_params: Path, +) -> Result, HttpError> { + let sa = request_context.context(); + let producer_id = path_params.into_inner().producer_id; + collect(&sa.metrics_registry(), producer_id).await +} diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 94614c23636..a6f022f5f20 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -17,7 +17,6 @@ use crate::params::{ InstanceMigrationTargetParams, InstanceStateRequested, VpcFirewallRule, }; use crate::profile::*; -use crate::storage_manager::StorageResources; use crate::zone_bundle::BundleError; use crate::zone_bundle::ZoneBundler; use anyhow::anyhow; @@ -32,7 +31,6 @@ use illumos_utils::svc::wait_for_service; use illumos_utils::zone::Zones; use illumos_utils::zone::PROPOLIS_ZONE_PREFIX; use omicron_common::address::NEXUS_INTERNAL_PORT; -use omicron_common::address::PROPOLIS_PORT; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, SledInstanceState, VmmRuntimeState, }; @@ -43,7 +41,8 @@ use omicron_common::backoff; use propolis_client::Client as PropolisClient; use rand::prelude::SliceRandom; use rand::SeedableRng; -use sled_hardware::disk::ZONE_DATASET; +use sled_storage::dataset::ZONE_DATASET; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::net::IpAddr; use std::net::{SocketAddr, SocketAddrV6}; @@ -191,13 +190,13 @@ struct InstanceInner { log: Logger, // Properties visible to Propolis - properties: propolis_client::api::InstanceProperties, + properties: propolis_client::types::InstanceProperties, // The ID of the Propolis server (and zone) running this instance propolis_id: Uuid, - // The IP address of the Propolis server running this instance - propolis_ip: IpAddr, + // The socket address of the Propolis server running this instance + propolis_addr: SocketAddr, // NIC-related properties vnic_allocator: VnicAllocator, @@ -214,8 +213,7 @@ struct InstanceInner { dhcp_config: DhcpCfg, // Disk related properties - // TODO: replace `propolis_client::handmade::*` with properly-modeled local types - requested_disks: Vec, + requested_disks: Vec, cloud_init_bytes: Option, // Internal State management @@ -226,7 +224,7 @@ struct InstanceInner { nexus_client: NexusClientWithResolver, // Storage resources - storage: StorageResources, + storage: StorageHandle, // Object used to collect zone bundles from this instance when terminated. zone_bundler: ZoneBundler, @@ -380,7 +378,7 @@ impl InstanceInner { /// Sends an instance state PUT request to this instance's Propolis. async fn propolis_state_put( &self, - request: propolis_client::api::InstanceStateRequested, + request: propolis_client::types::InstanceStateRequested, ) -> Result<(), Error> { let res = self .running_state @@ -410,11 +408,11 @@ impl InstanceInner { ) -> Result<(), Error> { let nics = running_zone .opte_ports() - .map(|port| propolis_client::api::NetworkInterfaceRequest { + .map(|port| propolis_client::types::NetworkInterfaceRequest { // TODO-correctness: Remove `.vnic()` call when we use the port // directly. name: port.vnic_name().to_string(), - slot: propolis_client::api::Slot(port.slot()), + slot: propolis_client::types::Slot(port.slot()), }) .collect(); @@ -424,7 +422,7 @@ impl InstanceInner { self.state.instance().migration_id.ok_or_else(|| { Error::Migration(anyhow!("Missing Migration UUID")) })?; - Some(propolis_client::api::InstanceMigrateInitiateRequest { + Some(propolis_client::types::InstanceMigrateInitiateRequest { src_addr: params.src_propolis_addr.to_string(), src_uuid: params.src_propolis_id, migration_id, @@ -433,7 +431,7 @@ impl InstanceInner { None => None, }; - let request = propolis_client::api::InstanceEnsureRequest { + let request = propolis_client::types::InstanceEnsureRequest { properties: self.properties.clone(), nics, disks: self @@ -649,7 +647,7 @@ impl Instance { let instance = InstanceInner { log: log.new(o!("instance_id" => id.to_string())), // NOTE: Mostly lies. - properties: propolis_client::api::InstanceProperties { + properties: propolis_client::types::InstanceProperties { id, name: hardware.properties.hostname.clone(), description: "Test description".to_string(), @@ -662,7 +660,7 @@ impl Instance { vcpus: hardware.properties.ncpus.0 as u8, }, propolis_id, - propolis_ip: propolis_addr.ip(), + propolis_addr, vnic_allocator, port_manager, requested_nics: hardware.nics, @@ -790,7 +788,7 @@ impl Instance { &self, state: crate::params::InstanceStateRequested, ) -> Result { - use propolis_client::api::InstanceStateRequested as PropolisRequest; + use propolis_client::types::InstanceStateRequested as PropolisRequest; let mut inner = self.inner.lock().await; let (propolis_state, next_published) = match state { InstanceStateRequested::MigrationTarget(migration_params) => { @@ -900,8 +898,9 @@ impl Instance { let mut rng = rand::rngs::StdRng::from_entropy(); let root = inner .storage - .all_u2_mountpoints(ZONE_DATASET) + .get_latest_resources() .await + .all_u2_mountpoints(ZONE_DATASET) .choose(&mut rng) .ok_or_else(|| Error::U2NotFound)? .clone(); @@ -964,9 +963,13 @@ impl Instance { .add_property( "listen_addr", "astring", - &inner.propolis_ip.to_string(), + &inner.propolis_addr.ip().to_string(), + ) + .add_property( + "listen_port", + "astring", + &inner.propolis_addr.port().to_string(), ) - .add_property("listen_port", "astring", &PROPOLIS_PORT.to_string()) .add_property("metric_addr", "astring", &metric_addr.to_string()); let profile = ProfileBuilder::new("omicron").add_service( @@ -984,18 +987,16 @@ impl Instance { // but it helps distinguish "online in SMF" from "responding to HTTP // requests". let fmri = fmri_name(); - wait_for_service(Some(&zname), &fmri) + wait_for_service(Some(&zname), &fmri, inner.log.clone()) .await .map_err(|_| Error::Timeout(fmri.to_string()))?; info!(inner.log, "Propolis SMF service is online"); - let server_addr = SocketAddr::new(inner.propolis_ip, PROPOLIS_PORT); - // We use a custom client builder here because the default progenitor // one has a timeout of 15s but we want to be able to wait indefinitely. let reqwest_client = reqwest::ClientBuilder::new().build().unwrap(); let client = Arc::new(PropolisClient::new_with_client( - &format!("http://{}", server_addr), + &format!("http://{}", &inner.propolis_addr), reqwest_client, )); @@ -1034,7 +1035,9 @@ impl Instance { // known to Propolis. let response = client .instance_state_monitor() - .body(propolis_client::api::InstanceStateMonitorRequest { gen }) + .body(propolis_client::types::InstanceStateMonitorRequest { + gen, + }) .send() .await? .into_inner(); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 2860f0624ba..fa40a876f0f 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -12,7 +12,6 @@ use crate::params::{ InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse, InstanceStateRequested, InstanceUnregisterResponse, }; -use crate::storage_manager::StorageResources; use crate::zone_bundle::BundleError; use crate::zone_bundle::ZoneBundler; use illumos_utils::dladm::Etherstub; @@ -23,6 +22,7 @@ use omicron_common::api::external::ByteCount; use omicron_common::api::internal::nexus::InstanceRuntimeState; use omicron_common::api::internal::nexus::SledInstanceState; use omicron_common::api::internal::nexus::VmmRuntimeState; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::BTreeMap; use std::net::SocketAddr; @@ -43,6 +43,9 @@ pub enum Error { #[error("Failed to create reservoir: {0}")] Reservoir(#[from] vmm_reservoir::Error), + #[error("Invalid reservoir configuration: {0}")] + ReservoirConfig(String), + #[error("Cannot find data link: {0}")] Underlay(#[from] sled_hardware::underlay::Error), @@ -50,6 +53,12 @@ pub enum Error { ZoneBundle(#[from] BundleError), } +pub enum ReservoirMode { + None, + Size(u32), + Percentage(u8), +} + struct InstanceManagerInternal { log: Logger, nexus_client: NexusClientWithResolver, @@ -65,7 +74,7 @@ struct InstanceManagerInternal { vnic_allocator: VnicAllocator, port_manager: PortManager, - storage: StorageResources, + storage: StorageHandle, zone_bundler: ZoneBundler, } @@ -73,7 +82,7 @@ pub(crate) struct InstanceManagerServices { pub nexus_client: NexusClientWithResolver, pub vnic_allocator: VnicAllocator, pub port_manager: PortManager, - pub storage: StorageResources, + pub storage: StorageHandle, pub zone_bundler: ZoneBundler, } @@ -89,7 +98,7 @@ impl InstanceManager { nexus_client: NexusClientWithResolver, etherstub: Etherstub, port_manager: PortManager, - storage: StorageResources, + storage: StorageHandle, zone_bundler: ZoneBundler, ) -> Result { Ok(InstanceManager { @@ -108,44 +117,69 @@ impl InstanceManager { }) } - /// Sets the VMM reservoir size to the requested (nonzero) percentage of - /// usable physical RAM, rounded down to nearest aligned size required by - /// the control plane. + /// Sets the VMM reservoir to the requested percentage of usable physical + /// RAM or to a size in MiB. Either mode will round down to the nearest + /// aligned size required by the control plane. pub fn set_reservoir_size( &self, hardware: &sled_hardware::HardwareManager, - target_percent: u8, + mode: ReservoirMode, ) -> Result<(), Error> { - assert!( - target_percent > 0 && target_percent < 100, - "target_percent {} must be nonzero and < 100", - target_percent - ); + let hardware_physical_ram_bytes = hardware.usable_physical_ram_bytes(); + let req_bytes = match mode { + ReservoirMode::None => return Ok(()), + ReservoirMode::Size(mb) => { + let bytes = ByteCount::from_mebibytes_u32(mb).to_bytes(); + if bytes > hardware_physical_ram_bytes { + return Err(Error::ReservoirConfig(format!( + "cannot specify a reservoir of {bytes} bytes when \ + physical memory is {hardware_physical_ram_bytes} bytes", + ))); + } + bytes + } + ReservoirMode::Percentage(percent) => { + if !matches!(percent, 1..=99) { + return Err(Error::ReservoirConfig(format!( + "VMM reservoir percentage of {} must be between 0 and \ + 100", + percent + ))); + }; + (hardware_physical_ram_bytes as f64 * (percent as f64 / 100.0)) + .floor() as u64 + } + }; - let req_bytes = (hardware.usable_physical_ram_bytes() as f64 - * (target_percent as f64 / 100.0)) - .floor() as u64; let req_bytes_aligned = vmm_reservoir::align_reservoir_size(req_bytes); if req_bytes_aligned == 0 { warn!( self.inner.log, - "Requested reservoir size of {} bytes < minimum aligned size of {} bytes", - req_bytes, vmm_reservoir::RESERVOIR_SZ_ALIGN); + "Requested reservoir size of {} bytes < minimum aligned size \ + of {} bytes", + req_bytes, + vmm_reservoir::RESERVOIR_SZ_ALIGN + ); return Ok(()); } - // The max ByteCount value is i64::MAX, which is ~8 million TiB. As this - // value is a percentage of DRAM, constructing this should always work. + // The max ByteCount value is i64::MAX, which is ~8 million TiB. + // As this value is either a percentage of DRAM or a size in MiB + // represented as a u32, constructing this should always work. let reservoir_size = ByteCount::try_from(req_bytes_aligned).unwrap(); + if let ReservoirMode::Percentage(percent) = mode { + info!( + self.inner.log, + "{}% of {} physical ram = {} bytes)", + percent, + hardware_physical_ram_bytes, + req_bytes, + ); + } info!( self.inner.log, - "Setting reservoir size to {} bytes \ - ({}% of {} total = {} bytes requested)", - reservoir_size, - target_percent, - hardware.usable_physical_ram_bytes(), - req_bytes, + "Setting reservoir size to {reservoir_size} bytes" ); vmm_reservoir::ReservoirControl::set(reservoir_size)?; diff --git a/sled-agent/src/lib.rs b/sled-agent/src/lib.rs index 4e7921c605d..924fd4bd925 100644 --- a/sled-agent/src/lib.rs +++ b/sled-agent/src/lib.rs @@ -20,9 +20,13 @@ pub mod common; mod backing_fs; pub mod bootstrap; pub mod config; +pub(crate) mod dump_setup; +pub(crate) mod hardware_monitor; mod http_entrypoints; mod instance; mod instance_manager; +mod long_running_tasks; +mod metrics; mod nexus; pub mod params; mod profile; @@ -31,8 +35,7 @@ pub mod server; mod services; mod sled_agent; mod smf_helper; -pub(crate) mod storage; -mod storage_manager; +mod storage_monitor; mod swap_device; mod updates; mod zone_bundle; diff --git a/sled-agent/src/long_running_tasks.rs b/sled-agent/src/long_running_tasks.rs new file mode 100644 index 00000000000..f4a665c0983 --- /dev/null +++ b/sled-agent/src/long_running_tasks.rs @@ -0,0 +1,241 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This module is responsible for spawning, starting, and managing long running +//! tasks and task driven subsystems. These tasks run for the remainder of the +//! sled-agent process from the moment they begin. Primarily they include the +//! "managers", like `StorageManager`, `InstanceManager`, etc..., and are used +//! by both the bootstrap agent and the sled-agent. +//! +//! We don't bother keeping track of the spawned tasks handles because we know +//! these tasks are supposed to run forever, and they can shutdown if their +//! handles are dropped. + +use crate::bootstrap::bootstore_setup::{ + new_bootstore_config, poll_ddmd_for_bootstore_peer_update, +}; +use crate::bootstrap::secret_retriever::LrtqOrHardcodedSecretRetriever; +use crate::config::Config; +use crate::hardware_monitor::HardwareMonitor; +use crate::services::ServiceManager; +use crate::sled_agent::SledAgent; +use crate::storage_monitor::{StorageMonitor, UnderlayAccess}; +use crate::zone_bundle::{CleanupContext, ZoneBundler}; +use bootstore::schemes::v0 as bootstore; +use key_manager::{KeyManager, StorageKeyRequester}; +use sled_hardware::{HardwareManager, SledMode}; +use sled_storage::disk::SyntheticDisk; +use sled_storage::manager::{StorageHandle, StorageManager}; +use slog::{info, Logger}; +use std::net::Ipv6Addr; +use tokio::sync::oneshot; + +/// A mechanism for interacting with all long running tasks that can be shared +/// between the bootstrap-agent and sled-agent code. +#[derive(Clone)] +pub struct LongRunningTaskHandles { + /// A mechanism for retrieving storage keys. This interacts with the + /// [`KeyManager`] task. In the future, there may be other handles for + /// retrieving different types of keys. Separating the handles limits the + /// access for a given key type to the code that holds the handle. + pub storage_key_requester: StorageKeyRequester, + + /// A mechanism for talking to the [`StorageManager`] which is responsible + /// for establishing zpools on disks and managing their datasets. + pub storage_manager: StorageHandle, + + /// A mechanism for interacting with the hardware device tree + pub hardware_manager: HardwareManager, + + // A handle for interacting with the bootstore + pub bootstore: bootstore::NodeHandle, + + // A reference to the object used to manage zone bundles + pub zone_bundler: ZoneBundler, +} + +/// Spawn all long running tasks +pub async fn spawn_all_longrunning_tasks( + log: &Logger, + sled_mode: SledMode, + global_zone_bootstrap_ip: Ipv6Addr, + config: &Config, +) -> ( + LongRunningTaskHandles, + oneshot::Sender, + oneshot::Sender, + oneshot::Sender, +) { + let storage_key_requester = spawn_key_manager(log); + let mut storage_manager = + spawn_storage_manager(log, storage_key_requester.clone()); + + let underlay_available_tx = + spawn_storage_monitor(log, storage_manager.clone()); + + let hardware_manager = spawn_hardware_manager(log, sled_mode).await; + + // Start monitoring for hardware changes + let (sled_agent_started_tx, service_manager_ready_tx) = + spawn_hardware_monitor(log, &hardware_manager, &storage_manager); + + // Add some synthetic disks if necessary. + upsert_synthetic_zpools_if_needed(&log, &storage_manager, &config).await; + + // Wait for the boot disk so that we can work with any ledgers, + // such as those needed by the bootstore and sled-agent + info!(log, "Waiting for boot disk"); + let (disk_id, _) = storage_manager.wait_for_boot_disk().await; + info!(log, "Found boot disk {:?}", disk_id); + + let bootstore = spawn_bootstore_tasks( + log, + &mut storage_manager, + &hardware_manager, + global_zone_bootstrap_ip, + ) + .await; + + let zone_bundler = spawn_zone_bundler_tasks(log, &mut storage_manager); + + ( + LongRunningTaskHandles { + storage_key_requester, + storage_manager, + hardware_manager, + bootstore, + zone_bundler, + }, + sled_agent_started_tx, + service_manager_ready_tx, + underlay_available_tx, + ) +} + +fn spawn_key_manager(log: &Logger) -> StorageKeyRequester { + info!(log, "Starting KeyManager"); + let secret_retriever = LrtqOrHardcodedSecretRetriever::new(); + let (mut key_manager, storage_key_requester) = + KeyManager::new(log, secret_retriever); + tokio::spawn(async move { key_manager.run().await }); + storage_key_requester +} + +fn spawn_storage_manager( + log: &Logger, + key_requester: StorageKeyRequester, +) -> StorageHandle { + info!(log, "Starting StorageManager"); + let (manager, handle) = StorageManager::new(log, key_requester); + tokio::spawn(async move { + manager.run().await; + }); + handle +} + +fn spawn_storage_monitor( + log: &Logger, + storage_handle: StorageHandle, +) -> oneshot::Sender { + info!(log, "Starting StorageMonitor"); + let (storage_monitor, underlay_available_tx) = + StorageMonitor::new(log, storage_handle); + tokio::spawn(async move { + storage_monitor.run().await; + }); + underlay_available_tx +} + +async fn spawn_hardware_manager( + log: &Logger, + sled_mode: SledMode, +) -> HardwareManager { + // The `HardwareManager` does not use the the "task/handle" pattern + // and spawns its worker task inside `HardwareManager::new`. Instead of returning + // a handle to send messages to that task, the "Inner/Mutex" pattern is used + // which shares data between the task, the manager itself, and the users of the manager + // since the manager can be freely cloned and passed around. + // + // There are pros and cons to both methods, but the reason to mention it here is that + // the handle in this case is the `HardwareManager` itself. + info!(log, "Starting HardwareManager"; "sled_mode" => ?sled_mode); + let log = log.clone(); + tokio::task::spawn_blocking(move || { + HardwareManager::new(&log, sled_mode).unwrap() + }) + .await + .unwrap() +} + +fn spawn_hardware_monitor( + log: &Logger, + hardware_manager: &HardwareManager, + storage_handle: &StorageHandle, +) -> (oneshot::Sender, oneshot::Sender) { + info!(log, "Starting HardwareMonitor"); + let (mut monitor, sled_agent_started_tx, service_manager_ready_tx) = + HardwareMonitor::new(log, hardware_manager, storage_handle); + tokio::spawn(async move { + monitor.run().await; + }); + (sled_agent_started_tx, service_manager_ready_tx) +} + +async fn spawn_bootstore_tasks( + log: &Logger, + storage_handle: &mut StorageHandle, + hardware_manager: &HardwareManager, + global_zone_bootstrap_ip: Ipv6Addr, +) -> bootstore::NodeHandle { + let storage_resources = storage_handle.get_latest_resources().await; + let config = new_bootstore_config( + &storage_resources, + hardware_manager.baseboard(), + global_zone_bootstrap_ip, + ) + .unwrap(); + + // Create and spawn the bootstore + info!(log, "Starting Bootstore"); + let (mut node, node_handle) = bootstore::Node::new(config, log).await; + tokio::spawn(async move { node.run().await }); + + // Spawn a task for polling DDMD and updating bootstore with peer addresses + info!(log, "Starting Bootstore DDMD poller"); + let log = log.new(o!("component" => "bootstore_ddmd_poller")); + let node_handle2 = node_handle.clone(); + tokio::spawn(async move { + poll_ddmd_for_bootstore_peer_update(log, node_handle2).await + }); + + node_handle +} + +// `ZoneBundler::new` spawns a periodic cleanup task that runs indefinitely +fn spawn_zone_bundler_tasks( + log: &Logger, + storage_handle: &mut StorageHandle, +) -> ZoneBundler { + info!(log, "Starting ZoneBundler related tasks"); + let log = log.new(o!("component" => "ZoneBundler")); + ZoneBundler::new(log, storage_handle.clone(), CleanupContext::default()) +} + +async fn upsert_synthetic_zpools_if_needed( + log: &Logger, + storage_manager: &StorageHandle, + config: &Config, +) { + if let Some(pools) = &config.zpools { + for pool in pools { + info!( + log, + "Upserting synthetic zpool to Storage Manager: {}", + pool.to_string() + ); + let disk = SyntheticDisk::new(pool.clone()).into(); + storage_manager.upsert_disk(disk).await; + } + } +} diff --git a/sled-agent/src/metrics.rs b/sled-agent/src/metrics.rs new file mode 100644 index 00000000000..6c3383c88f0 --- /dev/null +++ b/sled-agent/src/metrics.rs @@ -0,0 +1,260 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Metrics produced by the sled-agent for collection by oximeter. + +use oximeter::types::MetricsError; +use oximeter::types::ProducerRegistry; +use sled_hardware::Baseboard; +use slog::Logger; +use std::time::Duration; +use uuid::Uuid; + +cfg_if::cfg_if! { + if #[cfg(target_os = "illumos")] { + use oximeter_instruments::kstat::link; + use oximeter_instruments::kstat::CollectionDetails; + use oximeter_instruments::kstat::Error as KstatError; + use oximeter_instruments::kstat::KstatSampler; + use oximeter_instruments::kstat::TargetId; + use std::collections::BTreeMap; + } else { + use anyhow::anyhow; + } +} + +/// The interval on which we ask `oximeter` to poll us for metric data. +pub(crate) const METRIC_COLLECTION_INTERVAL: Duration = Duration::from_secs(30); + +/// The interval on which we sample link metrics. +pub(crate) const LINK_SAMPLE_INTERVAL: Duration = Duration::from_secs(10); + +/// An error during sled-agent metric production. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[cfg(target_os = "illumos")] + #[error("Kstat-based metric failure")] + Kstat(#[source] KstatError), + + #[cfg(not(target_os = "illumos"))] + #[error("Kstat-based metric failure")] + Kstat(#[source] anyhow::Error), + + #[error("Failed to insert metric producer into registry")] + Registry(#[source] MetricsError), + + #[error("Failed to fetch hostname")] + Hostname(#[source] std::io::Error), +} + +/// Type managing all oximeter metrics produced by the sled-agent. +// +// TODO-completeness: We probably want to get kstats or other metrics in to this +// type from other parts of the code, possibly before the `SledAgent` itself +// exists. This is similar to the storage resources or other objects, most of +// which are essentially an `Arc>`. It would be nice to avoid that +// pattern, but until we have more statistics, it's not clear whether that's +// worth it right now. +#[derive(Clone, Debug)] +// NOTE: The ID fields aren't used on non-illumos systems, rather than changing +// the name of fields that are not yet used. +#[cfg_attr(not(target_os = "illumos"), allow(dead_code))] +pub struct MetricsManager { + sled_id: Uuid, + rack_id: Uuid, + baseboard: Baseboard, + hostname: Option, + _log: Logger, + #[cfg(target_os = "illumos")] + kstat_sampler: KstatSampler, + // TODO-scalability: We may want to generalize this to store any kind of + // tracked target, and use a naming scheme that allows us pick out which + // target we're interested in from the arguments. + // + // For example, we can use the link name to do this, for any physical or + // virtual link, because they need to be unique. We could also do the same + // for disks or memory. If we wanted to guarantee uniqueness, we could + // namespace them internally, e.g., `"datalink:{link_name}"` would be the + // real key. + #[cfg(target_os = "illumos")] + tracked_links: BTreeMap, + registry: ProducerRegistry, +} + +impl MetricsManager { + /// Construct a new metrics manager. + /// + /// This takes a few key pieces of identifying information that are used + /// when reporting sled-specific metrics. + pub fn new( + sled_id: Uuid, + rack_id: Uuid, + baseboard: Baseboard, + log: Logger, + ) -> Result { + let registry = ProducerRegistry::with_id(sled_id); + + cfg_if::cfg_if! { + if #[cfg(target_os = "illumos")] { + let kstat_sampler = KstatSampler::new(&log).map_err(Error::Kstat)?; + registry + .register_producer(kstat_sampler.clone()) + .map_err(Error::Registry)?; + let tracked_links = BTreeMap::new(); + } + } + Ok(Self { + sled_id, + rack_id, + baseboard, + hostname: None, + _log: log, + #[cfg(target_os = "illumos")] + kstat_sampler, + #[cfg(target_os = "illumos")] + tracked_links, + registry, + }) + } + + /// Return a reference to the contained producer registry. + pub fn registry(&self) -> &ProducerRegistry { + &self.registry + } +} + +#[cfg(target_os = "illumos")] +impl MetricsManager { + /// Track metrics for a physical datalink. + pub async fn track_physical_link( + &mut self, + link_name: impl AsRef, + interval: Duration, + ) -> Result<(), Error> { + let hostname = self.hostname().await?; + let link = link::PhysicalDataLink { + rack_id: self.rack_id, + sled_id: self.sled_id, + serial: self.serial_number(), + hostname, + link_name: link_name.as_ref().to_string(), + }; + let details = CollectionDetails::never(interval); + let id = self + .kstat_sampler + .add_target(link, details) + .await + .map_err(Error::Kstat)?; + self.tracked_links.insert(link_name.as_ref().to_string(), id); + Ok(()) + } + + /// Stop tracking metrics for a datalink. + /// + /// This works for both physical and virtual links. + #[allow(dead_code)] + pub async fn stop_tracking_link( + &mut self, + link_name: impl AsRef, + ) -> Result<(), Error> { + if let Some(id) = self.tracked_links.remove(link_name.as_ref()) { + self.kstat_sampler.remove_target(id).await.map_err(Error::Kstat) + } else { + Ok(()) + } + } + + /// Track metrics for a virtual datalink. + #[allow(dead_code)] + pub async fn track_virtual_link( + &self, + link_name: impl AsRef, + hostname: impl AsRef, + interval: Duration, + ) -> Result<(), Error> { + let link = link::VirtualDataLink { + rack_id: self.rack_id, + sled_id: self.sled_id, + serial: self.serial_number(), + hostname: hostname.as_ref().to_string(), + link_name: link_name.as_ref().to_string(), + }; + let details = CollectionDetails::never(interval); + self.kstat_sampler + .add_target(link, details) + .await + .map(|_| ()) + .map_err(Error::Kstat) + } + + // Return the serial number out of the baseboard, if one exists. + fn serial_number(&self) -> String { + match &self.baseboard { + Baseboard::Gimlet { identifier, .. } => identifier.clone(), + Baseboard::Unknown => String::from("unknown"), + Baseboard::Pc { identifier, .. } => identifier.clone(), + } + } + + // Return the system's hostname. + // + // If we've failed to get it previously, we try again. If _that_ fails, + // return an error. + // + // TODO-cleanup: This will become much simpler once + // `OnceCell::get_or_try_init` is stabilized. + async fn hostname(&mut self) -> Result { + if let Some(hn) = &self.hostname { + return Ok(hn.clone()); + } + let hn = tokio::process::Command::new("hostname") + .env_clear() + .output() + .await + .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string()) + .map_err(Error::Hostname)?; + self.hostname.replace(hn.clone()); + Ok(hn) + } +} + +#[cfg(not(target_os = "illumos"))] +impl MetricsManager { + /// Track metrics for a physical datalink. + pub async fn track_physical_link( + &mut self, + _link_name: impl AsRef, + _interval: Duration, + ) -> Result<(), Error> { + Err(Error::Kstat(anyhow!( + "kstat metrics are not supported on this platform" + ))) + } + + /// Stop tracking metrics for a datalink. + /// + /// This works for both physical and virtual links. + #[allow(dead_code)] + pub async fn stop_tracking_link( + &mut self, + _link_name: impl AsRef, + ) -> Result<(), Error> { + Err(Error::Kstat(anyhow!( + "kstat metrics are not supported on this platform" + ))) + } + + /// Track metrics for a virtual datalink. + #[allow(dead_code)] + pub async fn track_virtual_link( + &self, + _link_name: impl AsRef, + _hostname: impl AsRef, + _interval: Duration, + ) -> Result<(), Error> { + Err(Error::Kstat(anyhow!( + "kstat metrics are not supported on this platform" + ))) + } +} diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs index 2af6fa00232..cc715f4010b 100644 --- a/sled-agent/src/nexus.rs +++ b/sled-agent/src/nexus.rs @@ -154,3 +154,51 @@ fn d2n_record( } } } + +// Although it is a bit awkward to define these conversions here, it frees us +// from depending on sled_storage/sled_hardware in the nexus_client crate. + +pub(crate) trait ConvertInto: Sized { + fn convert(self) -> T; +} + +impl ConvertInto + for sled_hardware::DiskVariant +{ + fn convert(self) -> nexus_client::types::PhysicalDiskKind { + use nexus_client::types::PhysicalDiskKind; + + match self { + sled_hardware::DiskVariant::U2 => PhysicalDiskKind::U2, + sled_hardware::DiskVariant::M2 => PhysicalDiskKind::M2, + } + } +} + +impl ConvertInto for sled_hardware::Baseboard { + fn convert(self) -> nexus_client::types::Baseboard { + nexus_client::types::Baseboard { + serial_number: self.identifier().to_string(), + part_number: self.model().to_string(), + revision: self.revision(), + } + } +} + +impl ConvertInto + for sled_storage::dataset::DatasetKind +{ + fn convert(self) -> nexus_client::types::DatasetKind { + use nexus_client::types::DatasetKind; + use sled_storage::dataset::DatasetKind::*; + + match self { + CockroachDb => DatasetKind::Cockroach, + Crucible => DatasetKind::Crucible, + Clickhouse => DatasetKind::Clickhouse, + ClickhouseKeeper => DatasetKind::ClickhouseKeeper, + ExternalDns => DatasetKind::ExternalDns, + InternalDns => DatasetKind::InternalDns, + } + } +} diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 5fda3c1ae68..b22bd849750 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -20,6 +20,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sled_hardware::Baseboard; pub use sled_hardware::DendriteAsic; +use sled_storage::dataset::DatasetName; use std::fmt::{Debug, Display, Formatter, Result as FormatResult}; use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; use std::time::Duration; @@ -70,8 +71,8 @@ pub struct InstanceHardware { pub external_ips: Vec, pub firewall_rules: Vec, pub dhcp_config: DhcpConfig, - // TODO: replace `propolis_client::handmade::*` with locally-modeled request type - pub disks: Vec, + // TODO: replace `propolis_client::*` with locally-modeled request type + pub disks: Vec, pub cloud_init_bytes: Option, } @@ -228,64 +229,6 @@ pub struct Zpool { pub disk_type: DiskType, } -/// The type of a dataset, and an auxiliary information necessary -/// to successfully launch a zone managing the associated data. -#[derive( - Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, -)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum DatasetKind { - CockroachDb, - Crucible, - Clickhouse, - ClickhouseKeeper, - ExternalDns, - InternalDns, -} - -impl From for sled_agent_client::types::DatasetKind { - fn from(k: DatasetKind) -> Self { - use DatasetKind::*; - match k { - CockroachDb => Self::CockroachDb, - Crucible => Self::Crucible, - Clickhouse => Self::Clickhouse, - ClickhouseKeeper => Self::ClickhouseKeeper, - ExternalDns => Self::ExternalDns, - InternalDns => Self::InternalDns, - } - } -} - -impl From for nexus_client::types::DatasetKind { - fn from(k: DatasetKind) -> Self { - use DatasetKind::*; - match k { - CockroachDb => Self::Cockroach, - Crucible => Self::Crucible, - Clickhouse => Self::Clickhouse, - ClickhouseKeeper => Self::ClickhouseKeeper, - ExternalDns => Self::ExternalDns, - InternalDns => Self::InternalDns, - } - } -} - -impl std::fmt::Display for DatasetKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use DatasetKind::*; - let s = match self { - Crucible => "crucible", - CockroachDb { .. } => "cockroachdb", - Clickhouse => "clickhouse", - ClickhouseKeeper => "clickhouse_keeper", - ExternalDns { .. } => "external_dns", - InternalDns { .. } => "internal_dns", - }; - write!(f, "{}", s) - } -} - /// Describes service-specific parameters. #[derive( Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, @@ -347,6 +290,7 @@ pub enum ServiceType { #[serde(skip)] Tfport { pkt_source: String, + asic: DendriteAsic, }, #[serde(skip)] Uplink, @@ -593,7 +537,7 @@ impl std::fmt::Display for ZoneType { )] pub struct DatasetRequest { pub id: Uuid, - pub name: crate::storage::dataset::DatasetName, + pub name: DatasetName, pub service_address: SocketAddrV6, } diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 3dac5d7d1e9..980f5b6ebd1 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -6,12 +6,10 @@ use crate::bootstrap::params::StartSledAgentRequest; use crate::params::{ - DatasetKind, DatasetRequest, ServiceType, ServiceZoneRequest, - ServiceZoneService, ZoneType, + DatasetRequest, ServiceType, ServiceZoneRequest, ServiceZoneService, + ZoneType, }; use crate::rack_setup::config::SetupServiceConfig as Config; -use crate::storage::dataset::DatasetName; -use crate::storage_manager::StorageResources; use camino::Utf8PathBuf; use dns_service_client::types::DnsConfigParams; use illumos_utils::zpool::ZpoolName; @@ -36,6 +34,8 @@ use serde::{Deserialize, Serialize}; use sled_agent_client::{ types as SledAgentTypes, Client as SledAgentClient, Error as SledAgentError, }; +use sled_storage::dataset::{DatasetKind, DatasetName, CONFIG_DATASET}; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::{BTreeSet, HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; @@ -125,11 +125,12 @@ const RSS_SERVICE_PLAN_FILENAME: &str = "rss-service-plan.json"; impl Plan { pub async fn load( log: &Logger, - storage: &StorageResources, + storage_manager: &StorageHandle, ) -> Result, PlanError> { - let paths: Vec = storage - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + let paths: Vec = storage_manager + .get_latest_resources() .await + .all_m2_mountpoints(CONFIG_DATASET) .into_iter() .map(|p| p.join(RSS_SERVICE_PLAN_FILENAME)) .collect(); @@ -237,7 +238,7 @@ impl Plan { pub async fn create( log: &Logger, config: &Config, - storage: &StorageResources, + storage_manager: &StorageHandle, sleds: &HashMap, ) -> Result { let mut dns_builder = internal_dns::DnsConfigBuilder::new(); @@ -249,7 +250,7 @@ impl Plan { let result: Result, PlanError> = futures::future::try_join_all(sleds.values().map( |sled_request| async { - let subnet = sled_request.subnet; + let subnet = sled_request.body.subnet; let sled_address = get_sled_address(subnet); let u2_zpools = Self::get_u2_zpools_from_sled(log, sled_address) @@ -257,7 +258,7 @@ impl Plan { let is_scrimlet = Self::is_sled_scrimlet(log, sled_address).await?; Ok(SledInfo::new( - sled_request.id, + sled_request.body.id, subnet, sled_address, u2_zpools, @@ -737,9 +738,10 @@ impl Plan { let plan = Self { services, dns_config }; // Once we've constructed a plan, write it down to durable storage. - let paths: Vec = storage - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + let paths: Vec = storage_manager + .get_latest_resources() .await + .all_m2_mountpoints(CONFIG_DATASET) .into_iter() .map(|p| p.join(RSS_SERVICE_PLAN_FILENAME)) .collect(); diff --git a/sled-agent/src/rack_setup/plan/sled.rs b/sled-agent/src/rack_setup/plan/sled.rs index ea12f0db32e..07f33893fcb 100644 --- a/sled-agent/src/rack_setup/plan/sled.rs +++ b/sled-agent/src/rack_setup/plan/sled.rs @@ -4,15 +4,17 @@ //! Plan generation for "how should sleds be initialized". +use crate::bootstrap::params::StartSledAgentRequestBody; use crate::bootstrap::{ config::BOOTSTRAP_AGENT_RACK_INIT_PORT, params::StartSledAgentRequest, }; use crate::rack_setup::config::SetupServiceConfig as Config; -use crate::storage_manager::StorageResources; use camino::Utf8PathBuf; use omicron_common::ledger::{self, Ledger, Ledgerable}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use sled_storage::dataset::CONFIG_DATASET; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::{HashMap, HashSet}; use std::net::{Ipv6Addr, SocketAddrV6}; @@ -54,11 +56,12 @@ pub struct Plan { impl Plan { pub async fn load( log: &Logger, - storage: &StorageResources, + storage: &StorageHandle, ) -> Result, PlanError> { let paths: Vec = storage - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + .get_latest_resources() .await + .all_m2_mountpoints(CONFIG_DATASET) .into_iter() .map(|p| p.join(RSS_SLED_PLAN_FILENAME)) .collect(); @@ -77,7 +80,7 @@ impl Plan { pub async fn create( log: &Logger, config: &Config, - storage: &StorageResources, + storage_manager: &StorageHandle, bootstrap_addrs: HashSet, use_trust_quorum: bool, ) -> Result { @@ -99,12 +102,15 @@ impl Plan { ( bootstrap_addr, StartSledAgentRequest { - id: Uuid::new_v4(), - subnet, - ntp_servers: config.ntp_servers.clone(), - dns_servers: config.dns_servers.clone(), - use_trust_quorum, - rack_id, + generation: 0, + schema_version: 1, + body: StartSledAgentRequestBody { + id: Uuid::new_v4(), + subnet, + use_trust_quorum, + is_lrtq_learner: false, + rack_id, + }, }, ) }); @@ -119,9 +125,10 @@ impl Plan { let plan = Self { rack_id, sleds, config: config.clone() }; // Once we've constructed a plan, write it down to durable storage. - let paths: Vec = storage - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + let paths: Vec = storage_manager + .get_latest_resources() .await + .all_m2_mountpoints(CONFIG_DATASET) .into_iter() .map(|p| p.join(RSS_SLED_PLAN_FILENAME)) .collect(); diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 7f6469d2c02..7dcbfa70451 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -63,7 +63,7 @@ use crate::bootstrap::early_networking::{ use crate::bootstrap::params::BootstrapAddressDiscovery; use crate::bootstrap::params::StartSledAgentRequest; use crate::bootstrap::rss_handle::BootstrapAgentHandle; -use crate::nexus::d2n_params; +use crate::nexus::{d2n_params, ConvertInto}; use crate::params::{ AutonomousServiceOnlyError, ServiceType, ServiceZoneRequest, ServiceZoneService, TimeSync, ZoneType, @@ -74,7 +74,6 @@ use crate::rack_setup::plan::service::{ use crate::rack_setup::plan::sled::{ Plan as SledPlan, PlanError as SledPlanError, }; -use crate::storage_manager::StorageResources; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use ddm_admin_client::{Client as DdmAdminClient, DdmError}; @@ -94,6 +93,8 @@ use sled_agent_client::{ types as SledAgentTypes, Client as SledAgentClient, Error as SledAgentError, }; use sled_hardware::underlay::BootstrapInterface; +use sled_storage::dataset::CONFIG_DATASET; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::BTreeSet; use std::collections::{HashMap, HashSet}; @@ -187,7 +188,7 @@ impl RackSetupService { pub(crate) fn new( log: Logger, config: Config, - storage_resources: StorageResources, + storage_manager: StorageHandle, local_bootstrap_agent: BootstrapAgentHandle, bootstore: bootstore::NodeHandle, ) -> Self { @@ -196,7 +197,7 @@ impl RackSetupService { if let Err(e) = svc .run( &config, - &storage_resources, + &storage_manager, local_bootstrap_agent, bootstore, ) @@ -535,8 +536,10 @@ impl ServiceInner { // We need the ID when passing info to Nexus. let mut id_map = HashMap::new(); for (_, sled_request) in sled_plan.sleds.iter() { - id_map - .insert(get_sled_address(sled_request.subnet), sled_request.id); + id_map.insert( + get_sled_address(sled_request.body.subnet), + sled_request.body.id, + ); } // Convert all the information we have about services and datasets into @@ -561,7 +564,7 @@ impl ServiceInner { dataset_id: dataset.id, request: NexusTypes::DatasetPutRequest { address: dataset.service_address.to_string(), - kind: dataset.name.dataset().clone().into(), + kind: dataset.name.dataset().clone().convert(), }, }) } @@ -610,6 +613,11 @@ impl ServiceInner { addr: b.addr, asn: b.asn, port: b.port.clone(), + hold_time: b.hold_time, + connect_retry: b.connect_retry, + delay_open: b.delay_open, + idle_hold_time: b.idle_hold_time, + keepalive: b.keepalive, }) .collect(), }) @@ -766,7 +774,7 @@ impl ServiceInner { async fn run( &self, config: &Config, - storage_resources: &StorageResources, + storage_manager: &StorageHandle, local_bootstrap_agent: BootstrapAgentHandle, bootstore: bootstore::NodeHandle, ) -> Result<(), SetupServiceError> { @@ -777,9 +785,10 @@ impl ServiceInner { config.az_subnet(), )?; - let marker_paths: Vec = storage_resources - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) + let marker_paths: Vec = storage_manager + .get_latest_resources() .await + .all_m2_mountpoints(CONFIG_DATASET) .into_iter() .map(|p| p.join(RSS_COMPLETED_FILENAME)) .collect(); @@ -800,7 +809,7 @@ impl ServiceInner { "RSS configuration looks like it has already been applied", ); - let sled_plan = SledPlan::load(&self.log, storage_resources) + let sled_plan = SledPlan::load(&self.log, storage_manager) .await? .expect("Sled plan should exist if completed marker exists"); if &sled_plan.config != config { @@ -808,7 +817,7 @@ impl ServiceInner { "Configuration changed".to_string(), )); } - let service_plan = ServicePlan::load(&self.log, storage_resources) + let service_plan = ServicePlan::load(&self.log, storage_manager) .await? .expect("Service plan should exist if completed marker exists"); @@ -842,7 +851,7 @@ impl ServiceInner { BootstrapAddressDiscovery::OnlyThese { addrs } => addrs.clone(), }; let maybe_sled_plan = - SledPlan::load(&self.log, storage_resources).await?; + SledPlan::load(&self.log, storage_manager).await?; if let Some(plan) = &maybe_sled_plan { let stored_peers: HashSet = plan.sleds.keys().map(|a| *a.ip()).collect(); @@ -874,7 +883,7 @@ impl ServiceInner { SledPlan::create( &self.log, config, - &storage_resources, + &storage_manager, bootstrap_addrs, config.trust_quorum_peers.is_some(), ) @@ -925,18 +934,18 @@ impl ServiceInner { .sleds .values() .map(|initialization_request| { - get_sled_address(initialization_request.subnet) + get_sled_address(initialization_request.body.subnet) }) .collect(); let service_plan = if let Some(plan) = - ServicePlan::load(&self.log, storage_resources).await? + ServicePlan::load(&self.log, storage_manager).await? { plan } else { ServicePlan::create( &self.log, &config, - &storage_resources, + &storage_manager, &plan.sleds, ) .await? diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index 156547627c2..903c8dabaa0 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -8,14 +8,15 @@ use super::config::Config; use super::http_entrypoints::api as http_api; use super::sled_agent::SledAgent; use crate::bootstrap::params::StartSledAgentRequest; +use crate::long_running_tasks::LongRunningTaskHandles; use crate::nexus::NexusClientWithResolver; use crate::services::ServiceManager; -use crate::storage_manager::StorageManager; -use bootstore::schemes::v0 as bootstore; +use crate::storage_monitor::UnderlayAccess; use internal_dns::resolver::Resolver; use slog::Logger; use std::net::SocketAddr; use std::sync::Arc; +use tokio::sync::oneshot; use uuid::Uuid; /// Packages up a [`SledAgent`], running the sled agent API under a Dropshot @@ -39,9 +40,9 @@ impl Server { config: &Config, log: Logger, request: StartSledAgentRequest, + long_running_tasks_handles: LongRunningTaskHandles, services: ServiceManager, - storage: StorageManager, - bootstore: bootstore::NodeHandle, + underlay_available_tx: oneshot::Sender, ) -> Result { info!(log, "setting up sled agent server"); @@ -63,8 +64,8 @@ impl Server { nexus_client, request, services, - storage, - bootstore, + long_running_tasks_handles, + underlay_available_tx, ) .await .map_err(|e| e.to_string())?; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a9be0e7c4a2..b87c91768ba 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -5,7 +5,7 @@ //! Sled-local service management. //! //! For controlling zone-based storage services, refer to -//! [crate::storage_manager::StorageManager]. +//! [sled_storage::manager::StorageManager]. //! //! For controlling virtual machine instances, refer to //! [crate::instance_manager::InstanceManager]. @@ -38,7 +38,6 @@ use crate::params::{ use crate::profile::*; use crate::smf_helper::Service; use crate::smf_helper::SmfHelper; -use crate::storage_manager::StorageResources; use crate::zone_bundle::BundleError; use crate::zone_bundle::ZoneBundler; use anyhow::anyhow; @@ -91,12 +90,13 @@ use omicron_common::nexus_config::{ use once_cell::sync::OnceCell; use rand::prelude::SliceRandom; use rand::SeedableRng; -use sled_hardware::disk::ZONE_DATASET; use sled_hardware::is_gimlet; use sled_hardware::underlay; use sled_hardware::underlay::BOOTSTRAP_PREFIX; use sled_hardware::Baseboard; use sled_hardware::SledMode; +use sled_storage::dataset::{CONFIG_DATASET, INSTALL_DATASET, ZONE_DATASET}; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::BTreeMap; use std::collections::HashSet; @@ -373,7 +373,7 @@ pub struct ServiceManagerInner { advertised_prefixes: Mutex>>, sled_info: OnceCell, switch_zone_bootstrap_address: Ipv6Addr, - storage: StorageResources, + storage: StorageHandle, zone_bundler: ZoneBundler, ledger_directory_override: OnceCell, image_directory_override: OnceCell, @@ -418,10 +418,11 @@ impl ServiceManager { skip_timesync: Option, sidecar_revision: SidecarRevision, switch_zone_maghemite_links: Vec, - storage: StorageResources, + storage: StorageHandle, zone_bundler: ZoneBundler, ) -> Self { let log = log.new(o!("component" => "ServiceManager")); + info!(log, "Creating ServiceManager"); Self { inner: Arc::new(ServiceManagerInner { log: log.clone(), @@ -476,10 +477,9 @@ impl ServiceManager { if let Some(dir) = self.inner.ledger_directory_override.get() { return vec![dir.join(SERVICES_LEDGER_FILENAME)]; } - self.inner - .storage - .all_m2_mountpoints(sled_hardware::disk::CONFIG_DATASET) - .await + let resources = self.inner.storage.get_latest_resources().await; + resources + .all_m2_mountpoints(CONFIG_DATASET) .into_iter() .map(|p| p.join(SERVICES_LEDGER_FILENAME)) .collect() @@ -676,6 +676,11 @@ impl ServiceManager { device: "tofino".to_string(), }); } + ServiceType::Dendrite { + asic: DendriteAsic::SoftNpuPropolisDevice, + } => { + devices.push("/dev/tty03".into()); + } _ => (), } } @@ -741,7 +746,7 @@ impl ServiceManager { for svc in &req.services { match &svc.details { - ServiceType::Tfport { pkt_source } => { + ServiceType::Tfport { pkt_source, asic: _ } => { // The tfport service requires a MAC device to/from which sidecar // packets may be multiplexed. If the link isn't present, don't // bother trying to start the zone. @@ -772,9 +777,13 @@ impl ServiceManager { } Err(_) => { - return Err(Error::MissingDevice { - device: link.to_string(), - }); + if let SidecarRevision::SoftZone(_) = + self.inner.sidecar_revision + { + return Err(Error::MissingDevice { + device: link.to_string(), + }); + } } } } @@ -1087,11 +1096,11 @@ impl ServiceManager { // If the boot disk exists, look for the image in the "install" dataset // there too. - if let Some((_, boot_zpool)) = self.inner.storage.boot_disk().await { - zone_image_paths.push( - boot_zpool - .dataset_mountpoint(sled_hardware::disk::INSTALL_DATASET), - ); + if let Some((_, boot_zpool)) = + self.inner.storage.get_latest_resources().await.boot_disk() + { + zone_image_paths + .push(boot_zpool.dataset_mountpoint(INSTALL_DATASET)); } let installed_zone = InstalledZone::install( @@ -1815,14 +1824,21 @@ impl ServiceManager { "config/port_config", "/opt/oxide/dendrite/misc/model_config.toml", )?, - DendriteAsic::SoftNpu => { - smfh.setprop("config/mgmt", "uds")?; - smfh.setprop( - "config/uds_path", - "/opt/softnpu/stuff", - )?; + asic @ (DendriteAsic::SoftNpuZone + | DendriteAsic::SoftNpuPropolisDevice) => { + if asic == &DendriteAsic::SoftNpuZone { + smfh.setprop("config/mgmt", "uds")?; + smfh.setprop( + "config/uds_path", + "/opt/softnpu/stuff", + )?; + } + if asic == &DendriteAsic::SoftNpuPropolisDevice { + smfh.setprop("config/mgmt", "uart")?; + } let s = match self.inner.sidecar_revision { - SidecarRevision::Soft(ref s) => s, + SidecarRevision::SoftZone(ref s) => s, + SidecarRevision::SoftPropolis(ref s) => s, _ => { return Err(Error::SidecarRevision( anyhow::anyhow!( @@ -1847,7 +1863,7 @@ impl ServiceManager { }; smfh.refresh()?; } - ServiceType::Tfport { pkt_source } => { + ServiceType::Tfport { pkt_source, asic } => { info!(self.inner.log, "Setting up tfport service"); let is_gimlet = is_gimlet().map_err(|e| { @@ -1882,6 +1898,12 @@ impl ServiceManager { } smfh.setprop("config/pkt_source", pkt_source)?; } + if asic == &DendriteAsic::SoftNpuZone { + smfh.setprop("config/flags", "--sync-only")?; + } + if asic == &DendriteAsic::SoftNpuPropolisDevice { + smfh.setprop("config/pkt_source", pkt_source)?; + } smfh.setprop( "config/host", &format!("[{}]", Ipv6Addr::LOCALHOST), @@ -2230,8 +2252,12 @@ impl ServiceManager { // Create zones that should be running let mut zone_requests = AllZoneRequests::default(); - let all_u2_roots = - self.inner.storage.all_u2_mountpoints(ZONE_DATASET).await; + let all_u2_roots = self + .inner + .storage + .get_latest_resources() + .await + .all_u2_mountpoints(ZONE_DATASET); for zone in zones_to_be_added { // Check if we think the zone should already be running let name = zone.zone_name(); @@ -2509,19 +2535,42 @@ impl ServiceManager { vec![ ServiceType::Dendrite { asic: DendriteAsic::TofinoAsic }, ServiceType::ManagementGatewayService, - ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, + ServiceType::Tfport { + pkt_source: "tfpkt0".to_string(), + asic: DendriteAsic::TofinoAsic, + }, + ServiceType::Uplink, + ServiceType::Wicketd { baseboard }, + ServiceType::Mgd, + ServiceType::MgDdm { mode: "transit".to_string() }, + ] + } + + SledMode::Scrimlet { + asic: asic @ DendriteAsic::SoftNpuPropolisDevice, + } => { + data_links = vec!["vioif0".to_owned()]; + vec![ + ServiceType::Dendrite { asic }, + ServiceType::ManagementGatewayService, ServiceType::Uplink, ServiceType::Wicketd { baseboard }, ServiceType::Mgd, ServiceType::MgDdm { mode: "transit".to_string() }, + ServiceType::Tfport { + pkt_source: "vioif0".to_string(), + asic, + }, + ServiceType::SpSim, ] } // Sled is a scrimlet but is not running the real tofino driver. SledMode::Scrimlet { - asic: asic @ (DendriteAsic::TofinoStub | DendriteAsic::SoftNpu), + asic: + asic @ (DendriteAsic::TofinoStub | DendriteAsic::SoftNpuZone), } => { - if let DendriteAsic::SoftNpu = asic { + if let DendriteAsic::SoftNpuZone = asic { let softnpu_filesystem = zone::Fs { ty: "lofs".to_string(), dir: "/opt/softnpu/stuff".to_string(), @@ -2538,7 +2587,10 @@ impl ServiceManager { ServiceType::Wicketd { baseboard }, ServiceType::Mgd, ServiceType::MgDdm { mode: "transit".to_string() }, - ServiceType::Tfport { pkt_source: "tfpkt0".to_string() }, + ServiceType::Tfport { + pkt_source: "tfpkt0".to_string(), + asic, + }, ServiceType::SpSim, ] } @@ -2931,8 +2983,12 @@ impl ServiceManager { let root = if request.zone_type == ZoneType::Switch { Utf8PathBuf::from(ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT) } else { - let all_u2_roots = - self.inner.storage.all_u2_mountpoints(ZONE_DATASET).await; + let all_u2_roots = self + .inner + .storage + .get_latest_resources() + .await + .all_u2_mountpoints(ZONE_DATASET); let mut rng = rand::rngs::StdRng::from_entropy(); all_u2_roots .choose(&mut rng) @@ -2990,7 +3046,7 @@ impl ServiceManager { mod test { use super::*; use crate::params::{ServiceZoneService, ZoneType}; - use async_trait::async_trait; + use illumos_utils::zpool::ZpoolName; use illumos_utils::{ dladm::{ Etherstub, MockDladm, BOOTSTRAP_ETHERSTUB_NAME, @@ -2999,10 +3055,10 @@ mod test { svc, zone::MockZones, }; - use key_manager::{ - SecretRetriever, SecretRetrieverError, SecretState, VersionedIkm, - }; use omicron_common::address::OXIMETER_PORT; + use sled_storage::disk::{RawDisk, SyntheticDisk}; + + use sled_storage::manager::{FakeStorageManager, StorageHandle}; use std::net::{Ipv6Addr, SocketAddrV6}; use std::os::unix::process::ExitStatusExt; use uuid::Uuid; @@ -3030,6 +3086,7 @@ mod test { // Returns the expectations for a new service to be created. fn expect_new_service() -> Vec> { + illumos_utils::USE_MOCKS.store(true, Ordering::SeqCst); // Create a VNIC let create_vnic_ctx = MockDladm::create_vnic_context(); create_vnic_ctx.expect().return_once( @@ -3070,10 +3127,9 @@ mod test { // Wait for the networking service. let wait_ctx = svc::wait_for_service_context(); - wait_ctx.expect().return_once(|_, _| Ok(())); + wait_ctx.expect().return_once(|_, _, _| Ok(())); - // Import the manifest, enable the service - let execute_ctx = illumos_utils::execute_context(); + let execute_ctx = illumos_utils::execute_helper_context(); execute_ctx.expect().times(..).returning(|_| { Ok(std::process::Output { status: std::process::ExitStatus::from_raw(0), @@ -3195,29 +3251,24 @@ mod test { } } - pub struct TestSecretRetriever {} + async fn setup_storage() -> StorageHandle { + let (manager, handle) = FakeStorageManager::new(); - #[async_trait] - impl SecretRetriever for TestSecretRetriever { - async fn get_latest( - &self, - ) -> Result { - let epoch = 0; - let salt = [0u8; 32]; - let secret = [0x1d; 32]; + // Spawn the storage manager as done by sled-agent + tokio::spawn(async move { + manager.run().await; + }); - Ok(VersionedIkm::new(epoch, salt, &secret)) - } + let internal_zpool_name = ZpoolName::new_internal(Uuid::new_v4()); + let internal_disk: RawDisk = + SyntheticDisk::new(internal_zpool_name).into(); + handle.upsert_disk(internal_disk).await; + let external_zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let external_disk: RawDisk = + SyntheticDisk::new(external_zpool_name).into(); + handle.upsert_disk(external_disk).await; - async fn get( - &self, - epoch: u64, - ) -> Result { - if epoch != 0 { - return Err(SecretRetrieverError::NoSuchEpoch(epoch)); - } - Ok(SecretState::Current(self.get_latest().await?)) - } + handle } #[tokio::test] @@ -3228,10 +3279,10 @@ mod test { let log = logctx.log.clone(); let test_config = TestConfig::new().await; - let resources = StorageResources::new_for_test(); + let storage_handle = setup_storage().await; let zone_bundler = ZoneBundler::new( log.clone(), - resources.clone(), + storage_handle.clone(), Default::default(), ); let mgr = ServiceManager::new( @@ -3242,7 +3293,7 @@ mod test { Some(true), SidecarRevision::Physical("rev-test".to_string()), vec![], - resources, + storage_handle, zone_bundler, ); test_config.override_paths(&mgr); @@ -3276,10 +3327,10 @@ mod test { let log = logctx.log.clone(); let test_config = TestConfig::new().await; - let resources = StorageResources::new_for_test(); + let storage_handle = setup_storage().await; let zone_bundler = ZoneBundler::new( log.clone(), - resources.clone(), + storage_handle.clone(), Default::default(), ); let mgr = ServiceManager::new( @@ -3290,7 +3341,7 @@ mod test { Some(true), SidecarRevision::Physical("rev-test".to_string()), vec![], - resources, + storage_handle, zone_bundler, ); test_config.override_paths(&mgr); @@ -3329,10 +3380,10 @@ mod test { // First, spin up a ServiceManager, create a new service, and tear it // down. - let resources = StorageResources::new_for_test(); + let storage_handle = setup_storage().await; let zone_bundler = ZoneBundler::new( log.clone(), - resources.clone(), + storage_handle.clone(), Default::default(), ); let mgr = ServiceManager::new( @@ -3343,7 +3394,7 @@ mod test { Some(true), SidecarRevision::Physical("rev-test".to_string()), vec![], - resources.clone(), + storage_handle.clone(), zone_bundler.clone(), ); test_config.override_paths(&mgr); @@ -3376,7 +3427,7 @@ mod test { Some(true), SidecarRevision::Physical("rev-test".to_string()), vec![], - resources.clone(), + storage_handle.clone(), zone_bundler.clone(), ); test_config.override_paths(&mgr); @@ -3412,10 +3463,10 @@ mod test { // First, spin up a ServiceManager, create a new service, and tear it // down. - let resources = StorageResources::new_for_test(); + let storage_handle = setup_storage().await; let zone_bundler = ZoneBundler::new( log.clone(), - resources.clone(), + storage_handle.clone(), Default::default(), ); let mgr = ServiceManager::new( @@ -3426,7 +3477,7 @@ mod test { Some(true), SidecarRevision::Physical("rev-test".to_string()), vec![], - resources.clone(), + storage_handle.clone(), zone_bundler.clone(), ); test_config.override_paths(&mgr); @@ -3464,7 +3515,7 @@ mod test { Some(true), SidecarRevision::Physical("rev-test".to_string()), vec![], - resources.clone(), + storage_handle, zone_bundler.clone(), ); test_config.override_paths(&mgr); diff --git a/sled-agent/src/sim/disk.rs b/sled-agent/src/sim/disk.rs index 2d2c18be258..0f08289b74d 100644 --- a/sled-agent/src/sim/disk.rs +++ b/sled-agent/src/sim/disk.rs @@ -19,7 +19,7 @@ use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::ProducerEndpoint; use oximeter_producer::LogConfig; use oximeter_producer::Server as ProducerServer; -use propolis_client::api::DiskAttachmentState as PropolisDiskState; +use propolis_client::types::DiskAttachmentState as PropolisDiskState; use std::net::{Ipv6Addr, SocketAddr}; use std::sync::Arc; use std::time::Duration; diff --git a/sled-agent/src/sim/http_entrypoints_pantry.rs b/sled-agent/src/sim/http_entrypoints_pantry.rs index 46686a74e22..64b26a83a4f 100644 --- a/sled-agent/src/sim/http_entrypoints_pantry.rs +++ b/sled-agent/src/sim/http_entrypoints_pantry.rs @@ -4,11 +4,11 @@ //! HTTP entrypoint functions for simulating the crucible pantry API. -use crucible_client_types::VolumeConstructionRequest; use dropshot::{ endpoint, ApiDescription, HttpError, HttpResponseDeleted, HttpResponseOk, HttpResponseUpdatedNoContent, Path as TypedPath, RequestContext, TypedBody, }; +use propolis_client::types::VolumeConstructionRequest; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::sync::Arc; diff --git a/sled-agent/src/sim/instance.rs b/sled-agent/src/sim/instance.rs index 397a1980a58..15ff83c9699 100644 --- a/sled-agent/src/sim/instance.rs +++ b/sled-agent/src/sim/instance.rs @@ -19,9 +19,10 @@ use omicron_common::api::external::ResourceType; use omicron_common::api::internal::nexus::{ InstanceRuntimeState, SledInstanceState, }; -use propolis_client::api::InstanceMigrateStatusResponse as PropolisMigrateStatus; -use propolis_client::api::InstanceState as PropolisInstanceState; -use propolis_client::api::InstanceStateMonitorResponse; +use propolis_client::types::{ + InstanceMigrateStatusResponse as PropolisMigrateStatus, + InstanceState as PropolisInstanceState, InstanceStateMonitorResponse, +}; use std::collections::VecDeque; use std::sync::Arc; use std::sync::Mutex; @@ -131,11 +132,11 @@ impl SimInstanceInner { }); self.queue_migration_status(PropolisMigrateStatus { migration_id, - state: propolis_client::api::MigrationState::Sync, + state: propolis_client::types::MigrationState::Sync, }); self.queue_migration_status(PropolisMigrateStatus { migration_id, - state: propolis_client::api::MigrationState::Finish, + state: propolis_client::types::MigrationState::Finish, }); self.queue_propolis_state(PropolisInstanceState::Running); } diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index e4dac2f4b98..c06ae96f2e0 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -35,15 +35,16 @@ use uuid::Uuid; use std::collections::HashMap; use std::str::FromStr; -use crucible_client_types::VolumeConstructionRequest; use dropshot::HttpServer; use illumos_utils::opte::params::{ DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, }; use nexus_client::types::PhysicalDiskKind; use omicron_common::address::PROPOLIS_PORT; -use propolis_client::Client as PropolisClient; -use propolis_server::mock_server::Context as PropolisContext; +use propolis_client::{ + types::VolumeConstructionRequest, Client as PropolisClient, +}; +use propolis_mock_server::Context as PropolisContext; /// Simulates management of the control plane on a sled /// @@ -70,13 +71,14 @@ pub struct SledAgent { } fn extract_targets_from_volume_construction_request( - vec: &mut Vec, vcr: &VolumeConstructionRequest, -) { +) -> Result, std::net::AddrParseError> { // A snapshot is simply a flush with an extra parameter, and flushes are // only sent to sub volumes, not the read only parent. Flushes are only // processed by regions, so extract each region that would be affected by a // flush. + + let mut res = vec![]; match vcr { VolumeConstructionRequest::Volume { id: _, @@ -85,9 +87,9 @@ fn extract_targets_from_volume_construction_request( read_only_parent: _, } => { for sub_volume in sub_volumes.iter() { - extract_targets_from_volume_construction_request( - vec, sub_volume, - ); + res.extend(extract_targets_from_volume_construction_request( + sub_volume, + )?); } } @@ -103,7 +105,7 @@ fn extract_targets_from_volume_construction_request( gen: _, } => { for target in &opts.target { - vec.push(*target); + res.push(SocketAddr::from_str(target)?); } } @@ -111,6 +113,7 @@ fn extract_targets_from_volume_construction_request( // noop } } + Ok(res) } impl SledAgent { @@ -171,23 +174,19 @@ impl SledAgent { volume_construction_request: &VolumeConstructionRequest, ) -> Result<(), Error> { let disk_id = match volume_construction_request { - VolumeConstructionRequest::Volume { - id, - block_size: _, - sub_volumes: _, - read_only_parent: _, - } => id, + VolumeConstructionRequest::Volume { id, .. } => id, _ => { panic!("root of volume construction request not a volume!"); } }; - let mut targets = Vec::new(); - extract_targets_from_volume_construction_request( - &mut targets, + let targets = extract_targets_from_volume_construction_request( &volume_construction_request, - ); + ) + .map_err(|e| { + Error::invalid_request(&format!("bad socketaddr: {e:?}")) + })?; let mut region_ids = Vec::new(); @@ -640,11 +639,10 @@ impl SledAgent { ..Default::default() }; let propolis_log = log.new(o!("component" => "propolis-server-mock")); - let private = - Arc::new(PropolisContext::new(Default::default(), propolis_log)); + let private = Arc::new(PropolisContext::new(propolis_log)); info!(log, "Starting mock propolis-server..."); let dropshot_log = log.new(o!("component" => "dropshot")); - let mock_api = propolis_server::mock_server::api(); + let mock_api = propolis_mock_server::api(); let srv = dropshot::HttpServerStarter::new( &dropshot_config, diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 89b1f59c7af..2528a258d71 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -16,13 +16,13 @@ use chrono::prelude::*; use crucible_agent_client::types::{ CreateRegion, Region, RegionId, RunningSnapshot, Snapshot, State, }; -use crucible_client_types::VolumeConstructionRequest; use dropshot::HandlerTaskMode; use dropshot::HttpError; use futures::lock::Mutex; use nexus_client::types::{ ByteCount, PhysicalDiskKind, PhysicalDiskPutRequest, ZpoolPutRequest, }; +use propolis_client::types::VolumeConstructionRequest; use slog::Logger; use std::collections::HashMap; use std::collections::HashSet; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 52513f081d1..9826a987d45 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -4,13 +4,16 @@ //! Sled agent implementation +use crate::bootstrap::config::BOOTSTRAP_AGENT_RACK_INIT_PORT; use crate::bootstrap::early_networking::{ EarlyNetworkConfig, EarlyNetworkSetupError, }; use crate::bootstrap::params::StartSledAgentRequest; use crate::config::Config; -use crate::instance_manager::InstanceManager; -use crate::nexus::{NexusClientWithResolver, NexusRequestQueue}; +use crate::instance_manager::{InstanceManager, ReservoirMode}; +use crate::long_running_tasks::LongRunningTaskHandles; +use crate::metrics::MetricsManager; +use crate::nexus::{ConvertInto, NexusClientWithResolver, NexusRequestQueue}; use crate::params::{ DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse, InstanceStateRequested, @@ -18,13 +21,17 @@ use crate::params::{ VpcFirewallRule, ZoneBundleMetadata, Zpool, }; use crate::services::{self, ServiceManager}; -use crate::storage_manager::{self, StorageManager}; +use crate::storage_monitor::UnderlayAccess; use crate::updates::{ConfigUpdates, UpdateManager}; use crate::zone_bundle; use crate::zone_bundle::BundleError; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; +use ddm_admin_client::Client as DdmAdminClient; +use derive_more::From; use dropshot::HttpError; +use futures::stream::FuturesUnordered; +use futures::StreamExt; use illumos_utils::opte::params::{ DeleteVirtualNetworkInterfaceHost, SetVirtualNetworkInterfaceHost, }; @@ -35,6 +42,7 @@ use omicron_common::address::{ get_sled_address, get_switch_zone_address, Ipv6Subnet, SLED_PREFIX, }; use omicron_common::api::external::Vni; +use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_common::api::internal::nexus::{ SledInstanceState, VmmRuntimeState, }; @@ -46,15 +54,17 @@ use omicron_common::api::{ internal::nexus::UpdateArtifactId, }; use omicron_common::backoff::{ - retry_notify, retry_notify_ext, retry_policy_internal_service_aggressive, - BackoffError, + retry_notify, retry_notify_ext, retry_policy_internal_service, + retry_policy_internal_service_aggressive, BackoffError, }; -use sled_hardware::underlay; -use sled_hardware::HardwareManager; +use oximeter::types::ProducerRegistry; +use sled_hardware::{underlay, Baseboard, HardwareManager}; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::collections::BTreeMap; use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; +use tokio::sync::oneshot; use uuid::Uuid; #[cfg(not(test))] @@ -101,7 +111,7 @@ pub enum Error { Instance(#[from] crate::instance_manager::Error), #[error("Error managing storage: {0}")] - Storage(#[from] crate::storage_manager::Error), + Storage(#[from] sled_storage::error::Error), #[error("Error updating: {0}")] Download(#[from] crate::updates::Error), @@ -129,6 +139,9 @@ pub enum Error { #[error("Zone bundle error: {0}")] ZoneBundle(#[from] BundleError), + + #[error("Metrics error: {0}")] + Metrics(#[from] crate::metrics::Error), } impl From for omicron_common::api::external::Error { @@ -210,8 +223,12 @@ struct SledAgentInner { // The Sled Agent's address can be derived from this value. subnet: Ipv6Subnet, + // The request that was used to start the sled-agent + // This is used for idempotence checks during RSS/Add-Sled internal APIs + start_request: StartSledAgentRequest, + // Component of Sled Agent responsible for storage and dataset management. - storage: StorageManager, + storage: StorageHandle, // Component of Sled Agent responsible for managing Propolis instances. instances: InstanceManager, @@ -242,6 +259,9 @@ struct SledAgentInner { // A handle to the bootstore. bootstore: bootstore::NodeHandle, + + // Object handling production of metrics for oximeter. + metrics_manager: MetricsManager, } impl SledAgentInner { @@ -268,8 +288,8 @@ impl SledAgent { nexus_client: NexusClientWithResolver, request: StartSledAgentRequest, services: ServiceManager, - storage: StorageManager, - bootstore: bootstore::NodeHandle, + long_running_task_handles: LongRunningTaskHandles, + underlay_available_tx: oneshot::Sender, ) -> Result { // Pass the "parent_log" to all subcomponents that want to set their own // "component" value. @@ -278,18 +298,18 @@ impl SledAgent { // Use "log" for ourself. let log = log.new(o!( "component" => "SledAgent", - "sled_id" => request.id.to_string(), + "sled_id" => request.body.id.to_string(), )); info!(&log, "SledAgent::new(..) starting"); - let boot_disk = storage - .resources() - .boot_disk() + let storage_manager = &long_running_task_handles.storage_manager; + let boot_disk = storage_manager + .get_latest_resources() .await + .boot_disk() .ok_or_else(|| Error::BootDiskNotFound)?; - // Configure a swap device of the configured size before other system - // setup. + // Configure a swap device of the configured size before other system setup. match config.swap_device_size_gb { Some(sz) if sz > 0 => { info!(log, "Requested swap device of size {} GiB", sz); @@ -299,7 +319,7 @@ impl SledAgent { sz, )?; } - Some(sz) if sz == 0 => { + Some(0) => { panic!("Invalid requested swap device size of 0 GiB"); } None | Some(_) => { @@ -344,45 +364,55 @@ impl SledAgent { *sled_address.ip(), ); - storage - .setup_underlay_access(storage_manager::UnderlayAccess { + // Inform the `StorageMonitor` that the underlay is available so that + // it can try to contact nexus. + underlay_available_tx + .send(UnderlayAccess { nexus_client: nexus_client.clone(), - sled_id: request.id, + sled_id: request.body.id, }) - .await?; - - // TODO-correctness The bootstrap agent _also_ has a `HardwareManager`. - // We only use it for reading properties, but it's not `Clone`able - // because it's holding an inner task handle. Could we add a way to get - // a read-only handle to it, and have bootstrap agent give us that - // instead of creating a new full one ourselves? - let hardware = HardwareManager::new(&parent_log, services.sled_mode()) - .map_err(|e| Error::Hardware(e))?; + .map_err(|_| ()) + .expect("Failed to send to StorageMonitor"); let instances = InstanceManager::new( parent_log.clone(), nexus_client.clone(), etherstub.clone(), port_manager.clone(), - storage.resources().clone(), - storage.zone_bundler().clone(), + storage_manager.clone(), + long_running_task_handles.zone_bundler.clone(), )?; - match config.vmm_reservoir_percentage { - Some(sz) if sz > 0 && sz < 100 => { - instances.set_reservoir_size(&hardware, sz).map_err(|e| { - error!(log, "Failed to set VMM reservoir size: {e}"); - e - })?; - } - Some(sz) if sz == 0 => { - warn!(log, "Not using VMM reservoir (size 0 bytes requested)"); - } - None => { - warn!(log, "Not using VMM reservoir"); + // Configure the VMM reservoir as either a percentage of DRAM or as an + // exact size in MiB. + let reservoir_mode = match ( + config.vmm_reservoir_percentage, + config.vmm_reservoir_size_mb, + ) { + (None, None) => ReservoirMode::None, + (Some(p), None) => ReservoirMode::Percentage(p), + (None, Some(mb)) => ReservoirMode::Size(mb), + (Some(_), Some(_)) => panic!( + "only one of vmm_reservoir_percentage and \ + vmm_reservoir_size_mb is allowed" + ), + }; + + match reservoir_mode { + ReservoirMode::None => warn!(log, "Not using VMM reservoir"), + ReservoirMode::Size(0) | ReservoirMode::Percentage(0) => { + warn!(log, "Not using VMM reservoir (size 0 bytes requested)") } - Some(sz) => { - panic!("invalid requested VMM reservoir percentage: {}", sz); + _ => { + instances + .set_reservoir_size( + &long_running_task_handles.hardware_manager, + reservoir_mode, + ) + .map_err(|e| { + error!(log, "Failed to setup VMM reservoir: {e}"); + e + })?; } } @@ -391,14 +421,17 @@ impl SledAgent { }; let updates = UpdateManager::new(update_config); - let svc_config = - services::Config::new(request.id, config.sidecar_revision.clone()); + let svc_config = services::Config::new( + request.body.id, + config.sidecar_revision.clone(), + ); // Get our rack network config from the bootstore; we cannot proceed // until we have this, as we need to know which switches have uplinks to // correctly set up services. let get_network_config = || async { - let serialized_config = bootstore + let serialized_config = long_running_task_handles + .bootstore .get_network_config() .await .map_err(|err| BackoffError::transient(err.to_string()))? @@ -437,18 +470,58 @@ impl SledAgent { svc_config, port_manager.clone(), *sled_address.ip(), - request.rack_id, + request.body.rack_id, rack_network_config.clone(), )?; - let zone_bundler = storage.zone_bundler().clone(); + let mut metrics_manager = MetricsManager::new( + request.body.id, + request.body.rack_id, + long_running_task_handles.hardware_manager.baseboard(), + log.new(o!("component" => "MetricsManager")), + )?; + + // Start tracking the underlay physical links. + for nic in underlay::find_nics(&config.data_links)? { + let link_name = nic.interface(); + if let Err(e) = metrics_manager + .track_physical_link( + link_name, + crate::metrics::LINK_SAMPLE_INTERVAL, + ) + .await + { + error!( + log, + "failed to start tracking physical link metrics"; + "link_name" => link_name, + "error" => ?e, + ); + } + } + + // Spawn a task in the background to register our metric producer with + // Nexus. This should not block progress here. + let endpoint = ProducerEndpoint { + id: request.body.id, + address: sled_address.into(), + base_route: String::from("/metrics/collect"), + interval: crate::metrics::METRIC_COLLECTION_INTERVAL, + }; + tokio::task::spawn(register_metric_producer_with_nexus( + log.clone(), + nexus_client.clone(), + endpoint, + )); + let sled_agent = SledAgent { inner: Arc::new(SledAgentInner { - id: request.id, - subnet: request.subnet, - storage, + id: request.body.id, + subnet: request.body.subnet, + start_request: request, + storage: long_running_task_handles.storage_manager.clone(), instances, - hardware, + hardware: long_running_task_handles.hardware_manager.clone(), updates, port_manager, services, @@ -462,8 +535,9 @@ impl SledAgent { // request queue? nexus_request_queue: NexusRequestQueue::new(), rack_network_config, - zone_bundler, - bootstore: bootstore.clone(), + zone_bundler: long_running_task_handles.zone_bundler.clone(), + bootstore: long_running_task_handles.bootstore.clone(), + metrics_manager, }), log: log.clone(), }; @@ -483,6 +557,7 @@ impl SledAgent { /// Blocks until all services have started, retrying indefinitely on /// failure. pub(crate) async fn cold_boot_load_services(&self) { + info!(self.log, "Loading cold boot services"); retry_notify( retry_policy_internal_service_aggressive(), || async { @@ -522,15 +597,17 @@ impl SledAgent { &self.log } + pub fn start_request(&self) -> &StartSledAgentRequest { + &self.inner.start_request + } + // Sends a request to Nexus informing it that the current sled exists. pub(crate) fn notify_nexus_about_self(&self, log: &Logger) { let sled_id = self.inner.id; let nexus_client = self.inner.nexus_client.clone(); let sled_address = self.inner.sled_address(); let is_scrimlet = self.inner.hardware.is_scrimlet(); - let baseboard = nexus_client::types::Baseboard::from( - self.inner.hardware.baseboard(), - ); + let baseboard = self.inner.hardware.baseboard().convert(); let usable_hardware_threads = self.inner.hardware.online_processor_count(); let usable_physical_ram = @@ -585,12 +662,15 @@ impl SledAgent { if call_count == 0 { info!( log, - "failed to notify nexus about sled agent"; "error" => err, + "failed to notify nexus about sled agent"; + "error" => %err, ); } else if total_duration > std::time::Duration::from_secs(30) { warn!( log, - "failed to notify nexus about sled agent"; "error" => err, "total duration" => ?total_duration, + "failed to notify nexus about sled agent"; + "error" => %err, + "total duration" => ?total_duration, ); } }; @@ -759,9 +839,18 @@ impl SledAgent { } /// Gets the sled's current list of all zpools. - pub async fn zpools_get(&self) -> Result, Error> { - let zpools = self.inner.storage.get_zpools().await?; - Ok(zpools) + pub async fn zpools_get(&self) -> Vec { + self.inner + .storage + .get_latest_resources() + .await + .get_all_zpools() + .into_iter() + .map(|(name, variant)| Zpool { + id: name.id(), + disk_type: variant.into(), + }) + .collect() } /// Returns whether or not the sled believes itself to be a scrimlet @@ -939,4 +1028,126 @@ impl SledAgent { pub fn bootstore(&self) -> bootstore::NodeHandle { self.inner.bootstore.clone() } + + /// Return the metric producer registry. + pub fn metrics_registry(&self) -> &ProducerRegistry { + self.inner.metrics_manager.registry() + } +} + +async fn register_metric_producer_with_nexus( + log: Logger, + client: NexusClientWithResolver, + endpoint: ProducerEndpoint, +) { + let endpoint = nexus_client::types::ProducerEndpoint::from(&endpoint); + let register_with_nexus = || async { + client.client().cpapi_producers_post(&endpoint).await.map_err(|e| { + BackoffError::transient(format!("Metric registration error: {e}")) + }) + }; + retry_notify( + retry_policy_internal_service(), + register_with_nexus, + |error, delay| { + warn!( + log, + "failed to register as a metric producer with Nexus"; + "error" => ?error, + "retry_after" => ?delay, + ); + }, + ) + .await + .expect("Expected an infinite retry loop registering with Nexus"); +} + +#[derive(From, thiserror::Error, Debug)] +pub enum AddSledError { + #[error("Failed to learn bootstrap ip for {sled_id}")] + BootstrapAgentClient { + sled_id: Baseboard, + #[source] + err: bootstrap_agent_client::Error, + }, + #[error("Failed to connect to DDM")] + DdmAdminClient(#[source] ddm_admin_client::DdmError), + #[error("Failed to learn bootstrap ip for {0}")] + NotFound(Baseboard), + #[error("Failed to initialize {sled_id}: {err}")] + BootstrapTcpClient { + sled_id: Baseboard, + err: crate::bootstrap::client::Error, + }, +} + +/// Add a sled to an initialized rack. +pub async fn add_sled_to_initialized_rack( + log: Logger, + sled_id: Baseboard, + request: StartSledAgentRequest, +) -> Result<(), AddSledError> { + // Get all known bootstrap addresses via DDM + let ddm_admin_client = DdmAdminClient::localhost(&log)?; + let addrs = ddm_admin_client + .derive_bootstrap_addrs_from_prefixes(&[ + underlay::BootstrapInterface::GlobalZone, + ]) + .await?; + + // Create a set of futures to concurrently map the baseboard to bootstrap ip + // for each sled + let mut addrs_to_sleds = addrs + .map(|ip| { + let log = log.clone(); + async move { + let client = bootstrap_agent_client::Client::new( + &format!("http://[{ip}]"), + log, + ); + let result = client.baseboard_get().await; + + (ip, result) + } + }) + .collect::>(); + + // Execute the futures until we find our matching sled or done searching + let mut target_ip = None; + while let Some((ip, result)) = addrs_to_sleds.next().await { + match result { + Ok(baseboard) => { + // Convert from progenitor type back to `sled-hardware` + // type. + let found = baseboard.into_inner().into(); + if sled_id == found { + target_ip = Some(ip); + break; + } + } + Err(err) => { + warn!( + log, "Failed to get baseboard for {ip}"; + "err" => #%err, + ); + } + } + } + + // Contact the sled and initialize it + let bootstrap_addr = + target_ip.ok_or_else(|| AddSledError::NotFound(sled_id.clone()))?; + let bootstrap_addr = + SocketAddrV6::new(bootstrap_addr, BOOTSTRAP_AGENT_RACK_INIT_PORT, 0, 0); + let client = crate::bootstrap::client::Client::new( + bootstrap_addr, + log.new(o!("BootstrapAgentClient" => bootstrap_addr.to_string())), + ); + + client.start_sled_agent(&request).await.map_err(|err| { + AddSledError::BootstrapTcpClient { sled_id: sled_id.clone(), err } + })?; + + info!(log, "Peer agent initialized"; "peer_bootstrap_addr" => %bootstrap_addr, "peer_id" => %sled_id); + Ok(()) } diff --git a/sled-agent/src/storage/dataset.rs b/sled-agent/src/storage/dataset.rs deleted file mode 100644 index 4efc0f320af..00000000000 --- a/sled-agent/src/storage/dataset.rs +++ /dev/null @@ -1,63 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use crate::params::DatasetKind; -use illumos_utils::zpool::ZpoolName; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::str::FromStr; - -#[derive( - Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone, JsonSchema, -)] -pub struct DatasetName { - // A unique identifier for the Zpool on which the dataset is stored. - pool_name: ZpoolName, - // A name for the dataset within the Zpool. - kind: DatasetKind, -} - -impl DatasetName { - pub fn new(pool_name: ZpoolName, kind: DatasetKind) -> Self { - Self { pool_name, kind } - } - - pub fn pool(&self) -> &ZpoolName { - &self.pool_name - } - - pub fn dataset(&self) -> &DatasetKind { - &self.kind - } - - pub fn full(&self) -> String { - format!("{}/{}", self.pool_name, self.kind) - } -} - -impl From for sled_agent_client::types::DatasetName { - fn from(n: DatasetName) -> Self { - Self { - pool_name: sled_agent_client::types::ZpoolName::from_str( - &n.pool().to_string(), - ) - .unwrap(), - kind: n.dataset().clone().into(), - } - } -} - -#[cfg(test)] -mod test { - use super::*; - use uuid::Uuid; - - #[test] - fn serialize_dataset_name() { - let pool = ZpoolName::new_internal(Uuid::new_v4()); - let kind = DatasetKind::Crucible; - let name = DatasetName::new(pool, kind); - toml::to_string(&name).unwrap(); - } -} diff --git a/sled-agent/src/storage/mod.rs b/sled-agent/src/storage/mod.rs deleted file mode 100644 index 74bd59a1511..00000000000 --- a/sled-agent/src/storage/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Management of local storage - -pub(crate) mod dataset; -pub(crate) mod dump_setup; diff --git a/sled-agent/src/storage_manager.rs b/sled-agent/src/storage_manager.rs deleted file mode 100644 index c31a4dc0bc4..00000000000 --- a/sled-agent/src/storage_manager.rs +++ /dev/null @@ -1,1432 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -//! Management of sled-local storage. - -use crate::nexus::NexusClientWithResolver; -use crate::storage::dataset::DatasetName; -use crate::storage::dump_setup::DumpSetup; -use crate::zone_bundle::ZoneBundler; -use camino::Utf8PathBuf; -use derive_more::From; -use futures::stream::FuturesOrdered; -use futures::FutureExt; -use futures::StreamExt; -use illumos_utils::zpool::{ZpoolKind, ZpoolName}; -use illumos_utils::{zfs::Mountpoint, zpool::ZpoolInfo}; -use key_manager::StorageKeyRequester; -use nexus_client::types::PhysicalDiskDeleteRequest; -use nexus_client::types::PhysicalDiskKind; -use nexus_client::types::PhysicalDiskPutRequest; -use nexus_client::types::ZpoolPutRequest; -use omicron_common::api::external::{ByteCount, ByteCountRangeError}; -use omicron_common::backoff; -use omicron_common::disk::DiskIdentity; -use sled_hardware::{Disk, DiskVariant, UnparsedDisk}; -use slog::Logger; -use std::collections::hash_map; -use std::collections::HashMap; -use std::collections::HashSet; -use std::convert::TryFrom; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::OnceLock; -use std::time::Duration; -use tokio::sync::{mpsc, oneshot, Mutex}; -use tokio::task::JoinHandle; -use tokio::time::{interval, MissedTickBehavior}; -use uuid::Uuid; - -use illumos_utils::dumpadm::DumpHdrError; -#[cfg(test)] -use illumos_utils::{zfs::MockZfs as Zfs, zpool::MockZpool as Zpool}; -#[cfg(not(test))] -use illumos_utils::{zfs::Zfs, zpool::Zpool}; - -// A key manager can only become ready once. This occurs during RSS or cold -// boot when the bootstore has detected it has a key share. -static KEY_MANAGER_READY: OnceLock<()> = OnceLock::new(); - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error(transparent)] - DiskError(#[from] sled_hardware::DiskError), - - // TODO: We could add the context of "why are we doint this op", maybe? - #[error(transparent)] - ZfsListDataset(#[from] illumos_utils::zfs::ListDatasetsError), - - #[error(transparent)] - ZfsEnsureFilesystem(#[from] illumos_utils::zfs::EnsureFilesystemError), - - #[error(transparent)] - ZfsSetValue(#[from] illumos_utils::zfs::SetValueError), - - #[error(transparent)] - ZfsGetValue(#[from] illumos_utils::zfs::GetValueError), - - #[error(transparent)] - GetZpoolInfo(#[from] illumos_utils::zpool::GetInfoError), - - #[error(transparent)] - Fstyp(#[from] illumos_utils::fstyp::Error), - - #[error(transparent)] - ZoneCommand(#[from] illumos_utils::running_zone::RunCommandError), - - #[error(transparent)] - ZoneBoot(#[from] illumos_utils::running_zone::BootError), - - #[error(transparent)] - ZoneEnsureAddress(#[from] illumos_utils::running_zone::EnsureAddressError), - - #[error(transparent)] - ZoneInstall(#[from] illumos_utils::running_zone::InstallZoneError), - - #[error("No U.2 Zpools found")] - NoU2Zpool, - - #[error("Failed to parse UUID from {path}: {err}")] - ParseUuid { - path: Utf8PathBuf, - #[source] - err: uuid::Error, - }, - - #[error("Dataset {name:?} exists with a different uuid (has {old}, requested {new})")] - UuidMismatch { name: Box, old: Uuid, new: Uuid }, - - #[error("Error parsing pool {name}'s size: {err}")] - BadPoolSize { - name: String, - #[source] - err: ByteCountRangeError, - }, - - #[error("Failed to parse the dataset {name}'s UUID: {err}")] - ParseDatasetUuid { - name: String, - #[source] - err: uuid::Error, - }, - - #[error("Zpool Not Found: {0}")] - ZpoolNotFound(String), - - #[error("Failed to serialize toml (intended for {path:?}): {err}")] - Serialize { - path: Utf8PathBuf, - #[source] - err: toml::ser::Error, - }, - - #[error("Failed to deserialize toml from {path:?}: {err}")] - Deserialize { - path: Utf8PathBuf, - #[source] - err: toml::de::Error, - }, - - #[error("Failed to perform I/O: {message}: {err}")] - Io { - message: String, - #[source] - err: std::io::Error, - }, - - #[error("Underlay not yet initialized")] - UnderlayNotInitialized, - - #[error("Encountered error checking dump device flags: {0}")] - DumpHdr(#[from] DumpHdrError), -} - -/// A ZFS storage pool. -struct Pool { - name: ZpoolName, - info: ZpoolInfo, - parent: DiskIdentity, -} - -impl Pool { - /// Queries for an existing Zpool by name. - /// - /// Returns Ok if the pool exists. - fn new(name: ZpoolName, parent: DiskIdentity) -> Result { - let info = Zpool::get_info(&name.to_string())?; - Ok(Pool { name, info, parent }) - } - - fn parent(&self) -> &DiskIdentity { - &self.parent - } -} - -// The type of a future which is used to send a notification to Nexus. -type NotifyFut = - Pin> + Send>>; - -#[derive(Debug)] -struct NewFilesystemRequest { - dataset_id: Uuid, - dataset_name: DatasetName, - responder: oneshot::Sender>, -} - -struct UnderlayRequest { - underlay: UnderlayAccess, - responder: oneshot::Sender>, -} - -#[derive(PartialEq, Eq, Clone)] -pub(crate) enum DiskWrapper { - Real { disk: Disk, devfs_path: Utf8PathBuf }, - Synthetic { zpool_name: ZpoolName }, -} - -impl From for DiskWrapper { - fn from(disk: Disk) -> Self { - let devfs_path = disk.devfs_path().clone(); - Self::Real { disk, devfs_path } - } -} - -impl DiskWrapper { - fn identity(&self) -> DiskIdentity { - match self { - DiskWrapper::Real { disk, .. } => disk.identity().clone(), - DiskWrapper::Synthetic { zpool_name } => { - let id = zpool_name.id(); - DiskIdentity { - vendor: "synthetic-vendor".to_string(), - serial: format!("synthetic-serial-{id}"), - model: "synthetic-model".to_string(), - } - } - } - } - - fn variant(&self) -> DiskVariant { - match self { - DiskWrapper::Real { disk, .. } => disk.variant(), - DiskWrapper::Synthetic { zpool_name } => match zpool_name.kind() { - ZpoolKind::External => DiskVariant::U2, - ZpoolKind::Internal => DiskVariant::M2, - }, - } - } - - fn zpool_name(&self) -> &ZpoolName { - match self { - DiskWrapper::Real { disk, .. } => disk.zpool_name(), - DiskWrapper::Synthetic { zpool_name } => zpool_name, - } - } -} - -#[derive(Clone)] -pub struct StorageResources { - // All disks, real and synthetic, being managed by this sled - disks: Arc>>, - - // A map of "Uuid" to "pool". - pools: Arc>>, -} - -// The directory within the debug dataset in which bundles are created. -const BUNDLE_DIRECTORY: &str = "bundle"; - -// The directory for zone bundles. -const ZONE_BUNDLE_DIRECTORY: &str = "zone"; - -impl StorageResources { - /// Creates a fabricated view of storage resources. - /// - /// Use this only when you want to reference the disks, but not actually - /// access them. Creates one internal and one external disk. - #[cfg(test)] - pub fn new_for_test() -> Self { - let new_disk_identity = || DiskIdentity { - vendor: "vendor".to_string(), - serial: Uuid::new_v4().to_string(), - model: "model".to_string(), - }; - - Self { - disks: Arc::new(Mutex::new(HashMap::from([ - ( - new_disk_identity(), - DiskWrapper::Synthetic { - zpool_name: ZpoolName::new_internal(Uuid::new_v4()), - }, - ), - ( - new_disk_identity(), - DiskWrapper::Synthetic { - zpool_name: ZpoolName::new_external(Uuid::new_v4()), - }, - ), - ]))), - pools: Arc::new(Mutex::new(HashMap::new())), - } - } - - /// Returns the identity of the boot disk. - /// - /// If this returns `None`, we have not processed the boot disk yet. - pub async fn boot_disk(&self) -> Option<(DiskIdentity, ZpoolName)> { - let disks = self.disks.lock().await; - disks.iter().find_map(|(id, disk)| { - match disk { - // This is the "real" use-case: if we have real disks, query - // their properties to identify if they truly are the boot disk. - DiskWrapper::Real { disk, .. } => { - if disk.is_boot_disk() { - return Some((id.clone(), disk.zpool_name().clone())); - } - } - // This is the "less real" use-case: if we have synthetic disks, - // just label the first M.2-looking one as a "boot disk". - DiskWrapper::Synthetic { .. } => { - if matches!(disk.variant(), DiskVariant::M2) { - return Some((id.clone(), disk.zpool_name().clone())); - } - } - }; - None - }) - } - - // TODO: Could be generic over DiskVariant - - /// Returns all M.2 zpools - pub async fn all_m2_zpools(&self) -> Vec { - self.all_zpools(DiskVariant::M2).await - } - - /// Returns all U.2 zpools - pub async fn all_u2_zpools(&self) -> Vec { - self.all_zpools(DiskVariant::U2).await - } - - /// Returns all mountpoints within all M.2s for a particular dataset. - pub async fn all_m2_mountpoints(&self, dataset: &str) -> Vec { - let m2_zpools = self.all_m2_zpools().await; - m2_zpools - .iter() - .map(|zpool| zpool.dataset_mountpoint(dataset)) - .collect() - } - - /// Returns all mountpoints within all U.2s for a particular dataset. - pub async fn all_u2_mountpoints(&self, dataset: &str) -> Vec { - let u2_zpools = self.all_u2_zpools().await; - u2_zpools - .iter() - .map(|zpool| zpool.dataset_mountpoint(dataset)) - .collect() - } - - /// Returns all zpools of a particular variant - pub async fn all_zpools(&self, variant: DiskVariant) -> Vec { - let disks = self.disks.lock().await; - disks - .values() - .filter_map(|disk| { - if disk.variant() == variant { - return Some(disk.zpool_name().clone()); - } - None - }) - .collect() - } - - /// Return the directories for storing zone service bundles. - pub async fn all_zone_bundle_directories(&self) -> Vec { - self.all_m2_mountpoints(sled_hardware::disk::M2_DEBUG_DATASET) - .await - .into_iter() - .map(|p| p.join(BUNDLE_DIRECTORY).join(ZONE_BUNDLE_DIRECTORY)) - .collect() - } -} - -/// Describes the access to the underlay used by the StorageManager. -pub struct UnderlayAccess { - pub nexus_client: NexusClientWithResolver, - pub sled_id: Uuid, -} - -// A worker that starts zones for pools as they are received. -struct StorageWorker { - log: Logger, - nexus_notifications: FuturesOrdered, - rx: mpsc::Receiver, - underlay: Arc>>, - - // A mechanism for requesting disk encryption keys from the - // [`key_manager::KeyManager`] - key_requester: StorageKeyRequester, - - // Invokes dumpadm(8) and savecore(8) when new disks are encountered - dump_setup: Arc, -} - -#[derive(Clone, Debug)] -enum NotifyDiskRequest { - Add { identity: DiskIdentity, variant: DiskVariant }, - Remove(DiskIdentity), -} - -#[derive(From, Clone, Debug, PartialEq, Eq, Hash)] -enum QueuedDiskCreate { - Real(UnparsedDisk), - Synthetic(ZpoolName), -} - -impl QueuedDiskCreate { - fn is_synthetic(&self) -> bool { - if let QueuedDiskCreate::Synthetic(_) = self { - true - } else { - false - } - } -} - -impl StorageWorker { - // Ensures the named dataset exists as a filesystem with a UUID, optionally - // creating it if `do_format` is true. - // - // Returns the UUID attached to the ZFS filesystem. - fn ensure_dataset( - &mut self, - dataset_id: Uuid, - dataset_name: &DatasetName, - ) -> Result<(), Error> { - let zoned = true; - let fs_name = &dataset_name.full(); - let do_format = true; - let encryption_details = None; - let size_details = None; - Zfs::ensure_filesystem( - &dataset_name.full(), - Mountpoint::Path(Utf8PathBuf::from("/data")), - zoned, - do_format, - encryption_details, - size_details, - None, - )?; - // Ensure the dataset has a usable UUID. - if let Ok(id_str) = Zfs::get_oxide_value(&fs_name, "uuid") { - if let Ok(id) = id_str.parse::() { - if id != dataset_id { - return Err(Error::UuidMismatch { - name: Box::new(dataset_name.clone()), - old: id, - new: dataset_id, - }); - } - return Ok(()); - } - } - Zfs::set_oxide_value(&fs_name, "uuid", &dataset_id.to_string())?; - Ok(()) - } - - // Adds a "notification to nexus" to `nexus_notifications`, - // informing it about the addition of `pool_id` to this sled. - async fn add_zpool_notify(&mut self, pool: &Pool, size: ByteCount) { - // The underlay network is setup once at sled-agent startup. Before - // there is an underlay we want to avoid sending notifications to nexus for - // two reasons: - // 1. They can't possibly succeed - // 2. They increase the backoff time exponentially, so that once - // sled-agent does start it may take much longer to notify nexus - // than it would if we avoid this. This goes especially so for rack - // setup, when bootstrap agent is waiting an aribtrary time for RSS - // initialization. - if self.underlay.lock().await.is_none() { - return; - } - - let pool_id = pool.name.id(); - let DiskIdentity { vendor, serial, model } = pool.parent.clone(); - let underlay = self.underlay.clone(); - - let notify_nexus = move || { - let zpool_request = ZpoolPutRequest { - size: size.into(), - disk_vendor: vendor.clone(), - disk_serial: serial.clone(), - disk_model: model.clone(), - }; - let underlay = underlay.clone(); - - async move { - let underlay_guard = underlay.lock().await; - let Some(underlay) = underlay_guard.as_ref() else { - return Err(backoff::BackoffError::transient( - Error::UnderlayNotInitialized.to_string(), - )); - }; - let sled_id = underlay.sled_id; - let nexus_client = underlay.nexus_client.client().clone(); - drop(underlay_guard); - - nexus_client - .zpool_put(&sled_id, &pool_id, &zpool_request) - .await - .map_err(|e| { - backoff::BackoffError::transient(e.to_string()) - })?; - Ok(()) - } - }; - let log = self.log.clone(); - let name = pool.name.clone(); - let disk = pool.parent().clone(); - let log_post_failure = move |_, call_count, total_duration| { - if call_count == 0 { - info!(log, "failed to notify nexus about a new pool {name} on disk {disk:?}"); - } else if total_duration > std::time::Duration::from_secs(30) { - warn!(log, "failed to notify nexus about a new pool {name} on disk {disk:?}"; - "total duration" => ?total_duration); - } - }; - self.nexus_notifications.push_back( - backoff::retry_notify_ext( - backoff::retry_policy_internal_service_aggressive(), - notify_nexus, - log_post_failure, - ) - .boxed(), - ); - } - - async fn ensure_using_exactly_these_disks( - &mut self, - resources: &StorageResources, - unparsed_disks: Vec, - queued_u2_drives: &mut Option>, - ) -> Result<(), Error> { - // Queue U.2 drives if necessary - // We clear all existing queued drives that are not synthetic and add - // new ones in the loop below - if let Some(queued) = queued_u2_drives { - info!( - self.log, - "Ensure exact disks: clearing non-synthetic queued disks." - ); - queued.retain(|d| d.is_synthetic()); - } - - let mut new_disks = HashMap::new(); - - // We may encounter errors while parsing any of the disks; keep track of - // any errors that occur and return any of them if something goes wrong. - // - // That being said, we should not prevent access to the other disks if - // only one failure occurs. - let mut err: Option = None; - - // Ensure all disks conform to the expected partition layout. - for disk in unparsed_disks.into_iter() { - if disk.variant() == DiskVariant::U2 { - if let Some(queued) = queued_u2_drives { - info!(self.log, "Queuing disk for upsert: {disk:?}"); - queued.insert(disk.into()); - continue; - } - } - match self.add_new_disk(disk, queued_u2_drives).await.map_err( - |err| { - warn!(self.log, "Could not ensure partitions: {err}"); - err - }, - ) { - Ok(disk) => { - new_disks.insert(disk.identity().clone(), disk); - } - Err(e) => { - warn!(self.log, "Cannot parse disk: {e}"); - err = Some(e.into()); - } - }; - } - - let mut disks = resources.disks.lock().await; - - // Remove disks that don't appear in the "new_disks" set. - // - // This also accounts for zpools and notifies Nexus. - let disks_to_be_removed = disks - .iter_mut() - .filter(|(key, old_disk)| { - // If this disk appears in the "new" and "old" set, it should - // only be removed if it has changed. - // - // This treats a disk changing in an unexpected way as a - // "removal and re-insertion". - match old_disk { - DiskWrapper::Real { disk, .. } => { - if let Some(new_disk) = new_disks.get(*key) { - // Changed Disk -> Disk should be removed. - new_disk != disk - } else { - // Real disk, not in the new set -> Disk should be removed. - true - } - } - // Synthetic disk -> Disk should NOT be removed. - DiskWrapper::Synthetic { .. } => false, - } - }) - .map(|(_key, disk)| disk.clone()) - .collect::>(); - - for disk in disks_to_be_removed { - if let Err(e) = self - .delete_disk_locked(&resources, &mut disks, &disk.identity()) - .await - { - warn!(self.log, "Failed to delete disk: {e}"); - err = Some(e); - } - } - - // Add new disks to `resources.disks`. - // - // This also accounts for zpools and notifies Nexus. - for (key, new_disk) in new_disks { - if let Some(old_disk) = disks.get(&key) { - // In this case, the disk should be unchanged. - // - // This assertion should be upheld by the filter above, which - // should remove disks that changed. - assert!(old_disk == &new_disk.into()); - } else { - let disk = DiskWrapper::Real { - disk: new_disk.clone(), - devfs_path: new_disk.devfs_path().clone(), - }; - if let Err(e) = - self.upsert_disk_locked(&resources, &mut disks, disk).await - { - warn!(self.log, "Failed to upsert disk: {e}"); - err = Some(e); - } - } - } - - if let Some(err) = err { - Err(err) - } else { - Ok(()) - } - } - - // Attempt to create a new disk via `sled_hardware::Disk::new()`. If the - // disk addition fails because the the key manager cannot load a secret, - // this indicates a transient error, and so we queue the disk so we can - // try again. - async fn add_new_disk( - &mut self, - unparsed_disk: UnparsedDisk, - queued_u2_drives: &mut Option>, - ) -> Result { - match sled_hardware::Disk::new( - &self.log, - unparsed_disk.clone(), - Some(&self.key_requester), - ) - .await - { - Ok(disk) => Ok(disk), - Err(sled_hardware::DiskError::KeyManager(err)) => { - warn!( - self.log, - "Transient error: {err} - queuing disk {:?}", unparsed_disk - ); - if let Some(queued) = queued_u2_drives { - queued.insert(unparsed_disk.into()); - } else { - *queued_u2_drives = - Some(HashSet::from([unparsed_disk.into()])); - } - Err(sled_hardware::DiskError::KeyManager(err)) - } - Err(err) => { - error!( - self.log, - "Persistent error: {err} - not queueing disk {:?}", - unparsed_disk - ); - Err(err) - } - } - } - - // Attempt to create a new synthetic disk via - // `sled_hardware::Disk::ensure_zpool_ready()`. If the disk addition fails - // because the the key manager cannot load a secret, this indicates a - // transient error, and so we queue the disk so we can try again. - async fn add_new_synthetic_disk( - &mut self, - zpool_name: ZpoolName, - queued_u2_drives: &mut Option>, - ) -> Result<(), sled_hardware::DiskError> { - let synthetic_id = DiskIdentity { - vendor: "fake_vendor".to_string(), - serial: "fake_serial".to_string(), - model: zpool_name.id().to_string(), - }; - match sled_hardware::Disk::ensure_zpool_ready( - &self.log, - &zpool_name, - &synthetic_id, - Some(&self.key_requester), - ) - .await - { - Ok(()) => Ok(()), - Err(sled_hardware::DiskError::KeyManager(err)) => { - warn!( - self.log, - "Transient error: {err} - queuing synthetic disk: {:?}", - zpool_name - ); - if let Some(queued) = queued_u2_drives { - queued.insert(zpool_name.into()); - } else { - *queued_u2_drives = - Some(HashSet::from([zpool_name.into()])); - } - Err(sled_hardware::DiskError::KeyManager(err)) - } - Err(err) => { - error!( - self.log, - "Persistent error: {} - not queueing synthetic disk {:?}", - err, - zpool_name - ); - Err(err) - } - } - } - - async fn upsert_disk( - &mut self, - resources: &StorageResources, - disk: UnparsedDisk, - queued_u2_drives: &mut Option>, - ) -> Result<(), Error> { - // Queue U.2 drives if necessary - if let Some(queued) = queued_u2_drives { - if disk.variant() == DiskVariant::U2 { - info!(self.log, "Queuing disk for upsert: {disk:?}"); - queued.insert(disk.into()); - return Ok(()); - } - } - - info!(self.log, "Upserting disk: {disk:?}"); - - // Ensure the disk conforms to an expected partition layout. - let disk = - self.add_new_disk(disk, queued_u2_drives).await.map_err(|err| { - warn!(self.log, "Could not ensure partitions: {err}"); - err - })?; - - let mut disks = resources.disks.lock().await; - let disk = DiskWrapper::Real { - disk: disk.clone(), - devfs_path: disk.devfs_path().clone(), - }; - self.upsert_disk_locked(resources, &mut disks, disk).await - } - - async fn upsert_synthetic_disk( - &mut self, - resources: &StorageResources, - zpool_name: ZpoolName, - queued_u2_drives: &mut Option>, - ) -> Result<(), Error> { - // Queue U.2 drives if necessary - if let Some(queued) = queued_u2_drives { - if zpool_name.kind() == ZpoolKind::External { - info!( - self.log, - "Queuing synthetic disk for upsert: {zpool_name:?}" - ); - queued.insert(zpool_name.into()); - return Ok(()); - } - } - - info!(self.log, "Upserting synthetic disk for: {zpool_name:?}"); - - self.add_new_synthetic_disk(zpool_name.clone(), queued_u2_drives) - .await?; - let disk = DiskWrapper::Synthetic { zpool_name }; - let mut disks = resources.disks.lock().await; - self.upsert_disk_locked(resources, &mut disks, disk).await - } - - async fn upsert_disk_locked( - &mut self, - resources: &StorageResources, - disks: &mut tokio::sync::MutexGuard< - '_, - HashMap, - >, - disk: DiskWrapper, - ) -> Result<(), Error> { - disks.insert(disk.identity(), disk.clone()); - self.physical_disk_notify(NotifyDiskRequest::Add { - identity: disk.identity(), - variant: disk.variant(), - }) - .await; - self.upsert_zpool(&resources, disk.identity(), disk.zpool_name()) - .await?; - - self.dump_setup.update_dumpdev_setup(disks).await; - - Ok(()) - } - - async fn delete_disk( - &mut self, - resources: &StorageResources, - disk: UnparsedDisk, - ) -> Result<(), Error> { - info!(self.log, "Deleting disk: {disk:?}"); - // TODO: Don't we need to do some accounting, e.g. for all the information - // that's no longer accessible? Or is that up to Nexus to figure out at - // a later point-in-time? - // - // If we're storing zone images on the M.2s for internal services, how - // do we reconcile them? - let mut disks = resources.disks.lock().await; - self.delete_disk_locked(resources, &mut disks, disk.identity()).await - } - - async fn delete_disk_locked( - &mut self, - resources: &StorageResources, - disks: &mut tokio::sync::MutexGuard< - '_, - HashMap, - >, - key: &DiskIdentity, - ) -> Result<(), Error> { - if let Some(parsed_disk) = disks.remove(key) { - resources.pools.lock().await.remove(&parsed_disk.zpool_name().id()); - self.physical_disk_notify(NotifyDiskRequest::Remove(key.clone())) - .await; - } - - self.dump_setup.update_dumpdev_setup(disks).await; - - Ok(()) - } - - /// When the underlay becomes available, we need to notify nexus about any - /// discovered disks and pools, since we don't attempt to notify until there - /// is an underlay available. - async fn notify_nexus_about_existing_resources( - &mut self, - resources: &StorageResources, - ) -> Result<(), Error> { - let disks = resources.disks.lock().await; - for disk in disks.values() { - self.physical_disk_notify(NotifyDiskRequest::Add { - identity: disk.identity(), - variant: disk.variant(), - }) - .await; - } - - // We may encounter errors while processing any of the pools; keep track of - // any errors that occur and return any of them if something goes wrong. - // - // That being said, we should not prevent notification to nexus of the - // other pools if only one failure occurs. - let mut err: Option = None; - - let pools = resources.pools.lock().await; - for pool in pools.values() { - match ByteCount::try_from(pool.info.size()).map_err(|err| { - Error::BadPoolSize { name: pool.name.to_string(), err } - }) { - Ok(size) => self.add_zpool_notify(pool, size).await, - Err(e) => { - warn!(self.log, "Failed to notify nexus about pool: {e}"); - err = Some(e) - } - } - } - - if let Some(err) = err { - Err(err) - } else { - Ok(()) - } - } - - // Adds a "notification to nexus" to `self.nexus_notifications`, informing it - // about the addition/removal of a physical disk to this sled. - async fn physical_disk_notify(&mut self, disk: NotifyDiskRequest) { - // The underlay network is setup once at sled-agent startup. Before - // there is an underlay we want to avoid sending notifications to nexus for - // two reasons: - // 1. They can't possibly succeed - // 2. They increase the backoff time exponentially, so that once - // sled-agent does start it may take much longer to notify nexus - // than it would if we avoid this. This goes especially so for rack - // setup, when bootstrap agent is waiting an aribtrary time for RSS - // initialization. - if self.underlay.lock().await.is_none() { - return; - } - let underlay = self.underlay.clone(); - let disk2 = disk.clone(); - let notify_nexus = move || { - let disk = disk.clone(); - let underlay = underlay.clone(); - async move { - let underlay_guard = underlay.lock().await; - let Some(underlay) = underlay_guard.as_ref() else { - return Err(backoff::BackoffError::transient( - Error::UnderlayNotInitialized.to_string(), - )); - }; - let sled_id = underlay.sled_id; - let nexus_client = underlay.nexus_client.client().clone(); - drop(underlay_guard); - - match &disk { - NotifyDiskRequest::Add { identity, variant } => { - let request = PhysicalDiskPutRequest { - model: identity.model.clone(), - serial: identity.serial.clone(), - vendor: identity.vendor.clone(), - variant: match variant { - DiskVariant::U2 => PhysicalDiskKind::U2, - DiskVariant::M2 => PhysicalDiskKind::M2, - }, - sled_id, - }; - nexus_client - .physical_disk_put(&request) - .await - .map_err(|e| { - backoff::BackoffError::transient(e.to_string()) - })?; - } - NotifyDiskRequest::Remove(disk_identity) => { - let request = PhysicalDiskDeleteRequest { - model: disk_identity.model.clone(), - serial: disk_identity.serial.clone(), - vendor: disk_identity.vendor.clone(), - sled_id, - }; - nexus_client - .physical_disk_delete(&request) - .await - .map_err(|e| { - backoff::BackoffError::transient(e.to_string()) - })?; - } - } - Ok(()) - } - }; - let log = self.log.clone(); - // This notification is often invoked before Nexus has started - // running, so avoid flagging any errors as concerning until some - // time has passed. - let log_post_failure = move |_, call_count, total_duration| { - if call_count == 0 { - info!(log, "failed to notify nexus about {disk2:?}"); - } else if total_duration > std::time::Duration::from_secs(30) { - warn!(log, "failed to notify nexus about {disk2:?}"; - "total duration" => ?total_duration); - } - }; - self.nexus_notifications.push_back( - backoff::retry_notify_ext( - backoff::retry_policy_internal_service_aggressive(), - notify_nexus, - log_post_failure, - ) - .boxed(), - ); - } - - async fn upsert_zpool( - &mut self, - resources: &StorageResources, - parent: DiskIdentity, - pool_name: &ZpoolName, - ) -> Result<(), Error> { - let mut pools = resources.pools.lock().await; - let zpool = Pool::new(pool_name.clone(), parent)?; - - let pool = match pools.entry(pool_name.id()) { - hash_map::Entry::Occupied(mut entry) => { - // The pool already exists. - entry.get_mut().info = zpool.info; - return Ok(()); - } - hash_map::Entry::Vacant(entry) => entry.insert(zpool), - }; - info!(&self.log, "Storage manager processing zpool: {:#?}", pool.info); - - let size = ByteCount::try_from(pool.info.size()).map_err(|err| { - Error::BadPoolSize { name: pool_name.to_string(), err } - })?; - // Notify Nexus of the zpool. - self.add_zpool_notify(&pool, size).await; - Ok(()) - } - - // Attempts to add a dataset within a zpool, according to `request`. - async fn add_dataset( - &mut self, - resources: &StorageResources, - request: &NewFilesystemRequest, - ) -> Result { - info!(self.log, "add_dataset: {:?}", request); - let mut pools = resources.pools.lock().await; - let pool = pools - .get_mut(&request.dataset_name.pool().id()) - .ok_or_else(|| { - Error::ZpoolNotFound(format!( - "{}, looked up while trying to add dataset", - request.dataset_name.pool(), - )) - })?; - let dataset_name = DatasetName::new( - pool.name.clone(), - request.dataset_name.dataset().clone(), - ); - self.ensure_dataset(request.dataset_id, &dataset_name)?; - Ok(dataset_name) - } - - // Small wrapper around `Self::do_work_internal` that ensures we always - // emit info to the log when we exit. - async fn do_work( - &mut self, - resources: StorageResources, - ) -> Result<(), Error> { - // We queue U.2 sleds until the StorageKeyRequester is ready to use. - let mut queued_u2_drives = Some(HashSet::new()); - loop { - match self.do_work_internal(&resources, &mut queued_u2_drives).await - { - Ok(()) => { - info!(self.log, "StorageWorker exited successfully"); - return Ok(()); - } - Err(e) => { - warn!( - self.log, - "StorageWorker encountered unexpected error: {}", e - ); - // ... for now, keep trying. - } - } - } - } - - async fn do_work_internal( - &mut self, - resources: &StorageResources, - queued_u2_drives: &mut Option>, - ) -> Result<(), Error> { - const QUEUED_DISK_RETRY_TIMEOUT: Duration = Duration::from_secs(5); - let mut interval = interval(QUEUED_DISK_RETRY_TIMEOUT); - interval.set_missed_tick_behavior(MissedTickBehavior::Delay); - loop { - tokio::select! { - _ = self.nexus_notifications.next(), - if !self.nexus_notifications.is_empty() => {}, - Some(request) = self.rx.recv() => { - // We want to queue failed requests related to the key manager - match self.handle_storage_worker_request( - resources, queued_u2_drives, request) - .await { - Err(Error::DiskError(_)) => { - // We already handle and log disk errors, no need to - // return here. - } - Err(e) => return Err(e), - Ok(()) => {} - } - } - _ = interval.tick(), if queued_u2_drives.is_some() && - KEY_MANAGER_READY.get().is_some()=> - { - self.upsert_queued_disks(resources, queued_u2_drives).await; - } - } - } - } - - async fn handle_storage_worker_request( - &mut self, - resources: &StorageResources, - queued_u2_drives: &mut Option>, - request: StorageWorkerRequest, - ) -> Result<(), Error> { - use StorageWorkerRequest::*; - match request { - AddDisk(disk) => { - self.upsert_disk(&resources, disk, queued_u2_drives).await?; - } - AddSyntheticDisk(zpool_name) => { - self.upsert_synthetic_disk( - &resources, - zpool_name, - queued_u2_drives, - ) - .await?; - } - RemoveDisk(disk) => { - self.delete_disk(&resources, disk).await?; - } - NewFilesystem(request) => { - let result = self.add_dataset(&resources, &request).await; - let _ = request.responder.send(result); - } - DisksChanged(disks) => { - self.ensure_using_exactly_these_disks( - &resources, - disks, - queued_u2_drives, - ) - .await?; - } - SetupUnderlayAccess(UnderlayRequest { underlay, responder }) => { - // If this is the first time establishing an - // underlay we should notify nexus of all existing - // disks and zpools. - // - // Instead of individual notifications, we should - // send a bulk notification as described in https:// - // github.com/oxidecomputer/omicron/issues/1917 - if self.underlay.lock().await.replace(underlay).is_none() { - self.notify_nexus_about_existing_resources(&resources) - .await?; - } - let _ = responder.send(Ok(())); - } - KeyManagerReady => { - let _ = KEY_MANAGER_READY.set(()); - self.upsert_queued_disks(resources, queued_u2_drives).await; - } - } - Ok(()) - } - - async fn upsert_queued_disks( - &mut self, - resources: &StorageResources, - queued_u2_drives: &mut Option>, - ) { - let queued = queued_u2_drives.take(); - if let Some(queued) = queued { - for disk in queued { - if let Some(saved) = queued_u2_drives { - // We already hit a transient error and recreated our queue. - // Add any remaining queued disks back on the queue so we - // can try again later. - saved.insert(disk); - } else { - match self.upsert_queued_disk(disk, resources).await { - Ok(()) => {} - Err((_, None)) => { - // We already logged this as a persistent error in - // `add_new_disk` or `add_new_synthetic_disk` - } - Err((_, Some(disk))) => { - // We already logged this as a transient error in - // `add_new_disk` or `add_new_synthetic_disk` - *queued_u2_drives = Some(HashSet::from([disk])); - } - } - } - } - } - if queued_u2_drives.is_none() { - info!(self.log, "upserted all queued disks"); - } else { - warn!( - self.log, - "failed to upsert all queued disks - will try again" - ); - } - } - - // Attempt to upsert a queued disk. Return the disk and error if the upsert - // fails due to a transient error. Examples of transient errors are key - // manager errors which indicate that there are not enough sleds available - // to unlock the rack. - async fn upsert_queued_disk( - &mut self, - disk: QueuedDiskCreate, - resources: &StorageResources, - ) -> Result<(), (Error, Option)> { - let mut temp: Option> = None; - let res = match disk { - QueuedDiskCreate::Real(disk) => { - self.upsert_disk(&resources, disk, &mut temp).await - } - QueuedDiskCreate::Synthetic(zpool_name) => { - self.upsert_synthetic_disk(&resources, zpool_name, &mut temp) - .await - } - }; - if let Some(mut disks) = temp.take() { - assert!(res.is_err()); - assert_eq!(disks.len(), 1); - return Err(( - res.unwrap_err(), - disks.drain().next().unwrap().into(), - )); - } - // Any error at this point is not transient. - // We don't requeue the disk. - res.map_err(|e| (e, None)) - } -} - -enum StorageWorkerRequest { - AddDisk(UnparsedDisk), - AddSyntheticDisk(ZpoolName), - RemoveDisk(UnparsedDisk), - DisksChanged(Vec), - NewFilesystem(NewFilesystemRequest), - SetupUnderlayAccess(UnderlayRequest), - KeyManagerReady, -} - -struct StorageManagerInner { - log: Logger, - - resources: StorageResources, - - tx: mpsc::Sender, - - // A handle to a worker which updates "pools". - task: JoinHandle>, -} - -/// A sled-local view of all attached storage. -#[derive(Clone)] -pub struct StorageManager { - inner: Arc, - zone_bundler: ZoneBundler, -} - -impl StorageManager { - /// Creates a new [`StorageManager`] which should manage local storage. - pub async fn new(log: &Logger, key_requester: StorageKeyRequester) -> Self { - let log = log.new(o!("component" => "StorageManager")); - let resources = StorageResources { - disks: Arc::new(Mutex::new(HashMap::new())), - pools: Arc::new(Mutex::new(HashMap::new())), - }; - let (tx, rx) = mpsc::channel(30); - - let zb_log = log.new(o!("component" => "ZoneBundler")); - let zone_bundler = - ZoneBundler::new(zb_log, resources.clone(), Default::default()); - - StorageManager { - inner: Arc::new(StorageManagerInner { - log: log.clone(), - resources: resources.clone(), - tx, - task: tokio::task::spawn(async move { - let dump_setup = Arc::new(DumpSetup::new(&log)); - let mut worker = StorageWorker { - log, - nexus_notifications: FuturesOrdered::new(), - rx, - underlay: Arc::new(Mutex::new(None)), - key_requester, - dump_setup, - }; - - worker.do_work(resources).await - }), - }), - zone_bundler, - } - } - - /// Return a reference to the object used to manage zone bundles. - /// - /// This can be cloned by other code wishing to create and manage their own - /// zone bundles. - pub fn zone_bundler(&self) -> &ZoneBundler { - &self.zone_bundler - } - - /// Ensures that the storage manager tracks exactly the provided disks. - /// - /// This acts similar to a batch [Self::upsert_disk] for all new disks, and - /// [Self::delete_disk] for all removed disks. - /// - /// If errors occur, an arbitrary "one" of them will be returned, but a - /// best-effort attempt to add all disks will still be attempted. - // Receiver implemented by [StorageWorker::ensure_using_exactly_these_disks] - pub async fn ensure_using_exactly_these_disks(&self, unparsed_disks: I) - where - I: IntoIterator, - { - self.inner - .tx - .send(StorageWorkerRequest::DisksChanged( - unparsed_disks.into_iter().collect::>(), - )) - .await - .map_err(|e| e.to_string()) - .expect("Failed to send DisksChanged request"); - } - - /// Adds a disk and associated zpool to the storage manager. - // Receiver implemented by [StorageWorker::upsert_disk]. - pub async fn upsert_disk(&self, disk: UnparsedDisk) { - info!(self.inner.log, "Upserting disk: {disk:?}"); - self.inner - .tx - .send(StorageWorkerRequest::AddDisk(disk)) - .await - .map_err(|e| e.to_string()) - .expect("Failed to send AddDisk request"); - } - - /// Removes a disk, if it's tracked by the storage manager, as well - /// as any associated zpools. - // Receiver implemented by [StorageWorker::delete_disk]. - pub async fn delete_disk(&self, disk: UnparsedDisk) { - info!(self.inner.log, "Deleting disk: {disk:?}"); - self.inner - .tx - .send(StorageWorkerRequest::RemoveDisk(disk)) - .await - .map_err(|e| e.to_string()) - .expect("Failed to send RemoveDisk request"); - } - - /// Adds a synthetic zpool to the storage manager. - // Receiver implemented by [StorageWorker::upsert_synthetic_disk]. - pub async fn upsert_synthetic_disk(&self, name: ZpoolName) { - self.inner - .tx - .send(StorageWorkerRequest::AddSyntheticDisk(name)) - .await - .map_err(|e| e.to_string()) - .expect("Failed to send AddSyntheticDisk request"); - } - - /// Adds underlay access to the storage manager. - pub async fn setup_underlay_access( - &self, - underlay: UnderlayAccess, - ) -> Result<(), Error> { - let (tx, rx) = oneshot::channel(); - self.inner - .tx - .send(StorageWorkerRequest::SetupUnderlayAccess(UnderlayRequest { - underlay, - responder: tx, - })) - .await - .map_err(|e| e.to_string()) - .expect("Failed to send SetupUnderlayAccess request"); - rx.await.expect("Failed to await underlay setup") - } - - pub async fn get_zpools(&self) -> Result, Error> { - let disks = self.inner.resources.disks.lock().await; - let pools = self.inner.resources.pools.lock().await; - - let mut zpools = Vec::with_capacity(pools.len()); - - for (id, pool) in pools.iter() { - let disk_identity = &pool.parent; - let disk_type = if let Some(disk) = disks.get(&disk_identity) { - disk.variant().into() - } else { - // If the zpool claims to be attached to a disk that we - // don't know about, that's an error. - return Err(Error::ZpoolNotFound( - format!("zpool: {id} claims to be from unknown disk: {disk_identity:#?}") - )); - }; - zpools.push(crate::params::Zpool { id: *id, disk_type }); - } - - Ok(zpools) - } - - pub async fn upsert_filesystem( - &self, - dataset_id: Uuid, - dataset_name: DatasetName, - ) -> Result { - let (tx, rx) = oneshot::channel(); - let request = - NewFilesystemRequest { dataset_id, dataset_name, responder: tx }; - - self.inner - .tx - .send(StorageWorkerRequest::NewFilesystem(request)) - .await - .map_err(|e| e.to_string()) - .expect("Storage worker bug (not alive)"); - let dataset_name = rx.await.expect( - "Storage worker bug (dropped responder without responding)", - )?; - - Ok(dataset_name) - } - - /// Inform the storage worker that the KeyManager is capable of retrieving - /// secrets now and that any queued disks can be upserted. - pub async fn key_manager_ready(&self) { - info!(self.inner.log, "KeyManger ready"); - self.inner - .tx - .send(StorageWorkerRequest::KeyManagerReady) - .await - .map_err(|e| e.to_string()) - .expect("Failed to send KeyManagerReady request"); - } - - pub fn resources(&self) -> &StorageResources { - &self.inner.resources - } -} - -impl Drop for StorageManagerInner { - fn drop(&mut self) { - // NOTE: Ideally, with async drop, we'd await completion of the worker - // somehow. - // - // Without that option, we instead opt to simply cancel the worker - // task to ensure it does not remain alive beyond the StorageManager - // itself. - self.task.abort(); - } -} diff --git a/sled-agent/src/storage_monitor.rs b/sled-agent/src/storage_monitor.rs new file mode 100644 index 00000000000..0c9b2873966 --- /dev/null +++ b/sled-agent/src/storage_monitor.rs @@ -0,0 +1,373 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! A task that listens for storage events from [`sled_storage::manager::StorageManager`] +//! and dispatches them to other parst of the bootstrap agent and sled agent +//! code. + +use crate::dump_setup::DumpSetup; +use crate::nexus::{ConvertInto, NexusClientWithResolver}; +use derive_more::From; +use futures::stream::FuturesOrdered; +use futures::FutureExt; +use futures::StreamExt; +use nexus_client::types::PhysicalDiskDeleteRequest; +use nexus_client::types::PhysicalDiskPutRequest; +use nexus_client::types::ZpoolPutRequest; +use omicron_common::api::external::ByteCount; +use omicron_common::backoff; +use omicron_common::disk::DiskIdentity; +use sled_storage::manager::StorageHandle; +use sled_storage::pool::Pool; +use sled_storage::resources::StorageResources; +use slog::Logger; +use std::fmt::Debug; +use std::pin::Pin; +use tokio::sync::oneshot; +use uuid::Uuid; + +#[derive(From, Clone, Debug)] +enum NexusDiskRequest { + Put(PhysicalDiskPutRequest), + Delete(PhysicalDiskDeleteRequest), +} + +/// Describes the access to the underlay used by the StorageManager. +#[derive(Clone)] +pub struct UnderlayAccess { + pub nexus_client: NexusClientWithResolver, + pub sled_id: Uuid, +} + +impl Debug for UnderlayAccess { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UnderlayAccess") + .field("sled_id", &self.sled_id) + .finish() + } +} + +pub struct StorageMonitor { + log: Logger, + storage_manager: StorageHandle, + + // Receive a onetime notification that the underlay is available + underlay_available_rx: oneshot::Receiver, + + // A cached copy of the `StorageResources` from the last update + storage_resources: StorageResources, + + // Ability to access the underlay network + underlay: Option, + + // A queue for sending nexus notifications in order + nexus_notifications: FuturesOrdered, + + // Invokes dumpadm(8) and savecore(8) when new disks are encountered + dump_setup: DumpSetup, +} + +impl StorageMonitor { + pub fn new( + log: &Logger, + storage_manager: StorageHandle, + ) -> (StorageMonitor, oneshot::Sender) { + let (underlay_available_tx, underlay_available_rx) = oneshot::channel(); + let storage_resources = StorageResources::default(); + let dump_setup = DumpSetup::new(&log); + let log = log.new(o!("component" => "StorageMonitor")); + ( + StorageMonitor { + log, + storage_manager, + underlay_available_rx, + storage_resources, + underlay: None, + nexus_notifications: FuturesOrdered::new(), + dump_setup, + }, + underlay_available_tx, + ) + } + + /// Run the main receive loop of the `StorageMonitor` + /// + /// This should be spawned into a tokio task + pub async fn run(mut self) { + loop { + tokio::select! { + res = self.nexus_notifications.next(), + if !self.nexus_notifications.is_empty() => + { + match res { + Some(Ok(s)) => { + info!(self.log, "Nexus notification complete: {s}"); + } + e => error!(self.log, "Nexus notification error: {e:?}") + } + } + resources = self.storage_manager.wait_for_changes() => { + info!( + self.log, + "Received storage manager update"; + "resources" => ?resources + ); + self.handle_resource_update(resources).await; + } + Ok(underlay) = &mut self.underlay_available_rx, + if self.underlay.is_none() => + { + let sled_id = underlay.sled_id; + info!( + self.log, + "Underlay Available"; "sled_id" => %sled_id + ); + self.underlay = Some(underlay); + self.notify_nexus_about_existing_resources(sled_id).await; + } + } + } + } + + /// When the underlay becomes available, we need to notify nexus about any + /// discovered disks and pools, since we don't attempt to notify until there + /// is an underlay available. + async fn notify_nexus_about_existing_resources(&mut self, sled_id: Uuid) { + let current = StorageResources::default(); + let updated = &self.storage_resources; + let nexus_updates = + compute_resource_diffs(&self.log, &sled_id, ¤t, updated); + for put in nexus_updates.disk_puts { + self.physical_disk_notify(put.into()).await; + } + for (pool, put) in nexus_updates.zpool_puts { + self.add_zpool_notify(pool, put).await; + } + } + + async fn handle_resource_update( + &mut self, + updated_resources: StorageResources, + ) { + // If the underlay isn't available, we only record the changes. Nexus + // isn't yet reachable to notify. + if self.underlay.is_some() { + let nexus_updates = compute_resource_diffs( + &self.log, + &self.underlay.as_ref().unwrap().sled_id, + &self.storage_resources, + &updated_resources, + ); + + for put in nexus_updates.disk_puts { + self.physical_disk_notify(put.into()).await; + } + for del in nexus_updates.disk_deletes { + self.physical_disk_notify(del.into()).await; + } + for (pool, put) in nexus_updates.zpool_puts { + self.add_zpool_notify(pool, put).await; + } + } + self.dump_setup.update_dumpdev_setup(updated_resources.disks()).await; + + // Save the updated `StorageResources` + self.storage_resources = updated_resources; + } + + // Adds a "notification to nexus" to `self.nexus_notifications`, informing it + // about the addition/removal of a physical disk to this sled. + async fn physical_disk_notify(&mut self, disk: NexusDiskRequest) { + let underlay = self.underlay.as_ref().unwrap().clone(); + let disk2 = disk.clone(); + let notify_nexus = move || { + let underlay = underlay.clone(); + let disk = disk.clone(); + async move { + let nexus_client = underlay.nexus_client.client().clone(); + + match &disk { + NexusDiskRequest::Put(request) => { + nexus_client + .physical_disk_put(&request) + .await + .map_err(|e| { + backoff::BackoffError::transient(e.to_string()) + })?; + } + NexusDiskRequest::Delete(request) => { + nexus_client + .physical_disk_delete(&request) + .await + .map_err(|e| { + backoff::BackoffError::transient(e.to_string()) + })?; + } + } + let msg = format!("{:?}", disk); + Ok(msg) + } + }; + + let log = self.log.clone(); + // This notification is often invoked before Nexus has started + // running, so avoid flagging any errors as concerning until some + // time has passed. + let log_post_failure = move |err, call_count, total_duration| { + if call_count == 0 { + info!(log, "failed to notify nexus about {disk2:?}"; + "err" => ?err + ); + } else if total_duration > std::time::Duration::from_secs(30) { + warn!(log, "failed to notify nexus about {disk2:?}"; + "err" => ?err, + "total duration" => ?total_duration); + } + }; + self.nexus_notifications.push_back( + backoff::retry_notify_ext( + backoff::retry_policy_internal_service_aggressive(), + notify_nexus, + log_post_failure, + ) + .boxed(), + ); + } + + // Adds a "notification to nexus" to `nexus_notifications`, + // informing it about the addition of `pool_id` to this sled. + async fn add_zpool_notify( + &mut self, + pool: Pool, + zpool_request: ZpoolPutRequest, + ) { + let pool_id = pool.name.id(); + let underlay = self.underlay.as_ref().unwrap().clone(); + + let notify_nexus = move || { + let underlay = underlay.clone(); + let zpool_request = zpool_request.clone(); + async move { + let sled_id = underlay.sled_id; + let nexus_client = underlay.nexus_client.client().clone(); + nexus_client + .zpool_put(&sled_id, &pool_id, &zpool_request) + .await + .map_err(|e| { + backoff::BackoffError::transient(e.to_string()) + })?; + let msg = format!("{:?}", zpool_request); + Ok(msg) + } + }; + + let log = self.log.clone(); + let name = pool.name.clone(); + let disk = pool.parent.clone(); + let log_post_failure = move |err, call_count, total_duration| { + if call_count == 0 { + info!(log, "failed to notify nexus about a new pool {name} on disk {disk:?}"; + "err" => ?err); + } else if total_duration > std::time::Duration::from_secs(30) { + warn!(log, "failed to notify nexus about a new pool {name} on disk {disk:?}"; + "err" => ?err, + "total duration" => ?total_duration); + } + }; + self.nexus_notifications.push_back( + backoff::retry_notify_ext( + backoff::retry_policy_internal_service_aggressive(), + notify_nexus, + log_post_failure, + ) + .boxed(), + ); + } +} + +// The type of a future which is used to send a notification to Nexus. +type NotifyFut = + Pin> + Send>>; + +struct NexusUpdates { + disk_puts: Vec, + disk_deletes: Vec, + zpool_puts: Vec<(Pool, ZpoolPutRequest)>, +} + +fn compute_resource_diffs( + log: &Logger, + sled_id: &Uuid, + current: &StorageResources, + updated: &StorageResources, +) -> NexusUpdates { + let mut disk_puts = vec![]; + let mut disk_deletes = vec![]; + let mut zpool_puts = vec![]; + + let mut put_pool = |disk_id: &DiskIdentity, updated_pool: &Pool| { + match ByteCount::try_from(updated_pool.info.size()) { + Ok(size) => zpool_puts.push(( + updated_pool.clone(), + ZpoolPutRequest { + size: size.into(), + disk_model: disk_id.model.clone(), + disk_serial: disk_id.serial.clone(), + disk_vendor: disk_id.vendor.clone(), + }, + )), + Err(err) => { + error!( + log, + "Error parsing pool size"; + "name" => updated_pool.name.to_string(), + "err" => ?err); + } + } + }; + + // Diff the existing resources with the update to see what has changed + // This loop finds disks and pools that were modified or deleted + for (disk_id, (disk, pool)) in current.disks().iter() { + match updated.disks().get(disk_id) { + Some((updated_disk, updated_pool)) => { + if disk != updated_disk { + disk_puts.push(PhysicalDiskPutRequest { + sled_id: *sled_id, + model: disk_id.model.clone(), + serial: disk_id.serial.clone(), + vendor: disk_id.vendor.clone(), + variant: updated_disk.variant().convert(), + }); + } + if pool != updated_pool { + put_pool(disk_id, updated_pool); + } + } + None => disk_deletes.push(PhysicalDiskDeleteRequest { + model: disk_id.model.clone(), + serial: disk_id.serial.clone(), + vendor: disk_id.vendor.clone(), + sled_id: *sled_id, + }), + } + } + + // Diff the existing resources with the update to see what has changed + // This loop finds new disks and pools + for (disk_id, (updated_disk, updated_pool)) in updated.disks().iter() { + if !current.disks().contains_key(disk_id) { + disk_puts.push(PhysicalDiskPutRequest { + sled_id: *sled_id, + model: disk_id.model.clone(), + serial: disk_id.serial.clone(), + vendor: disk_id.vendor.clone(), + variant: updated_disk.variant().convert(), + }); + put_pool(disk_id, updated_pool); + } + } + + NexusUpdates { disk_puts, disk_deletes, zpool_puts } +} diff --git a/sled-agent/src/zone_bundle.rs b/sled-agent/src/zone_bundle.rs index 55661371c34..70b9da77080 100644 --- a/sled-agent/src/zone_bundle.rs +++ b/sled-agent/src/zone_bundle.rs @@ -6,7 +6,6 @@ //! Tools for collecting and inspecting service bundles for zones. -use crate::storage_manager::StorageResources; use anyhow::anyhow; use anyhow::Context; use camino::FromPathBufError; @@ -33,6 +32,8 @@ use illumos_utils::zone::AdmError; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; +use sled_storage::dataset::U2_DEBUG_DATASET; +use sled_storage::manager::StorageHandle; use slog::Logger; use std::cmp::Ord; use std::cmp::Ordering; @@ -221,20 +222,12 @@ pub struct ZoneBundler { inner: Arc>, // Channel for notifying the cleanup task that it should reevaluate. notify_cleanup: Arc, - // Tokio task handle running the period cleanup operation. - cleanup_task: Arc>, -} - -impl Drop for ZoneBundler { - fn drop(&mut self) { - self.cleanup_task.abort(); - } } // State shared between tasks, e.g., used when creating a bundle in different // tasks or between a creation and cleanup. struct Inner { - resources: StorageResources, + storage_handle: StorageHandle, cleanup_context: CleanupContext, last_cleanup_at: Instant, } @@ -262,7 +255,8 @@ impl Inner { // that can exist but do not, i.e., those whose parent datasets already // exist; and returns those. async fn bundle_directories(&self) -> Vec { - let expected = self.resources.all_zone_bundle_directories().await; + let resources = self.storage_handle.get_latest_resources().await; + let expected = resources.all_zone_bundle_directories(); let mut out = Vec::with_capacity(expected.len()); for each in expected.into_iter() { if tokio::fs::create_dir_all(&each).await.is_ok() { @@ -322,11 +316,11 @@ impl ZoneBundler { /// Create a new zone bundler. /// /// This creates an object that manages zone bundles on the system. It can - /// be used to create bundles from running zones, and runs a period task to - /// clean them up to free up space. + /// be used to create bundles from running zones, and runs a periodic task + /// to clean them up to free up space. pub fn new( log: Logger, - resources: StorageResources, + storage_handle: StorageHandle, cleanup_context: CleanupContext, ) -> Self { // This is compiled out in tests because there's no way to set our @@ -336,17 +330,19 @@ impl ZoneBundler { .expect("Failed to initialize existing ZFS resources"); let notify_cleanup = Arc::new(Notify::new()); let inner = Arc::new(Mutex::new(Inner { - resources, + storage_handle, cleanup_context, last_cleanup_at: Instant::now(), })); let cleanup_log = log.new(slog::o!("component" => "auto-cleanup-task")); let notify_clone = notify_cleanup.clone(); let inner_clone = inner.clone(); - let cleanup_task = Arc::new(tokio::task::spawn( - Self::periodic_cleanup(cleanup_log, inner_clone, notify_clone), + tokio::task::spawn(Self::periodic_cleanup( + cleanup_log, + inner_clone, + notify_clone, )); - Self { log, inner, notify_cleanup, cleanup_task } + Self { log, inner, notify_cleanup } } /// Trigger an immediate cleanup of low-priority zone bundles. @@ -431,10 +427,9 @@ impl ZoneBundler { ) -> Result { let inner = self.inner.lock().await; let storage_dirs = inner.bundle_directories().await; - let extra_log_dirs = inner - .resources - .all_u2_mountpoints(sled_hardware::disk::U2_DEBUG_DATASET) - .await + let resources = inner.storage_handle.get_latest_resources().await; + let extra_log_dirs = resources + .all_u2_mountpoints(U2_DEBUG_DATASET) .into_iter() .collect(); let context = ZoneBundleContext { cause, storage_dirs, extra_log_dirs }; @@ -904,7 +899,7 @@ async fn find_service_log_files( if path != current_log_file && path_ref.starts_with(current_log_file_ref) { - log_files.push(path.clone().into()); + log_files.push(path.into()); } } @@ -2165,7 +2160,6 @@ mod illumos_tests { use super::CleanupPeriod; use super::PriorityOrder; use super::StorageLimit; - use super::StorageResources; use super::Utf8Path; use super::Utf8PathBuf; use super::Uuid; @@ -2178,6 +2172,10 @@ mod illumos_tests { use anyhow::Context; use chrono::TimeZone; use chrono::Utc; + use illumos_utils::zpool::ZpoolName; + use sled_storage::disk::RawDisk; + use sled_storage::disk::SyntheticDisk; + use sled_storage::manager::{FakeStorageManager, StorageHandle}; use slog::Drain; use slog::Logger; use tokio::process::Command; @@ -2219,22 +2217,43 @@ mod illumos_tests { // system, that creates the directories implied by the `StorageResources` // expected disk structure. struct ResourceWrapper { - resources: StorageResources, + storage_handle: StorageHandle, dirs: Vec, } + async fn setup_storage() -> StorageHandle { + let (manager, handle) = FakeStorageManager::new(); + + // Spawn the storage manager as done by sled-agent + tokio::spawn(async move { + manager.run().await; + }); + + // These must be internal zpools + for _ in 0..2 { + let internal_zpool_name = ZpoolName::new_internal(Uuid::new_v4()); + let internal_disk: RawDisk = + SyntheticDisk::new(internal_zpool_name.clone()).into(); + handle.upsert_disk(internal_disk).await; + } + handle + } + impl ResourceWrapper { // Create new storage resources, and mount fake datasets at the required // locations. async fn new() -> Self { - let resources = StorageResources::new_for_test(); - let dirs = resources.all_zone_bundle_directories().await; + // Spawn the storage related tasks required for testing and insert + // synthetic disks. + let storage_handle = setup_storage().await; + let resources = storage_handle.get_latest_resources().await; + let dirs = resources.all_zone_bundle_directories(); for d in dirs.iter() { let id = d.components().nth(3).unwrap().as_str().parse().unwrap(); create_test_dataset(&id, d).await.unwrap(); } - Self { resources, dirs } + Self { storage_handle, dirs } } } @@ -2261,8 +2280,11 @@ mod illumos_tests { let log = test_logger(); let context = CleanupContext::default(); let resource_wrapper = ResourceWrapper::new().await; - let bundler = - ZoneBundler::new(log, resource_wrapper.resources.clone(), context); + let bundler = ZoneBundler::new( + log, + resource_wrapper.storage_handle.clone(), + context, + ); Ok(CleanupTestContext { resource_wrapper, context, bundler }) } diff --git a/sled-hardware/Cargo.toml b/sled-hardware/Cargo.toml index 14ae15996b9..36ba6330674 100644 --- a/sled-hardware/Cargo.toml +++ b/sled-hardware/Cargo.toml @@ -11,10 +11,8 @@ camino.workspace = true cfg-if.workspace = true futures.workspace = true illumos-utils.workspace = true -key-manager.workspace = true libc.workspace = true macaddr.workspace = true -nexus-client.workspace = true omicron-common.workspace = true rand.workspace = true schemars.workspace = true diff --git a/sled-hardware/src/disk.rs b/sled-hardware/src/disk.rs index e3078cbeea2..44658658be1 100644 --- a/sled-hardware/src/disk.rs +++ b/sled-hardware/src/disk.rs @@ -4,34 +4,14 @@ use camino::{Utf8Path, Utf8PathBuf}; use illumos_utils::fstyp::Fstyp; -use illumos_utils::zfs; -use illumos_utils::zfs::DestroyDatasetErrorVariant; -use illumos_utils::zfs::EncryptionDetails; -use illumos_utils::zfs::Keypath; -use illumos_utils::zfs::Mountpoint; -use illumos_utils::zfs::SizeDetails; -use illumos_utils::zfs::Zfs; use illumos_utils::zpool::Zpool; use illumos_utils::zpool::ZpoolKind; use illumos_utils::zpool::ZpoolName; -use key_manager::StorageKeyRequester; use omicron_common::disk::DiskIdentity; -use rand::distributions::{Alphanumeric, DistString}; use slog::Logger; use slog::{info, warn}; -use std::sync::OnceLock; -use tokio::fs::{remove_file, File}; -use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom}; use uuid::Uuid; -/// This path is intentionally on a `tmpfs` to prevent copy-on-write behavior -/// and to ensure it goes away on power off. -/// -/// We want minimize the time the key files are in memory, and so we rederive -/// the keys and recreate the files on demand when creating and mounting -/// encrypted filesystems. We then zero them and unlink them. -pub const KEYPATH_ROOT: &str = "/var/run/oxide/"; - cfg_if::cfg_if! { if #[cfg(target_os = "illumos")] { use crate::illumos::*; @@ -41,7 +21,7 @@ cfg_if::cfg_if! { } #[derive(Debug, thiserror::Error)] -pub enum DiskError { +pub enum PooledDiskError { #[error("Cannot open {path} due to {error}")] IoError { path: Utf8PathBuf, error: std::io::Error }, #[error("Failed to open partition at {path} due to {error}")] @@ -51,10 +31,6 @@ pub enum DiskError { #[error("Requested partition {partition:?} not found on device {path}")] NotFound { path: Utf8PathBuf, partition: Partition }, #[error(transparent)] - DestroyFilesystem(#[from] illumos_utils::zfs::DestroyDatasetError), - #[error(transparent)] - EnsureFilesystem(#[from] illumos_utils::zfs::EnsureFilesystemError), - #[error(transparent)] ZpoolCreate(#[from] illumos_utils::zpool::CreateError), #[error("Cannot import zpool: {0}")] ZpoolImport(illumos_utils::zpool::Error), @@ -62,18 +38,6 @@ pub enum DiskError { CannotFormatMissingDevPath { path: Utf8PathBuf }, #[error("Formatting M.2 devices is not yet implemented")] CannotFormatM2NotImplemented, - #[error("KeyManager error: {0}")] - KeyManager(#[from] key_manager::Error), - #[error("Missing StorageKeyRequester when creating U.2 disk")] - MissingStorageKeyRequester, - #[error("Encrypted filesystem '{0}' missing 'oxide:epoch' property")] - CannotParseEpochProperty(String), - #[error("Encrypted dataset '{dataset}' cannot set 'oxide:agent' property: {err}")] - CannotSetAgentProperty { - dataset: String, - #[source] - err: Box, - }, } /// A partition (or 'slice') of a disk. @@ -126,17 +90,17 @@ impl DiskPaths { } // Finds the first 'variant' partition, and returns the path to it. - fn partition_device_path( + pub fn partition_device_path( &self, partitions: &[Partition], expected_partition: Partition, raw: bool, - ) -> Result { + ) -> Result { for (index, partition) in partitions.iter().enumerate() { if &expected_partition == partition { let path = self.partition_path(index, raw).ok_or_else(|| { - DiskError::NotFound { + PooledDiskError::NotFound { path: self.devfs_path.clone(), partition: expected_partition, } @@ -144,7 +108,7 @@ impl DiskPaths { return Ok(path); } } - Err(DiskError::NotFound { + Err(PooledDiskError::NotFound { path: self.devfs_path.clone(), partition: expected_partition, }) @@ -154,9 +118,9 @@ impl DiskPaths { /// A disk which has been observed by monitoring hardware. /// /// No guarantees are made about the partitions which exist within this disk. -/// This exists as a distinct entity from [Disk] because it may be desirable to -/// monitor for hardware in one context, and conform disks to partition layouts -/// in a different context. +/// This exists as a distinct entity from `Disk` in `sled-storage` because it +/// may be desirable to monitor for hardware in one context, and conform disks +/// to partition layouts in a different context. #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct UnparsedDisk { paths: DiskPaths, @@ -202,127 +166,34 @@ impl UnparsedDisk { } } -/// A physical disk conforming to the expected partition layout. +/// A physical disk that is partitioned to contain exactly one zpool +/// +/// A PooledDisk relies on hardware specific information to be constructed +/// and is the highest level disk structure in the `sled-hardware` package. +/// The `sled-storage` package contains `Disk`s whose zpool and datasets can be +/// manipulated. This separation exists to remove the hardware dependent logic +/// from the ZFS related logic which can also operate on file backed zpools. +/// Doing things this way allows us to not put higher level concepts like +/// storage keys into this hardware related package. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct Disk { - paths: DiskPaths, - slot: i64, - variant: DiskVariant, - identity: DiskIdentity, - is_boot_disk: bool, - partitions: Vec, - +pub struct PooledDisk { + pub paths: DiskPaths, + pub slot: i64, + pub variant: DiskVariant, + pub identity: DiskIdentity, + pub is_boot_disk: bool, + pub partitions: Vec, // This embeds the assumtion that there is exactly one parsed zpool per // disk. - zpool_name: ZpoolName, -} - -// Helper type for describing expected datasets and their optional quota. -#[derive(Clone, Copy, Debug)] -struct ExpectedDataset { - // Name for the dataset - name: &'static str, - // Optional quota, in _bytes_ - quota: Option, - // Identifies if the dataset should be deleted on boot - wipe: bool, - // Optional compression mode - compression: Option<&'static str>, + pub zpool_name: ZpoolName, } -impl ExpectedDataset { - const fn new(name: &'static str) -> Self { - ExpectedDataset { name, quota: None, wipe: false, compression: None } - } - - const fn quota(mut self, quota: usize) -> Self { - self.quota = Some(quota); - self - } - - const fn wipe(mut self) -> Self { - self.wipe = true; - self - } - - const fn compression(mut self, compression: &'static str) -> Self { - self.compression = Some(compression); - self - } -} - -pub const INSTALL_DATASET: &'static str = "install"; -pub const CRASH_DATASET: &'static str = "crash"; -pub const CLUSTER_DATASET: &'static str = "cluster"; -pub const CONFIG_DATASET: &'static str = "config"; -pub const M2_DEBUG_DATASET: &'static str = "debug"; -pub const M2_BACKING_DATASET: &'static str = "backing"; -// TODO-correctness: This value of 100GiB is a pretty wild guess, and should be -// tuned as needed. -pub const DEBUG_DATASET_QUOTA: usize = 100 * (1 << 30); -// ditto. -pub const DUMP_DATASET_QUOTA: usize = 100 * (1 << 30); -// passed to zfs create -o compression= -pub const DUMP_DATASET_COMPRESSION: &'static str = "gzip-9"; - -// U.2 datasets live under the encrypted dataset and inherit encryption -pub const ZONE_DATASET: &'static str = "crypt/zone"; -pub const DUMP_DATASET: &'static str = "crypt/debug"; -pub const U2_DEBUG_DATASET: &'static str = "crypt/debug"; - -// This is the root dataset for all U.2 drives. Encryption is inherited. -pub const CRYPT_DATASET: &'static str = "crypt"; - -const U2_EXPECTED_DATASET_COUNT: usize = 2; -static U2_EXPECTED_DATASETS: [ExpectedDataset; U2_EXPECTED_DATASET_COUNT] = [ - // Stores filesystems for zones - ExpectedDataset::new(ZONE_DATASET).wipe(), - // For storing full kernel RAM dumps - ExpectedDataset::new(DUMP_DATASET) - .quota(DUMP_DATASET_QUOTA) - .compression(DUMP_DATASET_COMPRESSION), -]; - -const M2_EXPECTED_DATASET_COUNT: usize = 6; -static M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ - // Stores software images. - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(INSTALL_DATASET), - // Stores crash dumps. - ExpectedDataset::new(CRASH_DATASET), - // Backing store for OS data that should be persisted across reboots. - // Its children are selectively overlay mounted onto parts of the ramdisk - // root. - ExpectedDataset::new(M2_BACKING_DATASET), - // Stores cluster configuration information. - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(CLUSTER_DATASET), - // Stores configuration data, including: - // - What services should be launched on this sled - // - Information about how to initialize the Sled Agent - // - (For scrimlets) RSS setup information - // - // Should be duplicated to both M.2s. - ExpectedDataset::new(CONFIG_DATASET), - // Store debugging data, such as service bundles. - ExpectedDataset::new(M2_DEBUG_DATASET).quota(DEBUG_DATASET_QUOTA), -]; - -impl Disk { - /// Create a new Disk - /// - /// WARNING: In all cases where a U.2 is a possible `DiskVariant`, a - /// `StorageKeyRequester` must be passed so that disk encryption can - /// be used. The `StorageManager` for the sled-agent always has a - /// `StorageKeyRequester` available, and so the only place we should pass - /// `None` is for the M.2s touched by the Installinator. - pub async fn new( +impl PooledDisk { + /// Create a new PooledDisk + pub fn new( log: &Logger, unparsed_disk: UnparsedDisk, - key_requester: Option<&StorageKeyRequester>, - ) -> Result { + ) -> Result { let paths = &unparsed_disk.paths; let variant = unparsed_disk.variant; // Ensure the GPT has the right format. This does not necessarily @@ -340,13 +211,8 @@ impl Disk { )?; let zpool_name = Self::ensure_zpool_exists(log, variant, &zpool_path)?; - Self::ensure_zpool_ready( - log, - &zpool_name, - &unparsed_disk.identity, - key_requester, - ) - .await?; + Self::ensure_zpool_imported(log, &zpool_name)?; + Self::ensure_zpool_failmode_is_continue(log, &zpool_name)?; Ok(Self { paths: unparsed_disk.paths, @@ -359,29 +225,11 @@ impl Disk { }) } - pub async fn ensure_zpool_ready( - log: &Logger, - zpool_name: &ZpoolName, - disk_identity: &DiskIdentity, - key_requester: Option<&StorageKeyRequester>, - ) -> Result<(), DiskError> { - Self::ensure_zpool_imported(log, &zpool_name)?; - Self::ensure_zpool_failmode_is_continue(log, &zpool_name)?; - Self::ensure_zpool_has_datasets( - log, - &zpool_name, - disk_identity, - key_requester, - ) - .await?; - Ok(()) - } - fn ensure_zpool_exists( log: &Logger, variant: DiskVariant, zpool_path: &Utf8Path, - ) -> Result { + ) -> Result { let zpool_name = match Fstyp::get_zpool(&zpool_path) { Ok(zpool_name) => zpool_name, Err(_) => { @@ -406,13 +254,13 @@ impl Disk { DiskVariant::M2 => ZpoolName::new_internal(Uuid::new_v4()), DiskVariant::U2 => ZpoolName::new_external(Uuid::new_v4()), }; - Zpool::create(zpool_name.clone(), &zpool_path)?; + Zpool::create(&zpool_name, &zpool_path)?; zpool_name } }; - Zpool::import(zpool_name.clone()).map_err(|e| { + Zpool::import(&zpool_name).map_err(|e| { warn!(log, "Failed to import zpool {zpool_name}: {e}"); - DiskError::ZpoolImport(e) + PooledDiskError::ZpoolImport(e) })?; Ok(zpool_name) @@ -421,10 +269,10 @@ impl Disk { fn ensure_zpool_imported( log: &Logger, zpool_name: &ZpoolName, - ) -> Result<(), DiskError> { - Zpool::import(zpool_name.clone()).map_err(|e| { + ) -> Result<(), PooledDiskError> { + Zpool::import(&zpool_name).map_err(|e| { warn!(log, "Failed to import zpool {zpool_name}: {e}"); - DiskError::ZpoolImport(e) + PooledDiskError::ZpoolImport(e) })?; Ok(()) } @@ -432,7 +280,7 @@ impl Disk { fn ensure_zpool_failmode_is_continue( log: &Logger, zpool_name: &ZpoolName, - ) -> Result<(), DiskError> { + ) -> Result<(), PooledDiskError> { // Ensure failmode is set to `continue`. See // https://github.com/oxidecomputer/omicron/issues/2766 for details. The // short version is, each pool is only backed by one vdev. There is no @@ -445,214 +293,10 @@ impl Disk { log, "Failed to set failmode=continue on zpool {zpool_name}: {e}" ); - DiskError::ZpoolImport(e) + PooledDiskError::ZpoolImport(e) })?; Ok(()) } - - // Ensure that the zpool contains all the datasets we would like it to - // contain. - async fn ensure_zpool_has_datasets( - log: &Logger, - zpool_name: &ZpoolName, - disk_identity: &DiskIdentity, - key_requester: Option<&StorageKeyRequester>, - ) -> Result<(), DiskError> { - let (root, datasets) = match zpool_name.kind().into() { - DiskVariant::M2 => (None, M2_EXPECTED_DATASETS.iter()), - DiskVariant::U2 => { - (Some(CRYPT_DATASET), U2_EXPECTED_DATASETS.iter()) - } - }; - - let zoned = false; - let do_format = true; - - // Ensure the root encrypted filesystem exists - // Datasets below this in the hierarchy will inherit encryption - if let Some(dataset) = root { - let Some(key_requester) = key_requester else { - return Err(DiskError::MissingStorageKeyRequester); - }; - let mountpoint = zpool_name.dataset_mountpoint(dataset); - let keypath: Keypath = disk_identity.into(); - - let epoch = - if let Ok(epoch_str) = Zfs::get_oxide_value(dataset, "epoch") { - if let Ok(epoch) = epoch_str.parse::() { - epoch - } else { - return Err(DiskError::CannotParseEpochProperty( - dataset.to_string(), - )); - } - } else { - // We got an error trying to call `Zfs::get_oxide_value` - // which indicates that the dataset doesn't exist or there - // was a problem running the command. - // - // Note that `Zfs::get_oxide_value` will succeed even if - // the epoch is missing. `epoch_str` will show up as a dash - // (`-`) and will not parse into a `u64`. So we don't have - // to worry about that case here as it is handled above. - // - // If the error indicated that the command failed for some - // other reason, but the dataset actually existed, we will - // try to create the dataset below and that will fail. So - // there is no harm in just loading the latest secret here. - key_requester.load_latest_secret().await? - }; - - let key = - key_requester.get_key(epoch, disk_identity.clone()).await?; - - let mut keyfile = - KeyFile::create(keypath.clone(), key.expose_secret(), log) - .await - .map_err(|error| DiskError::IoError { - path: keypath.0.clone(), - error, - })?; - - let encryption_details = EncryptionDetails { keypath, epoch }; - - info!( - log, - "Ensuring encrypted filesystem: {} for epoch {}", - dataset, - epoch - ); - let result = Zfs::ensure_filesystem( - &format!("{}/{}", zpool_name, dataset), - Mountpoint::Path(mountpoint), - zoned, - do_format, - Some(encryption_details), - None, - None, - ); - - keyfile.zero_and_unlink().await.map_err(|error| { - DiskError::IoError { path: keyfile.path().0.clone(), error } - })?; - - result?; - }; - - for dataset in datasets.into_iter() { - let mountpoint = zpool_name.dataset_mountpoint(dataset.name); - let name = &format!("{}/{}", zpool_name, dataset.name); - - // Use a value that's alive for the duration of this sled agent - // to answer the question: should we wipe this disk, or have - // we seen it before? - // - // If this value comes from a prior iteration of the sled agent, - // we opt to remove the corresponding dataset. - static AGENT_LOCAL_VALUE: OnceLock = OnceLock::new(); - let agent_local_value = AGENT_LOCAL_VALUE.get_or_init(|| { - Alphanumeric.sample_string(&mut rand::thread_rng(), 20) - }); - - if dataset.wipe { - match Zfs::get_oxide_value(name, "agent") { - Ok(v) if &v == agent_local_value => { - info!( - log, - "Skipping automatic wipe for dataset: {}", name - ); - } - Ok(_) | Err(_) => { - info!( - log, - "Automatically destroying dataset: {}", name - ); - Zfs::destroy_dataset(name).or_else(|err| { - // If we can't find the dataset, that's fine -- it - // might not have been formatted yet. - if let DestroyDatasetErrorVariant::NotFound = - err.err - { - Ok(()) - } else { - Err(err) - } - })?; - } - } - } - - let encryption_details = None; - let size_details = Some(SizeDetails { - quota: dataset.quota, - compression: dataset.compression, - }); - Zfs::ensure_filesystem( - name, - Mountpoint::Path(mountpoint), - zoned, - do_format, - encryption_details, - size_details, - None, - )?; - - if dataset.wipe { - Zfs::set_oxide_value(name, "agent", agent_local_value) - .map_err(|err| DiskError::CannotSetAgentProperty { - dataset: name.clone(), - err: Box::new(err), - })?; - } - } - Ok(()) - } - - pub fn is_boot_disk(&self) -> bool { - self.is_boot_disk - } - - pub fn identity(&self) -> &DiskIdentity { - &self.identity - } - - pub fn variant(&self) -> DiskVariant { - self.variant - } - - pub fn devfs_path(&self) -> &Utf8PathBuf { - &self.paths.devfs_path - } - - pub fn zpool_name(&self) -> &ZpoolName { - &self.zpool_name - } - - pub fn boot_image_devfs_path( - &self, - raw: bool, - ) -> Result { - self.paths.partition_device_path( - &self.partitions, - Partition::BootImage, - raw, - ) - } - - pub fn dump_device_devfs_path( - &self, - raw: bool, - ) -> Result { - self.paths.partition_device_path( - &self.partitions, - Partition::DumpDevice, - raw, - ) - } - - pub fn slot(&self) -> i64 { - self.slot - } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -671,56 +315,6 @@ impl From for DiskVariant { } } -/// A file that wraps a zfs encryption key. -/// -/// We put this in a RAM backed filesystem and zero and delete it when we are -/// done with it. Unfortunately we cannot do this inside `Drop` because there is no -/// equivalent async drop. -pub struct KeyFile { - path: Keypath, - file: File, - log: Logger, -} - -impl KeyFile { - pub async fn create( - path: Keypath, - key: &[u8; 32], - log: &Logger, - ) -> std::io::Result { - // TODO: fix this to not truncate - // We want to overwrite any existing contents. - // If we truncate we may leave dirty pages around - // containing secrets. - let mut file = tokio::fs::OpenOptions::new() - .create(true) - .write(true) - .open(&path.0) - .await?; - file.write_all(key).await?; - info!(log, "Created keyfile {}", path); - Ok(KeyFile { path, file, log: log.clone() }) - } - - /// These keyfiles live on a tmpfs and we zero the file so the data doesn't - /// linger on the page in memory. - /// - /// It'd be nice to `impl Drop for `KeyFile` and then call `zero` - /// from within the drop handler, but async `Drop` isn't supported. - pub async fn zero_and_unlink(&mut self) -> std::io::Result<()> { - let zeroes = [0u8; 32]; - let _ = self.file.seek(SeekFrom::Start(0)).await?; - self.file.write_all(&zeroes).await?; - info!(self.log, "Zeroed and unlinked keyfile {}", self.path); - remove_file(&self.path().0).await?; - Ok(()) - } - - pub fn path(&self) -> &Keypath { - &self.path - } -} - #[cfg(test)] mod test { use super::*; @@ -832,7 +426,7 @@ mod test { paths .partition_device_path(&[], Partition::ZfsPool, false) .expect_err("Should not have found partition"), - DiskError::NotFound { .. }, + PooledDiskError::NotFound { .. }, )); } } diff --git a/sled-hardware/src/illumos/mod.rs b/sled-hardware/src/illumos/mod.rs index 0364c98f14f..19111c6cda6 100644 --- a/sled-hardware/src/illumos/mod.rs +++ b/sled-hardware/src/illumos/mod.rs @@ -19,7 +19,6 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::Mutex; use tokio::sync::broadcast; -use tokio::task::JoinHandle; use uuid::Uuid; mod gpt; @@ -589,11 +588,11 @@ async fn hardware_tracking_task( /// /// This structure provides interfaces for both querying and for receiving new /// events. +#[derive(Clone)] pub struct HardwareManager { log: Logger, inner: Arc>, tx: broadcast::Sender, - _worker: JoinHandle<()>, } impl HardwareManager { @@ -611,34 +610,37 @@ impl HardwareManager { // receiver will receive a tokio::sync::broadcast::error::RecvError::Lagged // error, indicating they should re-scan the hardware themselves. let (tx, _) = broadcast::channel(1024); - let hw = match sled_mode { - // Treat as a possible scrimlet and setup to scan for real Tofino device. - SledMode::Auto - | SledMode::Scrimlet { asic: DendriteAsic::TofinoAsic } => { - HardwareView::new() - } + let hw = + match sled_mode { + // Treat as a possible scrimlet and setup to scan for real Tofino device. + SledMode::Auto + | SledMode::Scrimlet { asic: DendriteAsic::TofinoAsic } => { + HardwareView::new() + } - // Treat sled as gimlet and ignore any attached Tofino device. - SledMode::Gimlet => HardwareView::new_stub_tofino( - // active= - false, - ), + // Treat sled as gimlet and ignore any attached Tofino device. + SledMode::Gimlet => HardwareView::new_stub_tofino( + // active= + false, + ), - // Treat as scrimlet and use the stub Tofino device. - SledMode::Scrimlet { asic: DendriteAsic::TofinoStub } => { - HardwareView::new_stub_tofino(true) - } + // Treat as scrimlet and use the stub Tofino device. + SledMode::Scrimlet { asic: DendriteAsic::TofinoStub } => { + HardwareView::new_stub_tofino(true) + } - // Treat as scrimlet (w/ SoftNPU) and use the stub Tofino device. - // TODO-correctness: - // I'm not sure whether or not we should be treating softnpu - // as a stub or treating it as a different HardwareView variant, - // so this might change. - SledMode::Scrimlet { asic: DendriteAsic::SoftNpu } => { - HardwareView::new_stub_tofino(true) + // Treat as scrimlet (w/ SoftNPU) and use the stub Tofino device. + // TODO-correctness: + // I'm not sure whether or not we should be treating softnpu + // as a stub or treating it as a different HardwareView variant, + // so this might change. + SledMode::Scrimlet { + asic: + DendriteAsic::SoftNpuZone + | DendriteAsic::SoftNpuPropolisDevice, + } => HardwareView::new_stub_tofino(true), } - } - .map_err(|e| e.to_string())?; + .map_err(|e| e.to_string())?; let inner = Arc::new(Mutex::new(hw)); // Force the device tree to be polled at least once before returning. @@ -660,11 +662,11 @@ impl HardwareManager { let log2 = log.clone(); let inner2 = inner.clone(); let tx2 = tx.clone(); - let _worker = tokio::task::spawn(async move { + tokio::task::spawn(async move { hardware_tracking_task(log2, inner2, tx2).await }); - Ok(Self { log, inner, tx, _worker }) + Ok(Self { log, inner, tx }) } pub fn baseboard(&self) -> Baseboard { diff --git a/sled-hardware/src/illumos/partitions.rs b/sled-hardware/src/illumos/partitions.rs index 950074bd3ad..4b7e69057dc 100644 --- a/sled-hardware/src/illumos/partitions.rs +++ b/sled-hardware/src/illumos/partitions.rs @@ -5,7 +5,7 @@ //! illumos-specific mechanisms for parsing disk info. use crate::illumos::gpt; -use crate::{DiskError, DiskPaths, DiskVariant, Partition}; +use crate::{DiskPaths, DiskVariant, Partition, PooledDiskError}; use camino::Utf8Path; use illumos_utils::zpool::ZpoolName; use slog::info; @@ -41,9 +41,9 @@ fn parse_partition_types( path: &Utf8Path, partitions: &Vec, expected_partitions: &[Partition; N], -) -> Result, DiskError> { +) -> Result, PooledDiskError> { if partitions.len() != N { - return Err(DiskError::BadPartitionLayout { + return Err(PooledDiskError::BadPartitionLayout { path: path.to_path_buf(), why: format!( "Expected {} partitions, only saw {}", @@ -54,7 +54,7 @@ fn parse_partition_types( } for i in 0..N { if partitions[i].index() != i { - return Err(DiskError::BadPartitionLayout { + return Err(PooledDiskError::BadPartitionLayout { path: path.to_path_buf(), why: format!( "The {i}-th partition has index {}", @@ -80,7 +80,7 @@ pub fn ensure_partition_layout( log: &Logger, paths: &DiskPaths, variant: DiskVariant, -) -> Result, DiskError> { +) -> Result, PooledDiskError> { internal_ensure_partition_layout::(log, paths, variant) } @@ -90,7 +90,7 @@ fn internal_ensure_partition_layout( log: &Logger, paths: &DiskPaths, variant: DiskVariant, -) -> Result, DiskError> { +) -> Result, PooledDiskError> { // Open the "Whole Disk" as a raw device to be parsed by the // libefi-illumos library. This lets us peek at the GPT before // making too many assumptions about it. @@ -114,14 +114,16 @@ fn internal_ensure_partition_layout( let dev_path = if let Some(dev_path) = &paths.dev_path { dev_path } else { - return Err(DiskError::CannotFormatMissingDevPath { path }); + return Err(PooledDiskError::CannotFormatMissingDevPath { + path, + }); }; match variant { DiskVariant::U2 => { info!(log, "Formatting zpool on disk {}", paths.devfs_path); // If a zpool does not already exist, create one. let zpool_name = ZpoolName::new_external(Uuid::new_v4()); - Zpool::create(zpool_name, dev_path)?; + Zpool::create(&zpool_name, dev_path)?; return Ok(vec![Partition::ZfsPool]); } DiskVariant::M2 => { @@ -129,12 +131,12 @@ fn internal_ensure_partition_layout( // the expected partitions? Or would it be wiser to infer // that this indicates an unexpected error conditions that // needs mitigation? - return Err(DiskError::CannotFormatM2NotImplemented); + return Err(PooledDiskError::CannotFormatM2NotImplemented); } } } Err(err) => { - return Err(DiskError::Gpt { + return Err(PooledDiskError::Gpt { path, error: anyhow::Error::new(err), }); @@ -197,7 +199,7 @@ mod test { DiskVariant::U2, ); match result { - Err(DiskError::CannotFormatMissingDevPath { .. }) => {} + Err(PooledDiskError::CannotFormatMissingDevPath { .. }) => {} _ => panic!("Should have failed with a missing dev path error"), } @@ -373,7 +375,7 @@ mod test { DiskVariant::M2, ) .expect_err("Should have failed parsing empty GPT"), - DiskError::BadPartitionLayout { .. } + PooledDiskError::BadPartitionLayout { .. } )); logctx.cleanup_successful(); @@ -398,7 +400,7 @@ mod test { DiskVariant::U2, ) .expect_err("Should have failed parsing empty GPT"), - DiskError::BadPartitionLayout { .. } + PooledDiskError::BadPartitionLayout { .. } )); logctx.cleanup_successful(); diff --git a/sled-hardware/src/lib.rs b/sled-hardware/src/lib.rs index c81bcddbfbb..2e3fd4a576f 100644 --- a/sled-hardware/src/lib.rs +++ b/sled-hardware/src/lib.rs @@ -44,7 +44,8 @@ pub enum HardwareUpdate { pub enum DendriteAsic { TofinoAsic, TofinoStub, - SoftNpu, + SoftNpuZone, + SoftNpuPropolisDevice, } impl std::fmt::Display for DendriteAsic { @@ -55,7 +56,9 @@ impl std::fmt::Display for DendriteAsic { match self { DendriteAsic::TofinoAsic => "tofino_asic", DendriteAsic::TofinoStub => "tofino_stub", - DendriteAsic::SoftNpu => "soft_npu", + DendriteAsic::SoftNpuZone => "soft_npu_zone", + DendriteAsic::SoftNpuPropolisDevice => + "soft_npu_propolis_device", } ) } @@ -160,13 +163,3 @@ impl std::fmt::Display for Baseboard { } } } - -impl From for nexus_client::types::Baseboard { - fn from(b: Baseboard) -> nexus_client::types::Baseboard { - nexus_client::types::Baseboard { - serial_number: b.identifier().to_string(), - part_number: b.model().to_string(), - revision: b.revision(), - } - } -} diff --git a/sled-hardware/src/non_illumos/mod.rs b/sled-hardware/src/non_illumos/mod.rs index 6e36330df08..d8372dd8aa6 100644 --- a/sled-hardware/src/non_illumos/mod.rs +++ b/sled-hardware/src/non_illumos/mod.rs @@ -2,7 +2,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::disk::{DiskError, DiskPaths, DiskVariant, Partition, UnparsedDisk}; +use crate::disk::{ + DiskPaths, DiskVariant, Partition, PooledDiskError, UnparsedDisk, +}; use crate::{Baseboard, SledMode}; use slog::Logger; use std::collections::HashSet; @@ -16,6 +18,7 @@ use tokio::sync::broadcast; /// /// If you're actually trying to run the Sled Agent on non-illumos platforms, /// use the simulated sled agent, which does not attempt to abstract hardware. +#[derive(Clone)] pub struct HardwareManager {} impl HardwareManager { @@ -56,7 +59,7 @@ pub fn ensure_partition_layout( _log: &Logger, _paths: &DiskPaths, _variant: DiskVariant, -) -> Result, DiskError> { +) -> Result, PooledDiskError> { unimplemented!("Accessing hardware unsupported on non-illumos"); } diff --git a/sled-storage/Cargo.toml b/sled-storage/Cargo.toml new file mode 100644 index 00000000000..cb3a7906316 --- /dev/null +++ b/sled-storage/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "sled-storage" +version = "0.1.0" +edition = "2021" + +[dependencies] +async-trait.workspace = true +camino.workspace = true +cfg-if.workspace = true +derive_more.workspace = true +glob.workspace = true +illumos-utils.workspace = true +key-manager.workspace = true +omicron-common.workspace = true +rand.workspace = true +schemars = { workspace = true, features = [ "chrono", "uuid1" ] } +serde.workspace = true +serde_json.workspace = true +sled-hardware.workspace = true +slog.workspace = true +thiserror.workspace = true +tokio.workspace = true +uuid.workspace = true +omicron-workspace-hack.workspace = true + +[dev-dependencies] +illumos-utils = { workspace = true, features = ["tmp_keypath", "testing"] } +omicron-test-utils.workspace = true +camino-tempfile.workspace = true + +[features] +# Quotas and the like can be shrunk via this feature +testing = [] diff --git a/sled-storage/src/dataset.rs b/sled-storage/src/dataset.rs new file mode 100644 index 00000000000..a2878af7f6f --- /dev/null +++ b/sled-storage/src/dataset.rs @@ -0,0 +1,379 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! ZFS dataset related functionality + +use crate::keyfile::KeyFile; +use camino::Utf8PathBuf; +use cfg_if::cfg_if; +use illumos_utils::zfs::{ + self, DestroyDatasetErrorVariant, EncryptionDetails, Keypath, Mountpoint, + SizeDetails, Zfs, +}; +use illumos_utils::zpool::ZpoolName; +use key_manager::StorageKeyRequester; +use omicron_common::disk::DiskIdentity; +use rand::distributions::{Alphanumeric, DistString}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sled_hardware::DiskVariant; +use slog::{info, Logger}; +use std::sync::OnceLock; + +pub const INSTALL_DATASET: &'static str = "install"; +pub const CRASH_DATASET: &'static str = "crash"; +pub const CLUSTER_DATASET: &'static str = "cluster"; +pub const CONFIG_DATASET: &'static str = "config"; +pub const M2_DEBUG_DATASET: &'static str = "debug"; +pub const M2_BACKING_DATASET: &'static str = "backing"; + +cfg_if! { + if #[cfg(any(test, feature = "testing"))] { + // Tuned for zone_bundle tests + pub const DEBUG_DATASET_QUOTA: usize = 100 * (1 << 10); + } else { + // TODO-correctness: This value of 100GiB is a pretty wild guess, and should be + // tuned as needed. + pub const DEBUG_DATASET_QUOTA: usize = 100 * (1 << 30); + } +} +// TODO-correctness: This value of 100GiB is a pretty wild guess, and should be +// tuned as needed. +pub const DUMP_DATASET_QUOTA: usize = 100 * (1 << 30); +// passed to zfs create -o compression= +pub const DUMP_DATASET_COMPRESSION: &'static str = "gzip-9"; + +// U.2 datasets live under the encrypted dataset and inherit encryption +pub const ZONE_DATASET: &'static str = "crypt/zone"; +pub const DUMP_DATASET: &'static str = "crypt/debug"; +pub const U2_DEBUG_DATASET: &'static str = "crypt/debug"; + +// This is the root dataset for all U.2 drives. Encryption is inherited. +pub const CRYPT_DATASET: &'static str = "crypt"; + +const U2_EXPECTED_DATASET_COUNT: usize = 2; +static U2_EXPECTED_DATASETS: [ExpectedDataset; U2_EXPECTED_DATASET_COUNT] = [ + // Stores filesystems for zones + ExpectedDataset::new(ZONE_DATASET).wipe(), + // For storing full kernel RAM dumps + ExpectedDataset::new(DUMP_DATASET) + .quota(DUMP_DATASET_QUOTA) + .compression(DUMP_DATASET_COMPRESSION), +]; + +const M2_EXPECTED_DATASET_COUNT: usize = 6; +static M2_EXPECTED_DATASETS: [ExpectedDataset; M2_EXPECTED_DATASET_COUNT] = [ + // Stores software images. + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(INSTALL_DATASET), + // Stores crash dumps. + ExpectedDataset::new(CRASH_DATASET), + // Backing store for OS data that should be persisted across reboots. + // Its children are selectively overlay mounted onto parts of the ramdisk + // root. + ExpectedDataset::new(M2_BACKING_DATASET), + // Stores cluter configuration information. + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(CLUSTER_DATASET), + // Stores configuration data, including: + // - What services should be launched on this sled + // - Information about how to initialize the Sled Agent + // - (For scrimlets) RSS setup information + // + // Should be duplicated to both M.2s. + ExpectedDataset::new(CONFIG_DATASET), + // Store debugging data, such as service bundles. + ExpectedDataset::new(M2_DEBUG_DATASET).quota(DEBUG_DATASET_QUOTA), +]; + +// Helper type for describing expected datasets and their optional quota. +#[derive(Clone, Copy, Debug)] +struct ExpectedDataset { + // Name for the dataset + name: &'static str, + // Optional quota, in _bytes_ + quota: Option, + // Identifies if the dataset should be deleted on boot + wipe: bool, + // Optional compression mode + compression: Option<&'static str>, +} + +impl ExpectedDataset { + const fn new(name: &'static str) -> Self { + ExpectedDataset { name, quota: None, wipe: false, compression: None } + } + + const fn quota(mut self, quota: usize) -> Self { + self.quota = Some(quota); + self + } + + const fn wipe(mut self) -> Self { + self.wipe = true; + self + } + + const fn compression(mut self, compression: &'static str) -> Self { + self.compression = Some(compression); + self + } +} + +/// The type of a dataset, and an auxiliary information necessary +/// to successfully launch a zone managing the associated data. +#[derive( + Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, +)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum DatasetKind { + CockroachDb, + Crucible, + Clickhouse, + ClickhouseKeeper, + ExternalDns, + InternalDns, +} + +impl std::fmt::Display for DatasetKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use DatasetKind::*; + let s = match self { + Crucible => "crucible", + CockroachDb { .. } => "cockroachdb", + Clickhouse => "clickhouse", + ClickhouseKeeper => "clickhouse_keeper", + ExternalDns { .. } => "external_dns", + InternalDns { .. } => "internal_dns", + }; + write!(f, "{}", s) + } +} + +#[derive( + Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Clone, JsonSchema, +)] +pub struct DatasetName { + // A unique identifier for the Zpool on which the dataset is stored. + pool_name: ZpoolName, + // A name for the dataset within the Zpool. + kind: DatasetKind, +} + +impl DatasetName { + pub fn new(pool_name: ZpoolName, kind: DatasetKind) -> Self { + Self { pool_name, kind } + } + + pub fn pool(&self) -> &ZpoolName { + &self.pool_name + } + + pub fn dataset(&self) -> &DatasetKind { + &self.kind + } + + pub fn full(&self) -> String { + format!("{}/{}", self.pool_name, self.kind) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum DatasetError { + #[error("Cannot open {path} due to {error}")] + IoError { path: Utf8PathBuf, error: std::io::Error }, + #[error(transparent)] + DestroyFilesystem(#[from] illumos_utils::zfs::DestroyDatasetError), + #[error(transparent)] + EnsureFilesystem(#[from] illumos_utils::zfs::EnsureFilesystemError), + #[error("KeyManager error: {0}")] + KeyManager(#[from] key_manager::Error), + #[error("Missing StorageKeyRequester when creating U.2 disk")] + MissingStorageKeyRequester, + #[error("Encrypted filesystem '{0}' missing 'oxide:epoch' property")] + CannotParseEpochProperty(String), + #[error("Encrypted dataset '{dataset}' cannot set 'oxide:agent' property: {err}")] + CannotSetAgentProperty { + dataset: String, + #[source] + err: Box, + }, +} + +/// Ensure that the zpool contains all the datasets we would like it to +/// contain. +/// +/// WARNING: In all cases where a U.2 is a possible `DiskVariant`, a +/// `StorageKeyRequester` must be passed so that disk encryption can +/// be used. The `StorageManager` for the sled-agent always has a +/// `StorageKeyRequester` available, and so the only place we should pass +/// `None` is for the M.2s touched by the Installinator. +pub(crate) async fn ensure_zpool_has_datasets( + log: &Logger, + zpool_name: &ZpoolName, + disk_identity: &DiskIdentity, + key_requester: Option<&StorageKeyRequester>, +) -> Result<(), DatasetError> { + let (root, datasets) = match zpool_name.kind().into() { + DiskVariant::M2 => (None, M2_EXPECTED_DATASETS.iter()), + DiskVariant::U2 => (Some(CRYPT_DATASET), U2_EXPECTED_DATASETS.iter()), + }; + + let zoned = false; + let do_format = true; + + // Ensure the root encrypted filesystem exists + // Datasets below this in the hierarchy will inherit encryption + if let Some(dataset) = root { + let Some(key_requester) = key_requester else { + return Err(DatasetError::MissingStorageKeyRequester); + }; + let mountpoint = zpool_name.dataset_mountpoint(dataset); + let keypath: Keypath = disk_identity.into(); + + let epoch = if let Ok(epoch_str) = + Zfs::get_oxide_value(dataset, "epoch") + { + if let Ok(epoch) = epoch_str.parse::() { + epoch + } else { + return Err(DatasetError::CannotParseEpochProperty( + dataset.to_string(), + )); + } + } else { + // We got an error trying to call `Zfs::get_oxide_value` + // which indicates that the dataset doesn't exist or there + // was a problem running the command. + // + // Note that `Zfs::get_oxide_value` will succeed even if + // the epoch is missing. `epoch_str` will show up as a dash + // (`-`) and will not parse into a `u64`. So we don't have + // to worry about that case here as it is handled above. + // + // If the error indicated that the command failed for some + // other reason, but the dataset actually existed, we will + // try to create the dataset below and that will fail. So + // there is no harm in just loading the latest secret here. + info!(log, "Loading latest secret"; "disk_id"=>#?disk_identity); + let epoch = key_requester.load_latest_secret().await?; + info!(log, "Loaded latest secret"; "epoch"=>%epoch, "disk_id"=>#?disk_identity); + epoch + }; + + info!(log, "Retrieving key"; "epoch"=>%epoch, "disk_id"=>#?disk_identity); + let key = key_requester.get_key(epoch, disk_identity.clone()).await?; + info!(log, "Got key"; "epoch"=>%epoch, "disk_id"=>#?disk_identity); + + let mut keyfile = + KeyFile::create(keypath.clone(), key.expose_secret(), log) + .await + .map_err(|error| DatasetError::IoError { + path: keypath.0.clone(), + error, + })?; + + let encryption_details = EncryptionDetails { keypath, epoch }; + + info!( + log, + "Ensuring encrypted filesystem: {} for epoch {}", dataset, epoch + ); + let result = Zfs::ensure_filesystem( + &format!("{}/{}", zpool_name, dataset), + Mountpoint::Path(mountpoint), + zoned, + do_format, + Some(encryption_details), + None, + None, + ); + + keyfile.zero_and_unlink().await.map_err(|error| { + DatasetError::IoError { path: keyfile.path().0.clone(), error } + })?; + + result?; + }; + + for dataset in datasets.into_iter() { + let mountpoint = zpool_name.dataset_mountpoint(dataset.name); + let name = &format!("{}/{}", zpool_name, dataset.name); + + // Use a value that's alive for the duration of this sled agent + // to answer the question: should we wipe this disk, or have + // we seen it before? + // + // If this value comes from a prior iteration of the sled agent, + // we opt to remove the corresponding dataset. + static AGENT_LOCAL_VALUE: OnceLock = OnceLock::new(); + let agent_local_value = AGENT_LOCAL_VALUE.get_or_init(|| { + Alphanumeric.sample_string(&mut rand::thread_rng(), 20) + }); + + if dataset.wipe { + match Zfs::get_oxide_value(name, "agent") { + Ok(v) if &v == agent_local_value => { + info!(log, "Skipping automatic wipe for dataset: {}", name); + } + Ok(_) | Err(_) => { + info!(log, "Automatically destroying dataset: {}", name); + Zfs::destroy_dataset(name).or_else(|err| { + // If we can't find the dataset, that's fine -- it might + // not have been formatted yet. + if matches!( + err.err, + DestroyDatasetErrorVariant::NotFound + ) { + Ok(()) + } else { + Err(err) + } + })?; + } + } + } + + let encryption_details = None; + let size_details = Some(SizeDetails { + quota: dataset.quota, + compression: dataset.compression, + }); + Zfs::ensure_filesystem( + name, + Mountpoint::Path(mountpoint), + zoned, + do_format, + encryption_details, + size_details, + None, + )?; + + if dataset.wipe { + Zfs::set_oxide_value(name, "agent", agent_local_value).map_err( + |err| DatasetError::CannotSetAgentProperty { + dataset: name.clone(), + err: Box::new(err), + }, + )?; + } + } + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + use uuid::Uuid; + + #[test] + fn serialize_dataset_name() { + let pool = ZpoolName::new_internal(Uuid::new_v4()); + let kind = DatasetKind::Crucible; + let name = DatasetName::new(pool, kind); + serde_json::to_string(&name).unwrap(); + } +} diff --git a/sled-storage/src/disk.rs b/sled-storage/src/disk.rs new file mode 100644 index 00000000000..f5209def772 --- /dev/null +++ b/sled-storage/src/disk.rs @@ -0,0 +1,243 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Disk related types + +use camino::{Utf8Path, Utf8PathBuf}; +use derive_more::From; +use illumos_utils::zpool::{Zpool, ZpoolKind, ZpoolName}; +use key_manager::StorageKeyRequester; +use omicron_common::disk::DiskIdentity; +use sled_hardware::{ + DiskVariant, Partition, PooledDisk, PooledDiskError, UnparsedDisk, +}; +use slog::Logger; +use std::fs::File; + +use crate::dataset; + +#[derive(Debug, thiserror::Error)] +pub enum DiskError { + #[error(transparent)] + Dataset(#[from] crate::dataset::DatasetError), + #[error(transparent)] + PooledDisk(#[from] sled_hardware::PooledDiskError), +} + +// A synthetic disk that acts as one "found" by the hardware and that is backed +// by a zpool +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct SyntheticDisk { + pub identity: DiskIdentity, + pub zpool_name: ZpoolName, +} + +impl SyntheticDisk { + // Create a zpool and import it for the synthetic disk + // Zpools willl be set to the min size of 64Mib + pub fn create_zpool( + dir: &Utf8Path, + zpool_name: &ZpoolName, + ) -> SyntheticDisk { + // 64 MiB (min size of zpool) + const DISK_SIZE: u64 = 64 * 1024 * 1024; + let path = dir.join(zpool_name.to_string()); + let file = File::create(&path).unwrap(); + file.set_len(DISK_SIZE).unwrap(); + drop(file); + Zpool::create(zpool_name, &path).unwrap(); + Zpool::import(zpool_name).unwrap(); + Zpool::set_failmode_continue(zpool_name).unwrap(); + Self::new(zpool_name.clone()) + } + + pub fn new(zpool_name: ZpoolName) -> SyntheticDisk { + let id = zpool_name.id(); + let identity = DiskIdentity { + vendor: "synthetic-vendor".to_string(), + serial: format!("synthetic-serial-{id}"), + model: "synthetic-model".to_string(), + }; + SyntheticDisk { identity, zpool_name } + } +} + +// An [`UnparsedDisk`] disk learned about from the hardware or a wrapped zpool +#[derive(Debug, Clone, PartialEq, Eq, Hash, From)] +pub enum RawDisk { + Real(UnparsedDisk), + Synthetic(SyntheticDisk), +} + +impl RawDisk { + pub fn is_boot_disk(&self) -> bool { + match self { + Self::Real(disk) => disk.is_boot_disk(), + Self::Synthetic(disk) => { + // Just label any M.2 the boot disk. + disk.zpool_name.kind() == ZpoolKind::Internal + } + } + } + + pub fn identity(&self) -> &DiskIdentity { + match self { + Self::Real(disk) => &disk.identity(), + Self::Synthetic(disk) => &disk.identity, + } + } + + pub fn variant(&self) -> DiskVariant { + match self { + Self::Real(disk) => disk.variant(), + Self::Synthetic(disk) => match disk.zpool_name.kind() { + ZpoolKind::External => DiskVariant::U2, + ZpoolKind::Internal => DiskVariant::M2, + }, + } + } + + #[cfg(test)] + pub fn zpool_name(&self) -> &ZpoolName { + match self { + Self::Real(_) => unreachable!(), + Self::Synthetic(disk) => &disk.zpool_name, + } + } + + pub fn is_synthetic(&self) -> bool { + match self { + Self::Real(_) => false, + Self::Synthetic(_) => true, + } + } + + pub fn is_real(&self) -> bool { + !self.is_synthetic() + } + + pub fn devfs_path(&self) -> &Utf8PathBuf { + match self { + Self::Real(disk) => disk.devfs_path(), + Self::Synthetic(_) => unreachable!(), + } + } +} + +/// A physical [`PooledDisk`] or a [`SyntheticDisk`] that contains or is backed +/// by a single zpool and that has provisioned datasets. This disk is ready for +/// usage by higher level software. +#[derive(Debug, Clone, PartialEq, Eq, Hash, From)] +pub enum Disk { + Real(PooledDisk), + Synthetic(SyntheticDisk), +} + +impl Disk { + pub async fn new( + log: &Logger, + raw_disk: RawDisk, + key_requester: Option<&StorageKeyRequester>, + ) -> Result { + let disk = match raw_disk { + RawDisk::Real(disk) => PooledDisk::new(log, disk)?.into(), + RawDisk::Synthetic(disk) => Disk::Synthetic(disk), + }; + dataset::ensure_zpool_has_datasets( + log, + disk.zpool_name(), + disk.identity(), + key_requester, + ) + .await?; + Ok(disk) + } + + pub fn is_synthetic(&self) -> bool { + match self { + Self::Real(_) => false, + Self::Synthetic(_) => true, + } + } + + pub fn is_real(&self) -> bool { + !self.is_synthetic() + } + + pub fn is_boot_disk(&self) -> bool { + match self { + Self::Real(disk) => disk.is_boot_disk, + Self::Synthetic(disk) => { + // Just label any M.2 the boot disk. + disk.zpool_name.kind() == ZpoolKind::Internal + } + } + } + + pub fn identity(&self) -> &DiskIdentity { + match self { + Self::Real(disk) => &disk.identity, + Self::Synthetic(disk) => &disk.identity, + } + } + + pub fn variant(&self) -> DiskVariant { + match self { + Self::Real(disk) => disk.variant, + Self::Synthetic(disk) => match disk.zpool_name.kind() { + ZpoolKind::External => DiskVariant::U2, + ZpoolKind::Internal => DiskVariant::M2, + }, + } + } + + pub fn devfs_path(&self) -> &Utf8PathBuf { + match self { + Self::Real(disk) => &disk.paths.devfs_path, + Self::Synthetic(_) => unreachable!(), + } + } + + pub fn zpool_name(&self) -> &ZpoolName { + match self { + Self::Real(disk) => &disk.zpool_name, + Self::Synthetic(disk) => &disk.zpool_name, + } + } + + pub fn boot_image_devfs_path( + &self, + raw: bool, + ) -> Result { + match self { + Self::Real(disk) => disk.paths.partition_device_path( + &disk.partitions, + Partition::BootImage, + raw, + ), + Self::Synthetic(_) => unreachable!(), + } + } + + pub fn dump_device_devfs_path( + &self, + raw: bool, + ) -> Result { + match self { + Self::Real(disk) => disk.paths.partition_device_path( + &disk.partitions, + Partition::DumpDevice, + raw, + ), + Self::Synthetic(_) => unreachable!(), + } + } + + pub fn slot(&self) -> i64 { + match self { + Self::Real(disk) => disk.slot, + Self::Synthetic(_) => unreachable!(), + } + } +} diff --git a/sled-storage/src/error.rs b/sled-storage/src/error.rs new file mode 100644 index 00000000000..b9f97ee4287 --- /dev/null +++ b/sled-storage/src/error.rs @@ -0,0 +1,81 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Storage related errors + +use crate::dataset::{DatasetError, DatasetName}; +use crate::disk::DiskError; +use camino::Utf8PathBuf; +use omicron_common::api::external::ByteCountRangeError; +use uuid::Uuid; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error(transparent)] + DiskError(#[from] DiskError), + + #[error(transparent)] + DatasetError(#[from] DatasetError), + + // TODO: We could add the context of "why are we doint this op", maybe? + #[error(transparent)] + ZfsListDataset(#[from] illumos_utils::zfs::ListDatasetsError), + + #[error(transparent)] + ZfsEnsureFilesystem(#[from] illumos_utils::zfs::EnsureFilesystemError), + + #[error(transparent)] + ZfsSetValue(#[from] illumos_utils::zfs::SetValueError), + + #[error(transparent)] + ZfsGetValue(#[from] illumos_utils::zfs::GetValueError), + + #[error(transparent)] + GetZpoolInfo(#[from] illumos_utils::zpool::GetInfoError), + + #[error(transparent)] + Fstyp(#[from] illumos_utils::fstyp::Error), + + #[error(transparent)] + ZoneCommand(#[from] illumos_utils::running_zone::RunCommandError), + + #[error(transparent)] + ZoneBoot(#[from] illumos_utils::running_zone::BootError), + + #[error(transparent)] + ZoneEnsureAddress(#[from] illumos_utils::running_zone::EnsureAddressError), + + #[error(transparent)] + ZoneInstall(#[from] illumos_utils::running_zone::InstallZoneError), + + #[error("No U.2 Zpools found")] + NoU2Zpool, + + #[error("Failed to parse UUID from {path}: {err}")] + ParseUuid { + path: Utf8PathBuf, + #[source] + err: uuid::Error, + }, + + #[error("Dataset {name:?} exists with a different uuid (has {old}, requested {new})")] + UuidMismatch { name: Box, old: Uuid, new: Uuid }, + + #[error("Error parsing pool {name}'s size: {err}")] + BadPoolSize { + name: String, + #[source] + err: ByteCountRangeError, + }, + + #[error("Failed to parse the dataset {name}'s UUID: {err}")] + ParseDatasetUuid { + name: String, + #[source] + err: uuid::Error, + }, + + #[error("Zpool Not Found: {0}")] + ZpoolNotFound(String), +} diff --git a/sled-storage/src/keyfile.rs b/sled-storage/src/keyfile.rs new file mode 100644 index 00000000000..48e5d9a5282 --- /dev/null +++ b/sled-storage/src/keyfile.rs @@ -0,0 +1,76 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Key file support for ZFS dataset encryption + +use illumos_utils::zfs::Keypath; +use slog::{error, info, Logger}; +use tokio::fs::{remove_file, File}; +use tokio::io::{AsyncSeekExt, AsyncWriteExt, SeekFrom}; + +/// A file that wraps a zfs encryption key. +/// +/// We put this in a RAM backed filesystem and zero and delete it when we are +/// done with it. Unfortunately we cannot do this inside `Drop` because there is no +/// equivalent async drop. +pub struct KeyFile { + path: Keypath, + file: File, + log: Logger, + zero_and_unlink_called: bool, +} + +impl KeyFile { + pub async fn create( + path: Keypath, + key: &[u8; 32], + log: &Logger, + ) -> std::io::Result { + // We want to overwrite any existing contents. + let mut file = tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .open(&path.0) + .await?; + file.write_all(key).await?; + info!(log, "Created keyfile {}", path); + Ok(KeyFile { + path, + file, + log: log.clone(), + zero_and_unlink_called: false, + }) + } + + /// These keyfiles live on a tmpfs and we zero the file so the data doesn't + /// linger on the page in memory. + /// + /// It'd be nice to `impl Drop for `KeyFile` and then call `zero` + /// from within the drop handler, but async `Drop` isn't supported. + pub async fn zero_and_unlink(&mut self) -> std::io::Result<()> { + self.zero_and_unlink_called = true; + let zeroes = [0u8; 32]; + let _ = self.file.seek(SeekFrom::Start(0)).await?; + self.file.write_all(&zeroes).await?; + info!(self.log, "Zeroed and unlinked keyfile {}", self.path); + remove_file(&self.path().0).await?; + Ok(()) + } + + pub fn path(&self) -> &Keypath { + &self.path + } +} + +impl Drop for KeyFile { + fn drop(&mut self) { + if !self.zero_and_unlink_called { + error!( + self.log, + "Failed to call zero_and_unlink for keyfile"; + "path" => %self.path + ); + } + } +} diff --git a/sled-storage/src/lib.rs b/sled-storage/src/lib.rs new file mode 100644 index 00000000000..d4b64c55a5d --- /dev/null +++ b/sled-storage/src/lib.rs @@ -0,0 +1,17 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Local storage abstraction for use by sled-agent +//! +//! This abstraction operates at the ZFS level and relies on zpool setup on +//! hardware partitions from the `sled-hardware` crate. It utilizes the +//! `illumos-utils` crate to actually perform ZFS related OS calls. + +pub mod dataset; +pub mod disk; +pub mod error; +pub(crate) mod keyfile; +pub mod manager; +pub mod pool; +pub mod resources; diff --git a/sled-storage/src/manager.rs b/sled-storage/src/manager.rs new file mode 100644 index 00000000000..50b1c441481 --- /dev/null +++ b/sled-storage/src/manager.rs @@ -0,0 +1,1034 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The storage manager task + +use std::collections::HashSet; + +use crate::dataset::{DatasetError, DatasetName}; +use crate::disk::{Disk, DiskError, RawDisk}; +use crate::error::Error; +use crate::resources::{AddDiskResult, StorageResources}; +use camino::Utf8PathBuf; +use illumos_utils::zfs::{Mountpoint, Zfs}; +use illumos_utils::zpool::ZpoolName; +use key_manager::StorageKeyRequester; +use omicron_common::disk::DiskIdentity; +use sled_hardware::DiskVariant; +use slog::{error, info, o, warn, Logger}; +use tokio::sync::{mpsc, oneshot, watch}; +use tokio::time::{interval, Duration, MissedTickBehavior}; +use uuid::Uuid; + +// The size of the mpsc bounded channel used to communicate +// between the `StorageHandle` and `StorageManager`. +// +// How did we choose this bound, and why? +// +// Picking a bound can be tricky, but in general, you want the channel to act +// unbounded, such that sends never fail. This makes the channels reliable, +// such that we never drop messages inside the process, and the caller doesn't +// have to choose what to do when overloaded. This simplifies things drastically +// for developers. However, you also don't want to make the channel actually +// unbounded, because that can lead to run-away memory growth and pathological +// behaviors, such that requests get slower over time until the system crashes. +// +// Our team's chosen solution, and used elsewhere in the codebase, is is to +// choose a large enough bound such that we should never hit it in practice +// unless we are truly overloaded. If we hit the bound it means that beyond that +// requests will start to build up and we will eventually topple over. So when +// we hit this bound, we just go ahead and panic. +// +// Picking a channel bound is hard to do empirically, but practically, if +// requests are mostly mutating task local state, a bound of 1024 or even 8192 +// should be plenty. Tasks that must perform longer running ops can spawn helper +// tasks as necessary or include their own handles for replies rather than +// synchronously waiting. Memory for the queue can be kept small with boxing of +// large messages. +// +// Here we start relatively small so that we can evaluate our choice over time. +const QUEUE_SIZE: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageManagerState { + WaitingForKeyManager, + QueueingDisks, + Normal, +} + +#[derive(Debug)] +struct NewFilesystemRequest { + dataset_id: Uuid, + dataset_name: DatasetName, + responder: oneshot::Sender>, +} + +#[derive(Debug)] +enum StorageRequest { + AddDisk(RawDisk), + RemoveDisk(RawDisk), + DisksChanged(HashSet), + NewFilesystem(NewFilesystemRequest), + KeyManagerReady, + /// This will always grab the latest state after any new updates, as it + /// serializes through the `StorageManager` task after all prior requests. + /// This serialization is particularly useful for tests. + GetLatestResources(oneshot::Sender), + + /// Get the internal task state of the manager + GetManagerState(oneshot::Sender), +} + +/// Data managed internally to the StorageManagerTask that can be useful +/// to clients for debugging purposes, and that isn't exposed in other ways. +#[derive(Debug, Clone)] +pub struct StorageManagerData { + pub state: StorageManagerState, + pub queued_u2_drives: HashSet, +} + +/// A mechanism for interacting with the [`StorageManager`] +#[derive(Clone)] +pub struct StorageHandle { + tx: mpsc::Sender, + resource_updates: watch::Receiver, +} + +impl StorageHandle { + /// Adds a disk and associated zpool to the storage manager. + pub async fn upsert_disk(&self, disk: RawDisk) { + self.tx.send(StorageRequest::AddDisk(disk)).await.unwrap(); + } + + /// Removes a disk, if it's tracked by the storage manager, as well + /// as any associated zpools. + pub async fn delete_disk(&self, disk: RawDisk) { + self.tx.send(StorageRequest::RemoveDisk(disk)).await.unwrap(); + } + + /// Ensures that the storage manager tracks exactly the provided disks. + /// + /// This acts similar to a batch [Self::upsert_disk] for all new disks, and + /// [Self::delete_disk] for all removed disks. + /// + /// If errors occur, an arbitrary "one" of them will be returned, but a + /// best-effort attempt to add all disks will still be attempted. + pub async fn ensure_using_exactly_these_disks(&self, raw_disks: I) + where + I: IntoIterator, + { + self.tx + .send(StorageRequest::DisksChanged(raw_disks.into_iter().collect())) + .await + .unwrap(); + } + + /// Notify the [`StorageManager`] that the [`key_manager::KeyManager`] + /// has determined what [`key_manager::SecretRetriever`] to use and + /// it is now possible to retrieve secrets and construct keys. Note + /// that in cases of using the trust quorum, it is possible that the + /// [`key_manager::SecretRetriever`] is ready, but enough key shares cannot + /// be retrieved from other sleds. In this case, we still will be unable + /// to add the disks successfully. In the common case this is a transient + /// error. In other cases it may be fatal. However, that is outside the + /// scope of the cares of this module. + pub async fn key_manager_ready(&self) { + self.tx.send(StorageRequest::KeyManagerReady).await.unwrap(); + } + + /// Wait for a boot disk to be initialized + pub async fn wait_for_boot_disk(&mut self) -> (DiskIdentity, ZpoolName) { + loop { + let resources = self.resource_updates.borrow_and_update(); + if let Some((disk_id, zpool_name)) = resources.boot_disk() { + return (disk_id, zpool_name); + } + drop(resources); + // We panic if the sender is dropped, as this means + // the StorageManager has gone away, which it should not do. + self.resource_updates.changed().await.unwrap(); + } + } + + /// Wait for any storage resource changes + pub async fn wait_for_changes(&mut self) -> StorageResources { + self.resource_updates.changed().await.unwrap(); + self.resource_updates.borrow_and_update().clone() + } + + /// Retrieve the latest value of `StorageResources` from the + /// `StorageManager` task. + pub async fn get_latest_resources(&self) -> StorageResources { + let (tx, rx) = oneshot::channel(); + self.tx.send(StorageRequest::GetLatestResources(tx)).await.unwrap(); + rx.await.unwrap() + } + + /// Return internal data useful for debugging and testing + pub async fn get_manager_state(&self) -> StorageManagerData { + let (tx, rx) = oneshot::channel(); + self.tx.send(StorageRequest::GetManagerState(tx)).await.unwrap(); + rx.await.unwrap() + } + + pub async fn upsert_filesystem( + &self, + dataset_id: Uuid, + dataset_name: DatasetName, + ) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + let request = + NewFilesystemRequest { dataset_id, dataset_name, responder: tx }; + self.tx.send(StorageRequest::NewFilesystem(request)).await.unwrap(); + rx.await.unwrap() + } +} + +// Some sled-agent tests cannot currently use the real StorageManager +// and want to fake the entire behavior, but still have access to the +// `StorageResources`. We allow this via use of the `FakeStorageManager` +// that will respond to real storage requests from a real `StorageHandle`. +#[cfg(feature = "testing")] +pub struct FakeStorageManager { + rx: mpsc::Receiver, + resources: StorageResources, + resource_updates: watch::Sender, +} + +#[cfg(feature = "testing")] +impl FakeStorageManager { + pub fn new() -> (Self, StorageHandle) { + let (tx, rx) = mpsc::channel(QUEUE_SIZE); + let resources = StorageResources::default(); + let (update_tx, update_rx) = watch::channel(resources.clone()); + ( + Self { rx, resources, resource_updates: update_tx }, + StorageHandle { tx, resource_updates: update_rx }, + ) + } + + /// Run the main receive loop of the `FakeStorageManager` + /// + /// This should be spawned into a tokio task + pub async fn run(mut self) { + loop { + match self.rx.recv().await { + Some(StorageRequest::AddDisk(raw_disk)) => { + if self.add_disk(raw_disk).disk_inserted() { + self.resource_updates + .send_replace(self.resources.clone()); + } + } + Some(StorageRequest::GetLatestResources(tx)) => { + let _ = tx.send(self.resources.clone()); + } + Some(_) => { + unreachable!(); + } + None => break, + } + } + } + + // Add a disk to `StorageResources` if it is new and return true if so + fn add_disk(&mut self, raw_disk: RawDisk) -> AddDiskResult { + let disk = match raw_disk { + RawDisk::Real(_) => { + panic!( + "Only synthetic disks can be used with `FakeStorageManager`" + ); + } + RawDisk::Synthetic(synthetic_disk) => { + Disk::Synthetic(synthetic_disk) + } + }; + self.resources.insert_fake_disk(disk) + } +} + +/// The storage manager responsible for the state of the storage +/// on a sled. The storage manager runs in its own task and is interacted +/// with via the [`StorageHandle`]. +pub struct StorageManager { + log: Logger, + state: StorageManagerState, + // Used to find the capacity of the channel for tracking purposes + tx: mpsc::Sender, + rx: mpsc::Receiver, + resources: StorageResources, + queued_u2_drives: HashSet, + key_requester: StorageKeyRequester, + resource_updates: watch::Sender, + last_logged_capacity: usize, +} + +impl StorageManager { + pub fn new( + log: &Logger, + key_requester: StorageKeyRequester, + ) -> (StorageManager, StorageHandle) { + let (tx, rx) = mpsc::channel(QUEUE_SIZE); + let resources = StorageResources::default(); + let (update_tx, update_rx) = watch::channel(resources.clone()); + ( + StorageManager { + log: log.new(o!("component" => "StorageManager")), + state: StorageManagerState::WaitingForKeyManager, + tx: tx.clone(), + rx, + resources, + queued_u2_drives: HashSet::new(), + key_requester, + resource_updates: update_tx, + last_logged_capacity: QUEUE_SIZE, + }, + StorageHandle { tx, resource_updates: update_rx }, + ) + } + + /// Run the main receive loop of the `StorageManager` + /// + /// This should be spawned into a tokio task + pub async fn run(mut self) { + loop { + const QUEUED_DISK_RETRY_TIMEOUT: Duration = Duration::from_secs(10); + let mut interval = interval(QUEUED_DISK_RETRY_TIMEOUT); + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + tokio::select! { + res = self.step() => { + if let Err(e) = res { + warn!(self.log, "{e}"); + } + } + _ = interval.tick(), + if self.state == StorageManagerState::QueueingDisks => + { + if self.add_queued_disks().await { + let _ = self.resource_updates.send_replace(self.resources.clone()); + } + } + } + } + } + + /// Process the next event + /// + /// This is useful for testing/debugging + pub async fn step(&mut self) -> Result<(), Error> { + const CAPACITY_LOG_THRESHOLD: usize = 10; + // We check the capacity and log it every time it changes by at least 10 + // entries in either direction. + let current = self.tx.capacity(); + if self.last_logged_capacity.saturating_sub(current) + >= CAPACITY_LOG_THRESHOLD + { + info!( + self.log, + "Channel capacity decreased"; + "previous" => ?self.last_logged_capacity, + "current" => ?current + ); + self.last_logged_capacity = current; + } else if current.saturating_sub(self.last_logged_capacity) + >= CAPACITY_LOG_THRESHOLD + { + info!( + self.log, + "Channel capacity increased"; + "previous" => ?self.last_logged_capacity, + "current" => ?current + ); + self.last_logged_capacity = current; + } + // The sending side never disappears because we hold a copy + let req = self.rx.recv().await.unwrap(); + info!(self.log, "Received {:?}", req); + let should_send_updates = match req { + StorageRequest::AddDisk(raw_disk) => { + self.add_disk(raw_disk).await?.disk_inserted() + } + StorageRequest::RemoveDisk(raw_disk) => self.remove_disk(raw_disk), + StorageRequest::DisksChanged(raw_disks) => { + self.ensure_using_exactly_these_disks(raw_disks).await + } + StorageRequest::NewFilesystem(request) => { + let result = self.add_dataset(&request).await; + if result.is_err() { + warn!(self.log, "{result:?}"); + } + let _ = request.responder.send(result); + false + } + StorageRequest::KeyManagerReady => { + self.state = StorageManagerState::Normal; + self.add_queued_disks().await + } + StorageRequest::GetLatestResources(tx) => { + let _ = tx.send(self.resources.clone()); + false + } + StorageRequest::GetManagerState(tx) => { + let _ = tx.send(StorageManagerData { + state: self.state, + queued_u2_drives: self.queued_u2_drives.clone(), + }); + false + } + }; + + if should_send_updates { + let _ = self.resource_updates.send_replace(self.resources.clone()); + } + + Ok(()) + } + + // Loop through all queued disks inserting them into [`StorageResources`] + // unless we hit a transient error. If we hit a transient error, we return + // and wait for the next retry window to re-call this method. If we hit a + // permanent error we log it, but we continue inserting queued disks. + // + // Return true if updates should be sent to watchers, false otherwise + async fn add_queued_disks(&mut self) -> bool { + info!( + self.log, + "Attempting to add queued disks"; + "num_disks" => %self.queued_u2_drives.len() + ); + self.state = StorageManagerState::Normal; + + let mut send_updates = false; + + // Disks that should be requeued. + let queued = self.queued_u2_drives.clone(); + let mut to_dequeue = HashSet::new(); + for disk in queued.iter() { + if self.state == StorageManagerState::QueueingDisks { + // We hit a transient error in a prior iteration. + break; + } else { + match self.add_u2_disk(disk.clone()).await { + Err(_) => { + // This is an unrecoverable error, so we don't queue the + // disk again. + to_dequeue.insert(disk); + } + Ok(AddDiskResult::DiskInserted) => { + send_updates = true; + to_dequeue.insert(disk); + } + Ok(AddDiskResult::DiskAlreadyInserted) => { + to_dequeue.insert(disk); + } + Ok(AddDiskResult::DiskQueued) => (), + } + } + } + // Dequeue any inserted disks + self.queued_u2_drives.retain(|k| !to_dequeue.contains(k)); + send_updates + } + + // Add a disk to `StorageResources` if it is new, + // updated, or its pool has been updated as determined by + // [`$crate::resources::StorageResources::insert_disk`] and we decide not to + // queue the disk for later addition. + async fn add_disk( + &mut self, + raw_disk: RawDisk, + ) -> Result { + match raw_disk.variant() { + DiskVariant::U2 => self.add_u2_disk(raw_disk).await, + DiskVariant::M2 => self.add_m2_disk(raw_disk).await, + } + } + + // Add a U.2 disk to [`StorageResources`] or queue it to be added later + async fn add_u2_disk( + &mut self, + raw_disk: RawDisk, + ) -> Result { + if self.state != StorageManagerState::Normal { + self.queued_u2_drives.insert(raw_disk); + return Ok(AddDiskResult::DiskQueued); + } + + match Disk::new(&self.log, raw_disk.clone(), Some(&self.key_requester)) + .await + { + Ok(disk) => self.resources.insert_disk(disk), + Err(err @ DiskError::Dataset(DatasetError::KeyManager(_))) => { + warn!( + self.log, + "Transient error: {err}: queuing disk"; + "disk_id" => ?raw_disk.identity() + ); + self.queued_u2_drives.insert(raw_disk); + self.state = StorageManagerState::QueueingDisks; + Ok(AddDiskResult::DiskQueued) + } + Err(err) => { + error!( + self.log, + "Persistent error: {err}: not queueing disk"; + "disk_id" => ?raw_disk.identity() + ); + Err(err.into()) + } + } + } + + // Add a U.2 disk to [`StorageResources`] if new and return `Ok(true)` if so + // + // + // We never queue M.2 drives, as they don't rely on [`KeyManager`] based + // encryption + async fn add_m2_disk( + &mut self, + raw_disk: RawDisk, + ) -> Result { + let disk = + Disk::new(&self.log, raw_disk.clone(), Some(&self.key_requester)) + .await?; + self.resources.insert_disk(disk) + } + + // Delete a real disk and return `true` if the disk was actually removed + fn remove_disk(&mut self, raw_disk: RawDisk) -> bool { + // If the disk is a U.2, we want to first delete it from any queued disks + let _ = self.queued_u2_drives.remove(&raw_disk); + self.resources.remove_disk(raw_disk.identity()) + } + + // Find all disks to remove that are not in raw_disks and remove them. Then + // take the remaining disks and try to add them all. `StorageResources` will + // inform us if anything changed, and if so we return true, otherwise we + // return false. + async fn ensure_using_exactly_these_disks( + &mut self, + raw_disks: HashSet, + ) -> bool { + let mut should_update = false; + + // Clear out any queued U.2 disks that are real. + // We keep synthetic disks, as they are only added once. + self.queued_u2_drives.retain(|d| d.is_synthetic()); + + let all_ids: HashSet<_> = + raw_disks.iter().map(|d| d.identity()).collect(); + + // Find all existing disks not in the current set + let to_remove: Vec = self + .resources + .disks() + .keys() + .filter_map(|id| { + if !all_ids.contains(id) { + Some(id.clone()) + } else { + None + } + }) + .collect(); + + for id in to_remove { + if self.resources.remove_disk(&id) { + should_update = true; + } + } + + for raw_disk in raw_disks { + let disk_id = raw_disk.identity().clone(); + match self.add_disk(raw_disk).await { + Ok(AddDiskResult::DiskInserted) => should_update = true, + Ok(_) => (), + Err(err) => { + warn!( + self.log, + "Failed to add disk to storage resources: {err}"; + "disk_id" => ?disk_id + ); + } + } + } + + should_update + } + + // Attempts to add a dataset within a zpool, according to `request`. + async fn add_dataset( + &mut self, + request: &NewFilesystemRequest, + ) -> Result<(), Error> { + info!(self.log, "add_dataset: {:?}", request); + if !self + .resources + .disks() + .values() + .any(|(_, pool)| &pool.name == request.dataset_name.pool()) + { + return Err(Error::ZpoolNotFound(format!( + "{}, looked up while trying to add dataset", + request.dataset_name.pool(), + ))); + } + + let zoned = true; + let fs_name = &request.dataset_name.full(); + let do_format = true; + let encryption_details = None; + let size_details = None; + Zfs::ensure_filesystem( + fs_name, + Mountpoint::Path(Utf8PathBuf::from("/data")), + zoned, + do_format, + encryption_details, + size_details, + None, + )?; + // Ensure the dataset has a usable UUID. + if let Ok(id_str) = Zfs::get_oxide_value(&fs_name, "uuid") { + if let Ok(id) = id_str.parse::() { + if id != request.dataset_id { + return Err(Error::UuidMismatch { + name: Box::new(request.dataset_name.clone()), + old: id, + new: request.dataset_id, + }); + } + return Ok(()); + } + } + Zfs::set_oxide_value( + &fs_name, + "uuid", + &request.dataset_id.to_string(), + )?; + + Ok(()) + } +} + +/// All tests only use synthetic disks, but are expected to be run on illumos +/// systems. +#[cfg(all(test, target_os = "illumos"))] +mod tests { + use crate::dataset::DatasetKind; + use crate::disk::SyntheticDisk; + + use super::*; + use async_trait::async_trait; + use camino_tempfile::tempdir; + use illumos_utils::zpool::Zpool; + use key_manager::{ + KeyManager, SecretRetriever, SecretRetrieverError, SecretState, + VersionedIkm, + }; + use omicron_test_utils::dev::test_setup_log; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + use uuid::Uuid; + + /// A [`key-manager::SecretRetriever`] that only returns hardcoded IKM for + /// epoch 0 + #[derive(Debug, Default)] + struct HardcodedSecretRetriever { + inject_error: Arc, + } + + #[async_trait] + impl SecretRetriever for HardcodedSecretRetriever { + async fn get_latest( + &self, + ) -> Result { + if self.inject_error.load(Ordering::SeqCst) { + return Err(SecretRetrieverError::Bootstore( + "Timeout".to_string(), + )); + } + + let epoch = 0; + let salt = [0u8; 32]; + let secret = [0x1d; 32]; + + Ok(VersionedIkm::new(epoch, salt, &secret)) + } + + /// We don't plan to do any key rotation before trust quorum is ready + async fn get( + &self, + epoch: u64, + ) -> Result { + if self.inject_error.load(Ordering::SeqCst) { + return Err(SecretRetrieverError::Bootstore( + "Timeout".to_string(), + )); + } + if epoch != 0 { + return Err(SecretRetrieverError::NoSuchEpoch(epoch)); + } + Ok(SecretState::Current(self.get_latest().await?)) + } + } + + #[tokio::test] + async fn add_u2_disk_while_not_in_normal_stage_and_ensure_it_gets_queued() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log( + "add_u2_disk_while_not_in_normal_stage_and_ensure_it_gets_queued", + ); + let (mut _key_manager, key_requester) = + KeyManager::new(&logctx.log, HardcodedSecretRetriever::default()); + let (mut manager, _) = StorageManager::new(&logctx.log, key_requester); + let zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let raw_disk: RawDisk = SyntheticDisk::new(zpool_name).into(); + assert_eq!(StorageManagerState::WaitingForKeyManager, manager.state); + manager.add_u2_disk(raw_disk.clone()).await.unwrap(); + assert!(manager.resources.all_u2_zpools().is_empty()); + assert_eq!(manager.queued_u2_drives, HashSet::from([raw_disk.clone()])); + + // Check other non-normal stages and ensure disk gets queued + manager.queued_u2_drives.clear(); + manager.state = StorageManagerState::QueueingDisks; + manager.add_u2_disk(raw_disk.clone()).await.unwrap(); + assert!(manager.resources.all_u2_zpools().is_empty()); + assert_eq!(manager.queued_u2_drives, HashSet::from([raw_disk])); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn ensure_u2_gets_added_to_resources() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("ensure_u2_gets_added_to_resources"); + let (mut key_manager, key_requester) = + KeyManager::new(&logctx.log, HardcodedSecretRetriever::default()); + let (mut manager, _) = StorageManager::new(&logctx.log, key_requester); + let zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let dir = tempdir().unwrap(); + let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into(); + + // Spawn the key_manager so that it will respond to requests for encryption keys + tokio::spawn(async move { key_manager.run().await }); + + // Set the stage to pretend we've progressed enough to have a key_manager available. + manager.state = StorageManagerState::Normal; + manager.add_u2_disk(disk).await.unwrap(); + assert_eq!(manager.resources.all_u2_zpools().len(), 1); + Zpool::destroy(&zpool_name).unwrap(); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn wait_for_bootdisk() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("wait_for_bootdisk"); + let (mut key_manager, key_requester) = + KeyManager::new(&logctx.log, HardcodedSecretRetriever::default()); + let (manager, mut handle) = + StorageManager::new(&logctx.log, key_requester); + // Spawn the key_manager so that it will respond to requests for encryption keys + tokio::spawn(async move { key_manager.run().await }); + + // Spawn the storage manager as done by sled-agent + tokio::spawn(async move { + manager.run().await; + }); + + // Create a synthetic internal disk + let zpool_name = ZpoolName::new_internal(Uuid::new_v4()); + let dir = tempdir().unwrap(); + let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into(); + + handle.upsert_disk(disk).await; + handle.wait_for_boot_disk().await; + Zpool::destroy(&zpool_name).unwrap(); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn queued_disks_get_added_as_resources() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("queued_disks_get_added_as_resources"); + let (mut key_manager, key_requester) = + KeyManager::new(&logctx.log, HardcodedSecretRetriever::default()); + let (manager, handle) = StorageManager::new(&logctx.log, key_requester); + + // Spawn the key_manager so that it will respond to requests for encryption keys + tokio::spawn(async move { key_manager.run().await }); + + // Spawn the storage manager as done by sled-agent + tokio::spawn(async move { + manager.run().await; + }); + + // Queue up a disks, as we haven't told the `StorageManager` that + // the `KeyManager` is ready yet. + let zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let dir = tempdir().unwrap(); + let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into(); + handle.upsert_disk(disk).await; + let resources = handle.get_latest_resources().await; + assert!(resources.all_u2_zpools().is_empty()); + + // Now inform the storage manager that the key manager is ready + // The queued disk should be successfully added + handle.key_manager_ready().await; + let resources = handle.get_latest_resources().await; + assert_eq!(resources.all_u2_zpools().len(), 1); + Zpool::destroy(&zpool_name).unwrap(); + logctx.cleanup_successful(); + } + + /// For this test, we are going to step through the msg recv loop directly + /// without running the `StorageManager` in a tokio task. + /// This allows us to control timing precisely. + #[tokio::test] + async fn queued_disks_get_requeued_on_secret_retriever_error() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log( + "queued_disks_get_requeued_on_secret_retriever_error", + ); + let inject_error = Arc::new(AtomicBool::new(false)); + let (mut key_manager, key_requester) = KeyManager::new( + &logctx.log, + HardcodedSecretRetriever { inject_error: inject_error.clone() }, + ); + let (mut manager, handle) = + StorageManager::new(&logctx.log, key_requester); + + // Spawn the key_manager so that it will respond to requests for encryption keys + tokio::spawn(async move { key_manager.run().await }); + + // Queue up a disks, as we haven't told the `StorageManager` that + // the `KeyManager` is ready yet. + let zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let dir = tempdir().unwrap(); + let disk = SyntheticDisk::create_zpool(dir.path(), &zpool_name).into(); + handle.upsert_disk(disk).await; + manager.step().await.unwrap(); + + // We can't wait for a reply through the handle as the storage manager task + // isn't actually running. We just check the resources directly. + assert!(manager.resources.all_u2_zpools().is_empty()); + + // Let's inject an error to the `SecretRetriever` to simulate a trust + // quorum timeout + inject_error.store(true, Ordering::SeqCst); + + // Now inform the storage manager that the key manager is ready + // The queued disk should not be added due to the error + handle.key_manager_ready().await; + manager.step().await.unwrap(); + assert!(manager.resources.all_u2_zpools().is_empty()); + + // Manually simulating a timer tick to add queued disks should also + // still hit the error + manager.add_queued_disks().await; + assert!(manager.resources.all_u2_zpools().is_empty()); + + // Clearing the injected error will cause the disk to get added + inject_error.store(false, Ordering::SeqCst); + manager.add_queued_disks().await; + assert_eq!(1, manager.resources.all_u2_zpools().len()); + + Zpool::destroy(&zpool_name).unwrap(); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn delete_disk_triggers_notification() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("delete_disk_triggers_notification"); + let (mut key_manager, key_requester) = + KeyManager::new(&logctx.log, HardcodedSecretRetriever::default()); + let (manager, mut handle) = + StorageManager::new(&logctx.log, key_requester); + + // Spawn the key_manager so that it will respond to requests for encryption keys + tokio::spawn(async move { key_manager.run().await }); + + // Spawn the storage manager as done by sled-agent + tokio::spawn(async move { + manager.run().await; + }); + + // Inform the storage manager that the key manager is ready, so disks + // don't get queued + handle.key_manager_ready().await; + + // Create and add a disk + let zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let dir = tempdir().unwrap(); + let disk: RawDisk = + SyntheticDisk::create_zpool(dir.path(), &zpool_name).into(); + handle.upsert_disk(disk.clone()).await; + + // Wait for the add disk notification + let resources = handle.wait_for_changes().await; + assert_eq!(resources.all_u2_zpools().len(), 1); + + // Delete the disk and wait for a notification + handle.delete_disk(disk).await; + let resources = handle.wait_for_changes().await; + assert!(resources.all_u2_zpools().is_empty()); + + Zpool::destroy(&zpool_name).unwrap(); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn ensure_using_exactly_these_disks() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("ensure_using_exactly_these_disks"); + let (mut key_manager, key_requester) = + KeyManager::new(&logctx.log, HardcodedSecretRetriever::default()); + let (manager, mut handle) = + StorageManager::new(&logctx.log, key_requester); + + // Spawn the key_manager so that it will respond to requests for encryption keys + tokio::spawn(async move { key_manager.run().await }); + + // Spawn the storage manager as done by sled-agent + tokio::spawn(async move { + manager.run().await; + }); + + // Create a bunch of file backed external disks with zpools + let dir = tempdir().unwrap(); + let zpools: Vec = + (0..10).map(|_| ZpoolName::new_external(Uuid::new_v4())).collect(); + let disks: Vec = zpools + .iter() + .map(|zpool_name| { + SyntheticDisk::create_zpool(dir.path(), zpool_name).into() + }) + .collect(); + + // Add the first 3 disks, and ensure they get queued, as we haven't + // marked our key manager ready yet + handle + .ensure_using_exactly_these_disks(disks.iter().take(3).cloned()) + .await; + let state = handle.get_manager_state().await; + assert_eq!(state.queued_u2_drives.len(), 3); + assert_eq!(state.state, StorageManagerState::WaitingForKeyManager); + assert!(handle.get_latest_resources().await.all_u2_zpools().is_empty()); + + // Mark the key manager ready and wait for the storage update + handle.key_manager_ready().await; + let resources = handle.wait_for_changes().await; + let expected: HashSet<_> = + disks.iter().take(3).map(|d| d.identity()).collect(); + let actual: HashSet<_> = resources.disks().keys().collect(); + assert_eq!(expected, actual); + + // Add first three disks after the initial one. The returned resources + // should not contain the first disk. + handle + .ensure_using_exactly_these_disks( + disks.iter().skip(1).take(3).cloned(), + ) + .await; + let resources = handle.wait_for_changes().await; + let expected: HashSet<_> = + disks.iter().skip(1).take(3).map(|d| d.identity()).collect(); + let actual: HashSet<_> = resources.disks().keys().collect(); + assert_eq!(expected, actual); + + // Ensure the same set of disks and make sure no change occurs + // Note that we directly request the resources this time so we aren't + // waiting forever for a change notification. + handle + .ensure_using_exactly_these_disks( + disks.iter().skip(1).take(3).cloned(), + ) + .await; + let resources2 = handle.get_latest_resources().await; + assert_eq!(resources, resources2); + + // Add a disjoint set of disks and see that only they come through + handle + .ensure_using_exactly_these_disks( + disks.iter().skip(4).take(5).cloned(), + ) + .await; + let resources = handle.wait_for_changes().await; + let expected: HashSet<_> = + disks.iter().skip(4).take(5).map(|d| d.identity()).collect(); + let actual: HashSet<_> = resources.disks().keys().collect(); + assert_eq!(expected, actual); + + // Finally, change the zpool backing of the 5th disk to be that of the 10th + // and ensure that disk changes. Note that we don't change the identity + // of the 5th disk. + let mut modified_disk = disks[4].clone(); + if let RawDisk::Synthetic(disk) = &mut modified_disk { + disk.zpool_name = disks[9].zpool_name().clone(); + } else { + panic!(); + } + let mut expected: HashSet<_> = + disks.iter().skip(5).take(4).cloned().collect(); + expected.insert(modified_disk); + + handle + .ensure_using_exactly_these_disks(expected.clone().into_iter()) + .await; + let resources = handle.wait_for_changes().await; + + // Ensure the one modified disk changed as we expected + assert_eq!(5, resources.disks().len()); + for raw_disk in expected { + let (disk, pool) = + resources.disks().get(raw_disk.identity()).unwrap(); + assert_eq!(disk.zpool_name(), raw_disk.zpool_name()); + assert_eq!(&pool.name, disk.zpool_name()); + assert_eq!(raw_disk.identity(), &pool.parent); + } + + // Cleanup + for zpool in zpools { + Zpool::destroy(&zpool).unwrap(); + } + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn upsert_filesystem() { + illumos_utils::USE_MOCKS.store(false, Ordering::SeqCst); + let logctx = test_setup_log("upsert_filesystem"); + let (mut key_manager, key_requester) = + KeyManager::new(&logctx.log, HardcodedSecretRetriever::default()); + let (manager, handle) = StorageManager::new(&logctx.log, key_requester); + + // Spawn the key_manager so that it will respond to requests for encryption keys + tokio::spawn(async move { key_manager.run().await }); + + // Spawn the storage manager as done by sled-agent + tokio::spawn(async move { + manager.run().await; + }); + + handle.key_manager_ready().await; + + // Create and add a disk + let zpool_name = ZpoolName::new_external(Uuid::new_v4()); + let dir = tempdir().unwrap(); + let disk: RawDisk = + SyntheticDisk::create_zpool(dir.path(), &zpool_name).into(); + handle.upsert_disk(disk.clone()).await; + + // Create a filesystem + let dataset_id = Uuid::new_v4(); + let dataset_name = + DatasetName::new(zpool_name.clone(), DatasetKind::Crucible); + handle.upsert_filesystem(dataset_id, dataset_name).await.unwrap(); + + Zpool::destroy(&zpool_name).unwrap(); + logctx.cleanup_successful(); + } +} diff --git a/sled-storage/src/pool.rs b/sled-storage/src/pool.rs new file mode 100644 index 00000000000..cc71aeb19d2 --- /dev/null +++ b/sled-storage/src/pool.rs @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! ZFS storage pool + +use crate::error::Error; +use illumos_utils::zpool::{Zpool, ZpoolInfo, ZpoolName}; +use omicron_common::disk::DiskIdentity; + +/// A ZFS storage pool wrapper that tracks information returned from +/// `zpool` commands +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Pool { + pub name: ZpoolName, + pub info: ZpoolInfo, + pub parent: DiskIdentity, +} + +impl Pool { + /// Queries for an existing Zpool by name. + /// + /// Returns Ok if the pool exists. + pub fn new(name: ZpoolName, parent: DiskIdentity) -> Result { + let info = Zpool::get_info(&name.to_string())?; + Ok(Pool { name, info, parent }) + } + + /// Return a Pool consisting of fake info + #[cfg(feature = "testing")] + pub fn new_with_fake_info(name: ZpoolName, parent: DiskIdentity) -> Pool { + let info = ZpoolInfo::new_hardcoded(name.to_string()); + Pool { name, info, parent } + } +} diff --git a/sled-storage/src/resources.rs b/sled-storage/src/resources.rs new file mode 100644 index 00000000000..c1f460dc925 --- /dev/null +++ b/sled-storage/src/resources.rs @@ -0,0 +1,206 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Discovered and usable disks and zpools + +use crate::dataset::M2_DEBUG_DATASET; +use crate::disk::Disk; +use crate::error::Error; +use crate::pool::Pool; +use camino::Utf8PathBuf; +use cfg_if::cfg_if; +use illumos_utils::zpool::ZpoolName; +use omicron_common::disk::DiskIdentity; +use sled_hardware::DiskVariant; +use std::collections::BTreeMap; +use std::sync::Arc; + +// The directory within the debug dataset in which bundles are created. +const BUNDLE_DIRECTORY: &str = "bundle"; + +// The directory for zone bundles. +const ZONE_BUNDLE_DIRECTORY: &str = "zone"; + +pub enum AddDiskResult { + DiskInserted, + DiskAlreadyInserted, + DiskQueued, +} + +impl AddDiskResult { + pub fn disk_inserted(&self) -> bool { + match self { + AddDiskResult::DiskInserted => true, + _ => false, + } + } +} + +/// Storage related resources: disks and zpools +/// +/// This state is internal to the [`crate::manager::StorageManager`] task. Clones +/// of this state can be retrieved by requests to the `StorageManager` task +/// from the [`crate::manager::StorageHandle`]. This state is not `Sync`, and +/// as such does not require any mutexes. However, we do expect to share it +/// relatively frequently, and we want copies of it to be as cheaply made +/// as possible. So any large state is stored inside `Arc`s. On the other +/// hand, we expect infrequent updates to this state, and as such, we use +/// [`std::sync::Arc::make_mut`] to implement clone on write functionality +/// inside the `StorageManager` task if there are any outstanding copies. +/// Therefore, we only pay the cost to update infrequently, and no locks are +/// required by callers when operating on cloned data. The only contention here +/// is for the reference counters of the internal Arcs when `StorageResources` +/// gets cloned or dropped. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StorageResources { + // All disks, real and synthetic, being managed by this sled + disks: Arc>, +} + +impl StorageResources { + /// Return a reference to the current snapshot of disks + pub fn disks(&self) -> &BTreeMap { + &self.disks + } + + /// Insert a disk and its zpool + /// + /// If the disk passed in is new or modified, or its pool size or pool + /// name changed, then insert the changed values and return `DiskInserted`. + /// Otherwise, do not insert anything and return `DiskAlreadyInserted`. + /// For instance, if only the pool health changes, because it is not one + /// of the checked values, we will not insert the update and will return + /// `DiskAlreadyInserted`. + pub(crate) fn insert_disk( + &mut self, + disk: Disk, + ) -> Result { + let disk_id = disk.identity().clone(); + let zpool_name = disk.zpool_name().clone(); + let zpool = Pool::new(zpool_name, disk_id.clone())?; + if let Some((stored_disk, stored_pool)) = self.disks.get(&disk_id) { + if stored_disk == &disk + && stored_pool.info.size() == zpool.info.size() + && stored_pool.name == zpool.name + { + return Ok(AddDiskResult::DiskAlreadyInserted); + } + } + // Either the disk or zpool changed + Arc::make_mut(&mut self.disks).insert(disk_id, (disk, zpool)); + Ok(AddDiskResult::DiskInserted) + } + + /// Insert a disk while creating a fake pool + /// This is a workaround for current mock based testing strategies + /// in the sled-agent. + #[cfg(feature = "testing")] + pub fn insert_fake_disk(&mut self, disk: Disk) -> AddDiskResult { + let disk_id = disk.identity().clone(); + let zpool_name = disk.zpool_name().clone(); + let zpool = Pool::new_with_fake_info(zpool_name, disk_id.clone()); + if self.disks.contains_key(&disk_id) { + return AddDiskResult::DiskAlreadyInserted; + } + // Either the disk or zpool changed + Arc::make_mut(&mut self.disks).insert(disk_id, (disk, zpool)); + AddDiskResult::DiskInserted + } + + /// Delete a disk and its zpool + /// + /// Return true, if data was changed, false otherwise + /// + /// Note: We never allow removal of synthetic disks in production as they + /// are only added once. + pub(crate) fn remove_disk(&mut self, id: &DiskIdentity) -> bool { + let Some((disk, _)) = self.disks.get(id) else { + return false; + }; + + cfg_if! { + if #[cfg(test)] { + // For testing purposes, we allow synthetic disks to be deleted. + // Silence an unused variable warning. + _ = disk; + } else { + // In production, we disallow removal of synthetic disks as they + // are only added once. + if disk.is_synthetic() { + return false; + } + } + } + + // Safe to unwrap as we just checked the key existed above + Arc::make_mut(&mut self.disks).remove(id).unwrap(); + true + } + + /// Returns the identity of the boot disk. + /// + /// If this returns `None`, we have not processed the boot disk yet. + pub fn boot_disk(&self) -> Option<(DiskIdentity, ZpoolName)> { + for (id, (disk, _)) in self.disks.iter() { + if disk.is_boot_disk() { + return Some((id.clone(), disk.zpool_name().clone())); + } + } + None + } + + /// Returns all M.2 zpools + pub fn all_m2_zpools(&self) -> Vec { + self.all_zpools(DiskVariant::M2) + } + + /// Returns all U.2 zpools + pub fn all_u2_zpools(&self) -> Vec { + self.all_zpools(DiskVariant::U2) + } + + /// Returns all mountpoints within all M.2s for a particular dataset. + pub fn all_m2_mountpoints(&self, dataset: &str) -> Vec { + self.all_m2_zpools() + .iter() + .map(|zpool| zpool.dataset_mountpoint(dataset)) + .collect() + } + + /// Returns all mountpoints within all U.2s for a particular dataset. + pub fn all_u2_mountpoints(&self, dataset: &str) -> Vec { + self.all_u2_zpools() + .iter() + .map(|zpool| zpool.dataset_mountpoint(dataset)) + .collect() + } + + pub fn get_all_zpools(&self) -> Vec<(ZpoolName, DiskVariant)> { + self.disks + .values() + .map(|(disk, _)| (disk.zpool_name().clone(), disk.variant())) + .collect() + } + + // Returns all zpools of a particular variant + fn all_zpools(&self, variant: DiskVariant) -> Vec { + self.disks + .values() + .filter_map(|(disk, _)| { + if disk.variant() == variant { + return Some(disk.zpool_name().clone()); + } + None + }) + .collect() + } + + /// Return the directories for storing zone service bundles. + pub fn all_zone_bundle_directories(&self) -> Vec { + self.all_m2_mountpoints(M2_DEBUG_DATASET) + .into_iter() + .map(|p| p.join(BUNDLE_DIRECTORY).join(ZONE_BUNDLE_DIRECTORY)) + .collect() + } +} diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index b4cb7e6cffc..176f4002a52 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -12,7 +12,7 @@ sled_mode = "scrimlet" # Identifies the revision of the sidecar that is attached, if one is attached. # TODO: This field should be removed once Gimlets have the ability to auto-detect # this information. -sidecar_revision.soft = { front_port_count = 1, rear_port_count = 1 } +sidecar_revision.soft_zone = { front_port_count = 1, rear_port_count = 1 } # Setting this to true causes sled-agent to always report that its time is # in-sync, rather than querying its NTP zone. @@ -45,6 +45,11 @@ zpools = [ # guest memory is pulled from. vmm_reservoir_percentage = 50 +# Optionally you can specify the size of the VMM reservoir in MiB. +# Note vmm_reservoir_percentage and vmm_reservoir_size_mb cannot be specified +# at the same time. +#vmm_reservoir_size_mb = 2048 + # Swap device size for the system. The device is a sparsely allocated zvol on # the internal zpool of the M.2 that we booted from. # diff --git a/sp-sim/src/bin/sp-sim.rs b/sp-sim/src/bin/sp-sim.rs index 76cb8515288..ee0dd3b70c1 100644 --- a/sp-sim/src/bin/sp-sim.rs +++ b/sp-sim/src/bin/sp-sim.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use anyhow::Result; +use anyhow::{anyhow, Result}; use clap::Parser; use omicron_common::cmd::{fatal, CmdError}; use sp_sim::config::Config; @@ -27,14 +27,12 @@ async fn main() { async fn do_run() -> Result<(), CmdError> { let args = Args::parse(); let config = Config::from_file(args.config_file_path) - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(|e| CmdError::Failure(anyhow!(e)))?; - let log = sp_sim::logger(&config) - .map_err(|e| CmdError::Failure(e.to_string()))?; + let log = sp_sim::logger(&config).map_err(CmdError::Failure)?; - let _rack = SimRack::start(&config, &log) - .await - .map_err(|e| CmdError::Failure(e.to_string()))?; + let _rack = + SimRack::start(&config, &log).await.map_err(CmdError::Failure)?; // for now, do nothing except let the spawned tasks run. in the future // (or when used as a library), the expectation is that a caller can diff --git a/sp-sim/src/gimlet.rs b/sp-sim/src/gimlet.rs index d1316965593..be8d903d3f0 100644 --- a/sp-sim/src/gimlet.rs +++ b/sp-sim/src/gimlet.rs @@ -9,12 +9,15 @@ use crate::helpers::rot_slot_id_to_u16; use crate::rot::RotSprocketExt; use crate::serial_number_padded; use crate::server; +use crate::server::SimSpHandler; use crate::server::UdpServer; +use crate::update::SimSpUpdate; use crate::Responsiveness; use crate::SimulatedSp; use anyhow::{anyhow, bail, Context, Result}; use async_trait::async_trait; use futures::future; +use futures::Future; use gateway_messages::ignition::{self, LinkEvents}; use gateway_messages::sp_impl::SpHandler; use gateway_messages::sp_impl::{BoundsChecked, DeviceDescription}; @@ -26,7 +29,6 @@ use gateway_messages::SpError; use gateway_messages::SpPort; use gateway_messages::SpRequest; use gateway_messages::SpStateV2; -use gateway_messages::UpdateId; use gateway_messages::{version, MessageKind}; use gateway_messages::{ComponentDetails, Message, MgsError, StartupOptions}; use gateway_messages::{DiscoverResponse, IgnitionState, PowerState}; @@ -36,27 +38,30 @@ use sprockets_rot::common::Ed25519PublicKey; use sprockets_rot::{RotSprocket, RotSprocketError}; use std::cell::Cell; use std::collections::HashMap; -use std::io::Cursor; use std::iter; use std::net::{SocketAddr, SocketAddrV6}; +use std::pin::Pin; use std::sync::{Arc, Mutex}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream, UdpSocket}; use tokio::select; use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; use tokio::sync::oneshot; +use tokio::sync::watch; use tokio::sync::Mutex as TokioMutex; use tokio::task::{self, JoinHandle}; +pub const SIM_GIMLET_BOARD: &str = "SimGimletSp"; + pub struct Gimlet { rot: Mutex, manufacturing_public_key: Ed25519PublicKey, local_addrs: Option<[SocketAddrV6; 2]>, handler: Option>>, serial_console_addrs: HashMap, - commands: - mpsc::UnboundedSender<(Command, oneshot::Sender)>, + commands: mpsc::UnboundedSender, inner_tasks: Vec>, + responses_sent_count: Option>, } impl Drop for Gimlet { @@ -90,8 +95,7 @@ impl SimulatedSp for Gimlet { async fn set_responsiveness(&self, r: Responsiveness) { let (tx, rx) = oneshot::channel(); - if let Ok(()) = self.commands.send((Command::SetResponsiveness(r), tx)) - { + if let Ok(()) = self.commands.send(Command::SetResponsiveness(r, tx)) { rx.await.unwrap(); } } @@ -102,6 +106,37 @@ impl SimulatedSp for Gimlet { ) -> Result { self.rot.lock().unwrap().handle_deserialized(request) } + + async fn last_update_data(&self) -> Option> { + let handler = self.handler.as_ref()?; + let handler = handler.lock().await; + handler.update_state.last_update_data() + } + + async fn current_update_status(&self) -> gateway_messages::UpdateStatus { + let Some(handler) = self.handler.as_ref() else { + return gateway_messages::UpdateStatus::None; + }; + + handler.lock().await.update_state.status() + } + + fn responses_sent_count(&self) -> Option> { + self.responses_sent_count.clone() + } + + async fn install_udp_accept_semaphore( + &self, + ) -> mpsc::UnboundedSender { + let (tx, rx) = mpsc::unbounded_channel(); + let (resp_tx, resp_rx) = oneshot::channel(); + if let Ok(()) = + self.commands.send(Command::SetThrottler(Some(rx), resp_tx)) + { + resp_rx.await.unwrap(); + } + tx + } } impl Gimlet { @@ -115,108 +150,108 @@ impl Gimlet { let mut inner_tasks = Vec::new(); let (commands, commands_rx) = mpsc::unbounded_channel(); - let (local_addrs, handler) = if let Some(bind_addrs) = - gimlet.common.bind_addrs - { - // bind to our two local "KSZ" ports - assert_eq!(bind_addrs.len(), 2); // gimlet SP always has 2 ports - let servers = future::try_join( - UdpServer::new( - bind_addrs[0], - gimlet.common.multicast_addr, - &log, - ), - UdpServer::new( - bind_addrs[1], - gimlet.common.multicast_addr, - &log, - ), - ) - .await?; - let servers = [servers.0, servers.1]; - - for component_config in &gimlet.common.components { - let id = component_config.id.as_str(); - let component = SpComponent::try_from(id) - .map_err(|_| anyhow!("component id {:?} too long", id))?; - - if let Some(addr) = component_config.serial_console { - let listener = - TcpListener::bind(addr).await.with_context(|| { - format!("failed to bind to {}", addr) - })?; - info!( - log, "bound fake serial console to TCP port"; - "addr" => %addr, - "component" => ?component, - ); + let (manufacturing_public_key, rot) = + RotSprocket::bootstrap_from_config(&gimlet.common); - serial_console_addrs.insert( - component - .as_str() - .with_context(|| "non-utf8 component")? - .to_string(), - listener - .local_addr() - .with_context(|| { - "failed to get local address of bound socket" - }) - .and_then(|addr| match addr { - SocketAddr::V4(addr) => { - bail!("bound IPv4 address {}", addr) - } - SocketAddr::V6(addr) => Ok(addr), - })?, - ); + // Weird case - if we don't have any bind addresses, we're only being + // created to simulate an RoT, so go ahead and return without actually + // starting a simulated SP. + let Some(bind_addrs) = gimlet.common.bind_addrs else { + return Ok(Self { + rot: Mutex::new(rot), + manufacturing_public_key, + local_addrs: None, + handler: None, + serial_console_addrs, + commands, + inner_tasks, + responses_sent_count: None, + }); + }; - let (tx, rx) = mpsc::unbounded_channel(); - incoming_console_tx.insert(component, tx); - - let serial_console = SerialConsoleTcpTask::new( - component, - listener, - rx, - [ - Arc::clone(servers[0].socket()), - Arc::clone(servers[1].socket()), - ], - Arc::clone(&attached_mgs), - log.new(slog::o!("serial-console" => id.to_string())), - ); - inner_tasks.push(task::spawn(async move { - serial_console.run().await - })); - } + // bind to our two local "KSZ" ports + assert_eq!(bind_addrs.len(), 2); // gimlet SP always has 2 ports + let servers = future::try_join( + UdpServer::new(bind_addrs[0], gimlet.common.multicast_addr, &log), + UdpServer::new(bind_addrs[1], gimlet.common.multicast_addr, &log), + ) + .await?; + let servers = [servers.0, servers.1]; + + for component_config in &gimlet.common.components { + let id = component_config.id.as_str(); + let component = SpComponent::try_from(id) + .map_err(|_| anyhow!("component id {:?} too long", id))?; + + if let Some(addr) = component_config.serial_console { + let listener = TcpListener::bind(addr) + .await + .with_context(|| format!("failed to bind to {}", addr))?; + info!( + log, "bound fake serial console to TCP port"; + "addr" => %addr, + "component" => ?component, + ); + + serial_console_addrs.insert( + component + .as_str() + .with_context(|| "non-utf8 component")? + .to_string(), + listener + .local_addr() + .with_context(|| { + "failed to get local address of bound socket" + }) + .and_then(|addr| match addr { + SocketAddr::V4(addr) => { + bail!("bound IPv4 address {}", addr) + } + SocketAddr::V6(addr) => Ok(addr), + })?, + ); + + let (tx, rx) = mpsc::unbounded_channel(); + incoming_console_tx.insert(component, tx); + + let serial_console = SerialConsoleTcpTask::new( + component, + listener, + rx, + [ + Arc::clone(servers[0].socket()), + Arc::clone(servers[1].socket()), + ], + Arc::clone(&attached_mgs), + log.new(slog::o!("serial-console" => id.to_string())), + ); + inner_tasks.push(task::spawn(async move { + serial_console.run().await + })); } - let local_addrs = - [servers[0].local_addr(), servers[1].local_addr()]; - let (inner, handler) = UdpTask::new( - servers, - gimlet.common.components.clone(), - attached_mgs, - gimlet.common.serial_number.clone(), - incoming_console_tx, - commands_rx, - log, - ); - inner_tasks - .push(task::spawn(async move { inner.run().await.unwrap() })); - - (Some(local_addrs), Some(handler)) - } else { - (None, None) - }; + } + let local_addrs = [servers[0].local_addr(), servers[1].local_addr()]; + let (inner, handler, responses_sent_count) = UdpTask::new( + servers, + gimlet.common.components.clone(), + attached_mgs, + gimlet.common.serial_number.clone(), + incoming_console_tx, + commands_rx, + log, + ); + inner_tasks + .push(task::spawn(async move { inner.run().await.unwrap() })); - let (manufacturing_public_key, rot) = - RotSprocket::bootstrap_from_config(&gimlet.common); Ok(Self { rot: Mutex::new(rot), manufacturing_public_key, - local_addrs, - handler, + local_addrs: Some(local_addrs), + handler: Some(handler), serial_console_addrs, commands, inner_tasks, + responses_sent_count: Some(responses_sent_count), }) } @@ -393,19 +428,18 @@ impl SerialConsoleTcpTask { } enum Command { - SetResponsiveness(Responsiveness), + SetResponsiveness(Responsiveness, oneshot::Sender), + SetThrottler(Option>, oneshot::Sender), } -enum CommandResponse { - SetResponsivenessAck, -} +struct Ack; struct UdpTask { udp0: UdpServer, udp1: UdpServer, handler: Arc>, - commands: - mpsc::UnboundedReceiver<(Command, oneshot::Sender)>, + commands: mpsc::UnboundedReceiver, + responses_sent_count: watch::Sender, } impl UdpTask { @@ -415,12 +449,9 @@ impl UdpTask { attached_mgs: Arc>>, serial_number: String, incoming_serial_console: HashMap>>, - commands: mpsc::UnboundedReceiver<( - Command, - oneshot::Sender, - )>, + commands: mpsc::UnboundedReceiver, log: Logger, - ) -> (Self, Arc>) { + ) -> (Self, Arc>, watch::Receiver) { let [udp0, udp1] = servers; let handler = Arc::new(TokioMutex::new(Handler::new( serial_number, @@ -429,15 +460,40 @@ impl UdpTask { incoming_serial_console, log, ))); - (Self { udp0, udp1, handler: Arc::clone(&handler), commands }, handler) + let responses_sent_count = watch::Sender::new(0); + let responses_sent_count_rx = responses_sent_count.subscribe(); + ( + Self { + udp0, + udp1, + handler: Arc::clone(&handler), + commands, + responses_sent_count, + }, + handler, + responses_sent_count_rx, + ) } async fn run(mut self) -> Result<()> { let mut out_buf = [0; gateway_messages::MAX_SERIALIZED_SIZE]; let mut responsiveness = Responsiveness::Responsive; + let mut throttle_count = usize::MAX; + let mut throttler: Option> = None; loop { + let incr_throttle_count: Pin< + Box> + Send>, + > = if let Some(throttler) = throttler.as_mut() { + Box::pin(throttler.recv()) + } else { + Box::pin(future::pending()) + }; select! { - recv0 = self.udp0.recv_from() => { + Some(n) = incr_throttle_count => { + throttle_count = throttle_count.saturating_add(n); + } + + recv0 = self.udp0.recv_from(), if throttle_count > 0 => { if let Some((resp, addr)) = server::handle_request( &mut *self.handler.lock().await, recv0, @@ -445,11 +501,13 @@ impl UdpTask { responsiveness, SpPort::One, ).await? { + throttle_count -= 1; self.udp0.send_to(resp, addr).await?; + self.responses_sent_count.send_modify(|n| *n += 1); } } - recv1 = self.udp1.recv_from() => { + recv1 = self.udp1.recv_from(), if throttle_count > 0 => { if let Some((resp, addr)) = server::handle_request( &mut *self.handler.lock().await, recv1, @@ -457,21 +515,36 @@ impl UdpTask { responsiveness, SpPort::Two, ).await? { + throttle_count -= 1; self.udp1.send_to(resp, addr).await?; + self.responses_sent_count.send_modify(|n| *n += 1); } } command = self.commands.recv() => { // if sending half is gone, we're about to be killed anyway - let (command, tx) = match command { - Some((command, tx)) => (command, tx), + let command = match command { + Some(command) => command, None => return Ok(()), }; match command { - Command::SetResponsiveness(r) => { + Command::SetResponsiveness(r, tx) => { responsiveness = r; - tx.send(CommandResponse::SetResponsivenessAck) + tx.send(Ack) + .map_err(|_| "receiving half died").unwrap(); + } + Command::SetThrottler(thr, tx) => { + throttler = thr; + + // Either immediately start throttling, or + // immediately stop throttling. + if throttler.is_some() { + throttle_count = 0; + } else { + throttle_count = usize::MAX; + } + tx.send(Ack) .map_err(|_| "receiving half died").unwrap(); } } @@ -499,7 +572,15 @@ struct Handler { rot_active_slot: RotSlotId, power_state: PowerState, startup_options: StartupOptions, - update_state: UpdateState, + update_state: SimSpUpdate, + reset_pending: bool, + + // To simulate an SP reset, we should (after doing whatever housekeeping we + // need to track the reset) intentionally _fail_ to respond to the request, + // simulating a `-> !` function on the SP that triggers a reset. To provide + // this, our caller will pass us a function to call if they should ignore + // whatever result we return and fail to respond at all. + should_fail_to_respond_signal: Option>, } impl Handler { @@ -533,7 +614,9 @@ impl Handler { rot_active_slot: RotSlotId::A, power_state: PowerState::A2, startup_options: StartupOptions::empty(), - update_state: UpdateState::NotPrepared, + update_state: SimSpUpdate::default(), + reset_pending: false, + should_fail_to_respond_signal: None, } } @@ -853,14 +936,18 @@ impl SpHandler for Handler { port: SpPort, update: gateway_messages::SpUpdatePrepare, ) -> Result<(), SpError> { - warn!( + debug!( &self.log, - "received update prepare request; not supported by simulated gimlet"; + "received SP update prepare request"; "sender" => %sender, "port" => ?port, "update" => ?update, ); - Err(SpError::RequestUnsupportedForSp) + self.update_state.prepare( + SpComponent::SP_ITSELF, + update.id, + update.sp_image_size.try_into().unwrap(), + ) } fn component_update_prepare( @@ -877,14 +964,11 @@ impl SpHandler for Handler { "update" => ?update, ); - self.update_state = UpdateState::Prepared { - component: update.component, - id: update.id, - data: Cursor::new( - vec![0u8; update.total_size as usize].into_boxed_slice(), - ), - }; - Ok(()) + self.update_state.prepare( + update.component, + update.id, + update.total_size.try_into().unwrap(), + ) } fn update_status( @@ -900,8 +984,7 @@ impl SpHandler for Handler { "port" => ?port, "component" => ?component, ); - // TODO: check that component matches - Ok(self.update_state.to_message()) + Ok(self.update_state.status()) } fn update_chunk( @@ -919,38 +1002,7 @@ impl SpHandler for Handler { "offset" => chunk.offset, "length" => chunk_data.len(), ); - match &mut self.update_state { - UpdateState::Prepared { id, data, .. } => { - // Ensure that the update ID is correct. - // TODO: component? - if chunk.id != *id { - return Err(SpError::InvalidUpdateId { sp_update_id: *id }); - }; - if data.position() != chunk.offset as u64 { - return Err(SpError::UpdateInProgress( - self.update_state.to_message(), - )); - } - - std::io::Write::write_all(data, chunk_data).map_err(|error| { - // Writing to an in-memory buffer can only fail if the update is too large. - warn!( - &self.log, - "update is too large"; - "sender" => %sender, - "port" => ?port, - "offset" => chunk.offset, - "length" => chunk_data.len(), - "total_size" => data.get_ref().len(), - "error" => %error, - ); - SpError::UpdateIsTooLarge - }) - } - UpdateState::NotPrepared | UpdateState::Aborted(_) => { - Err(SpError::UpdateNotPrepared) - } - } + self.update_state.ingest_chunk(chunk, chunk_data) } fn update_abort( @@ -968,21 +1020,7 @@ impl SpHandler for Handler { "component" => ?update_component, "id" => ?update_id, ); - match &self.update_state { - UpdateState::NotPrepared => { - // Ignore this. TODO error here? - } - UpdateState::Prepared { id, .. } => { - if update_id != *id { - return Err(SpError::InvalidUpdateId { sp_update_id: *id }); - } - // TODO: check for component equality? - self.update_state = UpdateState::Aborted(update_id); - } - UpdateState::Aborted(_) => {} - } - - Err(SpError::RequestUnsupportedForSp) + self.update_state.abort(update_id) } fn power_state( @@ -1021,13 +1059,18 @@ impl SpHandler for Handler { port: SpPort, component: SpComponent, ) -> Result<(), SpError> { - warn!( - &self.log, "received reset prepare request; not supported by simulated gimlet"; + debug!( + &self.log, "received reset prepare request"; "sender" => %sender, "port" => ?port, "component" => ?component, ); - Err(SpError::RequestUnsupportedForSp) + if component == SpComponent::SP_ITSELF { + self.reset_pending = true; + Ok(()) + } else { + Err(SpError::RequestUnsupportedForComponent) + } } fn reset_component_trigger( @@ -1036,13 +1079,29 @@ impl SpHandler for Handler { port: SpPort, component: SpComponent, ) -> Result<(), SpError> { - warn!( - &self.log, "received reset trigger request; not supported by simulated gimlet"; + debug!( + &self.log, "received reset trigger request"; "sender" => %sender, "port" => ?port, "component" => ?component, ); - Err(SpError::RequestUnsupportedForSp) + if component == SpComponent::SP_ITSELF { + if self.reset_pending { + self.update_state.sp_reset(); + self.reset_pending = false; + if let Some(signal) = self.should_fail_to_respond_signal.take() + { + // Instruct `server::handle_request()` to _not_ respond to + // this request at all, simulating an SP actually resetting. + signal(); + } + Ok(()) + } else { + Err(SpError::ResetComponentTriggerWithoutPrepare) + } + } else { + Err(SpError::RequestUnsupportedForComponent) + } } fn num_devices(&mut self, _: SocketAddrV6, _: SpPort) -> u32 { @@ -1258,7 +1317,7 @@ impl SpHandler for Handler { buf: &mut [u8], ) -> std::result::Result { static SP_GITC: &[u8] = b"ffffffff"; - static SP_BORD: &[u8] = b"SimGimletSp"; + static SP_BORD: &[u8] = SIM_GIMLET_BOARD.as_bytes(); static SP_NAME: &[u8] = b"SimGimlet"; static SP_VERS: &[u8] = b"0.0.1"; @@ -1303,42 +1362,11 @@ impl SpHandler for Handler { } } -enum UpdateState { - NotPrepared, - Prepared { - #[allow(unused)] - component: SpComponent, - id: UpdateId, - // data would ordinarily be a Cursor>, but that can grow and - // reallocate. We want to ensure that we don't receive any more data - // than originally promised, so use a Cursor> to ensure that - // it never grows. - data: Cursor>, - }, - Aborted(UpdateId), -} - -impl UpdateState { - fn to_message(&self) -> gateway_messages::UpdateStatus { - match self { - Self::NotPrepared => gateway_messages::UpdateStatus::None, - Self::Prepared { id, data, .. } => { - // If all the data has written, mark it as completed. - let bytes_received = data.position() as u32; - let total_size = data.get_ref().len() as u32; - if bytes_received == total_size { - gateway_messages::UpdateStatus::Complete(*id) - } else { - gateway_messages::UpdateStatus::InProgress( - gateway_messages::UpdateInProgressStatus { - id: *id, - bytes_received, - total_size, - }, - ) - } - } - Self::Aborted(id) => gateway_messages::UpdateStatus::Aborted(*id), - } +impl SimSpHandler for Handler { + fn set_sp_should_fail_to_respond_signal( + &mut self, + signal: Box, + ) { + self.should_fail_to_respond_signal = Some(signal); } } diff --git a/sp-sim/src/lib.rs b/sp-sim/src/lib.rs index 2dac0647178..668c7c33119 100644 --- a/sp-sim/src/lib.rs +++ b/sp-sim/src/lib.rs @@ -8,20 +8,25 @@ mod helpers; mod rot; mod server; mod sidecar; +mod update; pub use anyhow::Result; use async_trait::async_trait; pub use config::Config; use gateway_messages::SpPort; pub use gimlet::Gimlet; +pub use gimlet::SIM_GIMLET_BOARD; pub use server::logger; pub use sidecar::Sidecar; +pub use sidecar::SIM_SIDECAR_BOARD; pub use slog::Logger; pub use sprockets_rot::common::msgs::RotRequestV1; pub use sprockets_rot::common::msgs::RotResponseV1; use sprockets_rot::common::Ed25519PublicKey; pub use sprockets_rot::RotSprocketError; use std::net::SocketAddrV6; +use tokio::sync::mpsc; +use tokio::sync::watch; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Responsiveness { @@ -50,6 +55,46 @@ pub trait SimulatedSp { &self, request: RotRequestV1, ) -> Result; + + /// Get the last completed update delivered to this simulated SP. + /// + /// Only returns data after a simulated reset. + async fn last_update_data(&self) -> Option>; + + /// Get the current update status, just as would be returned by an MGS + /// request to get the update status. + async fn current_update_status(&self) -> gateway_messages::UpdateStatus; + + /// Get a watch channel on which this simulated SP will publish a + /// monotonically increasing count of how many responses it has successfully + /// sent. + /// + /// Returns `None` if called before the SP has set up its sockets to handle + /// requests. + fn responses_sent_count(&self) -> Option>; + + /// Inject a UDP-accept-level semaphore on this simualted SP. + /// + /// If this method is not called, the SP will handle requests as they come + /// in. + /// + /// When this method is called, it will set its lease count to zero. + /// When the lease count is zero, the SP will not accept incoming UDP + /// packets. When a value is sent on the channel returned by this method, + /// that value will be added to the lease count. When the SP successfully + /// sends a response to a message, the lease count will be decremented by + /// one. + /// + /// Two example use cases for this method are that a caller could: + /// + /// * Artificially slow down the simulated SP (e.g., throttle the SP to "N + /// requests per second" by incrementing the lease count periodically) + /// * Force the simulated SP to single-step message (e.g., by incrementing + /// the lease count by 1 and then waiting for a message to be received, + /// which can be observed via `responses_sent_count`) + async fn install_udp_accept_semaphore( + &self, + ) -> mpsc::UnboundedSender; } // Helper function to pad a simulated serial number (stored as a `String`) to diff --git a/sp-sim/src/server.rs b/sp-sim/src/server.rs index 11dff3e13f2..94ff75d15bb 100644 --- a/sp-sim/src/server.rs +++ b/sp-sim/src/server.rs @@ -17,6 +17,8 @@ use slog::Logger; use std::net::Ipv6Addr; use std::net::SocketAddr; use std::net::SocketAddrV6; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use std::sync::Arc; use tokio::net::UdpSocket; @@ -122,7 +124,7 @@ pub fn logger(config: &Config) -> Result { } // TODO: This doesn't need to return Result anymore -pub(crate) async fn handle_request<'a, H: SpHandler>( +pub(crate) async fn handle_request<'a, H: SimSpHandler>( handler: &mut H, recv: Result<(&[u8], SocketAddrV6)>, out: &'a mut [u8; gateway_messages::MAX_SERIALIZED_SIZE], @@ -140,8 +142,28 @@ pub(crate) async fn handle_request<'a, H: SpHandler>( let (data, addr) = recv.with_context(|| format!("recv on {:?}", port_num))?; + let should_respond = Arc::new(AtomicBool::new(true)); + + { + let should_respond = Arc::clone(&should_respond); + handler.set_sp_should_fail_to_respond_signal(Box::new(move || { + should_respond.store(false, Ordering::SeqCst); + })); + } + let response = sp_impl::handle_message(addr, port_num, data, handler, out) .map(|n| (&out[..n], addr)); - Ok(response) + if should_respond.load(Ordering::SeqCst) { + Ok(response) + } else { + Ok(None) + } +} + +pub(crate) trait SimSpHandler: SpHandler { + fn set_sp_should_fail_to_respond_signal( + &mut self, + signal: Box, + ); } diff --git a/sp-sim/src/sidecar.rs b/sp-sim/src/sidecar.rs index e56c610c9ca..c8fb4c54818 100644 --- a/sp-sim/src/sidecar.rs +++ b/sp-sim/src/sidecar.rs @@ -11,12 +11,15 @@ use crate::helpers::rot_slot_id_to_u16; use crate::rot::RotSprocketExt; use crate::serial_number_padded; use crate::server; +use crate::server::SimSpHandler; use crate::server::UdpServer; +use crate::update::SimSpUpdate; use crate::Responsiveness; use crate::SimulatedSp; use anyhow::Result; use async_trait::async_trait; use futures::future; +use futures::Future; use gateway_messages::ignition; use gateway_messages::ignition::IgnitionError; use gateway_messages::ignition::LinkEvents; @@ -47,23 +50,27 @@ use sprockets_rot::RotSprocket; use sprockets_rot::RotSprocketError; use std::iter; use std::net::SocketAddrV6; +use std::pin::Pin; use std::sync::Arc; use std::sync::Mutex; use tokio::select; use tokio::sync::mpsc; use tokio::sync::oneshot; +use tokio::sync::watch; use tokio::sync::Mutex as TokioMutex; use tokio::task; use tokio::task::JoinHandle; +pub const SIM_SIDECAR_BOARD: &str = "SimSidecarSp"; + pub struct Sidecar { rot: Mutex, manufacturing_public_key: Ed25519PublicKey, local_addrs: Option<[SocketAddrV6; 2]>, handler: Option>>, - commands: - mpsc::UnboundedSender<(Command, oneshot::Sender)>, + commands: mpsc::UnboundedSender, inner_task: Option>, + responses_sent_count: Option>, } impl Drop for Sidecar { @@ -98,7 +105,7 @@ impl SimulatedSp for Sidecar { async fn set_responsiveness(&self, r: Responsiveness) { let (tx, rx) = oneshot::channel(); self.commands - .send((Command::SetResponsiveness(r), tx)) + .send(Command::SetResponsiveness(r, tx)) .map_err(|_| "sidecar task died unexpectedly") .unwrap(); rx.await.unwrap(); @@ -110,6 +117,37 @@ impl SimulatedSp for Sidecar { ) -> Result { self.rot.lock().unwrap().handle_deserialized(request) } + + async fn last_update_data(&self) -> Option> { + let handler = self.handler.as_ref()?; + let handler = handler.lock().await; + handler.update_state.last_update_data() + } + + async fn current_update_status(&self) -> gateway_messages::UpdateStatus { + let Some(handler) = self.handler.as_ref() else { + return gateway_messages::UpdateStatus::None; + }; + + handler.lock().await.update_state.status() + } + + fn responses_sent_count(&self) -> Option> { + self.responses_sent_count.clone() + } + + async fn install_udp_accept_semaphore( + &self, + ) -> mpsc::UnboundedSender { + let (tx, rx) = mpsc::unbounded_channel(); + let (resp_tx, resp_rx) = oneshot::channel(); + if let Ok(()) = + self.commands.send(Command::SetThrottler(Some(rx), resp_tx)) + { + resp_rx.await.unwrap(); + } + tx + } } impl Sidecar { @@ -122,7 +160,7 @@ impl Sidecar { let (commands, commands_rx) = mpsc::unbounded_channel(); - let (local_addrs, inner_task, handler) = + let (local_addrs, inner_task, handler, responses_sent_count) = if let Some(bind_addrs) = sidecar.common.bind_addrs { // bind to our two local "KSZ" ports assert_eq!(bind_addrs.len(), 2); @@ -143,7 +181,7 @@ impl Sidecar { let local_addrs = [servers[0].local_addr(), servers[1].local_addr()]; - let (inner, handler) = Inner::new( + let (inner, handler, responses_sent_count) = Inner::new( servers, sidecar.common.components.clone(), sidecar.common.serial_number.clone(), @@ -154,9 +192,14 @@ impl Sidecar { let inner_task = task::spawn(async move { inner.run().await.unwrap() }); - (Some(local_addrs), Some(inner_task), Some(handler)) + ( + Some(local_addrs), + Some(inner_task), + Some(handler), + Some(responses_sent_count), + ) } else { - (None, None, None) + (None, None, None, None) }; let (manufacturing_public_key, rot) = @@ -168,40 +211,36 @@ impl Sidecar { handler, commands, inner_task, + responses_sent_count, }) } pub async fn current_ignition_state(&self) -> Vec { let (tx, rx) = oneshot::channel(); self.commands - .send((Command::CurrentIgnitionState, tx)) + .send(Command::CurrentIgnitionState(tx)) .map_err(|_| "sidecar task died unexpectedly") .unwrap(); - match rx.await.unwrap() { - CommandResponse::CurrentIgnitionState(state) => state, - other => panic!("unexpected response {:?}", other), - } + rx.await.unwrap() } } #[derive(Debug)] enum Command { - CurrentIgnitionState, - SetResponsiveness(Responsiveness), + CurrentIgnitionState(oneshot::Sender>), + SetResponsiveness(Responsiveness, oneshot::Sender), + SetThrottler(Option>, oneshot::Sender), } #[derive(Debug)] -enum CommandResponse { - CurrentIgnitionState(Vec), - SetResponsivenessAck, -} +struct Ack; struct Inner { handler: Arc>, udp0: UdpServer, udp1: UdpServer, - commands: - mpsc::UnboundedReceiver<(Command, oneshot::Sender)>, + commands: mpsc::UnboundedReceiver, + responses_sent_count: watch::Sender, } impl Inner { @@ -210,12 +249,9 @@ impl Inner { components: Vec, serial_number: String, ignition: FakeIgnition, - commands: mpsc::UnboundedReceiver<( - Command, - oneshot::Sender, - )>, + commands: mpsc::UnboundedReceiver, log: Logger, - ) -> (Self, Arc>) { + ) -> (Self, Arc>, watch::Receiver) { let [udp0, udp1] = servers; let handler = Arc::new(TokioMutex::new(Handler::new( serial_number, @@ -223,15 +259,40 @@ impl Inner { ignition, log, ))); - (Self { handler: Arc::clone(&handler), udp0, udp1, commands }, handler) + let responses_sent_count = watch::Sender::new(0); + let responses_sent_count_rx = responses_sent_count.subscribe(); + ( + Self { + handler: Arc::clone(&handler), + udp0, + udp1, + commands, + responses_sent_count, + }, + handler, + responses_sent_count_rx, + ) } async fn run(mut self) -> Result<()> { let mut out_buf = [0; gateway_messages::MAX_SERIALIZED_SIZE]; let mut responsiveness = Responsiveness::Responsive; + let mut throttle_count = usize::MAX; + let mut throttler: Option> = None; loop { + let incr_throttle_count: Pin< + Box> + Send>, + > = if let Some(throttler) = throttler.as_mut() { + Box::pin(throttler.recv()) + } else { + Box::pin(future::pending()) + }; select! { - recv0 = self.udp0.recv_from() => { + Some(n) = incr_throttle_count => { + throttle_count = throttle_count.saturating_add(n); + } + + recv0 = self.udp0.recv_from(), if throttle_count > 0 => { if let Some((resp, addr)) = server::handle_request( &mut *self.handler.lock().await, recv0, @@ -239,11 +300,13 @@ impl Inner { responsiveness, SpPort::One, ).await? { + throttle_count -= 1; self.udp0.send_to(resp, addr).await?; + self.responses_sent_count.send_modify(|n| *n += 1); } } - recv1 = self.udp1.recv_from() => { + recv1 = self.udp1.recv_from(), if throttle_count > 0 => { if let Some((resp, addr)) = server::handle_request( &mut *self.handler.lock().await, recv1, @@ -251,31 +314,45 @@ impl Inner { responsiveness, SpPort::Two, ).await? { + throttle_count -= 1; self.udp1.send_to(resp, addr).await?; + self.responses_sent_count.send_modify(|n| *n += 1); } } command = self.commands.recv() => { // if sending half is gone, we're about to be killed anyway - let (command, tx) = match command { - Some((command, tx)) => (command, tx), + let command = match command { + Some(command) => command, None => return Ok(()), }; match command { - Command::CurrentIgnitionState => { - tx.send(CommandResponse::CurrentIgnitionState( - self.handler - .lock() - .await - .ignition - .state - .clone() - )).map_err(|_| "receiving half died").unwrap(); + Command::CurrentIgnitionState(tx) => { + tx.send(self.handler + .lock() + .await + .ignition + .state + .clone() + ).map_err(|_| "receiving half died").unwrap(); } - Command::SetResponsiveness(r) => { + Command::SetResponsiveness(r, tx) => { responsiveness = r; - tx.send(CommandResponse::SetResponsivenessAck) + tx.send(Ack) + .map_err(|_| "receiving half died").unwrap(); + } + Command::SetThrottler(thr, tx) => { + throttler = thr; + + // Either immediately start throttling, or + // immediately stop throttling. + if throttler.is_some() { + throttle_count = 0; + } else { + throttle_count = usize::MAX; + } + tx.send(Ack) .map_err(|_| "receiving half died").unwrap(); } } @@ -301,6 +378,16 @@ struct Handler { ignition: FakeIgnition, rot_active_slot: RotSlotId, power_state: PowerState, + + update_state: SimSpUpdate, + reset_pending: bool, + + // To simulate an SP reset, we should (after doing whatever housekeeping we + // need to track the reset) intentionally _fail_ to respond to the request, + // simulating a `-> !` function on the SP that triggers a reset. To provide + // this, our caller will pass us a function to call if they should ignore + // whatever result we return and fail to respond at all. + should_fail_to_respond_signal: Option>, } impl Handler { @@ -331,6 +418,9 @@ impl Handler { ignition, rot_active_slot: RotSlotId::A, power_state: PowerState::A2, + update_state: SimSpUpdate::default(), + reset_pending: false, + should_fail_to_respond_signal: None, } } @@ -628,14 +718,18 @@ impl SpHandler for Handler { port: SpPort, update: gateway_messages::SpUpdatePrepare, ) -> Result<(), SpError> { - warn!( + debug!( &self.log, - "received update prepare request; not supported by simulated sidecar"; + "received update prepare request"; "sender" => %sender, "port" => ?port, "update" => ?update, ); - Err(SpError::RequestUnsupportedForSp) + self.update_state.prepare( + SpComponent::SP_ITSELF, + update.id, + update.sp_image_size.try_into().unwrap(), + ) } fn component_update_prepare( @@ -644,14 +738,18 @@ impl SpHandler for Handler { port: SpPort, update: gateway_messages::ComponentUpdatePrepare, ) -> Result<(), SpError> { - warn!( + debug!( &self.log, - "received update prepare request; not supported by simulated sidecar"; + "received update prepare request"; "sender" => %sender, "port" => ?port, "update" => ?update, ); - Err(SpError::RequestUnsupportedForSp) + self.update_state.prepare( + update.component, + update.id, + update.total_size.try_into().unwrap(), + ) } fn update_status( @@ -660,14 +758,14 @@ impl SpHandler for Handler { port: SpPort, component: SpComponent, ) -> Result { - warn!( + debug!( &self.log, - "received update status request; not supported by simulated sidecar"; + "received update status request"; "sender" => %sender, "port" => ?port, "component" => ?component, ); - Err(SpError::RequestUnsupportedForSp) + Ok(self.update_state.status()) } fn update_chunk( @@ -675,17 +773,17 @@ impl SpHandler for Handler { sender: SocketAddrV6, port: SpPort, chunk: gateway_messages::UpdateChunk, - data: &[u8], + chunk_data: &[u8], ) -> Result<(), SpError> { - warn!( + debug!( &self.log, - "received update chunk; not supported by simulated sidecar"; + "received update chunk"; "sender" => %sender, "port" => ?port, "offset" => chunk.offset, - "length" => data.len(), + "length" => chunk_data.len(), ); - Err(SpError::RequestUnsupportedForSp) + self.update_state.ingest_chunk(chunk, chunk_data) } fn update_abort( @@ -693,17 +791,17 @@ impl SpHandler for Handler { sender: SocketAddrV6, port: SpPort, component: SpComponent, - id: gateway_messages::UpdateId, + update_id: gateway_messages::UpdateId, ) -> Result<(), SpError> { - warn!( + debug!( &self.log, "received update abort; not supported by simulated sidecar"; "sender" => %sender, "port" => ?port, "component" => ?component, - "id" => ?id, + "id" => ?update_id, ); - Err(SpError::RequestUnsupportedForSp) + self.update_state.abort(update_id) } fn power_state( @@ -742,13 +840,18 @@ impl SpHandler for Handler { port: SpPort, component: SpComponent, ) -> Result<(), SpError> { - warn!( - &self.log, "received reset prepare request; not supported by simulated sidecar"; + debug!( + &self.log, "received reset prepare request"; "sender" => %sender, "port" => ?port, "component" => ?component, ); - Err(SpError::RequestUnsupportedForSp) + if component == SpComponent::SP_ITSELF { + self.reset_pending = true; + Ok(()) + } else { + Err(SpError::RequestUnsupportedForComponent) + } } fn reset_component_trigger( @@ -757,13 +860,29 @@ impl SpHandler for Handler { port: SpPort, component: SpComponent, ) -> Result<(), SpError> { - warn!( - &self.log, "received sys-reset trigger request; not supported by simulated sidecar"; + debug!( + &self.log, "received sys-reset trigger request"; "sender" => %sender, "port" => ?port, "component" => ?component, ); - Err(SpError::RequestUnsupportedForSp) + if component == SpComponent::SP_ITSELF { + if self.reset_pending { + self.update_state.sp_reset(); + self.reset_pending = false; + if let Some(signal) = self.should_fail_to_respond_signal.take() + { + // Instruct `server::handle_request()` to _not_ respond to + // this request at all, simulating an SP actually resetting. + signal(); + } + Ok(()) + } else { + Err(SpError::ResetComponentTriggerWithoutPrepare) + } + } else { + Err(SpError::RequestUnsupportedForComponent) + } } fn num_devices(&mut self, _: SocketAddrV6, _: SpPort) -> u32 { @@ -977,7 +1096,7 @@ impl SpHandler for Handler { buf: &mut [u8], ) -> std::result::Result { static SP_GITC: &[u8] = b"ffffffff"; - static SP_BORD: &[u8] = b"SimSidecarSp"; + static SP_BORD: &[u8] = SIM_SIDECAR_BOARD.as_bytes(); static SP_NAME: &[u8] = b"SimSidecar"; static SP_VERS: &[u8] = b"0.0.1"; @@ -1022,6 +1141,15 @@ impl SpHandler for Handler { } } +impl SimSpHandler for Handler { + fn set_sp_should_fail_to_respond_signal( + &mut self, + signal: Box, + ) { + self.should_fail_to_respond_signal = Some(signal); + } +} + struct FakeIgnition { state: Vec, link_events: Vec, diff --git a/sp-sim/src/update.rs b/sp-sim/src/update.rs new file mode 100644 index 00000000000..e57659ca1aa --- /dev/null +++ b/sp-sim/src/update.rs @@ -0,0 +1,170 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::io::Cursor; +use std::mem; + +use gateway_messages::SpComponent; +use gateway_messages::SpError; +use gateway_messages::UpdateChunk; +use gateway_messages::UpdateId; +use gateway_messages::UpdateInProgressStatus; + +pub(crate) struct SimSpUpdate { + state: UpdateState, + last_update_data: Option>, +} + +impl Default for SimSpUpdate { + fn default() -> Self { + Self { state: UpdateState::NotPrepared, last_update_data: None } + } +} + +impl SimSpUpdate { + // TODO-completeness Split into `sp_prepare` and `component_prepare` when we + // need to simulate aux flash-related (SP update only) things. + pub(crate) fn prepare( + &mut self, + component: SpComponent, + id: UpdateId, + total_size: usize, + ) -> Result<(), SpError> { + match &self.state { + state @ UpdateState::Prepared { .. } => { + Err(SpError::UpdateInProgress(state.to_message())) + } + UpdateState::NotPrepared + | UpdateState::Aborted(_) + | UpdateState::Completed { .. } => { + self.state = UpdateState::Prepared { + component, + id, + data: Cursor::new(vec![0u8; total_size].into_boxed_slice()), + }; + Ok(()) + } + } + } + + pub(crate) fn status(&self) -> gateway_messages::UpdateStatus { + self.state.to_message() + } + + pub(crate) fn ingest_chunk( + &mut self, + chunk: UpdateChunk, + chunk_data: &[u8], + ) -> Result<(), SpError> { + match &mut self.state { + UpdateState::Prepared { component, id, data } => { + // Ensure that the update ID and target component are correct. + if chunk.id != *id || chunk.component != *component { + return Err(SpError::InvalidUpdateId { sp_update_id: *id }); + }; + if data.position() != chunk.offset as u64 { + return Err(SpError::UpdateInProgress( + self.state.to_message(), + )); + } + + // We're writing to an in-memory buffer; the only failure + // possible is if there isn't enough space left (i.e., the + // update is larger than what the preparation we received + // claimed it would be). + std::io::Write::write_all(data, chunk_data) + .map_err(|_| SpError::UpdateIsTooLarge)?; + + if data.position() == data.get_ref().len() as u64 { + let mut stolen = Cursor::new(Box::default()); + mem::swap(data, &mut stolen); + self.state = UpdateState::Completed { + id: *id, + data: stolen.into_inner(), + }; + } + + Ok(()) + } + UpdateState::NotPrepared + | UpdateState::Aborted(_) + | UpdateState::Completed { .. } => Err(SpError::UpdateNotPrepared), + } + } + + pub(crate) fn abort(&mut self, update_id: UpdateId) -> Result<(), SpError> { + match &self.state { + UpdateState::NotPrepared => Err(SpError::UpdateNotPrepared), + UpdateState::Prepared { id, .. } => { + if *id == update_id { + self.state = UpdateState::Aborted(update_id); + Ok(()) + } else { + Err(SpError::UpdateInProgress(self.status())) + } + } + UpdateState::Aborted(_) => Ok(()), + UpdateState::Completed { .. } => { + Err(SpError::UpdateInProgress(self.status())) + } + } + } + + pub(crate) fn sp_reset(&mut self) { + self.last_update_data = match &self.state { + UpdateState::Completed { data, .. } => Some(data.clone()), + UpdateState::NotPrepared + | UpdateState::Prepared { .. } + | UpdateState::Aborted(_) => None, + }; + } + + pub(crate) fn last_update_data(&self) -> Option> { + self.last_update_data.clone() + } +} + +enum UpdateState { + NotPrepared, + Prepared { + component: SpComponent, + id: UpdateId, + // data would ordinarily be a Cursor>, but that can grow and + // reallocate. We want to ensure that we don't receive any more data + // than originally promised, so use a Cursor> to ensure that + // it never grows. + data: Cursor>, + }, + Aborted(UpdateId), + Completed { + id: UpdateId, + data: Box<[u8]>, + }, +} + +impl UpdateState { + fn to_message(&self) -> gateway_messages::UpdateStatus { + use gateway_messages::UpdateStatus; + + match self { + Self::NotPrepared => UpdateStatus::None, + Self::Prepared { id, data, .. } => { + // If all the data has written, mark it as completed. + let bytes_received = data.position() as u32; + let total_size = data.get_ref().len() as u32; + if bytes_received == total_size { + UpdateStatus::Complete(*id) + } else { + UpdateStatus::InProgress(UpdateInProgressStatus { + id: *id, + bytes_received, + total_size, + }) + } + } + Self::Aborted(id) => UpdateStatus::Aborted(*id), + Self::Completed { id, .. } => UpdateStatus::Complete(*id), + } + } +} diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index 7b1f70c79e6..7f210134a25 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -11,7 +11,6 @@ camino.workspace = true camino-tempfile.workspace = true dropshot.workspace = true filetime = { workspace = true, optional = true } -futures.workspace = true headers.workspace = true hex.workspace = true http.workspace = true diff --git a/test-utils/src/dev/clickhouse.rs b/test-utils/src/dev/clickhouse.rs index 6fb495627f5..011de576caf 100644 --- a/test-utils/src/dev/clickhouse.rs +++ b/test-utils/src/dev/clickhouse.rs @@ -23,6 +23,9 @@ use crate::dev::poll; // Timeout used when starting up ClickHouse subprocess. const CLICKHOUSE_TIMEOUT: Duration = Duration::from_secs(30); +// Timeout used when starting a ClickHouse keeper subprocess. +const CLICKHOUSE_KEEPER_TIMEOUT: Duration = Duration::from_secs(30); + /// A `ClickHouseInstance` is used to start and manage a ClickHouse single node server process. #[derive(Debug)] pub struct ClickHouseInstance { @@ -39,16 +42,6 @@ pub struct ClickHouseInstance { child: Option, } -/// A `ClickHouseCluster` is used to start and manage a 2 replica 3 keeper ClickHouse cluster. -#[derive(Debug)] -pub struct ClickHouseCluster { - pub replica_1: ClickHouseInstance, - pub replica_2: ClickHouseInstance, - pub keeper_1: ClickHouseInstance, - pub keeper_2: ClickHouseInstance, - pub keeper_3: ClickHouseInstance, -} - #[derive(Debug, Error)] pub enum ClickHouseError { #[error("Failed to open ClickHouse log file")] @@ -330,25 +323,32 @@ impl Drop for ClickHouseInstance { } } +/// A `ClickHouseCluster` is used to start and manage a 2 replica 3 keeper ClickHouse cluster. +#[derive(Debug)] +pub struct ClickHouseCluster { + pub replica_1: ClickHouseInstance, + pub replica_2: ClickHouseInstance, + pub keeper_1: ClickHouseInstance, + pub keeper_2: ClickHouseInstance, + pub keeper_3: ClickHouseInstance, + pub replica_config_path: PathBuf, + pub keeper_config_path: PathBuf, +} + impl ClickHouseCluster { - pub async fn new() -> Result { + pub async fn new( + replica_config: PathBuf, + keeper_config: PathBuf, + ) -> Result { // Start all Keeper coordinator nodes - let cur_dir = std::env::current_dir().unwrap(); - let keeper_config = - cur_dir.as_path().join("src/configs/keeper_config.xml"); - let keeper_amount = 3; let mut keepers = - Self::new_keeper_set(keeper_amount, keeper_config).await?; + Self::new_keeper_set(keeper_amount, &keeper_config).await?; // Start all replica nodes - let cur_dir = std::env::current_dir().unwrap(); - let replica_config = - cur_dir.as_path().join("src/configs/replica_config.xml"); - let replica_amount = 2; let mut replicas = - Self::new_replica_set(replica_amount, replica_config).await?; + Self::new_replica_set(replica_amount, &replica_config).await?; let r1 = replicas.swap_remove(0); let r2 = replicas.swap_remove(0); @@ -362,12 +362,14 @@ impl ClickHouseCluster { keeper_1: k1, keeper_2: k2, keeper_3: k3, + replica_config_path: replica_config, + keeper_config_path: keeper_config, }) } pub async fn new_keeper_set( keeper_amount: u16, - config_path: PathBuf, + config_path: &PathBuf, ) -> Result, anyhow::Error> { let mut keepers = vec![]; @@ -392,7 +394,7 @@ impl ClickHouseCluster { pub async fn new_replica_set( replica_amount: u16, - config_path: PathBuf, + config_path: &PathBuf, ) -> Result, anyhow::Error> { let mut replicas = vec![]; @@ -419,6 +421,14 @@ impl ClickHouseCluster { Ok(replicas) } + + pub fn replica_config_path(&self) -> &Path { + &self.replica_config_path + } + + pub fn keeper_config_path(&self) -> &Path { + &self.keeper_config_path + } } // Wait for the ClickHouse log file to become available, including the @@ -520,7 +530,8 @@ async fn find_clickhouse_port_in_log( pub async fn wait_for_ready(log_path: PathBuf) -> Result<(), anyhow::Error> { let p = poll::wait_for_condition( || async { - let result = discover_ready(&log_path, CLICKHOUSE_TIMEOUT).await; + let result = + discover_ready(&log_path, CLICKHOUSE_KEEPER_TIMEOUT).await; match result { Ok(ready) => Ok(ready), Err(e) => { @@ -540,7 +551,7 @@ pub async fn wait_for_ready(log_path: PathBuf) -> Result<(), anyhow::Error> { } }, &Duration::from_millis(500), - &CLICKHOUSE_TIMEOUT, + &CLICKHOUSE_KEEPER_TIMEOUT, ) .await .context("waiting to discover if ClickHouse is ready for connections")?; diff --git a/tools/ci_download_softnpu_machinery b/tools/ci_download_softnpu_machinery index cb5ea40210c..3efb0300639 100755 --- a/tools/ci_download_softnpu_machinery +++ b/tools/ci_download_softnpu_machinery @@ -15,7 +15,7 @@ OUT_DIR="out/npuzone" # Pinned commit for softnpu ASIC simulator SOFTNPU_REPO="softnpu" -SOFTNPU_COMMIT="c1c42398c82b0220c8b5fa3bfba9c7a3bcaa0943" +SOFTNPU_COMMIT="dec63e67156fe6e958991bbfa090629868115ab5" # This is the softnpu ASIC simulator echo "fetching npuzone" diff --git a/tools/console_version b/tools/console_version index 7e8d352efd0..811620e9e79 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="bd65b9da7019ad812dd056e7fc182df2cf4ec128" -SHA2="e4d4f33996a6e89b972fac61737acb7f1dbd21943d1f6bef776d4ee9bcccd2b0" +COMMIT="ae8218df707360a902133f4a96b48a3b5a62a09e" +SHA2="ae35b991d3ff835a59b59126298790cb7431a282b25ba4add4e7fb6ea6b98989" diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index 908cb752e91..884d356222d 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -16,6 +16,7 @@ set -x SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" OMICRON_TOP="$SOURCE_DIR/.." +SOFTNPU_MODE=${SOFTNPU_MODE:-zone}; . "$SOURCE_DIR/virtual_hardware.sh" @@ -62,8 +63,9 @@ function ensure_softnpu_zone { --omicron-zone \ --ports sc0_0,tfportrear0_0 \ --ports sc0_1,tfportqsfp0_0 \ - --sidecar-lite-branch main - } + --sidecar-lite-commit f0585a29fb0285f7a1220c1118856b0e5c1f75c5 \ + --softnpu-commit dec63e67156fe6e958991bbfa090629868115ab5 + } "$SOURCE_DIR"/scrimlet/softnpu-init.sh success "softnpu zone exists" } @@ -83,6 +85,9 @@ in the SoftNPU zone later to add those entries." ensure_run_as_root ensure_zpools -ensure_simulated_links "$PHYSICAL_LINK" -warn_if_no_proxy_arp -ensure_softnpu_zone + +if [[ "$SOFTNPU_MODE" == "zone" ]]; then + ensure_simulated_links "$PHYSICAL_LINK" + warn_if_no_proxy_arp + ensure_softnpu_zone +fi diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index c91d1c2e98a..aadf68da1b5 100644 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="343e3a572cc02efe3f8b68f9affd008623a33966" -SHA2="544ab42ccc7942d8ece9cdc80cd85d002bcf9d5646a291322bf2f79087ab6df0" +COMMIT="147b03901aa8305b5271e0133a09f628b8140949" +SHA2="82437c74afd4894aa5b9ea800d5777793e8777fe87471321dd22ad1a1c9c9ef3" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index 8fa98114fb4..81a957323c4 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="0808f331741e02d55e199847579dfd01f3658b21c7122cef8c3f9279f43dbab0" -CIDL_SHA256_LINUX_DPD="3e276dd553dd7cdb75c8ad023c2cd29b91485fafb94f27097a745b2b7ef5ecea" -CIDL_SHA256_LINUX_SWADM="645faf8a93bcae9814b2f116bccd66a54763332b56220e93b66316c853ce13d2" +CIDL_SHA256_ILLUMOS="14fe7f904f963b50188d6e060106b63df6d061ca64238f7b21623c432b5944e3" +CIDL_SHA256_LINUX_DPD="fff6c7484bbb06aa644e3fe41b200e4f7f8d7f65d067cbecd851c834c15fe2ec" +CIDL_SHA256_LINUX_SWADM="0449383a57468aec3b5a4ad26962cfc9e9a121bd13e777329e8a70767e6d9aae" diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version index b52dded7d0b..f7fef543f4b 100644 --- a/tools/dvt_dock_version +++ b/tools/dvt_dock_version @@ -1 +1 @@ -COMMIT=9cb2b40cea90ad40f5c0d2c3da96d26913253e06 +COMMIT=ad874c11ecd0c45bdc1e4c2ac35c2bcbe472d55f diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 89c3e461647..40db886f69d 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" +COMMIT="aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" SHA2="9737906555a60911636532f00f1dc2866dc7cd6553beb106e9e57beabad41cdf" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index a7e18285ae3..ad88fef13e4 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="d7169a61fd8833b3a1e6f46d897ca3295b2a28b6" -SHA2="d0f7611e5ecd049b0f83bcfa843942401f155a0be36d9a2dfd73b8341d5f816e" +COMMIT="aefdfd3a57e5ca1949d4a913b8e35ce8cd7dfa8b" +SHA2="b3f55fe24e54530fdf96c22a033f9edc0bad9c0a5e3344763a23e52b251d5113" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index e65e1fc0a2c..7c1644b031d 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="452dfb3491e1b6d4df6be1cb689921f59623aed082e47606a78c0f44d918f66a" -MGD_LINUX_SHA256="d4c48eb6374c0cc7812b7af2c0ac92acdcbc91b7718a9ce64d069da00ae5ae73" +CIDL_SHA256="aa7241cd35976f28f25aaf3ce2ce2af14dae1da9d67585c7de3b724dbcc55e60" +MGD_LINUX_SHA256="a39387c361ff2c2d0701d66c00b10e43c72fb5ddd1a5900b59ecccb832c80731" \ No newline at end of file diff --git a/tools/update_crucible.sh b/tools/update_crucible.sh index af834091ca2..020a33927ec 100755 --- a/tools/update_crucible.sh +++ b/tools/update_crucible.sh @@ -21,7 +21,6 @@ PACKAGES=( CRATES=( "crucible-agent-client" - "crucible-client-types" "crucible-pantry-client" "crucible-smf" ) diff --git a/tools/update_maghemite.sh b/tools/update_maghemite.sh index eebece1aa53..db7e4827763 100755 --- a/tools/update_maghemite.sh +++ b/tools/update_maghemite.sh @@ -59,7 +59,7 @@ function update_mgd { fi echo "Updating Maghemite mgd from: $TARGET_COMMIT" set -x - echo "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH + printf "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH set +x } diff --git a/update-engine/Cargo.toml b/update-engine/Cargo.toml index af988bf091b..5bc3672f54c 100644 --- a/update-engine/Cargo.toml +++ b/update-engine/Cargo.toml @@ -12,14 +12,18 @@ derive-where.workspace = true either.workspace = true futures.workspace = true indexmap.workspace = true +libsw.workspace = true linear-map.workspace = true +owo-colors.workspace = true petgraph.workspace = true serde.workspace = true serde_json.workspace = true serde_with.workspace = true schemars = { workspace = true, features = ["uuid1"] } slog.workspace = true +swrite.workspace = true tokio = { workspace = true, features = ["macros", "sync", "time", "rt-multi-thread"] } +unicode-width.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true @@ -28,8 +32,10 @@ buf-list.workspace = true bytes.workspace = true camino.workspace = true camino-tempfile.workspace = true +clap.workspace = true indicatif.workspace = true omicron-test-utils.workspace = true owo-colors.workspace = true +supports-color.workspace = true tokio = { workspace = true, features = ["io-util"] } tokio-stream.workspace = true diff --git a/update-engine/examples/update-engine-basic/display.rs b/update-engine/examples/update-engine-basic/display.rs index e6b80e36377..122777211b5 100644 --- a/update-engine/examples/update-engine-basic/display.rs +++ b/update-engine/examples/update-engine-basic/display.rs @@ -12,28 +12,135 @@ use indexmap::{map::Entry, IndexMap}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use owo_colors::OwoColorize; use tokio::{sync::mpsc, task::JoinHandle}; -use update_engine::events::ProgressCounter; +use update_engine::{ + display::{GroupDisplay, LineDisplay, LineDisplayStyles}, + events::ProgressCounter, +}; -use crate::spec::{ - Event, ExampleComponent, ExampleStepId, ExampleStepMetadata, ProgressEvent, - ProgressEventKind, StepEventKind, StepInfoWithMetadata, StepOutcome, +use crate::{ + spec::{ + Event, EventBuffer, ExampleComponent, ExampleStepId, + ExampleStepMetadata, ProgressEvent, ProgressEventKind, StepEventKind, + StepInfoWithMetadata, StepOutcome, + }, + DisplayStyle, }; /// An example that displays an event stream on the command line. pub(crate) fn make_displayer( log: &slog::Logger, + display_style: DisplayStyle, + prefix: Option, ) -> (JoinHandle>, mpsc::Sender) { let (sender, receiver) = mpsc::channel(512); let log = log.clone(); let join_handle = - tokio::task::spawn( - async move { display_messages(&log, receiver).await }, - ); + match display_style { + DisplayStyle::ProgressBar => tokio::task::spawn(async move { + display_progress_bar(&log, receiver).await + }), + DisplayStyle::Line => tokio::task::spawn(async move { + display_line(&log, receiver, prefix).await + }), + DisplayStyle::Group => tokio::task::spawn(async move { + display_group(&log, receiver).await + }), + }; (join_handle, sender) } -async fn display_messages( +async fn display_line( + log: &slog::Logger, + mut receiver: mpsc::Receiver, + prefix: Option, +) -> Result<()> { + slog::info!(log, "setting up display"); + let mut buffer = EventBuffer::new(8); + let mut display = LineDisplay::new(std::io::stdout()); + // For now, always colorize. TODO: figure out whether colorization should be + // done based on always/auto/never etc. + if supports_color::on(supports_color::Stream::Stdout).is_some() { + display.set_styles(LineDisplayStyles::colorized()); + } + if let Some(prefix) = prefix { + display.set_prefix(prefix); + } + display.set_progress_interval(Duration::from_millis(50)); + while let Some(event) = receiver.recv().await { + buffer.add_event(event); + display.write_event_buffer(&buffer)?; + } + + Ok(()) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, PartialOrd, Ord)] +enum GroupDisplayKey { + Example, + Other, +} + +async fn display_group( + log: &slog::Logger, + mut receiver: mpsc::Receiver, +) -> Result<()> { + slog::info!(log, "setting up display"); + + let mut display = GroupDisplay::new( + [ + (GroupDisplayKey::Example, "example"), + (GroupDisplayKey::Other, "other"), + ], + std::io::stdout(), + ); + // For now, always colorize. TODO: figure out whether colorization should be + // done based on always/auto/never etc. + if supports_color::on(supports_color::Stream::Stdout).is_some() { + display.set_styles(LineDisplayStyles::colorized()); + } + + display.set_progress_interval(Duration::from_millis(50)); + + let mut example_buffer = EventBuffer::default(); + let mut example_buffer_last_seen = None; + let mut other_buffer = EventBuffer::default(); + let mut other_buffer_last_seen = None; + + let mut interval = tokio::time::interval(Duration::from_secs(2)); + interval.tick().await; + + loop { + tokio::select! { + _ = interval.tick() => { + // Print out status lines every 2 seconds. + display.write_stats("Status")?; + } + event = receiver.recv() => { + let Some(event) = event else { break }; + example_buffer.add_event(event.clone()); + other_buffer.add_event(event); + + display.add_event_report( + &GroupDisplayKey::Example, + example_buffer.generate_report_since(&mut example_buffer_last_seen), + )?; + display.add_event_report( + &GroupDisplayKey::Other, + other_buffer.generate_report_since(&mut other_buffer_last_seen), + )?; + display.write_events()?; + } + } + } + + // Print status at the end. + display.write_stats("Summary")?; + + Ok(()) +} + +async fn display_progress_bar( log: &slog::Logger, mut receiver: mpsc::Receiver, ) -> Result<()> { diff --git a/update-engine/examples/update-engine-basic/main.rs b/update-engine/examples/update-engine-basic/main.rs index 260473edde3..95fe3c54dc3 100644 --- a/update-engine/examples/update-engine-basic/main.rs +++ b/update-engine/examples/update-engine-basic/main.rs @@ -4,85 +4,139 @@ // Copyright 2023 Oxide Computer Company -use std::time::Duration; +use std::{io::IsTerminal, time::Duration}; -use anyhow::{bail, Context}; +use anyhow::{bail, Context, Result}; use buf_list::BufList; use bytes::Buf; use camino::Utf8PathBuf; use camino_tempfile::Utf8TempDir; +use clap::{Parser, ValueEnum}; use display::make_displayer; use omicron_test_utils::dev::test_setup_log; use spec::{ - ComponentRegistrar, ExampleCompletionMetadata, ExampleComponent, - ExampleSpec, ExampleStepId, ExampleStepMetadata, ExampleWriteSpec, - ExampleWriteStepId, StepHandle, StepProgress, StepSkipped, StepWarning, - UpdateEngine, + ComponentRegistrar, EventBuffer, ExampleCompletionMetadata, + ExampleComponent, ExampleSpec, ExampleStepId, ExampleStepMetadata, + ExampleWriteSpec, ExampleWriteStepId, StepHandle, StepProgress, + StepSkipped, StepWarning, UpdateEngine, +}; +use tokio::{io::AsyncWriteExt, sync::mpsc}; +use update_engine::{ + events::{Event, ProgressUnits}, + StepContext, StepSuccess, }; -use tokio::io::AsyncWriteExt; -use update_engine::{events::ProgressUnits, StepContext, StepSuccess}; mod display; mod spec; #[tokio::main(worker_threads = 2)] -async fn main() { - let logctx = test_setup_log("update_engine_basic_example"); - - let context = ExampleContext::new(&logctx.log); - let (display_handle, sender) = make_displayer(&logctx.log); - - let engine = UpdateEngine::new(&logctx.log, sender); - - // Download component 1. - let component_1 = engine.for_component(ExampleComponent::Component1); - let download_handle_1 = context.register_download_step( - &component_1, - "https://www.example.org".to_owned(), - 1_048_576, - ); - - // An example of a skipped step for component 1. - context.register_skipped_step(&component_1); - - // Create temporary directories for component 1. - let temp_dirs_handle_1 = - context.register_create_temp_dirs_step(&component_1, 2); - - // Write component 1 out to disk. - context.register_write_step( - &component_1, - download_handle_1, - temp_dirs_handle_1, - None, - ); - - // Download component 2. - let component_2 = engine.for_component(ExampleComponent::Component2); - let download_handle_2 = context.register_download_step( - &component_2, - "https://www.example.com".to_owned(), - 1_048_576 * 8, - ); - - // Create temporary directories for component 2. - let temp_dirs_handle_2 = - context.register_create_temp_dirs_step(&component_2, 3); - - // Now write component 2 out to disk. - context.register_write_step( - &component_2, - download_handle_2, - temp_dirs_handle_2, - Some(1), - ); - - _ = engine.execute().await; - - // Wait until all messages have been received by the displayer. - _ = display_handle.await; - - // Do not clean up the log file so people can inspect it. +async fn main() -> Result<()> { + let app = App::parse(); + app.exec().await +} + +#[derive(Debug, Parser)] +struct App { + /// Display style to use. + #[clap(long, short = 's', default_value_t, value_enum)] + display_style: DisplayStyleOpt, + + /// Prefix to set on all log messages with display-style=line. + #[clap(long, short = 'p')] + prefix: Option, +} + +impl App { + async fn exec(self) -> Result<()> { + let logctx = test_setup_log("update_engine_basic_example"); + + let display_style = match self.display_style { + DisplayStyleOpt::ProgressBar => DisplayStyle::ProgressBar, + DisplayStyleOpt::Line => DisplayStyle::Line, + DisplayStyleOpt::Group => DisplayStyle::Group, + DisplayStyleOpt::Auto => { + if std::io::stdout().is_terminal() { + DisplayStyle::ProgressBar + } else { + DisplayStyle::Line + } + } + }; + + let context = ExampleContext::new(&logctx.log); + let (display_handle, sender) = + make_displayer(&logctx.log, display_style, self.prefix); + + let engine = UpdateEngine::new(&logctx.log, sender); + + // Download component 1. + let component_1 = engine.for_component(ExampleComponent::Component1); + let download_handle_1 = context.register_download_step( + &component_1, + "https://www.example.org".to_owned(), + 1_048_576, + ); + + // An example of a skipped step for component 1. + context.register_skipped_step(&component_1); + + // Create temporary directories for component 1. + let temp_dirs_handle_1 = + context.register_create_temp_dirs_step(&component_1, 2); + + // Write component 1 out to disk. + context.register_write_step( + &component_1, + download_handle_1, + temp_dirs_handle_1, + None, + ); + + // Download component 2. + let component_2 = engine.for_component(ExampleComponent::Component2); + let download_handle_2 = context.register_download_step( + &component_2, + "https://www.example.com".to_owned(), + 1_048_576 * 8, + ); + + // Create temporary directories for component 2. + let temp_dirs_handle_2 = + context.register_create_temp_dirs_step(&component_2, 3); + + // Now write component 2 out to disk. + context.register_write_step( + &component_2, + download_handle_2, + temp_dirs_handle_2, + Some(1), + ); + + _ = engine.execute().await; + + // Wait until all messages have been received by the displayer. + _ = display_handle.await; + + // Do not clean up the log file so people can inspect it. + + Ok(()) + } +} + +#[derive(Copy, Clone, Debug, Default, ValueEnum)] +enum DisplayStyleOpt { + ProgressBar, + Line, + Group, + #[default] + Auto, +} + +#[derive(Copy, Clone, Debug)] +enum DisplayStyle { + ProgressBar, + Line, + Group, } /// Context shared across steps. This forms the lifetime "'a" defined by the @@ -146,9 +200,30 @@ impl ExampleContext { ({num_bytes} bytes)", ); - // Try a second time, and this time go all the way to 100%. + // Try a second time, and this time go to 80%. let mut buf_list = BufList::new(); - for i in 0..10 { + for i in 0..8 { + tokio::time::sleep(Duration::from_millis(100)).await; + cx.send_progress(StepProgress::with_current_and_total( + num_bytes * i / 10, + num_bytes, + ProgressUnits::BYTES, + serde_json::Value::Null, + )) + .await; + buf_list.push_chunk(&b"downloaded-data"[..]); + } + + // Now indicate a progress reset. + cx.send_progress(StepProgress::reset( + serde_json::Value::Null, + "Progress reset", + )) + .await; + + // Try again, and this time succeed. + let mut buf_list = BufList::new(); + for i in 0..=10 { tokio::time::sleep(Duration::from_millis(100)).await; cx.send_progress(StepProgress::with_current_and_total( num_bytes * i / 10, @@ -243,6 +318,7 @@ impl ExampleContext { cx.with_nested_engine(|engine| { register_nested_write_steps( + &self.log, engine, component, &destinations, @@ -282,6 +358,7 @@ impl ExampleContext { } fn register_nested_write_steps<'a>( + log: &'a slog::Logger, engine: &mut UpdateEngine<'a, ExampleWriteSpec>, component: ExampleComponent, destinations: &'a [Utf8PathBuf], @@ -307,6 +384,38 @@ fn register_nested_write_steps<'a>( Default::default(), )) .await; + + let mut remote_engine_receiver = create_remote_engine( + log, + component, + buf_list.clone(), + destination.clone(), + ); + let mut buffer = EventBuffer::default(); + let mut last_seen = None; + while let Some(event) = remote_engine_receiver.recv().await + { + // Only send progress up to 50% to demonstrate + // not receiving full progress. + if let Event::Progress(event) = &event { + if let Some(counter) = event.kind.progress_counter() + { + if let Some(total) = counter.total { + if counter.current > total / 2 { + break; + } + } + } + } + + buffer.add_event(event); + let report = + buffer.generate_report_since(&mut last_seen); + cx.send_nested_report(report) + .await + .expect("this engine should never fail"); + } + let mut file = tokio::fs::File::create(destination) .await @@ -345,3 +454,50 @@ fn register_nested_write_steps<'a>( .register(); } } + +/// Sets up a remote engine that can be used to execute steps. +fn create_remote_engine( + log: &slog::Logger, + component: ExampleComponent, + mut buf_list: BufList, + destination: Utf8PathBuf, +) -> mpsc::Receiver> { + let (sender, receiver) = tokio::sync::mpsc::channel(128); + let engine = UpdateEngine::new(log, sender); + engine + .for_component(component) + .new_step( + ExampleWriteStepId::Write { destination: destination.clone() }, + format!("Writing to {destination} (remote, fake)"), + move |cx| async move { + let num_bytes = buf_list.num_bytes(); + let mut total_written = 0; + + while buf_list.has_remaining() { + tokio::time::sleep(Duration::from_millis(20)).await; + // Don't actually write these bytes -- this engine is just + // for demoing. + let written_bytes = + (num_bytes / 10).min(buf_list.num_bytes()); + total_written += written_bytes; + buf_list.advance(written_bytes); + cx.send_progress(StepProgress::with_current_and_total( + total_written as u64, + num_bytes as u64, + ProgressUnits::new_const("fake bytes"), + (), + )) + .await; + } + + StepSuccess::new(()).into() + }, + ) + .register(); + + tokio::spawn(async move { + engine.execute().await.expect("remote engine succeeded") + }); + + receiver +} diff --git a/update-engine/src/buffer.rs b/update-engine/src/buffer.rs index 2426814444d..6e0e66d6d05 100644 --- a/update-engine/src/buffer.rs +++ b/update-engine/src/buffer.rs @@ -7,6 +7,7 @@ use std::{ collections::{HashMap, VecDeque}, fmt, + sync::Arc, time::Duration, }; @@ -106,6 +107,15 @@ impl EventBuffer { self.event_store.root_execution_id } + /// Returns an execution summary for the root execution ID, if this event buffer is aware of any + /// events. + pub fn root_execution_summary(&self) -> Option { + // XXX: more efficient algorithm + let root_execution_id = self.root_execution_id()?; + let mut summary = self.steps().summarize(); + summary.remove(&root_execution_id) + } + /// Returns information about each step, as currently tracked by the buffer, /// in order of when the events were first defined. pub fn steps(&self) -> EventBufferSteps<'_, S> { @@ -248,6 +258,7 @@ impl EventStore { &event, 0, None, + None, root_event_index, event.total_elapsed, ); @@ -255,6 +266,26 @@ impl EventStore { if new_execution.nest_level == 0 { self.root_execution_id = Some(new_execution.execution_id); } + // If there's a parent key, then what's the child index? + let parent_key_and_child_index = + if let Some(parent_key) = new_execution.parent_key { + match self.map.get_mut(&parent_key) { + Some(parent_data) => { + let child_index = parent_data.child_executions_seen; + parent_data.child_executions_seen += 1; + Some((parent_key, child_index)) + } + None => { + // This should never happen -- it indicates that the + // parent key was unknown. This can happen if we + // didn't receive an event regarding a parent + // execution being started. + None + } + } + } else { + None + }; let total_steps = new_execution.steps_to_add.len(); for (new_step_key, new_step, sort_key) in new_execution.steps_to_add { @@ -263,6 +294,7 @@ impl EventStore { self.map.entry(new_step_key).or_insert_with(|| { EventBufferStepData::new( new_step, + parent_key_and_child_index, sort_key, new_execution.nest_level, total_steps, @@ -319,6 +351,7 @@ impl EventStore { &mut self, event: &StepEvent, nest_level: usize, + parent_key: Option, parent_sort_key: Option<&StepSortKey>, root_event_index: RootEventIndex, root_total_elapsed: Duration, @@ -347,6 +380,7 @@ impl EventStore { } new_execution = Some(NewExecutionAction { execution_id: event.execution_id, + parent_key, nest_level, steps_to_add, }); @@ -497,6 +531,7 @@ impl EventStore { let actions = self.recurse_for_step_event( nested_event, nest_level + 1, + Some(parent_key), parent_sort_key.as_ref(), root_event_index, root_total_elapsed, @@ -554,10 +589,14 @@ impl EventStore { info: CompletionInfo, root_event_index: RootEventIndex, ) { + let info = Arc::new(info); if let Some(value) = self.map.get_mut(&root_key) { // Completion status only applies to the root key. Nodes reachable // from this node are still marked as complete, but without status. - value.mark_completed(Some(info), root_event_index); + value.mark_completed( + CompletionReason::StepCompleted(info.clone()), + root_event_index, + ); } // Mark anything reachable from this node as completed. @@ -567,7 +606,13 @@ impl EventStore { if let EventTreeNode::Step(key) = key { if key != root_key { if let Some(value) = self.map.get_mut(&key) { - value.mark_completed(None, root_event_index); + value.mark_completed( + CompletionReason::ParentCompleted { + parent_step: root_key, + parent_info: info.clone(), + }, + root_event_index, + ); } } } @@ -580,9 +625,13 @@ impl EventStore { info: CompletionInfo, root_event_index: RootEventIndex, ) { + let info = Arc::new(info); if let Some(value) = self.map.get_mut(&root_key) { // Completion status only applies to the root key. - value.mark_completed(Some(info), root_event_index); + value.mark_completed( + CompletionReason::StepCompleted(info.clone()), + root_event_index, + ); } let mut dfs = DfsPostOrder::new( @@ -593,7 +642,27 @@ impl EventStore { if let EventTreeNode::Step(key) = key { if key != root_key { if let Some(value) = self.map.get_mut(&key) { - value.mark_completed(None, root_event_index); + // There's two kinds of nodes reachable from + // EventTreeNode::Root that could be marked as + // completed: subsequent steps within the same + // execution, and steps in child executions. + if key.execution_id == root_key.execution_id { + value.mark_completed( + CompletionReason::SubsequentStarted { + later_step: root_key, + root_total_elapsed: info.root_total_elapsed, + }, + root_event_index, + ); + } else { + value.mark_completed( + CompletionReason::ParentCompleted { + parent_step: root_key, + parent_info: info.clone(), + }, + root_event_index, + ); + } } } } @@ -606,7 +675,8 @@ impl EventStore { info: FailureInfo, root_event_index: RootEventIndex, ) { - self.mark_step_failed_impl(root_key, root_event_index, |value, kind| { + let info = Arc::new(info); + self.mark_step_failed_impl(root_key, |value, kind| { match kind { MarkStepFailedImplKind::Root => { value.mark_failed( @@ -616,11 +686,14 @@ impl EventStore { } MarkStepFailedImplKind::Descendant => { value.mark_failed( - FailureReason::ParentFailed { parent_step: root_key }, + FailureReason::ParentFailed { + parent_step: root_key, + parent_info: info.clone(), + }, root_event_index, ); } - MarkStepFailedImplKind::Future => { + MarkStepFailedImplKind::Subsequent => { value.mark_will_not_be_run( WillNotBeRunReason::PreviousStepFailed { step: root_key, @@ -628,6 +701,15 @@ impl EventStore { root_event_index, ); } + MarkStepFailedImplKind::PreviousCompleted => { + value.mark_completed( + CompletionReason::SubsequentStarted { + later_step: root_key, + root_total_elapsed: info.root_total_elapsed, + }, + root_event_index, + ); + } }; }) } @@ -638,42 +720,48 @@ impl EventStore { info: AbortInfo, root_event_index: RootEventIndex, ) { - self.mark_step_failed_impl( - root_key, - root_event_index, - |value, kind| { - match kind { - MarkStepFailedImplKind::Root => { - value.mark_aborted( - AbortReason::StepAborted(info.clone()), - root_event_index, - ); - } - MarkStepFailedImplKind::Descendant => { - value.mark_aborted( - AbortReason::ParentAborted { - parent_step: root_key, - }, - root_event_index, - ); - } - MarkStepFailedImplKind::Future => { - value.mark_will_not_be_run( - WillNotBeRunReason::PreviousStepAborted { - step: root_key, - }, - root_event_index, - ); - } - }; - }, - ); + let info = Arc::new(info); + self.mark_step_failed_impl(root_key, |value, kind| { + match kind { + MarkStepFailedImplKind::Root => { + value.mark_aborted( + AbortReason::StepAborted(info.clone()), + root_event_index, + ); + } + MarkStepFailedImplKind::Descendant => { + value.mark_aborted( + AbortReason::ParentAborted { + parent_step: root_key, + parent_info: info.clone(), + }, + root_event_index, + ); + } + MarkStepFailedImplKind::Subsequent => { + value.mark_will_not_be_run( + WillNotBeRunReason::PreviousStepAborted { + step: root_key, + }, + root_event_index, + ); + } + MarkStepFailedImplKind::PreviousCompleted => { + value.mark_completed( + CompletionReason::SubsequentStarted { + later_step: root_key, + root_total_elapsed: info.root_total_elapsed, + }, + root_event_index, + ); + } + }; + }); } fn mark_step_failed_impl( &mut self, root_key: StepKey, - root_event_index: RootEventIndex, mut cb: impl FnMut(&mut EventBufferStepData, MarkStepFailedImplKind), ) { if let Some(value) = self.map.get_mut(&root_key) { @@ -686,7 +774,7 @@ impl EventStore { for index in 0..root_key.index { let key = StepKey { execution_id: root_key.execution_id, index }; if let Some(value) = self.map.get_mut(&key) { - value.mark_completed(None, root_event_index); + (cb)(value, MarkStepFailedImplKind::PreviousCompleted); } } @@ -713,7 +801,7 @@ impl EventStore { while let Some(key) = dfs.next(&self.event_tree) { if let EventTreeNode::Step(key) = key { if let Some(value) = self.map.get_mut(&key) { - (cb)(value, MarkStepFailedImplKind::Future); + (cb)(value, MarkStepFailedImplKind::Subsequent); } } } @@ -723,7 +811,8 @@ impl EventStore { enum MarkStepFailedImplKind { Root, Descendant, - Future, + Subsequent, + PreviousCompleted, } /// Actions taken by a recursion step. @@ -741,6 +830,9 @@ struct NewExecutionAction { // An execution ID corresponding to a new run, if seen. execution_id: ExecutionId, + // The parent key for this execution, if this is a nested step. + parent_key: Option, + // The nest level for this execution. nest_level: usize, @@ -802,12 +894,16 @@ impl<'buf, S: StepSpec> EventBufferSteps<'buf, S> { #[derive_where(Clone, Debug)] pub struct EventBufferStepData { step_info: StepInfo, + sort_key: StepSortKey, - // XXX: nest_level and total_steps are common to each execution, but are - // stored separately here. Should we store them in a separate map - // indexed by execution ID? + + // TODO: These steps are common to each execution, but are stored separately + // here. These should likely move into EventBufferExecutionData. + parent_key_and_child_index: Option<(StepKey, usize)>, nest_level: usize, total_steps: usize, + child_executions_seen: usize, + // Invariant: stored in order sorted by leaf event index. high_priority: Vec>, step_status: StepStatus, @@ -819,6 +915,7 @@ pub struct EventBufferStepData { impl EventBufferStepData { fn new( step_info: StepInfo, + parent_key_and_child_index: Option<(StepKey, usize)>, sort_key: StepSortKey, nest_level: usize, total_steps: usize, @@ -826,9 +923,11 @@ impl EventBufferStepData { ) -> Self { Self { step_info, + parent_key_and_child_index, sort_key, nest_level, total_steps, + child_executions_seen: 0, high_priority: Vec::new(), step_status: StepStatus::NotStarted, last_root_event_index: root_event_index, @@ -840,6 +939,11 @@ impl EventBufferStepData { &self.step_info } + #[inline] + pub fn parent_key_and_child_index(&self) -> Option<(StepKey, usize)> { + self.parent_key_and_child_index + } + #[inline] pub fn nest_level(&self) -> usize { self.nest_level @@ -850,6 +954,11 @@ impl EventBufferStepData { self.total_steps } + #[inline] + pub fn child_executions_seen(&self) -> usize { + self.child_executions_seen + } + #[inline] pub fn step_status(&self) -> &StepStatus { &self.step_status @@ -965,12 +1074,12 @@ impl EventBufferStepData { fn mark_completed( &mut self, - status: Option, + reason: CompletionReason, root_event_index: RootEventIndex, ) { match self.step_status { StepStatus::NotStarted | StepStatus::Running { .. } => { - self.step_status = StepStatus::Completed { info: status }; + self.step_status = StepStatus::Completed { reason }; self.update_root_event_index(root_event_index); } StepStatus::Completed { .. } @@ -1011,7 +1120,7 @@ impl EventBufferStepData { match &mut self.step_status { StepStatus::NotStarted => { match reason { - AbortReason::ParentAborted { parent_step } => { + AbortReason::ParentAborted { parent_step, .. } => { // A parent was aborted and this step hasn't been // started. self.step_status = StepStatus::WillNotBeRun { @@ -1116,10 +1225,8 @@ pub enum StepStatus { /// The step has completed execution. Completed { - /// Completion information. - /// - /// This might be unavailable in some cases. - info: Option, + /// The reason for completion. + reason: CompletionReason, }, /// The step has failed. @@ -1179,6 +1286,43 @@ impl StepStatus { } } +#[derive(Clone, Debug)] +pub enum CompletionReason { + /// This step completed. + StepCompleted(Arc), + /// A later step within the same execution was started and we don't have + /// information regarding this step. + SubsequentStarted { + /// The later step that was started. + later_step: StepKey, + + /// The root total elapsed time at the moment the later step was started. + root_total_elapsed: Duration, + }, + /// A parent step within the same execution completed and we don't have + /// information regarding this step. + ParentCompleted { + /// The parent step that completed. + parent_step: StepKey, + + /// Completion info associated with the parent step. + parent_info: Arc, + }, +} + +impl CompletionReason { + /// Returns the [`CompletionInfo`] for this step, if this is the + /// [`Self::StepCompleted`] variant. + pub fn step_completed_info(&self) -> Option<&Arc> { + match self { + Self::StepCompleted(info) => Some(info), + Self::SubsequentStarted { .. } | Self::ParentCompleted { .. } => { + None + } + } + } +} + #[derive(Clone, Debug)] pub struct CompletionInfo { pub attempt: usize, @@ -1192,17 +1336,21 @@ pub struct CompletionInfo { #[derive(Clone, Debug)] pub enum FailureReason { /// This step failed. - StepFailed(FailureInfo), + StepFailed(Arc), /// A parent step failed. ParentFailed { /// The parent step that failed. parent_step: StepKey, + + /// Failure info associated with the parent step. + parent_info: Arc, }, } impl FailureReason { - /// Returns the [`FailureInfo`] if present. - pub fn info(&self) -> Option<&FailureInfo> { + /// Returns the [`FailureInfo`] for this step, if this is the + /// [`Self::StepFailed`] variant. + pub fn step_failed_info(&self) -> Option<&Arc> { match self { Self::StepFailed(info) => Some(info), Self::ParentFailed { .. } => None, @@ -1224,17 +1372,21 @@ pub struct FailureInfo { #[derive(Clone, Debug)] pub enum AbortReason { /// This step was aborted. - StepAborted(AbortInfo), + StepAborted(Arc), /// A parent step was aborted. ParentAborted { /// The parent step key that was aborted. parent_step: StepKey, + + /// Abort info associated with the parent step. + parent_info: Arc, }, } impl AbortReason { - /// Returns the [`AbortInfo`] if present. - pub fn info(&self) -> Option<&AbortInfo> { + /// Returns the [`AbortInfo`] for this step, if this is the + /// [`Self::StepAborted`] variant. + pub fn step_aborted_info(&self) -> Option<&Arc> { match self { Self::StepAborted(info) => Some(info), Self::ParentAborted { .. } => None, @@ -1308,17 +1460,27 @@ impl ExecutionSummary { StepStatus::NotStarted => { // This step hasn't been started yet. Skip over it. } - StepStatus::Running { .. } => { - execution_status = ExecutionStatus::Running { step_key }; - } - StepStatus::Completed { info } => { - let (root_total_elapsed, leaf_total_elapsed) = match info { - Some(info) => ( - Some(info.root_total_elapsed), - Some(info.leaf_total_elapsed), - ), - None => (None, None), + StepStatus::Running { low_priority, progress_event } => { + let root_total_elapsed = low_priority + .iter() + .map(|event| event.total_elapsed) + .chain(std::iter::once(progress_event.total_elapsed)) + .max() + .expect("at least one value was provided"); + execution_status = ExecutionStatus::Running { + step_key, + root_total_elapsed, }; + } + StepStatus::Completed { reason } => { + let (root_total_elapsed, leaf_total_elapsed) = + match reason.step_completed_info() { + Some(info) => ( + Some(info.root_total_elapsed), + Some(info.leaf_total_elapsed), + ), + None => (None, None), + }; let terminal_status = ExecutionTerminalInfo { kind: TerminalKind::Completed, @@ -1331,7 +1493,7 @@ impl ExecutionSummary { } StepStatus::Failed { reason } => { let (root_total_elapsed, leaf_total_elapsed) = - match reason.info() { + match reason.step_failed_info() { Some(info) => ( Some(info.root_total_elapsed), Some(info.leaf_total_elapsed), @@ -1350,7 +1512,7 @@ impl ExecutionSummary { } StepStatus::Aborted { reason, .. } => { let (root_total_elapsed, leaf_total_elapsed) = - match reason.info() { + match reason.step_aborted_info() { Some(info) => ( Some(info.root_total_elapsed), Some(info.leaf_total_elapsed), @@ -1414,6 +1576,9 @@ pub enum ExecutionStatus { /// /// Use [`EventBuffer::get`] to get more information about this step. step_key: StepKey, + + /// The maximum root_total_elapsed seen. + root_total_elapsed: Duration, }, /// Execution has finished. @@ -1462,6 +1627,20 @@ pub enum TerminalKind { Aborted, } +impl ExecutionStatus { + /// Returns the terminal status and the total amount of time elapsed, or + /// None if the execution has not reached a terminal state. + /// + /// The time elapsed might be None if the execution was interrupted and + /// completion information wasn't available. + pub fn terminal_info(&self) -> Option<&ExecutionTerminalInfo> { + match self { + Self::NotStarted | Self::Running { .. } => None, + Self::Terminal(info) => Some(info), + } + } +} + /// Keys for the event tree. #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] enum EventTreeNode { @@ -1500,7 +1679,7 @@ mod tests { use tokio_stream::wrappers::ReceiverStream; use crate::{ - events::{ProgressUnits, StepProgress}, + events::{ProgressCounter, ProgressUnits, StepProgress}, test_utils::TestSpec, StepContext, StepSuccess, UpdateEngine, }; @@ -1735,6 +1914,36 @@ mod tests { } }; + // Ensure that nested step 2 produces progress events in the + // expected order and in succession. + let mut progress_check = NestedProgressCheck::new(); + for event in &generated_events { + if let Event::Progress(event) = event { + let progress_counter = event.kind.progress_counter(); + if progress_counter + == Some(&ProgressCounter::new(2, 3, "steps")) + { + progress_check.two_out_of_three_seen(); + } else if progress_check + == NestedProgressCheck::TwoOutOfThreeSteps + { + assert_eq!( + progress_counter, + Some(&ProgressCounter::current(50, "units")) + ); + progress_check.fifty_units_seen(); + } else if progress_check == NestedProgressCheck::FiftyUnits + { + assert_eq!( + progress_counter, + Some(&ProgressCounter::new(3, 3, "steps")) + ); + progress_check.three_out_of_three_seen(); + } + } + } + progress_check.assert_done(); + // Ensure that events are never seen twice. let mut event_indexes_seen = HashSet::new(); let mut leaf_event_indexes_seen = HashSet::new(); @@ -2241,6 +2450,7 @@ mod tests { 5, "Nested step 2 (fails)", move |cx| async move { + // This is used by NestedProgressCheck below. parent_cx .send_progress(StepProgress::with_current_and_total( 2, @@ -2251,18 +2461,76 @@ mod tests { .await; cx.send_progress(StepProgress::with_current( - 20, + 50, "units", Default::default(), )) .await; + parent_cx + .send_progress(StepProgress::with_current_and_total( + 3, + 3, + "steps", + Default::default(), + )) + .await; + bail!("failing step") }, ) .register(); } + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum NestedProgressCheck { + Initial, + TwoOutOfThreeSteps, + FiftyUnits, + ThreeOutOfThreeSteps, + } + + impl NestedProgressCheck { + fn new() -> Self { + Self::Initial + } + + fn two_out_of_three_seen(&mut self) { + assert_eq!( + *self, + Self::Initial, + "two_out_of_three_seen: expected Initial", + ); + *self = Self::TwoOutOfThreeSteps; + } + + fn fifty_units_seen(&mut self) { + assert_eq!( + *self, + Self::TwoOutOfThreeSteps, + "fifty_units_seen: expected TwoOutOfThreeSteps", + ); + *self = Self::FiftyUnits; + } + + fn three_out_of_three_seen(&mut self) { + assert_eq!( + *self, + Self::FiftyUnits, + "three_out_of_three_seen: expected FiftyUnits", + ); + *self = Self::ThreeOutOfThreeSteps; + } + + fn assert_done(&self) { + assert_eq!( + *self, + Self::ThreeOutOfThreeSteps, + "assert_done: expected ThreeOutOfThreeSteps", + ); + } + } + fn define_remote_nested_engine( engine: &mut UpdateEngine<'_, TestSpec>, start_id: usize, diff --git a/update-engine/src/context.rs b/update-engine/src/context.rs index c2c1e321194..b0666858bc3 100644 --- a/update-engine/src/context.rs +++ b/update-engine/src/context.rs @@ -11,6 +11,7 @@ use std::{collections::HashMap, fmt}; use derive_where::derive_where; use futures::FutureExt; use tokio::sync::{mpsc, oneshot}; +use tokio::time::Instant; use crate::errors::NestedEngineError; use crate::{ @@ -56,10 +57,13 @@ impl StepContext { /// Sends a progress update to the update engine. #[inline] pub async fn send_progress(&self, progress: StepProgress) { + let now = Instant::now(); + let (done, done_rx) = oneshot::channel(); self.payload_sender - .send(StepContextPayload::Progress(progress)) + .send(StepContextPayload::Progress { now, progress, done }) .await - .expect("our code always keeps the receiver open") + .expect("our code always keeps payload_receiver open"); + _ = done_rx.await; } /// Sends a report from a nested engine, typically one running on a remote @@ -71,6 +75,8 @@ impl StepContext { &self, report: EventReport, ) -> Result<(), NestedEngineError> { + let now = Instant::now(); + let mut res = Ok(()); let delta_report = if let Some(id) = report.root_execution_id { let mut nested_buffers = self.nested_buffers.lock().unwrap(); @@ -102,7 +108,7 @@ impl StepContext { component: failed_step.info.component.clone(), id: failed_step.info.id.clone(), description: failed_step.info.description.clone(), - error: NestedError::new( + error: NestedError::from_message_and_causes( message.clone(), causes.clone(), ), @@ -134,17 +140,32 @@ impl StepContext { } self.payload_sender - .send(StepContextPayload::Nested(Event::Step(event))) + .send(StepContextPayload::Nested { + now, + event: Event::Step(event), + }) .await - .expect("our code always keeps the receiver open"); + .expect("our code always keeps payload_receiver open"); } for event in delta_report.progress_events { self.payload_sender - .send(StepContextPayload::Nested(Event::Progress(event))) + .send(StepContextPayload::Nested { + now, + event: Event::Progress(event), + }) .await - .expect("our code always keeps the receiver open"); + .expect("our code always keeps payload_receiver open"); } + + // Ensure that all reports have been received by the engine before + // returning. + let (done, done_rx) = oneshot::channel(); + self.payload_sender + .send(StepContextPayload::Sync { done }) + .await + .expect("our code always keeps payload_receiver open"); + _ = done_rx.await; } res @@ -163,58 +184,75 @@ impl StepContext { F: FnOnce(&mut UpdateEngine<'a, S2>) -> Result<(), S2::Error> + Send, S2: StepSpec + 'a, { - let (sender, mut receiver) = mpsc::channel(128); - let mut engine = UpdateEngine::new(&self.log, sender); + // Previously, this code was of the form: + // + // let (sender, mut receiver) = mpsc::channel(128); + // let mut engine = UpdateEngine::new(&self.log, sender); + // + // And there was a loop below that selected over `engine` and + // `receiver`. + // + // That approach was abandoned because it had ordering issues, because + // it wasn't guaranteed that events were received in the order they were + // processed. For example, consider what happens if: + // + // 1. User code sent an event E1 through a child (nested) StepContext. + // 2. Then in quick succession, the same code sent an event E2 through + // self. + // + // What users would expect to happen is that E1 is received before E2. + // However, what actually happened was that: + // + // 1. `engine` was driven until the next suspend point. This caused E2 + // to be sent. + // 2. Then, `receiver` was polled. This caused E1 to be received. + // + // So the order of events was reversed. + // + // To fix this, we now use a single channel, and send events through it + // both from the nested engine and from self. + // + // An alternative would be to use a oneshot channel as a synchronization + // tool. However, just sharing a channel is easier. + let mut engine = UpdateEngine::::new_nested( + &self.log, + self.payload_sender.clone(), + ); + // Create the engine's steps. (engine_fn)(&mut engine) .map_err(|error| NestedEngineError::Creation { error })?; // Now run the engine. let engine = engine.execute(); - tokio::pin!(engine); - - let mut result = None; - let mut events_done = false; - - loop { - tokio::select! { - ret = &mut engine, if result.is_none() => { - match ret { - Ok(cx) => { - result = Some(Ok(cx)); - } - Err(ExecutionError::EventSendError(_)) => { - unreachable!("we always keep the receiver open") - } - Err(ExecutionError::StepFailed { component, id, description, error }) => { - result = Some(Err(NestedEngineError::StepFailed { component, id, description, error })); - } - Err(ExecutionError::Aborted { component, id, description, message }) => { - result = Some(Err(NestedEngineError::Aborted { component, id, description, message })); - } - } - } - event = receiver.recv(), if !events_done => { - match event { - Some(event) => { - self.payload_sender.send( - StepContextPayload::Nested(event.into_generic()) - ) - .await - .expect("we always keep the receiver open"); - } - None => { - events_done = true; - } - } - } - else => { - break; - } + match engine.await { + Ok(cx) => Ok(cx), + Err(ExecutionError::EventSendError(_)) => { + unreachable!("our code always keeps payload_receiver open") } + Err(ExecutionError::StepFailed { + component, + id, + description, + error, + }) => Err(NestedEngineError::StepFailed { + component, + id, + description, + error, + }), + Err(ExecutionError::Aborted { + component, + id, + description, + message, + }) => Err(NestedEngineError::Aborted { + component, + id, + description, + message, + }), } - - result.expect("the loop only exits if result is set") } /// Retrieves a token used to fetch the value out of a [`StepHandle`]. @@ -247,10 +285,32 @@ impl NestedEventBuffer { } } +/// An uninhabited type for oneshot channels, since we only care about them +/// being dropped. +#[derive(Debug)] +pub(crate) enum Never {} + #[derive_where(Debug)] pub(crate) enum StepContextPayload { - Progress(StepProgress), - Nested(Event), + Progress { + now: Instant, + progress: StepProgress, + done: oneshot::Sender, + }, + /// A single nested event with synchronization. + NestedSingle { + now: Instant, + event: Event, + done: oneshot::Sender, + }, + /// One out of a series of nested events sent in succession. + Nested { + now: Instant, + event: Event, + }, + Sync { + done: oneshot::Sender, + }, } /// Context for a step's metadata-generation function. diff --git a/update-engine/src/display/group_display.rs b/update-engine/src/display/group_display.rs new file mode 100644 index 00000000000..0d50489a9f5 --- /dev/null +++ b/update-engine/src/display/group_display.rs @@ -0,0 +1,512 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +use std::{borrow::Borrow, collections::BTreeMap, fmt, time::Duration}; + +use libsw::TokioSw; +use owo_colors::OwoColorize; +use swrite::{swrite, SWrite}; +use unicode_width::UnicodeWidthStr; + +use crate::{ + errors::UnknownReportKey, events::EventReport, EventBuffer, + ExecutionStatus, ExecutionTerminalInfo, StepSpec, TerminalKind, +}; + +use super::{ + line_display_shared::{LineDisplayFormatter, LineDisplayOutput}, + LineDisplayShared, LineDisplayStyles, HEADER_WIDTH, +}; + +/// A displayer that simultaneously manages and shows line-based output for +/// several event buffers. +/// +/// `K` is the key type for each element in the group. Its [`fmt::Display`] impl +/// is called to obtain the prefix, and `Eq + Ord` is used for keys. +#[derive(Debug)] +pub struct GroupDisplay { + // We don't need to add any buffering here because we already write data to + // the writer in a line-buffered fashion (see Self::write_events). + writer: W, + max_width: usize, + // This is set to the highest value of root_total_elapsed seen from any event reports. + start_sw: TokioSw, + single_states: BTreeMap>, + formatter: LineDisplayFormatter, + stats: GroupDisplayStats, +} + +impl GroupDisplay { + /// Creates a new `GroupDisplay` with the provided report keys and + /// prefixes. + /// + /// The function passed in is expected to create a writer. + pub fn new( + keys_and_prefixes: impl IntoIterator, + writer: W, + ) -> Self + where + Str: Into, + { + // Right-align prefixes to their maximum width -- this helps keep the + // output organized. + let mut max_width = 0; + let keys_and_prefixes: Vec<_> = keys_and_prefixes + .into_iter() + .map(|(k, prefix)| { + let prefix = prefix.into(); + max_width = + max_width.max(UnicodeWidthStr::width(prefix.as_str())); + (k, prefix) + }) + .collect(); + let single_states: BTreeMap<_, _> = keys_and_prefixes + .into_iter() + .map(|(k, prefix)| (k, SingleState::new(prefix, max_width))) + .collect(); + + let not_started = single_states.len(); + Self { + writer, + max_width, + // This creates the stopwatch in the stopped state with duration 0 -- i.e. a minimal + // value that will be replaced as soon as an event comes in. + start_sw: TokioSw::new(), + single_states, + formatter: LineDisplayFormatter::new(), + stats: GroupDisplayStats::new(not_started), + } + } + + /// Creates a new `GroupDisplay` with the provided report keys, using the + /// `Display` impl to obtain the respective prefixes. + pub fn new_with_display( + keys: impl IntoIterator, + writer: W, + ) -> Self + where + K: fmt::Display, + { + Self::new( + keys.into_iter().map(|k| { + let prefix = k.to_string(); + (k, prefix) + }), + writer, + ) + } + + /// Sets the styles for all future lines. + #[inline] + pub fn set_styles(&mut self, styles: LineDisplayStyles) { + self.formatter.set_styles(styles); + } + + /// Sets the amount of time before new progress events are shown. + #[inline] + pub fn set_progress_interval(&mut self, interval: Duration) { + self.formatter.set_progress_interval(interval); + } + + /// Returns true if this `GroupDisplay` is producing reports corresponding + /// to the given key. + pub fn contains_key(&self, key: &Q) -> bool + where + K: Borrow, + Q: Ord, + { + self.single_states.contains_key(key) + } + + /// Adds an event report to the display, keyed by the index, and updates + /// internal state. + /// + /// Returns `Ok(())` if the report was accepted because the key was + /// known to this `GroupDisplay`, and an error if it was not. + pub fn add_event_report( + &mut self, + key: &Q, + event_report: EventReport, + ) -> Result<(), UnknownReportKey> + where + K: Borrow, + Q: Ord, + { + if let Some(state) = self.single_states.get_mut(key) { + let result = state.add_event_report(event_report); + // Set self.start_sw to the max of root_total_elapsed and the current value. + if let Some(root_total_elapsed) = result.root_total_elapsed { + if self.start_sw.elapsed() < root_total_elapsed { + self.start_sw = + TokioSw::with_elapsed_started(root_total_elapsed); + } + } + self.stats.apply_result(result); + Ok(()) + } else { + Err(UnknownReportKey {}) + } + } + + /// Writes a "Status" or "Summary" line to the writer with statistics. + pub fn write_stats(&mut self, header: &str) -> std::io::Result<()> { + // Add a blank prefix which is equal to the maximum width of known prefixes. + let prefix = " ".repeat(self.max_width); + let mut line = + self.formatter.start_line(&prefix, Some(self.start_sw.elapsed())); + self.stats.format_line(&mut line, header, &self.formatter); + writeln!(self.writer, "{line}") + } + + /// Writes all pending events to the writer. + pub fn write_events(&mut self) -> std::io::Result<()> { + let mut out = LineDisplayOutput::new(); + for state in self.single_states.values_mut() { + state.format_events(&self.formatter, &mut out); + } + for line in out.iter() { + writeln!(self.writer, "{line}")?; + } + Ok(()) + } + + /// Returns the current statistics for this `GroupDisplay`. + pub fn stats(&self) -> &GroupDisplayStats { + &self.stats + } +} + +#[derive(Clone, Copy, Debug)] +pub struct GroupDisplayStats { + /// The total number of reports. + pub total: usize, + + /// The number of reports that have not yet started. + pub not_started: usize, + + /// The number of reports that are currently running. + pub running: usize, + + /// The number of reports that indicate successful completion. + pub completed: usize, + + /// The number of reports that indicate failure. + pub failed: usize, + + /// The number of reports that indicate being aborted. + pub aborted: usize, + + /// The number of reports where we didn't receive a final state and it got + /// overwritten by another report. + /// + /// Overwritten reports are considered failures since we don't know what + /// happened. + pub overwritten: usize, +} + +impl GroupDisplayStats { + fn new(total: usize) -> Self { + Self { + total, + not_started: total, + completed: 0, + failed: 0, + aborted: 0, + overwritten: 0, + running: 0, + } + } + + /// Returns the number of terminal reports. + pub fn terminal_count(&self) -> usize { + self.completed + self.failed + self.aborted + self.overwritten + } + + /// Returns true if all reports have reached a terminal state. + pub fn is_terminal(&self) -> bool { + self.not_started == 0 && self.running == 0 + } + + /// Returns true if there are any failures. + pub fn has_failures(&self) -> bool { + self.failed > 0 || self.aborted > 0 || self.overwritten > 0 + } + + fn apply_result(&mut self, result: AddEventReportResult) { + // Process result.after first to avoid integer underflow. + match result.after { + SingleStateTag::NotStarted => self.not_started += 1, + SingleStateTag::Running => self.running += 1, + SingleStateTag::Terminal(TerminalKind::Completed) => { + self.completed += 1 + } + SingleStateTag::Terminal(TerminalKind::Failed) => self.failed += 1, + SingleStateTag::Terminal(TerminalKind::Aborted) => { + self.aborted += 1 + } + SingleStateTag::Overwritten => self.overwritten += 1, + } + + match result.before { + SingleStateTag::NotStarted => self.not_started -= 1, + SingleStateTag::Running => self.running -= 1, + SingleStateTag::Terminal(TerminalKind::Completed) => { + self.completed -= 1 + } + SingleStateTag::Terminal(TerminalKind::Failed) => self.failed -= 1, + SingleStateTag::Terminal(TerminalKind::Aborted) => { + self.aborted -= 1 + } + SingleStateTag::Overwritten => self.overwritten -= 1, + } + } + + fn format_line( + &self, + line: &mut String, + header: &str, + formatter: &LineDisplayFormatter, + ) { + let header_style = if self.has_failures() { + formatter.styles().error_style + } else { + formatter.styles().progress_style + }; + + swrite!(line, "{:>HEADER_WIDTH$} ", header.style(header_style)); + let terminal_count = self.terminal_count(); + swrite!( + line, + "{terminal_count}/{}: {} running, {} {}", + self.total, + self.running.style(formatter.styles().meta_style), + self.completed.style(formatter.styles().meta_style), + "completed".style(formatter.styles().progress_style), + ); + if self.failed > 0 { + swrite!( + line, + ", {} {}", + self.failed.style(formatter.styles().meta_style), + "failed".style(formatter.styles().error_style), + ); + } + if self.aborted > 0 { + swrite!( + line, + ", {} {}", + self.aborted.style(formatter.styles().meta_style), + "aborted".style(formatter.styles().error_style), + ); + } + if self.overwritten > 0 { + swrite!( + line, + ", {} {}", + self.overwritten.style(formatter.styles().meta_style), + "overwritten".style(formatter.styles().error_style), + ); + } + } +} + +#[derive(Debug)] +struct SingleState { + shared: LineDisplayShared, + kind: SingleStateKind, + prefix: String, +} + +impl SingleState { + fn new(prefix: String, max_width: usize) -> Self { + // Right-align the prefix to the maximum width. + let prefix = format!("{:>max_width$}", prefix); + Self { + shared: LineDisplayShared::default(), + kind: SingleStateKind::NotStarted { displayed: false }, + prefix, + } + } + + /// Adds an event report and updates the internal state. + fn add_event_report( + &mut self, + event_report: EventReport, + ) -> AddEventReportResult { + let before = match &self.kind { + SingleStateKind::NotStarted { .. } => { + self.kind = SingleStateKind::Running { + event_buffer: EventBuffer::new(8), + }; + SingleStateTag::NotStarted + } + SingleStateKind::Running { .. } => SingleStateTag::Running, + + SingleStateKind::Terminal { info, .. } => { + // Once we've reached a terminal state, we don't record any more + // events. + return AddEventReportResult::unchanged( + SingleStateTag::Terminal(info.kind), + info.root_total_elapsed, + ); + } + SingleStateKind::Overwritten { .. } => { + // This update has already completed -- assume that the event + // buffer is for a new update, which we don't show. + return AddEventReportResult::unchanged( + SingleStateTag::Overwritten, + None, + ); + } + }; + + let SingleStateKind::Running { event_buffer } = &mut self.kind else { + unreachable!("other branches were handled above"); + }; + + if let Some(root_execution_id) = event_buffer.root_execution_id() { + if event_report.root_execution_id != Some(root_execution_id) { + // The report is for a different execution ID -- assume that + // this event is completed and mark our current execution as + // completed. + self.kind = SingleStateKind::Overwritten { displayed: false }; + return AddEventReportResult { + before, + after: SingleStateTag::Overwritten, + root_total_elapsed: None, + }; + } + } + + event_buffer.add_event_report(event_report); + let (after, max_total_elapsed) = + match event_buffer.root_execution_summary() { + Some(summary) => { + match summary.execution_status { + ExecutionStatus::NotStarted => { + (SingleStateTag::NotStarted, None) + } + ExecutionStatus::Running { + root_total_elapsed: max_total_elapsed, + .. + } => (SingleStateTag::Running, Some(max_total_elapsed)), + ExecutionStatus::Terminal(info) => { + // Grab the event buffer to store it in the terminal state. + let event_buffer = std::mem::replace( + event_buffer, + EventBuffer::new(0), + ); + let terminal_kind = info.kind; + let root_total_elapsed = info.root_total_elapsed; + self.kind = SingleStateKind::Terminal { + info, + pending_event_buffer: Some(event_buffer), + }; + ( + SingleStateTag::Terminal(terminal_kind), + root_total_elapsed, + ) + } + } + } + None => { + // We don't have a summary yet. + (SingleStateTag::NotStarted, None) + } + }; + + AddEventReportResult { + before, + after, + root_total_elapsed: max_total_elapsed, + } + } + + pub(super) fn format_events( + &mut self, + formatter: &LineDisplayFormatter, + out: &mut LineDisplayOutput, + ) { + let mut cx = self.shared.with_context(&self.prefix, formatter); + match &mut self.kind { + SingleStateKind::NotStarted { displayed } => { + if !*displayed { + let line = + cx.format_generic("Update not started, waiting..."); + out.add_line(line); + *displayed = true; + } + } + SingleStateKind::Running { event_buffer } => { + cx.format_event_buffer(event_buffer, out); + } + SingleStateKind::Terminal { info, pending_event_buffer } => { + // Are any remaining events left? This also sets pending_event_buffer + // to None after displaying remaining events. + if let Some(event_buffer) = pending_event_buffer.take() { + cx.format_event_buffer(&event_buffer, out); + // Also show a line to wrap up the terminal status. + let line = cx.format_terminal_info(info); + out.add_line(line); + } + + // Nothing to do, the terminal status was already printed above. + } + SingleStateKind::Overwritten { displayed } => { + if !*displayed { + let line = cx.format_generic( + "Update overwritten (a different update was started): \ + assuming failure", + ); + out.add_line(line); + *displayed = true; + } + } + } + } +} + +#[derive(Debug)] +enum SingleStateKind { + NotStarted { + displayed: bool, + }, + Running { + event_buffer: EventBuffer, + }, + Terminal { + info: ExecutionTerminalInfo, + // The event buffer is kept around so that we can display any remaining + // lines. + pending_event_buffer: Option>, + }, + Overwritten { + displayed: bool, + }, +} + +struct AddEventReportResult { + before: SingleStateTag, + after: SingleStateTag, + root_total_elapsed: Option, +} + +impl AddEventReportResult { + fn unchanged( + tag: SingleStateTag, + root_total_elapsed: Option, + ) -> Self { + Self { before: tag, after: tag, root_total_elapsed } + } +} + +#[derive(Copy, Clone, Debug)] +enum SingleStateTag { + NotStarted, + Running, + Terminal(TerminalKind), + Overwritten, +} diff --git a/update-engine/src/display/line_display.rs b/update-engine/src/display/line_display.rs new file mode 100644 index 00000000000..5321ec017c5 --- /dev/null +++ b/update-engine/src/display/line_display.rs @@ -0,0 +1,137 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +use debug_ignore::DebugIgnore; +use derive_where::derive_where; +use owo_colors::Style; +use std::time::Duration; + +use crate::{EventBuffer, ExecutionTerminalInfo, StepSpec}; + +use super::{ + line_display_shared::LineDisplayOutput, LineDisplayFormatter, + LineDisplayShared, +}; + +/// A line-oriented display. +/// +/// This display produces output to the provided writer. +#[derive_where(Debug)] +pub struct LineDisplay { + writer: DebugIgnore, + shared: LineDisplayShared, + formatter: LineDisplayFormatter, + prefix: String, +} + +impl LineDisplay { + /// Creates a new LineDisplay. + pub fn new(writer: W) -> Self { + Self { + writer: DebugIgnore(writer), + shared: LineDisplayShared::default(), + formatter: LineDisplayFormatter::new(), + prefix: String::new(), + } + } + + /// Sets the prefix for all future lines. + #[inline] + pub fn set_prefix(&mut self, prefix: impl Into) { + self.prefix = prefix.into(); + } + + /// Sets the styles for all future lines. + #[inline] + pub fn set_styles(&mut self, styles: LineDisplayStyles) { + self.formatter.set_styles(styles); + } + + /// Sets the amount of time before the next progress event is shown. + #[inline] + pub fn set_progress_interval(&mut self, interval: Duration) { + self.formatter.set_progress_interval(interval); + } + + /// Writes an event buffer to the writer, incrementally. + /// + /// This is a stateful method that will only display events that have not + /// been displayed before. + pub fn write_event_buffer( + &mut self, + buffer: &EventBuffer, + ) -> std::io::Result<()> { + let mut out = LineDisplayOutput::new(); + self.shared + .with_context(&self.prefix, &self.formatter) + .format_event_buffer(buffer, &mut out); + for line in out.iter() { + writeln!(self.writer, "{line}")?; + } + + Ok(()) + } + + /// Writes terminal information to the writer. + pub fn write_terminal_info( + &mut self, + info: &ExecutionTerminalInfo, + ) -> std::io::Result<()> { + let line = self + .shared + .with_context(&self.prefix, &self.formatter) + .format_terminal_info(info); + writeln!(self.writer, "{line}") + } + + /// Writes a generic line to the writer, with prefix attached if provided. + pub fn write_generic(&mut self, message: &str) -> std::io::Result<()> { + let line = self + .shared + .with_context(&self.prefix, &self.formatter) + .format_generic(message); + writeln!(self.writer, "{line}") + } +} + +/// Styles for [`LineDisplay`]. +/// +/// By default this isn't colorized, but it can be if so chosen. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct LineDisplayStyles { + pub prefix_style: Style, + pub meta_style: Style, + pub step_name_style: Style, + pub progress_style: Style, + pub progress_message_style: Style, + pub warning_style: Style, + pub warning_message_style: Style, + pub error_style: Style, + pub error_message_style: Style, + pub skipped_style: Style, + pub retry_style: Style, +} + +impl LineDisplayStyles { + /// Returns a default set of colorized styles with ANSI colors. + pub fn colorized() -> Self { + let mut ret = Self::default(); + ret.prefix_style = Style::new().bold(); + ret.meta_style = Style::new().bold(); + ret.step_name_style = Style::new().cyan(); + ret.progress_style = Style::new().bold().green(); + ret.progress_message_style = Style::new().green(); + ret.warning_style = Style::new().bold().yellow(); + ret.warning_message_style = Style::new().yellow(); + ret.error_style = Style::new().bold().red(); + ret.error_message_style = Style::new().red(); + ret.skipped_style = Style::new().bold().yellow(); + ret.retry_style = Style::new().bold().yellow(); + + ret + } +} diff --git a/update-engine/src/display/line_display_shared.rs b/update-engine/src/display/line_display_shared.rs new file mode 100644 index 00000000000..99b03b13f76 --- /dev/null +++ b/update-engine/src/display/line_display_shared.rs @@ -0,0 +1,1011 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +//! Types and code shared between `LineDisplay` and `GroupDisplay`. + +use std::{ + collections::HashMap, + fmt::{self, Write as _}, + time::Duration, +}; + +use owo_colors::OwoColorize; +use swrite::{swrite, SWrite as _}; + +use crate::{ + events::{ + ProgressCounter, ProgressEvent, ProgressEventKind, StepEvent, + StepEventKind, StepInfo, StepOutcome, + }, + EventBuffer, ExecutionId, ExecutionTerminalInfo, StepKey, StepSpec, + TerminalKind, +}; + +use super::LineDisplayStyles; + +// This is chosen to leave enough room for all possible headers: "Completed" at +// 9 characters is the longest. +pub(super) const HEADER_WIDTH: usize = 9; + +#[derive(Debug, Default)] +pub(super) struct LineDisplayShared { + // This is a map from root execution ID to data about it. + execution_data: HashMap, +} + +impl LineDisplayShared { + pub(super) fn with_context<'a>( + &'a mut self, + prefix: &'a str, + formatter: &'a LineDisplayFormatter, + ) -> LineDisplaySharedContext<'a> { + LineDisplaySharedContext { shared: self, prefix, formatter } + } +} + +#[derive(Debug)] +pub(super) struct LineDisplaySharedContext<'a> { + shared: &'a mut LineDisplayShared, + prefix: &'a str, + formatter: &'a LineDisplayFormatter, +} + +impl<'a> LineDisplaySharedContext<'a> { + /// Produces a generic line from the prefix and message. + /// + /// This line does not have a trailing newline; adding one is the caller's + /// responsibility. + pub(super) fn format_generic(&self, message: &str) -> String { + let mut line = self.formatter.start_line(self.prefix, None); + line.push_str(message); + line + } + + /// Produces lines for this event buffer, and advances internal state. + /// + /// Returned lines do not have a trailing newline; adding them is the + /// caller's responsibility. + pub(super) fn format_event_buffer( + &mut self, + buffer: &EventBuffer, + out: &mut LineDisplayOutput, + ) { + let Some(execution_id) = buffer.root_execution_id() else { + // No known events, so nothing to display. + return; + }; + let execution_data = + self.shared.execution_data.entry(execution_id).or_default(); + let prev_progress_event_at = execution_data.last_progress_event_at; + let mut current_progress_event_at = prev_progress_event_at; + + let report = + buffer.generate_report_since(&mut execution_data.last_seen); + + for event in &report.step_events { + self.format_step_event(buffer, event, out); + } + + // Update progress events. + for event in &report.progress_events { + if Some(event.total_elapsed) > prev_progress_event_at { + self.format_progress_event(buffer, event, out); + current_progress_event_at = + current_progress_event_at.max(Some(event.total_elapsed)); + } + } + + // Finally, write to last_progress_event_at. (Need to re-fetch execution data.) + let execution_data = self + .shared + .execution_data + .get_mut(&execution_id) + .expect("we created this execution data above"); + execution_data.last_progress_event_at = current_progress_event_at; + } + + /// Format this step event. + fn format_step_event( + &self, + buffer: &EventBuffer, + step_event: &StepEvent, + out: &mut LineDisplayOutput, + ) { + self.format_step_event_impl( + buffer, + step_event, + Default::default(), + step_event.total_elapsed, + out, + ); + } + + fn format_step_event_impl( + &self, + buffer: &EventBuffer, + step_event: &StepEvent, + mut nest_data: NestData, + root_total_elapsed: Duration, + out: &mut LineDisplayOutput, + ) { + match &step_event.kind { + StepEventKind::NoStepsDefined => { + let mut line = self + .formatter + .start_line(self.prefix, Some(step_event.total_elapsed)); + swrite!( + line, + "{}", + "No steps defined" + .style(self.formatter.styles.progress_style), + ); + out.add_line(line); + } + StepEventKind::ExecutionStarted { first_step, .. } => { + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &first_step.info, + &nest_data, + ); + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Running".style(self.formatter.styles.progress_style), + ); + self.formatter.add_step_info(&mut line, ld_step_info); + out.add_line(line); + } + StepEventKind::AttemptRetry { + step, + next_attempt, + attempt_elapsed, + message, + .. + } => { + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &step.info, + &nest_data, + ); + + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Retry".style(self.formatter.styles.warning_style) + ); + self.formatter.add_step_info(&mut line, ld_step_info); + swrite!( + line, + ": after {:.2?}", + attempt_elapsed.style(self.formatter.styles.meta_style), + ); + if *next_attempt > 1 { + swrite!( + line, + " (at attempt {})", + next_attempt + .saturating_sub(1) + .style(self.formatter.styles.meta_style), + ); + } + swrite!( + line, + " with message: {}", + message.style(self.formatter.styles.warning_message_style) + ); + + out.add_line(line); + } + StepEventKind::ProgressReset { + step, + attempt, + attempt_elapsed, + message, + .. + } => { + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &step.info, + &nest_data, + ); + + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Reset".style(self.formatter.styles.warning_style) + ); + self.formatter.add_step_info(&mut line, ld_step_info); + swrite!( + line, + ": after {:.2?}", + attempt_elapsed.style(self.formatter.styles.meta_style), + ); + if *attempt > 1 { + swrite!( + line, + " (at attempt {})", + attempt.style(self.formatter.styles.meta_style), + ); + } + swrite!( + line, + " with message: {}", + message.style(self.formatter.styles.warning_message_style) + ); + + out.add_line(line); + } + StepEventKind::StepCompleted { + step, + attempt, + outcome, + next_step, + attempt_elapsed, + .. + } => { + // --- Add completion info about this step. + + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &step.info, + &nest_data, + ); + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + self.formatter.add_completion_and_step_info( + &mut line, + ld_step_info, + *attempt_elapsed, + *attempt, + outcome, + ); + + out.add_line(line); + + // --- Add information about the next step. + + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &next_step.info, + &nest_data, + ); + + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + self.format_step_running(&mut line, ld_step_info); + + out.add_line(line); + } + StepEventKind::ExecutionCompleted { + last_step, + last_attempt, + last_outcome, + attempt_elapsed, + .. + } => { + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &last_step.info, + &nest_data, + ); + + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + self.formatter.add_completion_and_step_info( + &mut line, + ld_step_info, + *attempt_elapsed, + *last_attempt, + last_outcome, + ); + + out.add_line(line); + } + StepEventKind::ExecutionFailed { + failed_step, + total_attempts, + attempt_elapsed, + message, + causes, + .. + } => { + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &failed_step.info, + &nest_data, + ); + + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + // The prefix is used for "Caused by" lines below. Add + // the requisite amount of spacing here. + let mut caused_by_prefix = line.clone(); + swrite!(caused_by_prefix, "{:>HEADER_WIDTH$} ", ""); + nest_data.add_prefix(&mut caused_by_prefix); + + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Failed".style(self.formatter.styles.error_style) + ); + + self.formatter.add_step_info(&mut line, ld_step_info); + line.push_str(": "); + + self.formatter.add_failure_info( + &mut line, + &caused_by_prefix, + *attempt_elapsed, + *total_attempts, + message, + causes, + ); + + out.add_line(line); + } + StepEventKind::ExecutionAborted { + aborted_step, + attempt, + attempt_elapsed, + message, + .. + } => { + let ld_step_info = LineDisplayStepInfo::new( + buffer, + step_event.execution_id, + &aborted_step.info, + &nest_data, + ); + + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Aborted".style(self.formatter.styles.error_style) + ); + self.formatter.add_step_info(&mut line, ld_step_info); + line.push_str(": "); + + self.formatter.add_abort_info( + &mut line, + *attempt_elapsed, + *attempt, + message, + ); + + out.add_line(line); + } + StepEventKind::Nested { step, event, .. } => { + // Look up the child event's ID to add to the nest data. + let child_step_key = StepKey { + execution_id: event.execution_id, + // XXX: we currently look up index 0 because that should + // always exist (unless no steps are defined, in which case + // we skip this). The child index is actually shared by all + // steps within an execution. Fix this by changing + // EventBuffer to also track general per-execution data. + index: 0, + }; + let Some(child_step_data) = buffer.get(&child_step_key) else { + // This should only happen if no steps are defined. See TODO + // above. + return; + }; + let (_, child_index) = child_step_data + .parent_key_and_child_index() + .expect("child steps should have a child index"); + + nest_data.add_nest_level(step.info.index, child_index); + + self.format_step_event_impl( + buffer, + &**event, + nest_data, + root_total_elapsed, + out, + ); + } + StepEventKind::Unknown => {} + } + } + + fn format_step_running( + &self, + line: &mut String, + ld_step_info: LineDisplayStepInfo<'_, S>, + ) { + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Running".style(self.formatter.styles.progress_style), + ); + self.formatter.add_step_info(line, ld_step_info); + } + + /// Formats this terminal information. + /// + /// This line does not have a trailing newline; adding one is the caller's + /// responsibility. + pub(super) fn format_terminal_info( + &self, + info: &ExecutionTerminalInfo, + ) -> String { + let mut line = + self.formatter.start_line(self.prefix, info.leaf_total_elapsed); + match info.kind { + TerminalKind::Completed => { + swrite!( + line, + "{:>HEADER_WIDTH$} Execution {}", + "Terminal".style(self.formatter.styles.progress_style), + "completed".style(self.formatter.styles.progress_style), + ); + } + TerminalKind::Failed => { + swrite!( + line, + "{:>HEADER_WIDTH$} Execution {}", + "Terminal".style(self.formatter.styles.error_style), + "failed".style(self.formatter.styles.error_style), + ); + } + TerminalKind::Aborted => { + swrite!( + line, + "{:>HEADER_WIDTH$} Execution {}", + "Terminal".style(self.formatter.styles.error_style), + "aborted".style(self.formatter.styles.error_style), + ); + } + } + line + } + + fn format_progress_event( + &self, + buffer: &EventBuffer, + progress_event: &ProgressEvent, + out: &mut LineDisplayOutput, + ) { + self.format_progress_event_impl( + buffer, + progress_event, + NestData::default(), + progress_event.total_elapsed, + out, + ) + } + + fn format_progress_event_impl( + &self, + buffer: &EventBuffer, + progress_event: &ProgressEvent, + mut nest_data: NestData, + root_total_elapsed: Duration, + out: &mut LineDisplayOutput, + ) { + match &progress_event.kind { + ProgressEventKind::WaitingForProgress { .. } => { + // Don't need to show this because "Running" is shown within + // step events. + } + ProgressEventKind::Progress { + step, + progress, + attempt_elapsed, + .. + } => { + let step_key = StepKey { + execution_id: progress_event.execution_id, + index: step.info.index, + }; + let step_data = + buffer.get(&step_key).expect("step key must exist"); + let ld_step_info = LineDisplayStepInfo { + step_info: &step.info, + total_steps: step_data.total_steps(), + nest_data: &nest_data, + }; + + let mut line = self + .formatter + .start_line(self.prefix, Some(root_total_elapsed)); + + let (before, after) = match progress { + Some(counter) => { + let progress_str = format_progress_counter(counter); + ( + format!( + "{:>HEADER_WIDTH$} ", + "Progress".style( + self.formatter.styles.progress_style + ) + ), + format!( + "{progress_str} after {:.2?}", + attempt_elapsed + .style(self.formatter.styles.meta_style), + ), + ) + } + None => { + let before = format!( + "{:>HEADER_WIDTH$} ", + "Running" + .style(self.formatter.styles.progress_style), + ); + + // If the attempt elapsed is non-zero, show it. + let after = if *attempt_elapsed > Duration::ZERO { + format!( + "after {:.2?}", + attempt_elapsed + .style(self.formatter.styles.meta_style), + ) + } else { + String::new() + }; + + (before, after) + } + }; + + swrite!(line, "{}", before); + self.formatter.add_step_info(&mut line, ld_step_info); + if !after.is_empty() { + swrite!(line, ": {}", after); + } + + out.add_line(line); + } + ProgressEventKind::Nested { step, event, .. } => { + // Look up the child event's ID to add to the nest data. + let child_step_key = StepKey { + execution_id: event.execution_id, + // XXX: we currently look up index 0 because that should + // always exist (unless no steps are defined, in which case + // we skip this). The child index is actually shared by all + // steps within an execution. Fix this by changing + // EventBuffer to also track general per-execution data. + index: 0, + }; + let Some(child_step_data) = buffer.get(&child_step_key) else { + // This should only happen if no steps are defined. See TODO + // above. + return; + }; + let (_, child_index) = child_step_data + .parent_key_and_child_index() + .expect("child steps should have a child index"); + + nest_data.add_nest_level(step.info.index, child_index); + + self.format_progress_event_impl( + buffer, + &**event, + nest_data, + root_total_elapsed, + out, + ); + } + ProgressEventKind::Unknown => {} + } + } +} + +fn format_progress_counter(counter: &ProgressCounter) -> String { + match counter.total { + Some(total) => { + // Show a percentage value. Correct alignment requires converting to + // a string in the middle like this. + let percent = (counter.current as f64 / total as f64) * 100.0; + // <12.34> is 5 characters wide. + let percent_width = 5; + let counter_width = total.to_string().len(); + format!( + "{:>percent_width$.2}% ({:>counter_width$}/{} {})", + percent, counter.current, total, counter.units, + ) + } + None => format!("{} {}", counter.current, counter.units), + } +} + +/// State that tracks line display formatting. +/// +/// Each `LineDisplay` and `GroupDisplay` has one of these. +#[derive(Debug)] +pub(super) struct LineDisplayFormatter { + styles: LineDisplayStyles, + progress_interval: Duration, +} + +impl LineDisplayFormatter { + pub(super) fn new() -> Self { + Self { + styles: LineDisplayStyles::default(), + progress_interval: Duration::from_secs(1), + } + } + + #[inline] + pub(super) fn styles(&self) -> &LineDisplayStyles { + &self.styles + } + + #[inline] + pub(super) fn set_styles(&mut self, styles: LineDisplayStyles) { + self.styles = styles; + } + + #[inline] + pub(super) fn set_progress_interval(&mut self, interval: Duration) { + self.progress_interval = interval; + } + + // --- + // Internal helpers + // --- + + pub(super) fn start_line( + &self, + prefix: &str, + total_elapsed: Option, + ) -> String { + let mut line = format!("[{}", prefix.style(self.styles.prefix_style)); + + if !prefix.is_empty() { + line.push(' '); + } + + // Show total elapsed time in an hh:mm:ss format. + if let Some(total_elapsed) = total_elapsed { + let total_elapsed_secs = total_elapsed.as_secs(); + let hours = total_elapsed_secs / 3600; + let minutes = (total_elapsed_secs % 3600) / 60; + let seconds = total_elapsed_secs % 60; + swrite!(line, "{:02}:{:02}:{:02}", hours, minutes, seconds); + // To show total_elapsed more accurately, use: + // swrite!(line, "{:.2?}", total_elapsed); + } else { + // Add 8 spaces to align with hh:mm:ss. + line.push_str(" "); + } + + line.push_str("] "); + + line + } + + fn add_step_info( + &self, + line: &mut String, + ld_step_info: LineDisplayStepInfo<'_, S>, + ) { + ld_step_info.nest_data.add_prefix(line); + + // Print out "/)". Leave space such that we + // print out e.g. "1/8)" and " 3/14)". + // Add 1 to the index to make it 1-based. + let step_index = ld_step_info.step_info.index + 1; + let step_index_width = ld_step_info.total_steps.to_string().len(); + swrite!( + line, + "{:width$}/{:width$}) ", + step_index, + ld_step_info.total_steps, + width = step_index_width + ); + + swrite!( + line, + "{}", + ld_step_info + .step_info + .description + .style(self.styles.step_name_style) + ); + } + + pub(super) fn add_completion_and_step_info( + &self, + line: &mut String, + ld_step_info: LineDisplayStepInfo<'_, S>, + attempt_elapsed: Duration, + attempt: usize, + outcome: &StepOutcome, + ) { + let mut meta = format!( + "after {:.2?}", + attempt_elapsed.style(self.styles.meta_style) + ); + if attempt > 1 { + swrite!( + meta, + " (at attempt {})", + attempt.style(self.styles.meta_style) + ); + } + + match &outcome { + StepOutcome::Success { message, .. } => { + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Completed".style(self.styles.progress_style), + ); + self.add_step_info(line, ld_step_info); + match message { + Some(message) => { + swrite!( + line, + ": {meta} with message: {}", + message.style(self.styles.progress_message_style) + ); + } + None => { + swrite!(line, ": {meta}"); + } + } + } + StepOutcome::Warning { message, .. } => { + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Completed".style(self.styles.warning_style), + ); + self.add_step_info(line, ld_step_info); + swrite!( + line, + ": {meta} with warning: {}", + message.style(self.styles.warning_message_style) + ); + } + StepOutcome::Skipped { message, .. } => { + swrite!( + line, + "{:>HEADER_WIDTH$} ", + "Skipped".style(self.styles.skipped_style), + ); + self.add_step_info(line, ld_step_info); + swrite!( + line, + ": {}", + message.style(self.styles.warning_message_style) + ); + } + }; + } + + pub(super) fn add_failure_info( + &self, + line: &mut String, + line_prefix: &str, + attempt_elapsed: Duration, + total_attempts: usize, + message: &str, + causes: &[String], + ) { + let mut meta = format!( + "after {:.2?}", + attempt_elapsed.style(self.styles.meta_style) + ); + if total_attempts > 1 { + swrite!( + meta, + " (after {} attempts)", + total_attempts.style(self.styles.meta_style) + ); + } + + swrite!( + line, + "{meta}: {}", + message.style(self.styles.error_message_style) + ); + if !causes.is_empty() { + swrite!( + line, + "\n{line_prefix}{}", + " Caused by:".style(self.styles.meta_style) + ); + for cause in causes { + swrite!(line, "\n{line_prefix} - {}", cause); + } + } + + // The last newline is added by the caller. + } + + pub(super) fn add_abort_info( + &self, + line: &mut String, + attempt_elapsed: Duration, + attempt: usize, + message: &str, + ) { + let mut meta = format!( + "after {:.2?}", + attempt_elapsed.style(self.styles.meta_style) + ); + if attempt > 1 { + swrite!( + meta, + " (at attempt {})", + attempt.style(self.styles.meta_style) + ); + } + + swrite!(line, "{meta} with message \"{}\"", message); + } +} + +#[derive(Clone, Debug)] +pub(super) struct LineDisplayOutput { + lines: Vec, +} + +impl LineDisplayOutput { + pub(super) fn new() -> Self { + Self { lines: Vec::new() } + } + + pub(super) fn add_line(&mut self, line: String) { + self.lines.push(line); + } + + pub(super) fn iter(&self) -> impl Iterator { + self.lines.iter().map(|line| line.as_str()) + } +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct LineDisplayStepInfo<'a, S: StepSpec> { + pub(super) step_info: &'a StepInfo, + pub(super) total_steps: usize, + pub(super) nest_data: &'a NestData, +} + +impl<'a, S: StepSpec> LineDisplayStepInfo<'a, S> { + fn new( + buffer: &'a EventBuffer, + execution_id: ExecutionId, + step_info: &'a StepInfo, + nest_data: &'a NestData, + ) -> Self { + let step_key = StepKey { execution_id, index: step_info.index }; + let step_data = buffer.get(&step_key).expect("step key must exist"); + LineDisplayStepInfo { + step_info, + total_steps: step_data.total_steps(), + nest_data, + } + } +} + +/// Per-step stateful data tracked by the line displayer. +#[derive(Debug, Default)] +struct ExecutionData { + /// The last seen root event index. + /// + /// This is used to avoid displaying the same event twice. + last_seen: Option, + + /// The last `root_total_elapsed` at which a progress event was displayed for + /// this execution. + last_progress_event_at: Option, +} + +#[derive(Clone, Debug, Default)] +pub(super) struct NestData { + nest_indexes: Vec, +} + +impl NestData { + fn add_nest_level(&mut self, parent_step_index: usize, child_index: usize) { + self.nest_indexes.push(NestIndex { parent_step_index, child_index }); + } + + fn add_prefix(&self, line: &mut String) { + if !self.nest_indexes.is_empty() { + line.push_str(&"..".repeat(self.nest_indexes.len())); + line.push_str(" "); + } + + for nest_index in &self.nest_indexes { + swrite!( + line, + "{}{} ", + // Add 1 to the index to make it 1-based. + nest_index.parent_step_index + 1, + AsLetters(nest_index.child_index) + ); + } + } +} + +#[derive(Clone, Debug)] +struct NestIndex { + parent_step_index: usize, + // If a parent has multiple nested executions, this counts which execution + // this is, up from 0. + child_index: usize, +} + +/// A display impl that converts a 0-based index into a letter or a series of +/// letters. +/// +/// This is effectively a conversion to base 26. +struct AsLetters(usize); + +impl fmt::Display for AsLetters { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut index = self.0; + loop { + let letter = (b'a' + (index % 26) as u8) as char; + f.write_char(letter)?; + index /= 26; + if index == 0 { + break; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_progress_counter() { + let tests = vec![ + (ProgressCounter::new(5, 20, "units"), "25.00% ( 5/20 units)"), + (ProgressCounter::new(0, 20, "bytes"), " 0.00% ( 0/20 bytes)"), + (ProgressCounter::new(20, 20, "cubes"), "100.00% (20/20 cubes)"), + // NaN is a weird case that is a buggy update engine impl in practice + (ProgressCounter::new(0, 0, "units"), " NaN% (0/0 units)"), + (ProgressCounter::current(5, "units"), "5 units"), + ]; + for (input, output) in tests { + assert_eq!( + format_progress_counter(&input), + output, + "format matches for input: {:?}", + input + ); + } + } +} diff --git a/update-engine/src/display/mod.rs b/update-engine/src/display/mod.rs new file mode 100644 index 00000000000..c58a4535a08 --- /dev/null +++ b/update-engine/src/display/mod.rs @@ -0,0 +1,21 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +//! Displayers for the update engine. +//! +//! Currently implemented are: +//! +//! * [`LineDisplay`]: a line-oriented display suitable for the command line. +//! * [`GroupDisplay`]: manages state and shows the results of several +//! [`LineDisplay`]s at once. + +mod group_display; +mod line_display; +mod line_display_shared; + +pub use group_display::GroupDisplay; +pub use line_display::{LineDisplay, LineDisplayStyles}; +use line_display_shared::*; diff --git a/update-engine/src/engine.rs b/update-engine/src/engine.rs index 24f055858c5..6d59a82221e 100644 --- a/update-engine/src/engine.rs +++ b/update-engine/src/engine.rs @@ -5,7 +5,12 @@ // Copyright 2023 Oxide Computer Company use std::{ - borrow::Cow, fmt, ops::ControlFlow, pin::Pin, sync::Mutex, task::Poll, + borrow::Cow, + fmt, + ops::ControlFlow, + pin::Pin, + sync::{Arc, Mutex}, + task::Poll, }; use cancel_safe_futures::coop_cancel; @@ -28,7 +33,7 @@ use crate::{ StepEvent, StepEventKind, StepInfo, StepInfoWithMetadata, StepOutcome, StepProgress, }, - AsError, CompletionContext, MetadataContext, StepContext, + AsError, CompletionContext, MetadataContext, NestedSpec, StepContext, StepContextPayload, StepHandle, StepSpec, }; @@ -64,7 +69,7 @@ pub struct UpdateEngine<'a, S: StepSpec> { // be a graph in the future. log: slog::Logger, execution_id: ExecutionId, - sender: mpsc::Sender>, + sender: EngineSender, // This is set to None in Self::execute. canceler: Option>, @@ -82,6 +87,21 @@ pub struct UpdateEngine<'a, S: StepSpec> { impl<'a, S: StepSpec + 'a> UpdateEngine<'a, S> { /// Creates a new `UpdateEngine`. pub fn new(log: &slog::Logger, sender: mpsc::Sender>) -> Self { + let sender = Arc::new(DefaultSender { sender }); + Self::new_impl(log, EngineSender { sender }) + } + + // See the comment on `StepContext::with_nested_engine` for why this is + // necessary.`` + pub(crate) fn new_nested( + log: &slog::Logger, + sender: mpsc::Sender>, + ) -> Self { + let sender = Arc::new(NestedSender { sender }); + Self::new_impl(log, EngineSender { sender }) + } + + fn new_impl(log: &slog::Logger, sender: EngineSender) -> Self { let execution_id = ExecutionId(Uuid::new_v4()); let (canceler, cancel_receiver) = coop_cancel::new_pair(); Self { @@ -303,6 +323,91 @@ impl<'a, S: StepSpec + 'a> UpdateEngine<'a, S> { } } +/// Abstraction used to send events to whatever receiver is interested in them. +/// +/// # Why is this type so weird? +/// +/// `EngineSender` is a wrapper around a cloneable trait object. Why do we need +/// that? +/// +/// `SenderImpl` has two implementations: +/// +/// 1. `DefaultSender`, which is a wrapper around an `mpsc::Sender>`. +/// This is used when the receiver is user code. +/// 2. `NestedSender`, which is a more complex wrapper around an +/// `mpsc::Sender>`. +/// +/// You might imagine that we could just have `EngineSender` be an enum with +/// these two variants. But we actually want `NestedSender` to implement +/// `SenderImpl` for *any* StepSpec, not just `S`, to allow nested engines to +/// be a different StepSpec than the outer engine. +/// +/// In other words, `NestedSender` doesn't just represent a single +/// `mpsc::Sender>`, it represents the universe of all +/// possible StepSpecs S. This is an infinite number of variants, and requires a +/// trait object to represent. +#[derive_where(Clone, Debug)] +struct EngineSender { + sender: Arc>, +} + +impl EngineSender { + async fn send(&self, event: Event) -> Result<(), ExecutionError> { + self.sender.send(event).await + } +} + +trait SenderImpl: Send + Sync + fmt::Debug { + fn send( + &self, + event: Event, + ) -> BoxFuture<'_, Result<(), ExecutionError>>; +} + +#[derive_where(Debug)] +struct DefaultSender { + sender: mpsc::Sender>, +} + +impl SenderImpl for DefaultSender { + fn send( + &self, + event: Event, + ) -> BoxFuture<'_, Result<(), ExecutionError>> { + self.sender.send(event).map_err(|error| error.into()).boxed() + } +} + +#[derive_where(Debug)] +struct NestedSender { + sender: mpsc::Sender>, +} + +// Note that NestedSender implements SenderImpl for any S2: StepSpec. +// That is to allow nested engines to implement arbitrary StepSpecs. +impl SenderImpl for NestedSender { + fn send( + &self, + event: Event, + ) -> BoxFuture<'_, Result<(), ExecutionError>> { + let now = Instant::now(); + async move { + let (done, done_rx) = oneshot::channel(); + self.sender + .send(StepContextPayload::NestedSingle { + now, + event: event.into_generic(), + done, + }) + .await + .expect("our code always keeps payload_receiver open"); + _ = done_rx.await; + Ok(()) + } + .boxed() + } +} + /// A join handle for an UpdateEngine. /// /// This handle should be awaited to drive and obtain the result of an execution. @@ -819,6 +924,16 @@ impl<'a, S: StepSpec> StepExec<'a, S> { Ok(ControlFlow::Continue(())) } + // Note: payload_receiver is always kept open while step_fut + // is being driven. It is only dropped before completion if + // the step is aborted, in which case step_fut is also + // cancelled without being driven further. A bunch of + // expects with "our code always keeps payload_receiver + // open" rely on this. + // + // If we ever move the payload receiver to another task so + // it runs in parallel, this situation would have to be + // handled with care. payload = payload_receiver.recv(), if !payload_done => { match payload { Some(payload) => { @@ -868,14 +983,14 @@ struct ExecutionContext { execution_id: ExecutionId, next_event_index: DebugIgnore, total_start: Instant, - sender: mpsc::Sender>, + sender: EngineSender, } impl ExecutionContext { fn new( execution_id: ExecutionId, next_event_index: F, - sender: mpsc::Sender>, + sender: EngineSender, ) -> Self { let total_start = Instant::now(); Self { @@ -906,7 +1021,7 @@ struct StepExecutionContext { next_event_index: DebugIgnore, total_start: Instant, step_info: StepInfoWithMetadata, - sender: mpsc::Sender>, + sender: EngineSender, } type StepMetadataFn<'a, S> = Box< @@ -941,7 +1056,7 @@ struct StepProgressReporter { step_start: Instant, attempt: usize, attempt_start: Instant, - sender: mpsc::Sender>, + sender: EngineSender, } impl usize> StepProgressReporter { @@ -963,51 +1078,32 @@ impl usize> StepProgressReporter { async fn handle_payload( &mut self, payload: StepContextPayload, - ) -> Result<(), mpsc::error::SendError>> { + ) -> Result<(), ExecutionError> { match payload { - StepContextPayload::Progress(progress) => { - self.handle_progress(progress).await + StepContextPayload::Progress { now, progress, done } => { + self.handle_progress(now, progress).await?; + std::mem::drop(done); } - StepContextPayload::Nested(Event::Step(event)) => { - self.sender - .send(Event::Step(StepEvent { - spec: S::schema_name(), - execution_id: self.execution_id, - event_index: (self.next_event_index)(), - total_elapsed: self.total_start.elapsed(), - kind: StepEventKind::Nested { - step: self.step_info.clone(), - attempt: self.attempt, - event: Box::new(event), - step_elapsed: self.step_start.elapsed(), - attempt_elapsed: self.attempt_start.elapsed(), - }, - })) - .await + StepContextPayload::NestedSingle { now, event, done } => { + self.handle_nested(now, event).await?; + std::mem::drop(done); } - StepContextPayload::Nested(Event::Progress(event)) => { - self.sender - .send(Event::Progress(ProgressEvent { - spec: S::schema_name(), - execution_id: self.execution_id, - total_elapsed: self.total_start.elapsed(), - kind: ProgressEventKind::Nested { - step: self.step_info.clone(), - attempt: self.attempt, - event: Box::new(event), - step_elapsed: self.step_start.elapsed(), - attempt_elapsed: self.attempt_start.elapsed(), - }, - })) - .await + StepContextPayload::Nested { now, event } => { + self.handle_nested(now, event).await?; + } + StepContextPayload::Sync { done } => { + std::mem::drop(done); } } + + Ok(()) } async fn handle_progress( &mut self, + now: Instant, progress: StepProgress, - ) -> Result<(), mpsc::error::SendError>> { + ) -> Result<(), ExecutionError> { match progress { StepProgress::Progress { progress, metadata } => { // Send the progress to the sender. @@ -1015,14 +1111,14 @@ impl usize> StepProgressReporter { .send(Event::Progress(ProgressEvent { spec: S::schema_name(), execution_id: self.execution_id, - total_elapsed: self.total_start.elapsed(), + total_elapsed: now - self.total_start, kind: ProgressEventKind::Progress { step: self.step_info.clone(), attempt: self.attempt, progress, metadata, - step_elapsed: self.step_start.elapsed(), - attempt_elapsed: self.attempt_start.elapsed(), + step_elapsed: now - self.step_start, + attempt_elapsed: now - self.attempt_start, }, })) .await @@ -1034,13 +1130,13 @@ impl usize> StepProgressReporter { spec: S::schema_name(), execution_id: self.execution_id, event_index: (self.next_event_index)(), - total_elapsed: self.total_start.elapsed(), + total_elapsed: now - self.total_start, kind: StepEventKind::ProgressReset { step: self.step_info.clone(), attempt: self.attempt, metadata, - step_elapsed: self.step_start.elapsed(), - attempt_elapsed: self.attempt_start.elapsed(), + step_elapsed: now - self.step_start, + attempt_elapsed: now - self.attempt_start, message, }, })) @@ -1049,7 +1145,7 @@ impl usize> StepProgressReporter { StepProgress::Retry { message } => { // Retry this step. self.attempt += 1; - let attempt_elapsed = self.attempt_start.elapsed(); + let attempt_elapsed = now - self.attempt_start; self.attempt_start = Instant::now(); // Send the retry message. @@ -1058,11 +1154,11 @@ impl usize> StepProgressReporter { spec: S::schema_name(), execution_id: self.execution_id, event_index: (self.next_event_index)(), - total_elapsed: self.total_start.elapsed(), + total_elapsed: now - self.total_start, kind: StepEventKind::AttemptRetry { step: self.step_info.clone(), next_attempt: self.attempt, - step_elapsed: self.step_start.elapsed(), + step_elapsed: now - self.step_start, attempt_elapsed, message, }, @@ -1072,6 +1168,48 @@ impl usize> StepProgressReporter { } } + async fn handle_nested( + &mut self, + now: Instant, + event: Event, + ) -> Result<(), ExecutionError> { + match event { + Event::Step(event) => { + self.sender + .send(Event::Step(StepEvent { + spec: S::schema_name(), + execution_id: self.execution_id, + event_index: (self.next_event_index)(), + total_elapsed: now - self.total_start, + kind: StepEventKind::Nested { + step: self.step_info.clone(), + attempt: self.attempt, + event: Box::new(event), + step_elapsed: now - self.step_start, + attempt_elapsed: now - self.attempt_start, + }, + })) + .await + } + Event::Progress(event) => { + self.sender + .send(Event::Progress(ProgressEvent { + spec: S::schema_name(), + execution_id: self.execution_id, + total_elapsed: now - self.total_start, + kind: ProgressEventKind::Nested { + step: self.step_info.clone(), + attempt: self.attempt, + event: Box::new(event), + step_elapsed: now - self.step_start, + attempt_elapsed: now - self.attempt_start, + }, + })) + .await + } + } + } + async fn handle_abort(mut self, message: String) -> ExecutionError { // Send the abort message over the channel. // @@ -1102,7 +1240,7 @@ impl usize> StepProgressReporter { description: self.step_info.info.description.clone(), message: message, }, - Err(error) => error.into(), + Err(error) => error, } } @@ -1187,7 +1325,7 @@ impl usize> StepProgressReporter { async fn send_error( mut self, error: &S::Error, - ) -> Result<(), mpsc::error::SendError>> { + ) -> Result<(), ExecutionError> { // Stringify `error` into a message + list causes; this is written the // way it is to avoid `error` potentially living across the `.await` // below (which can cause lifetime issues in callers). diff --git a/update-engine/src/errors.rs b/update-engine/src/errors.rs index f40ce096d3b..0607ad6e271 100644 --- a/update-engine/src/errors.rs +++ b/update-engine/src/errors.rs @@ -48,13 +48,13 @@ impl fmt::Display for ExecutionError { ) } Self::EventSendError(_) => { - write!(f, "event receiver dropped") + write!(f, "while sending event, event receiver dropped") } } } } -impl error::Error for ExecutionError { +impl error::Error for ExecutionError { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match self { ExecutionError::StepFailed { error, .. } => Some(error.as_error()), @@ -112,7 +112,7 @@ impl fmt::Display for NestedEngineError { } } -impl error::Error for NestedEngineError { +impl error::Error for NestedEngineError { fn source(&self) -> Option<&(dyn error::Error + 'static)> { match self { Self::Creation { error } => Some(error.as_error()), @@ -185,3 +185,17 @@ pub enum ConvertGenericPathElement { Path(&'static str), ArrayIndex(&'static str, usize), } + +/// The +/// [`GroupDisplay::add_event_report`](crate::display::GroupDisplay::add_event_report) +/// method was called with an unknown key. +#[derive(Clone, Debug)] +pub struct UnknownReportKey {} + +impl fmt::Display for UnknownReportKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("unknown report key") + } +} + +impl error::Error for UnknownReportKey {} diff --git a/update-engine/src/events.rs b/update-engine/src/events.rs index 3816157d0dc..900a9776f5c 100644 --- a/update-engine/src/events.rs +++ b/update-engine/src/events.rs @@ -1143,6 +1143,8 @@ impl ProgressEventKind { /// Returns `step_elapsed` for the leaf event, recursing into nested events /// as necessary. + /// + /// Returns None for unknown events. pub fn leaf_step_elapsed(&self) -> Option { match self { ProgressEventKind::WaitingForProgress { step_elapsed, .. } @@ -1156,6 +1158,25 @@ impl ProgressEventKind { } } + /// Returns `attempt_elapsed` for the leaf event, recursing into nested + /// events as necessary. + /// + /// Returns None for unknown events. + pub fn leaf_attempt_elapsed(&self) -> Option { + match self { + ProgressEventKind::WaitingForProgress { + attempt_elapsed, .. + } + | ProgressEventKind::Progress { attempt_elapsed, .. } => { + Some(*attempt_elapsed) + } + ProgressEventKind::Nested { event, .. } => { + event.kind.leaf_attempt_elapsed() + } + ProgressEventKind::Unknown => None, + } + } + /// Converts a generic version into self. /// /// This version can be used to convert a generic type into a more concrete diff --git a/update-engine/src/lib.rs b/update-engine/src/lib.rs index f753fa738aa..fea92d3b73b 100644 --- a/update-engine/src/lib.rs +++ b/update-engine/src/lib.rs @@ -57,6 +57,7 @@ mod buffer; mod context; +pub mod display; mod engine; pub mod errors; pub mod events; diff --git a/update-engine/src/spec.rs b/update-engine/src/spec.rs index a569bcf14a5..0dfe6321813 100644 --- a/update-engine/src/spec.rs +++ b/update-engine/src/spec.rs @@ -15,7 +15,7 @@ use serde::{de::DeserializeOwned, Serialize}; /// /// NOTE: `StepSpec` is only required to implement `JsonSchema` to obtain the /// name of the schema. This is an upstream limitation in `JsonSchema`. -pub trait StepSpec: JsonSchema + Send { +pub trait StepSpec: JsonSchema + Send + 'static { /// A component associated with each step. type Component: Clone + fmt::Debug @@ -149,8 +149,19 @@ pub struct NestedError { } impl NestedError { + /// Creates a new `NestedError` from an error. + pub fn new(error: &dyn std::error::Error) -> Self { + Self { + message: format!("{}", error), + source: error.source().map(|s| Box::new(Self::new(s))), + } + } + /// Creates a new `NestedError` from a message and a list of causes. - pub fn new(message: String, causes: Vec) -> Self { + pub fn from_message_and_causes( + message: String, + causes: Vec, + ) -> Self { // Yes, this is an actual singly-linked list. You rarely ever see them // in Rust but they're required to implement Error::source. let mut next = None; @@ -174,6 +185,47 @@ impl std::error::Error for NestedError { } } +mod nested_error_serde { + use super::*; + use serde::Deserialize; + + #[derive(Serialize, Deserialize)] + struct SerializedNestedError { + message: String, + causes: Vec, + } + + impl Serialize for NestedError { + fn serialize( + &self, + serializer: S, + ) -> Result { + let mut causes = Vec::new(); + let mut cause = self.source.as_ref(); + while let Some(c) = cause { + causes.push(c.message.clone()); + cause = c.source.as_ref(); + } + + let serialized = + SerializedNestedError { message: self.message.clone(), causes }; + serialized.serialize(serializer) + } + } + + impl<'de> Deserialize<'de> for NestedError { + fn deserialize>( + deserializer: D, + ) -> Result { + let serialized = SerializedNestedError::deserialize(deserializer)?; + Ok(NestedError::from_message_and_causes( + serialized.message, + serialized.causes, + )) + } + } +} + impl AsError for NestedError { fn as_error(&self) -> &(dyn std::error::Error + 'static) { self @@ -183,7 +235,7 @@ impl AsError for NestedError { /// Trait that abstracts over concrete errors and `anyhow::Error`. /// /// This needs to be manually implemented for any custom error types. -pub trait AsError: fmt::Debug + Send + Sync { +pub trait AsError: fmt::Debug + Send + Sync + 'static { fn as_error(&self) -> &(dyn std::error::Error + 'static); } diff --git a/wicket-common/src/lib.rs b/wicket-common/src/lib.rs index 76095f14686..9e92d20c0ab 100644 --- a/wicket-common/src/lib.rs +++ b/wicket-common/src/lib.rs @@ -5,4 +5,5 @@ // Copyright 2023 Oxide Computer Company pub mod rack_setup; +pub mod rack_update; pub mod update_events; diff --git a/wicket-common/src/rack_update.rs b/wicket-common/src/rack_update.rs new file mode 100644 index 00000000000..1a24069bc9e --- /dev/null +++ b/wicket-common/src/rack_update.rs @@ -0,0 +1,70 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +use std::{collections::BTreeSet, fmt}; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +// TODO: unify this with the one in gateway http_entrypoints.rs. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Serialize, + Deserialize, + JsonSchema, +)] +pub struct SpIdentifier { + #[serde(rename = "type")] + pub type_: SpType, + pub slot: u32, +} + +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + JsonSchema, +)] +#[serde(rename_all = "lowercase")] +pub enum SpType { + Switch, + Sled, + Power, +} + +impl fmt::Display for SpType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + SpType::Switch => write!(f, "switch"), + SpType::Sled => write!(f, "sled"), + SpType::Power => write!(f, "power"), + } + } +} + +#[derive( + Clone, Debug, Default, PartialEq, Eq, JsonSchema, Serialize, Deserialize, +)] +pub struct ClearUpdateStateResponse { + /// The SPs for which update data was cleared. + pub cleared: BTreeSet, + + /// The SPs that had no update state to clear. + pub no_update_data: BTreeSet, +} diff --git a/wicket-common/src/update_events.rs b/wicket-common/src/update_events.rs index ac840f83ad0..e0f9d4b2285 100644 --- a/wicket-common/src/update_events.rs +++ b/wicket-common/src/update_events.rs @@ -169,7 +169,7 @@ pub enum UpdateTerminalError { #[source] error: anyhow::Error, }, - #[error("failed to find correctly-singed RoT image")] + #[error("failed to find correctly-signed RoT image")] FailedFindingSignedRotImage { #[source] error: anyhow::Error, diff --git a/wicket-dbg/Cargo.toml b/wicket-dbg/Cargo.toml index e7e8a584682..a00bcb9c1b7 100644 --- a/wicket-dbg/Cargo.toml +++ b/wicket-dbg/Cargo.toml @@ -11,7 +11,6 @@ camino.workspace = true ciborium.workspace = true clap.workspace = true crossterm.workspace = true -ratatui.workspace = true serde.workspace = true slog.workspace = true slog-async.workspace = true @@ -21,7 +20,7 @@ tokio = { workspace = true, features = ["full"] } wicket.workspace = true # used only by wicket-dbg binary -reedline = "0.23.0" +reedline = "0.25.0" omicron-workspace-hack.workspace = true [[bin]] diff --git a/wicket/Cargo.toml b/wicket/Cargo.toml index 5392e72e9f9..efb8e51dff9 100644 --- a/wicket/Cargo.toml +++ b/wicket/Cargo.toml @@ -14,7 +14,6 @@ ciborium.workspace = true clap.workspace = true crossterm.workspace = true futures.workspace = true -hex = { workspace = true, features = ["serde"] } humantime.workspace = true indexmap.workspace = true indicatif.workspace = true @@ -25,7 +24,6 @@ owo-colors.workspace = true ratatui.workspace = true reqwest.workspace = true rpassword.workspace = true -semver.workspace = true serde.workspace = true serde_json.workspace = true shell-words.workspace = true @@ -33,6 +31,7 @@ slog.workspace = true slog-async.workspace = true slog-envlogger.workspace = true slog-term.workspace = true +supports-color.workspace = true textwrap.workspace = true tokio = { workspace = true, features = ["full"] } tokio-util.workspace = true diff --git a/wicket/src/cli/command.rs b/wicket/src/cli/command.rs new file mode 100644 index 00000000000..54407e06ca6 --- /dev/null +++ b/wicket/src/cli/command.rs @@ -0,0 +1,101 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Code that manages command dispatch from a shell for wicket. + +use std::net::SocketAddrV6; + +use anyhow::Result; +use clap::{Args, ColorChoice, Parser, Subcommand}; + +use super::{ + preflight::PreflightArgs, rack_setup::SetupArgs, + rack_update::RackUpdateArgs, upload::UploadArgs, +}; + +pub(crate) struct CommandOutput<'a> { + pub(crate) stdout: &'a mut dyn std::io::Write, + pub(crate) stderr: &'a mut dyn std::io::Write, +} + +/// An app that represents wicket started with arguments over ssh. +#[derive(Debug, Parser)] +pub(crate) struct ShellApp { + /// Global options. + #[clap(flatten)] + pub(crate) global_opts: GlobalOpts, + + /// The command to run. + #[clap(subcommand)] + command: ShellCommand, +} + +impl ShellApp { + pub(crate) async fn exec( + self, + log: slog::Logger, + wicketd_addr: SocketAddrV6, + output: CommandOutput<'_>, + ) -> Result<()> { + match self.command { + ShellCommand::UploadRepo(args) => { + args.exec(log, wicketd_addr).await + } + ShellCommand::RackUpdate(args) => { + args.exec(log, wicketd_addr, self.global_opts, output).await + } + ShellCommand::Setup(args) => args.exec(log, wicketd_addr).await, + ShellCommand::Preflight(args) => args.exec(log, wicketd_addr).await, + } + } +} + +#[derive(Debug, Args)] +#[clap(next_help_heading = "Global options")] +pub(crate) struct GlobalOpts { + /// Color output + /// + /// This may not be obeyed everywhere at the moment. + #[clap(long, value_enum, global = true, default_value_t)] + pub(crate) color: ColorChoice, +} + +impl GlobalOpts { + /// Returns true if color should be used on standard error. + pub(crate) fn use_color(&self) -> bool { + match self.color { + ColorChoice::Auto => { + supports_color::on_cached(supports_color::Stream::Stderr) + .is_some() + } + ColorChoice::Always => true, + ColorChoice::Never => false, + } + } +} + +/// Arguments passed to wicket. +/// +/// Wicket is designed to be used as a captive shell, set up via sshd +/// ForceCommand. If no arguments are specified, wicket behaves like a TUI. +/// However, if arguments are specified via SSH_ORIGINAL_COMMAND, wicketd +/// accepts an upload command. +#[derive(Debug, Subcommand)] +enum ShellCommand { + /// Upload a TUF repository to wicketd. + #[command(visible_alias = "upload")] + UploadRepo(UploadArgs), + + /// Perform a rack update. + #[command(subcommand)] + RackUpdate(RackUpdateArgs), + + /// Interact with rack setup configuration. + #[command(subcommand)] + Setup(SetupArgs), + + /// Run checks prior to setting up the rack. + #[command(subcommand)] + Preflight(PreflightArgs), +} diff --git a/wicket/src/cli/mod.rs b/wicket/src/cli/mod.rs new file mode 100644 index 00000000000..e63ef467e73 --- /dev/null +++ b/wicket/src/cli/mod.rs @@ -0,0 +1,19 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +//! Command-line interface to wicket. +//! +//! By default, the main interface to wicket is via a TUI. However, some +//! commands and use cases must be done via the CLI, and this module contains +//! support for that. + +mod command; +mod preflight; +mod rack_setup; +mod rack_update; +mod upload; + +pub(super) use command::{CommandOutput, GlobalOpts, ShellApp}; diff --git a/wicket/src/preflight.rs b/wicket/src/cli/preflight.rs similarity index 96% rename from wicket/src/preflight.rs rename to wicket/src/cli/preflight.rs index ddbbf95c1a9..29b6d2a5cbc 100644 --- a/wicket/src/preflight.rs +++ b/wicket/src/cli/preflight.rs @@ -47,18 +47,7 @@ pub(crate) enum PreflightArgs { } impl PreflightArgs { - pub(crate) fn exec( - self, - log: Logger, - wicketd_addr: SocketAddrV6, - ) -> Result<()> { - let runtime = - tokio::runtime::Runtime::new().context("creating tokio runtime")?; - - runtime.block_on(self.exec_impl(log, wicketd_addr)) - } - - async fn exec_impl( + pub(crate) async fn exec( self, log: Logger, wicketd_addr: SocketAddrV6, diff --git a/wicket/src/rack_setup.rs b/wicket/src/cli/rack_setup.rs similarity index 96% rename from wicket/src/rack_setup.rs rename to wicket/src/cli/rack_setup.rs index c678d14e11c..88410468131 100644 --- a/wicket/src/rack_setup.rs +++ b/wicket/src/cli/rack_setup.rs @@ -61,18 +61,7 @@ pub(crate) enum SetupArgs { } impl SetupArgs { - pub(crate) fn exec( - self, - log: Logger, - wicketd_addr: SocketAddrV6, - ) -> Result<()> { - let runtime = - tokio::runtime::Runtime::new().context("creating tokio runtime")?; - - runtime.block_on(self.exec_impl(log, wicketd_addr)) - } - - async fn exec_impl( + pub(crate) async fn exec( self, log: Logger, wicketd_addr: SocketAddrV6, diff --git a/wicket/src/rack_setup/config_template.toml b/wicket/src/cli/rack_setup/config_template.toml similarity index 100% rename from wicket/src/rack_setup/config_template.toml rename to wicket/src/cli/rack_setup/config_template.toml diff --git a/wicket/src/rack_setup/config_toml.rs b/wicket/src/cli/rack_setup/config_toml.rs similarity index 90% rename from wicket/src/rack_setup/config_toml.rs rename to wicket/src/cli/rack_setup/config_toml.rs index e087c9aa7c2..9b1a25a50ea 100644 --- a/wicket/src/rack_setup/config_toml.rs +++ b/wicket/src/cli/rack_setup/config_toml.rs @@ -15,11 +15,11 @@ use toml_edit::InlineTable; use toml_edit::Item; use toml_edit::Table; use toml_edit::Value; +use wicket_common::rack_update::SpType; use wicketd_client::types::BootstrapSledDescription; use wicketd_client::types::CurrentRssUserConfigInsensitive; use wicketd_client::types::IpRange; use wicketd_client::types::RackNetworkConfigV1; -use wicketd_client::types::SpType; static TEMPLATE: &str = include_str!("config_template.toml"); @@ -274,6 +274,36 @@ fn populate_network_table( "port", Value::String(Formatted::new(p.port.to_string())), ); + if let Some(x) = p.hold_time { + peer.insert( + "hold_time", + Value::Integer(Formatted::new(x as i64)), + ); + } + if let Some(x) = p.connect_retry { + peer.insert( + "connect_retry", + Value::Integer(Formatted::new(x as i64)), + ); + } + if let Some(x) = p.delay_open { + peer.insert( + "delay_open", + Value::Integer(Formatted::new(x as i64)), + ); + } + if let Some(x) = p.idle_hold_time { + peer.insert( + "idle_hold_time", + Value::Integer(Formatted::new(x as i64)), + ); + } + if let Some(x) = p.keepalive { + peer.insert( + "keepalive", + Value::Integer(Formatted::new(x as i64)), + ); + } peers.push(Value::InlineTable(peer)); } uplink @@ -317,6 +347,7 @@ mod tests { use omicron_common::api::internal::shared::RackNetworkConfigV1 as InternalRackNetworkConfig; use std::net::Ipv6Addr; use wicket_common::rack_setup::PutRssUserConfigInsensitive; + use wicket_common::rack_update::SpIdentifier; use wicketd_client::types::Baseboard; use wicketd_client::types::BgpConfig; use wicketd_client::types::BgpPeerConfig; @@ -324,7 +355,6 @@ mod tests { use wicketd_client::types::PortFec; use wicketd_client::types::PortSpeed; use wicketd_client::types::RouteConfig; - use wicketd_client::types::SpIdentifier; use wicketd_client::types::SwitchLocation; fn put_config_from_current_config( @@ -389,6 +419,11 @@ mod tests { asn: p.asn, port: p.port.clone(), addr: p.addr, + hold_time: p.hold_time, + connect_retry: p.connect_retry, + delay_open: p.delay_open, + idle_hold_time: p.idle_hold_time, + keepalive: p.keepalive, }) .collect(), port: config.port.clone(), @@ -486,6 +521,11 @@ mod tests { asn: 47, addr: "10.2.3.4".parse().unwrap(), port: "port0".into(), + hold_time: Some(6), + connect_retry: Some(3), + delay_open: Some(0), + idle_hold_time: Some(3), + keepalive: Some(2), }], uplink_port_speed: PortSpeed::Speed400G, uplink_port_fec: PortFec::Firecode, diff --git a/wicket/src/cli/rack_update.rs b/wicket/src/cli/rack_update.rs new file mode 100644 index 00000000000..f539c22c35f --- /dev/null +++ b/wicket/src/cli/rack_update.rs @@ -0,0 +1,425 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Command-line driven rack update. +//! +//! This is an alternative to using the Wicket UI to perform a rack update. + +use std::{ + collections::{BTreeMap, BTreeSet}, + net::SocketAddrV6, + time::Duration, +}; + +use anyhow::{anyhow, bail, Context, Result}; +use clap::{Args, Subcommand, ValueEnum}; +use slog::Logger; +use tokio::{sync::watch, task::JoinHandle}; +use update_engine::{ + display::{GroupDisplay, LineDisplayStyles}, + NestedError, +}; +use wicket_common::{ + rack_update::ClearUpdateStateResponse, update_events::EventReport, +}; +use wicketd_client::types::{ClearUpdateStateParams, StartUpdateParams}; + +use crate::{ + cli::GlobalOpts, + state::{ + parse_event_report_map, ComponentId, CreateClearUpdateStateOptions, + CreateStartUpdateOptions, + }, + wicketd::{create_wicketd_client, WICKETD_TIMEOUT}, +}; + +use super::command::CommandOutput; + +#[derive(Debug, Subcommand)] +pub(crate) enum RackUpdateArgs { + /// Start one or more updates. + Start(StartRackUpdateArgs), + /// Attach to one or more running updates. + Attach(AttachArgs), + /// Clear updates. + Clear(ClearArgs), +} + +impl RackUpdateArgs { + pub(crate) async fn exec( + self, + log: Logger, + wicketd_addr: SocketAddrV6, + global_opts: GlobalOpts, + output: CommandOutput<'_>, + ) -> Result<()> { + match self { + RackUpdateArgs::Start(args) => { + args.exec(log, wicketd_addr, global_opts, output).await + } + RackUpdateArgs::Attach(args) => { + args.exec(log, wicketd_addr, global_opts, output).await + } + RackUpdateArgs::Clear(args) => { + args.exec(log, wicketd_addr, global_opts, output).await + } + } + } +} + +#[derive(Debug, Args)] +pub(crate) struct StartRackUpdateArgs { + #[clap(flatten)] + component_ids: ComponentIdSelector, + + /// Force update the RoT even if the version is the same. + #[clap(long, help_heading = "Update options")] + force_update_rot: bool, + + /// Force update the SP even if the version is the same. + #[clap(long, help_heading = "Update options")] + force_update_sp: bool, + + /// Detach after starting the update. + /// + /// The `attach` command can be used to reattach to the running update. + #[clap(short, long, help_heading = "Update options")] + detach: bool, +} + +impl StartRackUpdateArgs { + async fn exec( + self, + log: Logger, + wicketd_addr: SocketAddrV6, + global_opts: GlobalOpts, + output: CommandOutput<'_>, + ) -> Result<()> { + let client = create_wicketd_client(&log, wicketd_addr, WICKETD_TIMEOUT); + + let update_ids = self.component_ids.to_component_ids()?; + let options = CreateStartUpdateOptions { + force_update_rot: self.force_update_rot, + force_update_sp: self.force_update_sp, + } + .to_start_update_options()?; + + let num_update_ids = update_ids.len(); + + let params = StartUpdateParams { + targets: update_ids.iter().copied().map(Into::into).collect(), + options, + }; + + slog::debug!(log, "Sending post_start_update"; "num_update_ids" => num_update_ids); + match client.post_start_update(¶ms).await { + Ok(_) => { + slog::info!(log, "Update started for {num_update_ids} targets"); + } + Err(error) => { + // Error responses can be printed out more clearly. + if let wicketd_client::Error::ErrorResponse(rv) = &error { + slog::error!( + log, + "Error response from wicketd: {}", + rv.message + ); + bail!("Received error from wicketd while starting update"); + } else { + bail!(error); + } + } + } + + if self.detach { + return Ok(()); + } + + // Now, attach to the update by printing out update logs. + do_attach_to_updates(log, client, update_ids, global_opts, output) + .await?; + + Ok(()) + } +} + +#[derive(Debug, Args)] +pub(crate) struct AttachArgs { + #[clap(flatten)] + component_ids: ComponentIdSelector, +} + +impl AttachArgs { + async fn exec( + self, + log: Logger, + wicketd_addr: SocketAddrV6, + global_opts: GlobalOpts, + output: CommandOutput<'_>, + ) -> Result<()> { + let client = create_wicketd_client(&log, wicketd_addr, WICKETD_TIMEOUT); + + let update_ids = self.component_ids.to_component_ids()?; + do_attach_to_updates(log, client, update_ids, global_opts, output).await + } +} + +async fn do_attach_to_updates( + log: Logger, + client: wicketd_client::Client, + update_ids: BTreeSet, + global_opts: GlobalOpts, + output: CommandOutput<'_>, +) -> Result<()> { + let mut display = GroupDisplay::new_with_display( + update_ids.iter().copied(), + output.stderr, + ); + if global_opts.use_color() { + display.set_styles(LineDisplayStyles::colorized()); + } + + let (mut rx, handle) = start_fetch_reports_task(&log, client.clone()).await; + let mut status_timer = tokio::time::interval(Duration::from_secs(5)); + status_timer.tick().await; + + while !display.stats().is_terminal() { + tokio::select! { + res = rx.changed() => { + if res.is_err() { + // The sending end is closed, which means that the task + // created by start_fetch_reports_task died... this can + // happen either due to a panic or due to an error. + match handle.await { + Ok(Ok(())) => { + // The task exited normally, which means that the + // sending end was closed normally. This cannot + // happen. + bail!("fetch_reports task exited with Ok(()) \ + -- this should never happen here"); + } + Ok(Err(error)) => { + // The task exited with an error. + return Err(error).context("fetch_reports task errored out"); + } + Err(error) => { + // The task panicked. + return Err(anyhow!(error)).context("fetch_reports task panicked"); + } + } + } + + let event_reports = rx.borrow_and_update(); + // TODO: parallelize this computation? + for (id, event_report) in &*event_reports { + // If display.add_event_report errors out, it's for a report for a + // component we weren't interested in. Ignore it. + _ = display.add_event_report(&id, event_report.clone()); + } + + // Print out status for each component ID at the end -- do it here so + // that we also consider components for which we haven't seen status + // yet. + display.write_events()?; + } + _ = status_timer.tick() => { + display.write_stats("Status")?; + } + } + } + + // Show any remaining events. + display.write_events()?; + // And also show a summary. + display.write_stats("Summary")?; + + std::mem::drop(rx); + handle + .await + .context("fetch_reports task panicked after rx dropped")? + .context("fetch_reports task errored out after rx dropped")?; + + if display.stats().has_failures() { + bail!("one or more failures occurred"); + } + + Ok(()) +} + +async fn start_fetch_reports_task( + log: &Logger, + client: wicketd_client::Client, +) -> (watch::Receiver>, JoinHandle>) +{ + // Since reports are always cumulative, we can use a watch receiver here + // rather than an mpsc receiver. If we start using incremental reports at + // some point this would need to be changed to be an mpsc receiver. + let (tx, rx) = watch::channel(BTreeMap::new()); + let log = log.new(slog::o!("task" => "fetch_reports")); + + let handle = tokio::spawn(async move { + loop { + let response = client.get_artifacts_and_event_reports().await?; + let reports = response.into_inner().event_reports; + let reports = parse_event_report_map(&log, reports); + if tx.send(reports).is_err() { + // The receiving end is closed, exit. + break; + } + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(1)) => {}, + _ = tx.closed() => { + // The receiving end is closed, exit. + break; + } + } + } + + Ok(()) + }); + (rx, handle) +} + +#[derive(Debug, Args)] +pub(crate) struct ClearArgs { + #[clap(flatten)] + component_ids: ComponentIdSelector, + #[clap(long, value_name = "FORMAT", value_enum, default_value_t = MessageFormat::Human)] + message_format: MessageFormat, +} + +impl ClearArgs { + async fn exec( + self, + log: Logger, + wicketd_addr: SocketAddrV6, + global_opts: GlobalOpts, + output: CommandOutput<'_>, + ) -> Result<()> { + let client = create_wicketd_client(&log, wicketd_addr, WICKETD_TIMEOUT); + + let update_ids = self.component_ids.to_component_ids()?; + let response = + do_clear_update_state(client, update_ids, global_opts).await; + + match self.message_format { + MessageFormat::Human => { + let response = response?; + let cleared = response + .cleared + .iter() + .map(|sp| { + ComponentId::from_sp_type_and_slot(sp.type_, sp.slot) + .map(|id| id.to_string()) + }) + .collect::>>() + .context("unknown component ID returned in response")?; + let no_update_data = response + .no_update_data + .iter() + .map(|sp| { + ComponentId::from_sp_type_and_slot(sp.type_, sp.slot) + .map(|id| id.to_string()) + }) + .collect::>>() + .context("unknown component ID returned in response")?; + + if !cleared.is_empty() { + slog::info!( + log, + "cleared update state for {} components: {}", + cleared.len(), + cleared.join(", ") + ); + } + if !no_update_data.is_empty() { + slog::info!( + log, + "no update data found for {} components: {}", + no_update_data.len(), + no_update_data.join(", ") + ); + } + } + MessageFormat::Json => { + let response = + response.map_err(|error| NestedError::new(error.as_ref())); + // Return the response as a JSON object. + serde_json::to_writer_pretty(output.stdout, &response) + .context("error writing to output")?; + if response.is_err() { + bail!("error clearing update state"); + } + } + } + + Ok(()) + } +} + +async fn do_clear_update_state( + client: wicketd_client::Client, + update_ids: BTreeSet, + _global_opts: GlobalOpts, +) -> Result { + let options = + CreateClearUpdateStateOptions {}.to_clear_update_state_options()?; + let params = ClearUpdateStateParams { + targets: update_ids.iter().copied().map(Into::into).collect(), + options, + }; + + let result = client + .post_clear_update_state(¶ms) + .await + .context("error calling clear_update_state")?; + let response = result.into_inner(); + Ok(response) +} + +#[derive(Clone, Copy, Eq, PartialEq, Hash, Debug, ValueEnum)] +enum MessageFormat { + Human, + Json, +} + +/// Command-line arguments for selecting component IDs. +#[derive(Debug, Args)] +#[clap(next_help_heading = "Component selectors")] +struct ComponentIdSelector { + /// The sleds to operate on. + #[clap(long, value_delimiter = ',')] + sled: Vec, + + /// The switches to operate on. + #[clap(long, value_delimiter = ',')] + switch: Vec, + + /// The PSCs to operate on. + #[clap(long, value_delimiter = ',')] + psc: Vec, +} + +impl ComponentIdSelector { + /// Validates that all the sleds, switches, and PSCs are reasonable (though + /// they might not exist on the actual hardware), then return the set of + /// selected component IDs. + fn to_component_ids(&self) -> Result> { + let mut component_ids = BTreeSet::new(); + for sled in &self.sled { + component_ids.insert(ComponentId::new_sled(*sled)?); + } + for switch in &self.switch { + component_ids.insert(ComponentId::new_switch(*switch)?); + } + for psc in &self.psc { + component_ids.insert(ComponentId::new_psc(*psc)?); + } + if component_ids.is_empty() { + bail!("at least one component ID must be selected via --sled, --switch or --psc"); + } + + Ok(component_ids) + } +} diff --git a/wicket/src/upload.rs b/wicket/src/cli/upload.rs similarity index 94% rename from wicket/src/upload.rs rename to wicket/src/cli/upload.rs index 6e154fd4d98..c9ec4ea2bb1 100644 --- a/wicket/src/upload.rs +++ b/wicket/src/cli/upload.rs @@ -31,14 +31,12 @@ pub(crate) struct UploadArgs { } impl UploadArgs { - pub(crate) fn exec( + pub(crate) async fn exec( self, log: slog::Logger, wicketd_addr: SocketAddrV6, ) -> Result<()> { - let runtime = - tokio::runtime::Runtime::new().context("creating tokio runtime")?; - runtime.block_on(self.do_upload(log, wicketd_addr)) + self.do_upload(log, wicketd_addr).await } async fn do_upload( diff --git a/wicket/src/dispatch.rs b/wicket/src/dispatch.rs index e8191f59cba..fd6f7cd2900 100644 --- a/wicket/src/dispatch.rs +++ b/wicket/src/dispatch.rs @@ -13,7 +13,8 @@ use omicron_common::{address::WICKETD_PORT, FileKv}; use slog::Drain; use crate::{ - preflight::PreflightArgs, rack_setup::SetupArgs, upload::UploadArgs, Runner, + cli::{CommandOutput, ShellApp}, + Runner, }; pub fn exec() -> Result<()> { @@ -21,48 +22,73 @@ pub fn exec() -> Result<()> { SocketAddrV6::new(Ipv6Addr::LOCALHOST, WICKETD_PORT, 0, 0); // SSH_ORIGINAL_COMMAND contains additional arguments, if any. - if let Ok(ssh_args) = std::env::var("SSH_ORIGINAL_COMMAND") { - // The argument is in a quoted form, so split it using Unix shell semantics. - let args = shell_words::split(&ssh_args).with_context(|| { - format!("could not parse shell arguments from input {ssh_args}") - })?; - - let log = setup_log(&log_path()?, WithStderr::Yes)?; - // parse_from uses the the first argument as the command name. Insert "wicket" as - // the command name. - let args = ShellCommand::parse_from( - std::iter::once("wicket".to_owned()).chain(args), - ); - match args { - ShellCommand::UploadRepo(args) => args.exec(log, wicketd_addr), - ShellCommand::Setup(args) => args.exec(log, wicketd_addr), - ShellCommand::Preflight(args) => args.exec(log, wicketd_addr), + match std::env::var("SSH_ORIGINAL_COMMAND") { + Ok(ssh_args) => { + let args = shell_words::split(&ssh_args).with_context(|| { + format!("could not parse shell arguments from input {ssh_args}") + })?; + + let runtime = tokio::runtime::Runtime::new() + .context("creating tokio runtime")?; + runtime.block_on(exec_with_args( + wicketd_addr, + args, + OutputKind::Terminal, + )) + } + Err(_) => { + // Do not expose log messages via standard error since they'll show up + // on top of the TUI. + let log = setup_log(&log_path()?, WithStderr::No)?; + Runner::new(log, wicketd_addr).run() } - } else { - // Do not expose log messages via standard error since they'll show up - // on top of the TUI. - let log = setup_log(&log_path()?, WithStderr::No)?; - Runner::new(log, wicketd_addr).run() } } -/// Arguments passed to wicket. -/// -/// Wicket is designed to be used as a captive shell, set up via sshd -/// ForceCommand. If no arguments are specified, wicket behaves like a TUI. -/// However, if arguments are specified via SSH_ORIGINAL_COMMAND, wicketd -/// accepts an upload command. -#[derive(Debug, Parser)] -enum ShellCommand { - /// Upload a TUF repository to wicketd. - #[command(visible_alias = "upload")] - UploadRepo(UploadArgs), - /// Interact with rack setup configuration. - #[command(subcommand)] - Setup(SetupArgs), - /// Run checks prior to setting up the rack. - #[command(subcommand)] - Preflight(PreflightArgs), +/// Enables capturing of wicket's output. +pub enum OutputKind<'a> { + /// Captures output to the provided log, as well as a buffer. + Captured { + log: slog::Logger, + stdout: &'a mut Vec, + stderr: &'a mut Vec, + }, + + /// Writes output to a terminal. + Terminal, +} + +pub async fn exec_with_args( + wicketd_addr: SocketAddrV6, + args: Vec, + output: OutputKind<'_>, +) -> Result<()> +where + S: AsRef, +{ + // parse_from uses the the first argument as the command name. Insert "wicket" as + // the command name. + let app = ShellApp::parse_from( + std::iter::once("wicket").chain(args.iter().map(|s| s.as_ref())), + ); + + match output { + OutputKind::Captured { log, stdout, stderr } => { + let output = CommandOutput { stdout, stderr }; + app.exec(log, wicketd_addr, output).await + } + OutputKind::Terminal => { + let log = setup_log( + &log_path()?, + WithStderr::Yes { use_color: app.global_opts.use_color() }, + )?; + let mut stdout = std::io::stdout(); + let mut stderr = std::io::stderr(); + let output = + CommandOutput { stdout: &mut stdout, stderr: &mut stderr }; + app.exec(log, wicketd_addr, output).await + } + } } fn setup_log( @@ -80,8 +106,8 @@ fn setup_log( let drain = slog_term::FullFormat::new(decorator).build().fuse(); let drain = match with_stderr { - WithStderr::Yes => { - let stderr_drain = stderr_env_drain("RUST_LOG"); + WithStderr::Yes { use_color } => { + let stderr_drain = stderr_env_drain("RUST_LOG", use_color); let drain = slog::Duplicate::new(drain, stderr_drain).fuse(); slog_async::Async::new(drain).build().fuse() } @@ -93,7 +119,7 @@ fn setup_log( #[derive(Copy, Clone, Debug)] enum WithStderr { - Yes, + Yes { use_color: bool }, No, } @@ -107,8 +133,17 @@ fn log_path() -> Result { } } -fn stderr_env_drain(env_var: &str) -> impl Drain { - let stderr_decorator = slog_term::TermDecorator::new().build(); +fn stderr_env_drain( + env_var: &str, + use_color: bool, +) -> impl Drain { + let mut builder = slog_term::TermDecorator::new(); + if use_color { + builder = builder.force_color(); + } else { + builder = builder.force_plain(); + } + let stderr_decorator = builder.build(); let stderr_drain = slog_term::FullFormat::new(stderr_decorator).build().fuse(); let mut builder = slog_envlogger::LogBuilder::new(stderr_drain); diff --git a/wicket/src/helpers.rs b/wicket/src/helpers.rs new file mode 100644 index 00000000000..564b7e9348f --- /dev/null +++ b/wicket/src/helpers.rs @@ -0,0 +1,70 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Utility functions. + +use std::env::VarError; + +use anyhow::{bail, Context}; +use wicketd_client::types::{UpdateSimulatedResult, UpdateTestError}; + +pub(crate) fn get_update_test_error( + env_var: &str, +) -> Result, anyhow::Error> { + // 30 seconds should always be enough to cause a timeout. (The default + // timeout for progenitor is 15 seconds, and in wicket we set an even + // shorter timeout.) + const DEFAULT_TEST_TIMEOUT_SECS: u64 = 30; + + let test_error = match std::env::var(env_var) { + Ok(v) if v == "fail" => Some(UpdateTestError::Fail), + Ok(v) if v == "timeout" => { + Some(UpdateTestError::Timeout { secs: DEFAULT_TEST_TIMEOUT_SECS }) + } + Ok(v) if v.starts_with("timeout:") => { + // Extended start_timeout syntax with a custom + // number of seconds. + let suffix = v.strip_prefix("timeout:").unwrap(); + match suffix.parse::() { + Ok(secs) => Some(UpdateTestError::Timeout { secs }), + Err(error) => { + return Err(error).with_context(|| { + format!( + "could not parse {env_var} \ + in the form `timeout:`: {v}" + ) + }); + } + } + } + Ok(value) => { + bail!("unrecognized value for {env_var}: {value}"); + } + Err(VarError::NotPresent) => None, + Err(VarError::NotUnicode(value)) => { + bail!("invalid Unicode for {env_var}: {}", value.to_string_lossy()); + } + }; + Ok(test_error) +} + +pub(crate) fn get_update_simulated_result( + env_var: &str, +) -> Result, anyhow::Error> { + let result = match std::env::var(env_var) { + Ok(v) if v == "success" => Some(UpdateSimulatedResult::Success), + Ok(v) if v == "warning" => Some(UpdateSimulatedResult::Warning), + Ok(v) if v == "skipped" => Some(UpdateSimulatedResult::Skipped), + Ok(v) if v == "failure" => Some(UpdateSimulatedResult::Failure), + Ok(value) => { + bail!("unrecognized value for {env_var}: {value}"); + } + Err(VarError::NotPresent) => None, + Err(VarError::NotUnicode(value)) => { + bail!("invalid Unicode for {env_var}: {}", value.to_string_lossy()); + } + }; + + Ok(result) +} diff --git a/wicket/src/lib.rs b/wicket/src/lib.rs index a16ef2a3e1e..5e09cb91f45 100644 --- a/wicket/src/lib.rs +++ b/wicket/src/lib.rs @@ -7,15 +7,14 @@ use std::time::Duration; +mod cli; mod dispatch; mod events; +mod helpers; mod keymap; -mod preflight; -mod rack_setup; mod runner; mod state; mod ui; -mod upload; mod wicketd; pub const TICK_INTERVAL: Duration = Duration::from_millis(30); diff --git a/wicket/src/runner.rs b/wicket/src/runner.rs index c37b16d5d9f..32fabde53ee 100644 --- a/wicket/src/runner.rs +++ b/wicket/src/runner.rs @@ -2,8 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use anyhow::bail; -use anyhow::Context; use crossterm::event::Event as TermEvent; use crossterm::event::EventStream; use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; @@ -17,7 +15,6 @@ use ratatui::backend::CrosstermBackend; use ratatui::Terminal; use slog::Logger; use slog::{debug, error, info}; -use std::env::VarError; use std::io::{stdout, Stdout}; use std::net::SocketAddrV6; use std::time::Instant; @@ -26,12 +23,11 @@ use tokio::sync::mpsc::{ }; use tokio::time::{interval, Duration}; use wicketd_client::types::AbortUpdateOptions; -use wicketd_client::types::ClearUpdateStateOptions; -use wicketd_client::types::StartUpdateOptions; -use wicketd_client::types::UpdateSimulatedResult; -use wicketd_client::types::UpdateTestError; use crate::events::EventReportMap; +use crate::helpers::get_update_test_error; +use crate::state::CreateClearUpdateStateOptions; +use crate::state::CreateStartUpdateOptions; use crate::ui::Screen; use crate::wicketd::{self, WicketdHandle, WicketdManager}; use crate::{Action, Cmd, Event, KeyHandler, Recorder, State, TICK_INTERVAL}; @@ -180,43 +176,18 @@ impl RunnerCore { } Action::StartUpdate(component_id) => { if let Some(wicketd) = wicketd { - let test_error = get_update_test_error( - "WICKET_TEST_START_UPDATE_ERROR", - )?; - - // This is a debug environment variable used to - // add a test step. - let test_step_seconds = - std::env::var("WICKET_UPDATE_TEST_STEP_SECONDS") - .ok() - .map(|v| { - v.parse().expect( - "parsed WICKET_UPDATE_TEST_STEP_SECONDS \ - as a u64", - ) - }); - - let test_simulate_rot_result = get_update_simulated_result( - "WICKET_UPDATE_TEST_SIMULATE_ROT_RESULT", - )?; - let test_simulate_sp_result = get_update_simulated_result( - "WICKET_UPDATE_TEST_SIMULATE_SP_RESULT", - )?; - - let options = StartUpdateOptions { - test_error, - test_step_seconds, - test_simulate_rot_result, - test_simulate_sp_result, - skip_rot_version_check: self + let options = CreateStartUpdateOptions { + force_update_rot: self .state .force_update_state .force_update_rot, - skip_sp_version_check: self + force_update_sp: self .state .force_update_state .force_update_sp, - }; + } + .to_start_update_options()?; + wicketd.tx.blocking_send( wicketd::Request::StartUpdate { component_id, options }, )?; @@ -239,11 +210,8 @@ impl RunnerCore { } Action::ClearUpdateState(component_id) => { if let Some(wicketd) = wicketd { - let test_error = get_update_test_error( - "WICKET_TEST_CLEAR_UPDATE_STATE_ERROR", - )?; - - let options = ClearUpdateStateOptions { test_error }; + let options = CreateClearUpdateStateOptions {}; + let options = options.to_clear_update_state_options()?; wicketd.tx.blocking_send( wicketd::Request::ClearUpdateState { component_id, @@ -281,66 +249,6 @@ impl RunnerCore { } } -fn get_update_test_error( - env_var: &str, -) -> Result, anyhow::Error> { - // 30 seconds should always be enough to cause a timeout. (The default - // timeout for progenitor is 15 seconds, and in wicket we set an even - // shorter timeout.) - const DEFAULT_TEST_TIMEOUT_SECS: u64 = 30; - - let test_error = match std::env::var(env_var) { - Ok(v) if v == "fail" => Some(UpdateTestError::Fail), - Ok(v) if v == "timeout" => { - Some(UpdateTestError::Timeout { secs: DEFAULT_TEST_TIMEOUT_SECS }) - } - Ok(v) if v.starts_with("timeout:") => { - // Extended start_timeout syntax with a custom - // number of seconds. - let suffix = v.strip_prefix("timeout:").unwrap(); - match suffix.parse::() { - Ok(secs) => Some(UpdateTestError::Timeout { secs }), - Err(error) => { - return Err(error).with_context(|| { - format!( - "could not parse {env_var} \ - in the form `timeout:`: {v}" - ) - }); - } - } - } - Ok(value) => { - bail!("unrecognized value for {env_var}: {value}"); - } - Err(VarError::NotPresent) => None, - Err(VarError::NotUnicode(value)) => { - bail!("invalid Unicode for {env_var}: {}", value.to_string_lossy()); - } - }; - Ok(test_error) -} - -fn get_update_simulated_result( - env_var: &str, -) -> Result, anyhow::Error> { - let result = match std::env::var(env_var) { - Ok(v) if v == "success" => Some(UpdateSimulatedResult::Success), - Ok(v) if v == "warning" => Some(UpdateSimulatedResult::Warning), - Ok(v) if v == "skipped" => Some(UpdateSimulatedResult::Skipped), - Ok(v) if v == "failure" => Some(UpdateSimulatedResult::Failure), - Ok(value) => { - bail!("unrecognized value for {env_var}: {value}"); - } - Err(VarError::NotPresent) => None, - Err(VarError::NotUnicode(value)) => { - bail!("invalid Unicode for {env_var}: {}", value.to_string_lossy()); - } - }; - - Ok(result) -} - /// The `Runner` owns the main UI thread, and starts a tokio runtime /// for interaction with downstream services. pub struct Runner { diff --git a/wicket/src/state/inventory.rs b/wicket/src/state/inventory.rs index 23a0e244cff..5cfe536dfbe 100644 --- a/wicket/src/state/inventory.rs +++ b/wicket/src/state/inventory.rs @@ -4,15 +4,17 @@ //! Information about all top-level Oxide components (sleds, switches, PSCs) -use anyhow::anyhow; +use anyhow::{bail, Result}; +use omicron_common::api::internal::nexus::KnownArtifactKind; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt::Display; use std::iter::Iterator; +use wicket_common::rack_update::SpType; use wicketd_client::types::{ RackV1Inventory, RotInventory, RotSlot, SpComponentCaboose, - SpComponentInfo, SpIgnition, SpState, SpType, + SpComponentInfo, SpIgnition, SpState, }; pub static ALL_COMPONENT_IDS: Lazy> = Lazy::new(|| { @@ -64,26 +66,13 @@ impl Inventory { }; // Validate and get a ComponentId - let (id, component) = match type_ { - SpType::Sled => { - if i > 31 { - return Err(anyhow!("Invalid sled slot: {}", i)); - } - (ComponentId::Sled(i as u8), Component::Sled(sp)) - } - SpType::Switch => { - if i > 1 { - return Err(anyhow!("Invalid switch slot: {}", i)); - } - (ComponentId::Switch(i as u8), Component::Switch(sp)) - } - SpType::Power => { - if i > 1 { - return Err(anyhow!("Invalid power shelf slot: {}", i)); - } - (ComponentId::Psc(i as u8), Component::Psc(sp)) - } + let id = ComponentId::from_sp_type_and_slot(type_, i)?; + let component = match type_ { + SpType::Sled => Component::Sled(sp), + SpType::Switch => Component::Switch(sp), + SpType::Power => Component::Psc(sp), }; + new_inventory.inventory.insert(id, component); // TODO: Plumb through real power state @@ -204,6 +193,69 @@ pub enum ComponentId { } impl ComponentId { + /// The maximum possible sled ID. + pub const MAX_SLED_ID: u8 = 31; + + /// The maximum possible switch ID. + pub const MAX_SWITCH_ID: u8 = 1; + + /// The maximum possible power shelf ID. + /// + /// Currently shipping racks don't have PSC 1. + pub const MAX_PSC_ID: u8 = 0; + + pub fn new_sled(slot: u8) -> Result { + if slot > Self::MAX_SLED_ID { + bail!("Invalid sled slot: {}", slot); + } + Ok(Self::Sled(slot)) + } + + pub fn new_switch(slot: u8) -> Result { + if slot > Self::MAX_SWITCH_ID { + bail!("Invalid switch slot: {}", slot); + } + Ok(Self::Switch(slot)) + } + + pub fn new_psc(slot: u8) -> Result { + if slot > Self::MAX_PSC_ID { + bail!("Invalid power shelf slot: {}", slot); + } + Ok(Self::Psc(slot)) + } + + pub fn from_sp_type_and_slot(sp_type: SpType, slot: u32) -> Result { + let slot = slot.try_into().map_err(|_| { + anyhow::anyhow!("invalid slot (must fit in a u8): {}", slot) + })?; + match sp_type { + SpType::Sled => Self::new_sled(slot), + SpType::Switch => Self::new_switch(slot), + SpType::Power => Self::new_psc(slot), + } + } + + pub fn name(&self) -> String { + self.to_string() + } + + pub fn sp_known_artifact_kind(&self) -> KnownArtifactKind { + match self { + ComponentId::Sled(_) => KnownArtifactKind::GimletSp, + ComponentId::Switch(_) => KnownArtifactKind::SwitchSp, + ComponentId::Psc(_) => KnownArtifactKind::PscSp, + } + } + + pub fn rot_known_artifact_kind(&self) -> KnownArtifactKind { + match self { + ComponentId::Sled(_) => KnownArtifactKind::GimletRot, + ComponentId::Switch(_) => KnownArtifactKind::SwitchRot, + ComponentId::Psc(_) => KnownArtifactKind::PscRot, + } + } + pub fn to_string_uppercase(&self) -> String { let mut s = self.to_string(); s.make_ascii_uppercase(); diff --git a/wicket/src/state/mod.rs b/wicket/src/state/mod.rs index ac9cbcae2fb..e20ba1c9c32 100644 --- a/wicket/src/state/mod.rs +++ b/wicket/src/state/mod.rs @@ -18,8 +18,9 @@ pub use inventory::{ pub use rack::{KnightRiderMode, RackState}; pub use status::{Liveness, ServiceStatus}; pub use update::{ - update_component_title, RackUpdateState, UpdateItemState, - UpdateRunningState, + parse_event_report_map, update_component_title, + CreateClearUpdateStateOptions, CreateStartUpdateOptions, RackUpdateState, + UpdateItemState, UpdateRunningState, }; use serde::{Deserialize, Serialize}; diff --git a/wicket/src/state/update.rs b/wicket/src/state/update.rs index 1a0aafb9cf3..6d8a1686144 100644 --- a/wicket/src/state/update.rs +++ b/wicket/src/state/update.rs @@ -2,21 +2,25 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use anyhow::Result; use ratatui::style::Style; use wicket_common::update_events::{ EventReport, ProgressEventKind, StepEventKind, UpdateComponent, UpdateStepId, }; +use crate::helpers::{get_update_simulated_result, get_update_test_error}; use crate::{events::EventReportMap, ui::defaults::style}; use super::{ComponentId, ParsableComponentId, ALL_COMPONENT_IDS}; use omicron_common::api::internal::nexus::KnownArtifactKind; use serde::{Deserialize, Serialize}; -use slog::{warn, Logger}; -use std::collections::{BTreeMap, HashSet}; +use slog::Logger; +use std::collections::BTreeMap; use std::fmt::Display; -use wicketd_client::types::{ArtifactId, SemverVersion}; +use wicketd_client::types::{ + ArtifactId, ClearUpdateStateOptions, SemverVersion, StartUpdateOptions, +}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RackUpdateState { @@ -102,34 +106,18 @@ impl RackUpdateState { } } - let mut updated_component_ids = HashSet::new(); - - for (sp_type, logs) in reports { - for (i, log) in logs { - let Ok(id) = ComponentId::try_from(ParsableComponentId { - sp_type: &sp_type, - i: &i, - }) else { - warn!( - logger, - "Invalid ComponentId in EventReport: {} {}", - &sp_type, - &i - ); - continue; - }; - let item_state = self.items.get_mut(&id).unwrap(); - item_state.update(log); - updated_component_ids.insert(id); - } - } - - // Reset all component IDs that weren't updated. + let reports = parse_event_report_map(logger, reports); + // Reset all component IDs that aren't in the event report map. for (id, item) in &mut self.items { - if !updated_component_ids.contains(id) { + if !reports.contains_key(id) { item.reset(); } } + + for (id, report) in reports { + let item_state = self.items.get_mut(&id).unwrap(); + item_state.update(report); + } } } @@ -445,3 +433,81 @@ pub fn update_component_title(component: UpdateComponent) -> &'static str { UpdateComponent::Host => "HOST", } } + +pub struct CreateStartUpdateOptions { + pub(crate) force_update_rot: bool, + pub(crate) force_update_sp: bool, +} + +impl CreateStartUpdateOptions { + pub fn to_start_update_options(&self) -> Result { + let test_error = + get_update_test_error("WICKET_TEST_START_UPDATE_ERROR")?; + + // This is a debug environment variable used to + // add a test step. + let test_step_seconds = + std::env::var("WICKET_UPDATE_TEST_STEP_SECONDS").ok().map(|v| { + v.parse().expect( + "parsed WICKET_UPDATE_TEST_STEP_SECONDS \ + as a u64", + ) + }); + + let test_simulate_rot_result = get_update_simulated_result( + "WICKET_UPDATE_TEST_SIMULATE_ROT_RESULT", + )?; + let test_simulate_sp_result = get_update_simulated_result( + "WICKET_UPDATE_TEST_SIMULATE_SP_RESULT", + )?; + + Ok(StartUpdateOptions { + test_error, + test_step_seconds, + test_simulate_rot_result, + test_simulate_sp_result, + skip_rot_version_check: self.force_update_rot, + skip_sp_version_check: self.force_update_sp, + }) + } +} + +pub struct CreateClearUpdateStateOptions {} + +impl CreateClearUpdateStateOptions { + pub fn to_clear_update_state_options( + &self, + ) -> Result { + let test_error = + get_update_test_error("WICKET_TEST_CLEAR_UPDATE_STATE_ERROR")?; + + Ok(ClearUpdateStateOptions { test_error }) + } +} + +/// Converts an `EventReportMap` to a map by component ID. +pub fn parse_event_report_map( + log: &Logger, + reports: EventReportMap, +) -> BTreeMap { + let mut component_id_map = BTreeMap::new(); + for (sp_type, logs) in reports { + for (i, event_report) in logs { + let Ok(id) = ComponentId::try_from(ParsableComponentId { + sp_type: &sp_type, + i: &i, + }) else { + slog::warn!( + log, + "Invalid ComponentId in EventReportMap: {} {}", + &sp_type, + &i + ); + continue; + }; + component_id_map.insert(id, event_report); + } + } + + component_id_map +} diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index 2819b3ddda4..d14b90dfaba 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -29,8 +29,8 @@ use ratatui::widgets::{ use slog::{info, o, Logger}; use tui_tree_widget::{Tree, TreeItem, TreeState}; use update_engine::{ - AbortReason, ExecutionStatus, FailureReason, StepKey, TerminalKind, - WillNotBeRunReason, + AbortReason, CompletionReason, ExecutionStatus, FailureReason, StepKey, + TerminalKind, WillNotBeRunReason, }; use wicket_common::update_events::{ EventBuffer, EventReport, ProgressEvent, StepOutcome, StepStatus, @@ -282,7 +282,9 @@ impl UpdatePane { // TODO: show previous attempts } - StepStatus::Completed { info: Some(info) } => { + StepStatus::Completed { + reason: CompletionReason::StepCompleted(info), + } => { let mut spans = vec![Span::styled("Status: ", style::selected())]; @@ -333,9 +335,9 @@ impl UpdatePane { push_text_lines(&message, prefix, &mut body.lines); } } - StepStatus::Completed { info: None } => { - // No information is available, so all we can do is say that - // this step is completed. + StepStatus::Completed { reason: _ } => { + // No information about this step is available, so all we can do + // is say that this step is completed. body.lines.push(Line::from(vec![ Span::styled("Status: ", style::selected()), Span::styled("Completed", style::successful_update_bold()), @@ -383,7 +385,7 @@ impl UpdatePane { } } StepStatus::Failed { - reason: FailureReason::ParentFailed { parent_step }, + reason: FailureReason::ParentFailed { parent_step, .. }, } => { let mut spans = vec![ Span::styled("Status: ", style::selected()), @@ -442,7 +444,7 @@ impl UpdatePane { } } StepStatus::Aborted { - reason: AbortReason::ParentAborted { parent_step }, + reason: AbortReason::ParentAborted { parent_step, .. }, last_progress, } => { let mut spans = vec![ @@ -1893,7 +1895,7 @@ impl ComponentUpdateListState { )); None } - ExecutionStatus::Running { step_key } => { + ExecutionStatus::Running { step_key, .. } => { status_text .push(Span::styled("Update ", style::plain_text())); status_text.push(Span::styled( @@ -2017,24 +2019,25 @@ impl ComponentUpdateListState { } style::selected() } - StepStatus::Completed { info } => { - let (character, style) = if let Some(info) = info { - match info.outcome { - StepOutcome::Success { .. } => { - ('✔', style::successful_update()) - } - StepOutcome::Warning { .. } => { - ('⚠', style::warning_update()) - } - StepOutcome::Skipped { .. } => { - ('*', style::successful_update()) + StepStatus::Completed { reason } => { + let (character, style) = + if let Some(info) = reason.step_completed_info() { + match info.outcome { + StepOutcome::Success { .. } => { + ('✔', style::successful_update()) + } + StepOutcome::Warning { .. } => { + ('⚠', style::warning_update()) + } + StepOutcome::Skipped { .. } => { + ('*', style::successful_update()) + } } - } - } else { - // No information available for this step -- just mark - // it successful. - ('✔', style::successful_update()) - }; + } else { + // No information available for this step -- just mark + // it successful. + ('✔', style::successful_update()) + }; item_spans.push(Span::styled( format!("{:>5} ", character), style, diff --git a/wicket/src/wicketd.rs b/wicket/src/wicketd.rs index 24115424292..ec1130a5941 100644 --- a/wicket/src/wicketd.rs +++ b/wicket/src/wicketd.rs @@ -9,10 +9,11 @@ use std::convert::From; use std::net::SocketAddrV6; use tokio::sync::mpsc::{self, Sender, UnboundedSender}; use tokio::time::{interval, Duration, MissedTickBehavior}; +use wicket_common::rack_update::{SpIdentifier, SpType}; use wicketd_client::types::{ - AbortUpdateOptions, ClearUpdateStateOptions, GetInventoryParams, - GetInventoryResponse, GetLocationResponse, IgnitionCommand, SpIdentifier, - SpType, StartUpdateOptions, StartUpdateParams, + AbortUpdateOptions, ClearUpdateStateOptions, ClearUpdateStateParams, + GetInventoryParams, GetInventoryResponse, GetLocationResponse, + IgnitionCommand, StartUpdateOptions, StartUpdateParams, }; use crate::events::EventReportMap; @@ -40,7 +41,7 @@ const WICKETD_POLL_INTERVAL: Duration = Duration::from_millis(500); // WICKETD_TIMEOUT used to be 1 second, but that might be too short (and in // particular might be responsible for // https://github.com/oxidecomputer/omicron/issues/3103). -const WICKETD_TIMEOUT: Duration = Duration::from_secs(5); +pub(crate) const WICKETD_TIMEOUT: Duration = Duration::from_secs(5); // Assume that these requests are periodic on the order of seconds or the // result of human interaction. In either case, this buffer should be plenty @@ -199,7 +200,7 @@ impl WicketdManager { create_wicketd_client(&log, addr, WICKETD_TIMEOUT); let sp: SpIdentifier = component_id.into(); let response = match update_client - .post_abort_update(sp.type_, sp.slot, &options) + .post_abort_update(&sp.type_, sp.slot, &options) .await { Ok(_) => Ok(()), @@ -229,14 +230,15 @@ impl WicketdManager { tokio::spawn(async move { let update_client = create_wicketd_client(&log, addr, WICKETD_TIMEOUT); - let sp: SpIdentifier = component_id.into(); - let response = match update_client - .post_clear_update_state(sp.type_, sp.slot, &options) - .await - { - Ok(_) => Ok(()), - Err(error) => Err(error.to_string()), + let params = ClearUpdateStateParams { + targets: vec![component_id.into()], + options, }; + let response = + match update_client.post_clear_update_state(¶ms).await { + Ok(_) => Ok(()), + Err(error) => Err(error.to_string()), + }; slog::info!( log, @@ -265,7 +267,7 @@ impl WicketdManager { let client = create_wicketd_client(&log, addr, WICKETD_TIMEOUT); let sp: SpIdentifier = component_id.into(); let res = - client.post_ignition_command(sp.type_, sp.slot, command).await; + client.post_ignition_command(&sp.type_, sp.slot, command).await; // We don't return errors or success values, as there's nobody to // return them to. How do we relay this result to the user? slog::info!( diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index 655f3bb8030..db1ac9c04a8 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -71,6 +71,7 @@ gateway-test-utils.workspace = true http.workspace = true installinator.workspace = true installinator-artifact-client.workspace = true +maplit.workspace = true omicron-test-utils.workspace = true openapi-lint.workspace = true openapiv3.workspace = true @@ -80,4 +81,5 @@ subprocess.workspace = true tar.workspace = true tokio = { workspace = true, features = ["test-util"] } tufaceous.workspace = true +wicket.workspace = true wicketd-client.workspace = true diff --git a/wicketd/src/artifacts/error.rs b/wicketd/src/artifacts/error.rs index 626426ac488..ef81ec66f3f 100644 --- a/wicketd/src/artifacts/error.rs +++ b/wicketd/src/artifacts/error.rs @@ -5,6 +5,7 @@ use camino::Utf8PathBuf; use display_error_chain::DisplayErrorChain; use dropshot::HttpError; +use omicron_common::api::external::SemverVersion; use omicron_common::api::internal::nexus::KnownArtifactKind; use omicron_common::update::{ArtifactHashId, ArtifactId, ArtifactKind}; use slog::error; @@ -105,6 +106,15 @@ pub(super) enum RepositoryError { #[error("missing artifact of kind `{0:?}`")] MissingArtifactKind(KnownArtifactKind), + #[error( + "muliple versions present for artifact of kind `{kind:?}`: {v1}, {v2}" + )] + MultipleVersionsPresent { + kind: KnownArtifactKind, + v1: SemverVersion, + v2: SemverVersion, + }, + #[error( "duplicate hash entries found in artifacts.json for kind `{}`, hash `{}`", .0.kind, .0.hash )] @@ -134,7 +144,8 @@ impl RepositoryError { | RepositoryError::ParsingHubrisArchive { .. } | RepositoryError::ReadHubrisCaboose { .. } | RepositoryError::ReadHubrisCabooseBoard { .. } - | RepositoryError::ReadHubrisCabooseBoardUtf8(_) => { + | RepositoryError::ReadHubrisCabooseBoardUtf8(_) + | RepositoryError::MultipleVersionsPresent { .. } => { HttpError::for_bad_request(None, message) } diff --git a/wicketd/src/artifacts/extracted_artifacts.rs b/wicketd/src/artifacts/extracted_artifacts.rs index 352d8ad3d52..b796201936c 100644 --- a/wicketd/src/artifacts/extracted_artifacts.rs +++ b/wicketd/src/artifacts/extracted_artifacts.rs @@ -169,7 +169,7 @@ impl ExtractedArtifacts { /// /// As the returned file is written to, the data will be hashed; once /// writing is complete, call [`ExtractedArtifacts::store_tempfile()`] to - /// persist the temporary file into an [`ExtractedArtifactDataHandle()`]. + /// persist the temporary file into an [`ExtractedArtifactDataHandle`]. pub(super) fn new_tempfile( &self, ) -> Result { diff --git a/wicketd/src/artifacts/update_plan.rs b/wicketd/src/artifacts/update_plan.rs index 31a8a06ca26..5d7bee629a7 100644 --- a/wicketd/src/artifacts/update_plan.rs +++ b/wicketd/src/artifacts/update_plan.rs @@ -590,6 +590,59 @@ impl<'a> UpdatePlanBuilder<'a> { } } + // Ensure that all A/B RoT images for each board kind have the same + // version number. + for (kind, mut single_board_rot_artifacts) in [ + ( + KnownArtifactKind::GimletRot, + self.gimlet_rot_a.iter().chain(&self.gimlet_rot_b), + ), + ( + KnownArtifactKind::PscRot, + self.psc_rot_a.iter().chain(&self.psc_rot_b), + ), + ( + KnownArtifactKind::SwitchRot, + self.sidecar_rot_a.iter().chain(&self.sidecar_rot_b), + ), + ] { + // We know each of these iterators has at least 2 elements (one from + // the A artifacts and one from the B artifacts, checked above) so + // we can safely unwrap the first. + let version = + &single_board_rot_artifacts.next().unwrap().id.version; + for artifact in single_board_rot_artifacts { + if artifact.id.version != *version { + return Err(RepositoryError::MultipleVersionsPresent { + kind, + v1: version.clone(), + v2: artifact.id.version.clone(), + }); + } + } + } + + // Repeat the same version check for all SP images. (This is a separate + // loop because the types of the iterators don't match.) + for (kind, mut single_board_sp_artifacts) in [ + (KnownArtifactKind::GimletSp, self.gimlet_sp.values()), + (KnownArtifactKind::PscSp, self.psc_sp.values()), + (KnownArtifactKind::SwitchSp, self.sidecar_sp.values()), + ] { + // We know each of these iterators has at least 1 element (checked + // above) so we can safely unwrap the first. + let version = &single_board_sp_artifacts.next().unwrap().id.version; + for artifact in single_board_sp_artifacts { + if artifact.id.version != *version { + return Err(RepositoryError::MultipleVersionsPresent { + kind, + v1: version.clone(), + v2: artifact.id.version.clone(), + }); + } + } + } + Ok(UpdatePlan { system_version: self.system_version, gimlet_sp: self.gimlet_sp, // checked above diff --git a/wicketd/src/bin/wicketd.rs b/wicketd/src/bin/wicketd.rs index 2e6d51c0f0a..887ac496e07 100644 --- a/wicketd/src/bin/wicketd.rs +++ b/wicketd/src/bin/wicketd.rs @@ -4,6 +4,7 @@ //! Executable for wicketd: technician port based management service +use anyhow::{anyhow, Context}; use clap::Parser; use omicron_common::{ address::Ipv6Subnet, @@ -69,7 +70,9 @@ async fn do_run() -> Result<(), CmdError> { let args = Args::parse(); match args { - Args::Openapi => run_openapi().map_err(CmdError::Failure), + Args::Openapi => { + run_openapi().map_err(|err| CmdError::Failure(anyhow!(err))) + } Args::Run { config_file_path, address, @@ -82,10 +85,10 @@ async fn do_run() -> Result<(), CmdError> { } => { let baseboard = if let Some(baseboard_file) = baseboard_file { let baseboard_file = std::fs::read_to_string(baseboard_file) - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(|e| CmdError::Failure(anyhow!(e)))?; let baseboard: Baseboard = serde_json::from_str(&baseboard_file) - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(|e| CmdError::Failure(anyhow!(e)))?; // TODO-correctness `Baseboard::unknown()` is slated for removal // after some refactoring in sled-agent, at which point we'll @@ -100,19 +103,17 @@ async fn do_run() -> Result<(), CmdError> { None }; - let config = Config::from_file(&config_file_path).map_err(|e| { - CmdError::Failure(format!( - "failed to parse {}: {}", - config_file_path.display(), - e - )) - })?; + let config = Config::from_file(&config_file_path) + .with_context(|| { + format!("failed to parse {}", config_file_path.display()) + }) + .map_err(CmdError::Failure)?; let rack_subnet = match rack_subnet { Some(addr) => Some(Ipv6Subnet::new(addr)), None if read_smf_config => { let smf_values = SmfConfigValues::read_current() - .map_err(|e| CmdError::Failure(e.to_string()))?; + .map_err(CmdError::Failure)?; smf_values.rack_subnet } None => None, @@ -126,12 +127,18 @@ async fn do_run() -> Result<(), CmdError> { baseboard, rack_subnet, }; - let log = config.log.to_logger("wicketd").map_err(|msg| { - CmdError::Failure(format!("initializing logger: {}", msg)) - })?; - let server = - Server::start(log, args).await.map_err(CmdError::Failure)?; - server.wait_for_finish().await.map_err(CmdError::Failure) + let log = config + .log + .to_logger("wicketd") + .context("failed to initialize logger") + .map_err(CmdError::Failure)?; + let server = Server::start(log, args) + .await + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + server + .wait_for_finish() + .await + .map_err(|err| CmdError::Failure(anyhow!(err))) } } } diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index be0f681601d..d6cb6ebd6d0 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -708,6 +708,15 @@ pub(crate) enum UpdateSimulatedResult { Failure, } +#[derive(Clone, Debug, JsonSchema, Deserialize)] +pub(crate) struct ClearUpdateStateParams { + /// The SP identifiers to clear the update state for. Must be non-empty. + pub(crate) targets: BTreeSet, + + /// Options for clearing update state + pub(crate) options: ClearUpdateStateOptions, +} + #[derive(Clone, Debug, JsonSchema, Deserialize)] pub(crate) struct ClearUpdateStateOptions { /// If passed in, fails the clear update state operation with a simulated @@ -715,6 +724,15 @@ pub(crate) struct ClearUpdateStateOptions { pub(crate) test_error: Option, } +#[derive(Clone, Debug, Default, JsonSchema, Serialize)] +pub(crate) struct ClearUpdateStateResponse { + /// The SPs for which update data was cleared. + pub(crate) cleared: BTreeSet, + + /// The SPs that had no update state to clear. + pub(crate) no_update_data: BTreeSet, +} + #[derive(Clone, Debug, JsonSchema, Deserialize)] pub(crate) struct AbortUpdateOptions { /// The message to abort the update with. @@ -1080,25 +1098,31 @@ async fn post_abort_update( /// Use this to clear update state after a failed update. #[endpoint { method = POST, - path = "/clear-update-state/{type}/{slot}", + path = "/clear-update-state", }] async fn post_clear_update_state( rqctx: RequestContext, - target: Path, - opts: TypedBody, -) -> Result { + params: TypedBody, +) -> Result, HttpError> { let log = &rqctx.log; - let target = target.into_inner(); + let rqctx = rqctx.context(); + let params = params.into_inner(); - let opts = opts.into_inner(); - if let Some(test_error) = opts.test_error { + if params.targets.is_empty() { + return Err(HttpError::for_bad_request( + None, + "No targets specified".into(), + )); + } + + if let Some(test_error) = params.options.test_error { return Err(test_error .into_http_error(log, "clearing update state") .await); } - match rqctx.context().update_tracker.clear_update_state(target).await { - Ok(()) => Ok(HttpResponseUpdatedNoContent {}), + match rqctx.update_tracker.clear_update_state(params.targets).await { + Ok(response) => Ok(HttpResponseOk(response)), Err(err) => Err(err.to_http_error()), } } diff --git a/wicketd/src/preflight_check/uplink.rs b/wicketd/src/preflight_check/uplink.rs index 4d199d28b8f..d94baf19955 100644 --- a/wicketd/src/preflight_check/uplink.rs +++ b/wicketd/src/preflight_check/uplink.rs @@ -24,6 +24,7 @@ use omicron_common::api::internal::shared::PortFec as OmicronPortFec; use omicron_common::api::internal::shared::PortSpeed as OmicronPortSpeed; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::api::internal::shared::SwitchLocation; +use omicron_common::OMICRON_DPD_TAG; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; @@ -185,7 +186,11 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( // Create and configure the link. match dpd_client - .port_settings_apply(&port_id, &port_settings) + .port_settings_apply( + &port_id, + Some(OMICRON_DPD_TAG), + &port_settings, + ) .await { Ok(_response) => { @@ -714,8 +719,8 @@ fn add_steps_for_single_local_uplink_preflight_check<'a>( dpd_client .port_settings_apply( &port_id, + Some(OMICRON_DPD_TAG), &PortSettings { - tag: WICKETD_TAG.to_string(), links: HashMap::new(), v4_routes: HashMap::new(), v6_routes: HashMap::new(), @@ -758,7 +763,6 @@ fn build_port_settings( }; let mut port_settings = PortSettings { - tag: WICKETD_TAG.to_string(), links: HashMap::new(), v4_routes: HashMap::new(), v6_routes: HashMap::new(), @@ -777,6 +781,7 @@ fn build_port_settings( kr: false, fec, speed, + lane: Some(LinkId(0)), }, }, ); diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index a96acc56a0f..0aaea427f3f 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -521,6 +521,11 @@ fn validate_rack_network_config( addr: p.addr, asn: p.asn, port: p.port.clone(), + hold_time: p.hold_time, + connect_retry: p.connect_retry, + delay_open: p.delay_open, + idle_hold_time: p.idle_hold_time, + keepalive: p.keepalive, }) .collect(), switch: match config.switch { diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index bd8e187fe96..f4b5db24765 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -8,6 +8,7 @@ use crate::artifacts::ArtifactIdData; use crate::artifacts::UpdatePlan; use crate::artifacts::WicketdArtifactStore; use crate::helpers::sps_to_string; +use crate::http_entrypoints::ClearUpdateStateResponse; use crate::http_entrypoints::GetArtifactsAndEventReportsResponse; use crate::http_entrypoints::StartUpdateOptions; use crate::http_entrypoints::UpdateSimulatedResult; @@ -184,10 +185,10 @@ impl UpdateTracker { pub(crate) async fn clear_update_state( &self, - sp: SpIdentifier, - ) -> Result<(), ClearUpdateStateError> { + sps: BTreeSet, + ) -> Result { let mut update_data = self.sp_update_data.lock().await; - update_data.clear_update_state(sp) + update_data.clear_update_state(&sps) } pub(crate) async fn abort_update( @@ -226,24 +227,23 @@ impl UpdateTracker { let mut errors = Vec::new(); - // Check that we're not already updating any of these SPs. - let update_in_progress: Vec<_> = sps + // Check that we don't already have any update state for these SPs. + let existing_updates: Vec<_> = sps .iter() .filter(|sp| { // If we don't have any update data for this SP, it's not in // progress. // - // If we do, it's in progress if the task is not finished. - update_data - .sp_update_data - .get(sp) - .map_or(false, |data| !data.task.is_finished()) + // This used to check that the task was finished, but we changed + // that in favor of forcing users to clear update state before + // starting a new one. + update_data.sp_update_data.get(sp).is_some() }) .copied() .collect(); - if !update_in_progress.is_empty() { - errors.push(StartUpdateError::UpdateInProgress(update_in_progress)); + if !existing_updates.is_empty() { + errors.push(StartUpdateError::ExistingUpdates(existing_updates)); } let plan = update_data.artifact_store.current_plan(); @@ -609,19 +609,38 @@ impl UpdateTrackerData { fn clear_update_state( &mut self, - sp: SpIdentifier, - ) -> Result<(), ClearUpdateStateError> { - // Is an update currently running? If so, then reject the request. - let is_running = self - .sp_update_data - .get(&sp) - .map_or(false, |update_data| !update_data.task.is_finished()); - if is_running { - return Err(ClearUpdateStateError::UpdateInProgress); + sps: &BTreeSet, + ) -> Result { + // Are any updates currently running? If so, then reject the request. + let in_progress_updates = sps + .iter() + .filter_map(|sp| { + self.sp_update_data + .get(sp) + .map_or(false, |update_data| { + !update_data.task.is_finished() + }) + .then(|| *sp) + }) + .collect::>(); + + if !in_progress_updates.is_empty() { + return Err(ClearUpdateStateError::UpdateInProgress( + in_progress_updates, + )); } - self.sp_update_data.remove(&sp); - Ok(()) + let mut resp = ClearUpdateStateResponse::default(); + + for sp in sps { + if self.sp_update_data.remove(sp).is_some() { + resp.cleared.insert(*sp); + } else { + resp.no_update_data.insert(*sp); + } + } + + Ok(resp) } async fn abort_update( @@ -689,14 +708,14 @@ impl UpdateTrackerData { pub enum StartUpdateError { #[error("no TUF repository available")] TufRepositoryUnavailable, - #[error("targets are already being updated: {}", sps_to_string(.0))] - UpdateInProgress(Vec), + #[error("existing update data found (must clear state before starting): {}", sps_to_string(.0))] + ExistingUpdates(Vec), } #[derive(Debug, Clone, Error, Eq, PartialEq)] pub enum ClearUpdateStateError { - #[error("target is currently being updated")] - UpdateInProgress, + #[error("targets are currently being updated: {}", sps_to_string(.0))] + UpdateInProgress(Vec), } impl ClearUpdateStateError { @@ -704,7 +723,7 @@ impl ClearUpdateStateError { let message = DisplayErrorChain::new(self).to_string(); match self { - ClearUpdateStateError::UpdateInProgress => { + ClearUpdateStateError::UpdateInProgress(_) => { HttpError::for_bad_request(None, message) } } @@ -884,17 +903,24 @@ impl UpdateDriver { (), format!( "RoT active slot already at version {}", - rot_interrogation.artifact_to_apply.id.version + rot_interrogation.available_artifacts_version, ), ) .into(); } + let artifact_to_apply = rot_interrogation + .choose_artifact_to_apply( + &update_cx.mgs_client, + &update_cx.log, + ) + .await?; + cx.with_nested_engine(|engine| { inner_cx.register_steps( engine, rot_interrogation.slot_to_update, - &rot_interrogation.artifact_to_apply, + artifact_to_apply, ); Ok(()) }) @@ -908,7 +934,7 @@ impl UpdateDriver { (), format!( "RoT updated despite already having version {}", - rot_interrogation.artifact_to_apply.id.version + rot_interrogation.available_artifacts_version ), ) .into() @@ -1455,14 +1481,191 @@ fn define_test_steps(engine: &UpdateEngine, secs: u64) { #[derive(Debug)] struct RotInterrogation { + // Which RoT slot we need to update. slot_to_update: u16, - artifact_to_apply: ArtifactIdData, + // This is a `Vec<_>` because we may have the same version of the RoT + // artifact supplied with different keys. Once we decide whether we're going + // to apply this update at all, we'll ask the RoT for its CMPA and CFPA + // pages to filter this list down to the one matching artifact. + available_artifacts: Vec, + // We require all the artifacts in `available_artifacts` to have the same + // version, and we record that here for use in our methods below. + available_artifacts_version: SemverVersion, + // Identifier of the target RoT's SP. + sp: SpIdentifier, + // Version reported by the target RoT. active_version: Option, } impl RotInterrogation { fn active_version_matches_artifact_to_apply(&self) -> bool { - Some(&self.artifact_to_apply.id.version) == self.active_version.as_ref() + Some(&self.available_artifacts_version) == self.active_version.as_ref() + } + + /// Via `client`, ask the target RoT for its CMPA/CFPA pages, then loop + /// through our `available_artifacts` to find one that verifies. + /// + /// For backwards compatibility with RoTs that do not know how to return + /// their CMPA/CFPA pages, if we fail to fetch them _and_ + /// `available_artifacts` has exactly one item, we will return that one + /// item. + async fn choose_artifact_to_apply( + &self, + client: &gateway_client::Client, + log: &Logger, + ) -> Result<&ArtifactIdData, UpdateTerminalError> { + let cmpa = match client + .sp_rot_cmpa_get( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + ) + .await + { + Ok(response) => { + let data = response.into_inner().base64_data; + self.decode_rot_page(&data).map_err(|error| { + UpdateTerminalError::GetRotCmpaFailed { error } + })? + } + // TODO is there a better way to check the _specific_ error response + // we get here? We only have a couple of strings; we could check the + // error string contents for something like "WrongVersion", but + // that's pretty fragile. Instead we'll treat any error response + // here as a "fallback to previous behavior". + Err(err @ gateway_client::Error::ErrorResponse(_)) => { + if self.available_artifacts.len() == 1 { + info!( + log, + "Failed to get RoT CMPA page; \ + using only available RoT artifact"; + "err" => %err, + ); + return Ok(&self.available_artifacts[0]); + } else { + error!( + log, + "Failed to get RoT CMPA; unable to choose from \ + multiple available RoT artifacts"; + "err" => %err, + "num_rot_artifacts" => self.available_artifacts.len(), + ); + return Err(UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }); + } + } + // For any other error (e.g., comms failures), just fail as normal. + Err(err) => { + return Err(UpdateTerminalError::GetRotCmpaFailed { + error: err.into(), + }); + } + }; + + // We have a CMPA; we also need the CFPA, but we don't bother checking + // for an `ErrorResponse` as above because succeeding in getting the + // CMPA means the RoT is new enough to support returning both. + let cfpa = client + .sp_rot_cfpa_get( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + &gateway_client::types::GetCfpaParams { + slot: RotCfpaSlot::Active, + }, + ) + .await + .map_err(|err| UpdateTerminalError::GetRotCfpaFailed { + error: err.into(), + }) + .and_then(|response| { + let data = response.into_inner().base64_data; + self.decode_rot_page(&data).map_err(|error| { + UpdateTerminalError::GetRotCfpaFailed { error } + }) + })?; + + // Loop through our possible artifacts and find the first (we only + // expect one!) that verifies against the RoT's CMPA/CFPA. + for artifact in &self.available_artifacts { + let image = artifact + .data + .reader_stream() + .and_then(|stream| async { + let mut buf = Vec::with_capacity(artifact.data.file_size()); + StreamReader::new(stream) + .read_to_end(&mut buf) + .await + .context("I/O error reading extracted archive")?; + Ok(buf) + }) + .await + .map_err(|error| { + UpdateTerminalError::FailedFindingSignedRotImage { error } + })?; + let archive = RawHubrisArchive::from_vec(image).map_err(|err| { + UpdateTerminalError::FailedFindingSignedRotImage { + error: anyhow::Error::new(err).context(format!( + "failed to read hubris archive for {:?}", + artifact.id + )), + } + })?; + match archive.verify(&cmpa, &cfpa) { + Ok(()) => { + info!( + log, "RoT archive verification success"; + "name" => artifact.id.name.as_str(), + "version" => %artifact.id.version, + "kind" => ?artifact.id.kind, + ); + return Ok(artifact); + } + Err(err) => { + // We log this but don't fail - we want to continue + // looking for a verifiable artifact. + info!( + log, "RoT archive verification failed"; + "artifact" => ?artifact.id, + "err" => %DisplayErrorChain::new(&err), + ); + } + } + } + + // If the loop above didn't find a verifiable image, we cannot proceed. + Err(UpdateTerminalError::FailedFindingSignedRotImage { + error: anyhow!("no RoT image found with valid CMPA/CFPA"), + }) + } + + /// Decode a base64-encoded RoT page we received from MGS. + fn decode_rot_page( + &self, + data: &str, + ) -> anyhow::Result<[u8; ROT_PAGE_SIZE]> { + // Even though we know `data` should decode to exactly + // `ROT_PAGE_SIZE` bytes, the base64 crate requires an output buffer + // of at least `decoded_len_estimate`. Allocate such a buffer here, + // then we'll copy to the fixed-size array we need after confirming + // the number of decoded bytes; + let mut output_buf = vec![0; base64::decoded_len_estimate(data.len())]; + + let n = base64::engine::general_purpose::STANDARD + .decode_slice(&data, &mut output_buf) + .with_context(|| { + format!("failed to decode base64 string: {data:?}") + })?; + if n != ROT_PAGE_SIZE { + bail!( + "incorrect len ({n}, expected {ROT_PAGE_SIZE}) \ + after decoding base64 string: {data:?}", + ); + } + let mut page = [0; ROT_PAGE_SIZE]; + page.copy_from_slice(&output_buf[..n]); + Ok(page) } } @@ -1578,175 +1781,16 @@ impl UpdateContext { } }; - // Read the CMPA and currently-active CFPA so we can find the - // correctly-signed artifact. - let base64_decode_rot_page = |data: String| { - // Even though we know `data` should decode to exactly - // `ROT_PAGE_SIZE` bytes, the base64 crate requires an output buffer - // of at least `decoded_len_estimate`. Allocate such a buffer here, - // then we'll copy to the fixed-size array we need after confirming - // the number of decoded bytes; - let mut output_buf = - vec![0; base64::decoded_len_estimate(data.len())]; - - let n = base64::engine::general_purpose::STANDARD - .decode_slice(&data, &mut output_buf) - .with_context(|| { - format!("failed to decode base64 string: {data:?}") - })?; - if n != ROT_PAGE_SIZE { - bail!( - "incorrect len ({n}, expected {ROT_PAGE_SIZE}) \ - after decoding base64 string: {data:?}", - ); - } - let mut page = [0; ROT_PAGE_SIZE]; - page.copy_from_slice(&output_buf[..n]); - Ok(page) - }; - - // We may be talking to an SP / RoT that doesn't yet know how to give us - // its CMPA. If we hit a protocol version error here, we can fall back - // to our old behavior of requiring exactly 1 RoT artifact. - let mut artifact_to_apply = None; - - let cmpa = match self - .mgs_client - .sp_rot_cmpa_get( - self.sp.type_, - self.sp.slot, - SpComponent::ROT.const_as_str(), - ) - .await - { - Ok(response) => { - let data = response.into_inner().base64_data; - Some(base64_decode_rot_page(data).map_err(|error| { - UpdateTerminalError::GetRotCmpaFailed { error } - })?) - } - // TODO is there a better way to check the _specific_ error response - // we get here? We only have a couple of strings; we could check the - // error string contents for something like "WrongVersion", but - // that's pretty fragile. Instead we'll treat any error response - // here as a "fallback to previous behavior". - Err(err @ gateway_client::Error::ErrorResponse(_)) => { - if available_artifacts.len() == 1 { - info!( - self.log, - "Failed to get RoT CMPA page; \ - using only available RoT artifact"; - "err" => %err, - ); - artifact_to_apply = Some(available_artifacts[0].clone()); - None - } else { - error!( - self.log, - "Failed to get RoT CMPA; unable to choose from \ - multiple available RoT artifacts"; - "err" => %err, - "num_rot_artifacts" => available_artifacts.len(), - ); - return Err(UpdateTerminalError::GetRotCmpaFailed { - error: err.into(), - }); - } - } - // For any other error (e.g., comms failures), just fail as normal. - Err(err) => { - return Err(UpdateTerminalError::GetRotCmpaFailed { - error: err.into(), - }); - } - }; - - // If we were able to fetch the CMPA, we also need to fetch the CFPA and - // then find the correct RoT artifact. If we weren't able to fetch the - // CMPA, we either already set `artifact_to_apply` to the one-and-only - // RoT artifact available, or we returned a terminal error. - if let Some(cmpa) = cmpa { - let cfpa = self - .mgs_client - .sp_rot_cfpa_get( - self.sp.type_, - self.sp.slot, - SpComponent::ROT.const_as_str(), - &gateway_client::types::GetCfpaParams { - slot: RotCfpaSlot::Active, - }, - ) - .await - .map_err(|err| UpdateTerminalError::GetRotCfpaFailed { - error: err.into(), - }) - .and_then(|response| { - let data = response.into_inner().base64_data; - base64_decode_rot_page(data).map_err(|error| { - UpdateTerminalError::GetRotCfpaFailed { error } - }) - })?; - - // Loop through our possible artifacts and find the first (we only - // expect one!) that verifies against the RoT's CMPA/CFPA. - for artifact in available_artifacts { - let image = artifact - .data - .reader_stream() - .and_then(|stream| async { - let mut buf = - Vec::with_capacity(artifact.data.file_size()); - StreamReader::new(stream) - .read_to_end(&mut buf) - .await - .context("I/O error reading extracted archive")?; - Ok(buf) - }) - .await - .map_err(|error| { - UpdateTerminalError::FailedFindingSignedRotImage { - error, - } - })?; - let archive = - RawHubrisArchive::from_vec(image).map_err(|err| { - UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow::Error::new(err).context(format!( - "failed to read hubris archive for {:?}", - artifact.id - )), - } - })?; - match archive.verify(&cmpa, &cfpa) { - Ok(()) => { - info!( - self.log, "RoT archive verification success"; - "name" => artifact.id.name.as_str(), - "version" => %artifact.id.version, - "kind" => ?artifact.id.kind, - ); - artifact_to_apply = Some(artifact.clone()); - break; - } - Err(err) => { - // We log this but don't fail - we want to continue - // looking for a verifiable artifact. - info!( - self.log, "RoT archive verification failed"; - "artifact" => ?artifact.id, - "err" => %DisplayErrorChain::new(&err), - ); - } - } - } - } - - // Ensure we found a valid RoT artifact. - let artifact_to_apply = artifact_to_apply.ok_or_else(|| { - UpdateTerminalError::FailedFindingSignedRotImage { - error: anyhow!("no RoT image found with valid CMPA/CFPA"), - } - })?; + // We already validated at repo-upload time there is at least one RoT + // artifact available and that all available RoT artifacts are the same + // version, so we can unwrap the first artifact here and assume its + // version matches any subsequent artifacts. + let available_artifacts_version = available_artifacts + .get(0) + .expect("no RoT artifacts available") + .id + .version + .clone(); // Read the caboose of the currently-active slot. let caboose = self @@ -1768,9 +1812,12 @@ impl UpdateContext { caboose.version, caboose.git_commit ); + let available_artifacts = available_artifacts.to_vec(); let make_result = |active_version| RotInterrogation { slot_to_update, - artifact_to_apply, + available_artifacts, + available_artifacts_version, + sp: self.sp, active_version, }; diff --git a/wicketd/tests/integration_tests/updates.rs b/wicketd/tests/integration_tests/updates.rs index aa145a0f16a..fb1637f44ec 100644 --- a/wicketd/tests/integration_tests/updates.rs +++ b/wicketd/tests/integration_tests/updates.rs @@ -12,17 +12,23 @@ use clap::Parser; use gateway_messages::SpPort; use gateway_test_utils::setup as gateway_setup; use installinator::HOST_PHASE_2_FILE_NAME; +use maplit::btreeset; use omicron_common::{ api::internal::nexus::KnownArtifactKind, update::{ArtifactHashId, ArtifactKind}, }; use tokio::sync::watch; +use update_engine::NestedError; use uuid::Uuid; -use wicket_common::update_events::{StepEventKind, UpdateComponent}; +use wicket::OutputKind; +use wicket_common::{ + rack_update::{ClearUpdateStateResponse, SpIdentifier, SpType}, + update_events::{StepEventKind, UpdateComponent}, +}; use wicketd::{RunningUpdateState, StartUpdateError}; use wicketd_client::types::{ - GetInventoryParams, GetInventoryResponse, SpIdentifier, SpType, - StartUpdateOptions, StartUpdateParams, + GetInventoryParams, GetInventoryResponse, StartUpdateOptions, + StartUpdateParams, }; #[tokio::test] @@ -149,7 +155,7 @@ async fn test_updates() { let terminal_event = 'outer: loop { let event_report = wicketd_testctx .wicketd_client - .get_update_sp(target_sp.type_, target_sp.slot) + .get_update_sp(&target_sp.type_, target_sp.slot) .await .expect("get_update_sp successful") .into_inner(); @@ -176,6 +182,79 @@ async fn test_updates() { } } + // Try starting the update again -- this should fail because we require that update state is + // cleared before starting a new one. + { + let error = wicketd_testctx + .wicketd_client + .post_start_update(¶ms) + .await + .expect_err( + "post_start_update should fail \ + since update data is already present", + ); + let error_str = error.to_string(); + assert!( + // Errors lose type information across the OpenAPI boundary, so sadly we have to match on + // the error string. + error_str.contains("existing update data found"), + "unexpected error: {error_str}" + ); + } + + // Try clearing the update via the wicket CLI. + { + let args = vec![ + "rack-update", + "clear", + "--sled", + "0,1", + "--message-format", + "json", + ]; + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let output = OutputKind::Captured { + log: wicketd_testctx.log().clone(), + stdout: &mut stdout, + stderr: &mut stderr, + }; + + wicket::exec_with_args(wicketd_testctx.wicketd_addr, args, output) + .await + .expect("wicket rack-update clear failed"); + + // stdout should contain a JSON object. + let response: Result = + serde_json::from_slice(&stdout).expect("stdout is valid JSON"); + assert_eq!( + response.expect("expected Ok response"), + ClearUpdateStateResponse { + cleared: btreeset![SpIdentifier { + type_: SpType::Sled, + slot: 0 + }], + no_update_data: btreeset![SpIdentifier { + type_: SpType::Sled, + slot: 1 + }], + } + ); + } + + // Check to see that the update state for SP 0 was cleared. + let event_report = wicketd_testctx + .wicketd_client + .get_update_sp(&target_sp.type_, target_sp.slot) + .await + .expect("get_update_sp successful") + .into_inner(); + assert!( + event_report.step_events.is_empty(), + "update state should be cleared (instead got {:?}", + event_report + ); + wicketd_testctx.teardown().await; } @@ -367,8 +446,8 @@ async fn test_update_races() { .await .expect_err("failed because update is currently running"); - // Also try starting another fake update, which should fail -- we don't let - // updates be started in the middle of other updates. + // Also try starting another fake update, which should fail -- we don't let updates be started + // if there's current update state. { let (_, receiver) = watch::channel(()); let err = wicketd_testctx @@ -380,7 +459,7 @@ async fn test_update_races() { assert_eq!(err.len(), 1, "one error returned: {err:?}"); assert_eq!( err.first().unwrap(), - &StartUpdateError::UpdateInProgress(vec![sp]) + &StartUpdateError::ExistingUpdates(vec![sp]) ); } @@ -390,7 +469,7 @@ async fn test_update_races() { // Ensure that the event buffer indicates completion. let event_buffer = wicketd_testctx .wicketd_client - .get_update_sp(SpType::Sled, 0) + .get_update_sp(&SpType::Sled, 0) .await .expect("received event buffer successfully"); let last_event = @@ -412,7 +491,7 @@ async fn test_update_races() { // clean. let event_buffer = wicketd_testctx .wicketd_client - .get_update_sp(SpType::Sled, 0) + .get_update_sp(&SpType::Sled, 0) .await .expect("received event buffer successfully"); assert!( diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 72854ed29ad..c95226b9605 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -19,9 +19,9 @@ bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] } -bitvec = { version = "1.0.1" } bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } +byteorder = { version = "1.5.0" } bytes = { version = "1.5.0", features = ["serde"] } chrono = { version = "0.4.31", features = ["alloc", "serde"] } cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] } @@ -31,6 +31,7 @@ console = { version = "0.15.7" } const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] } crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } +crossterm = { version = "0.27.0", features = ["event-stream", "serde"] } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } diesel = { version = "2.1.3", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } @@ -46,17 +47,16 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } -hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hashbrown = { version = "0.13.2" } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } -indexmap = { version = "2.0.0", features = ["serde"] } +indexmap = { version = "2.1.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.149", features = ["extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } memchr = { version = "2.6.3" } @@ -64,21 +64,22 @@ num-bigint = { version = "0.4.4", features = ["rand"] } num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } -openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } +openapiv3 = { version = "2.0.0-rc.1", default-features = false, features = ["skip_serializing_defaults"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.4" } -rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } -rand_chacha = { version = "0.3.1" } +proc-macro2 = { version = "1.0.69" } +rand = { version = "0.8.5" } +rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] } regex = { version = "1.10.2" } regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8.2" } -reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } +reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } -serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +serde = { version = "1.0.192", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.108", features = ["raw_value"] } sha2 = { version = "0.10.8", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } @@ -94,12 +95,13 @@ tokio = { version = "1.33.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } +toml_edit-647d43efb71741da = { package = "toml_edit", version = "0.21.0", features = ["serde"] } tracing = { version = "0.1.37", features = ["log"] } trust-dns-proto = { version = "0.22.0" } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } usdt = { version = "0.3.5" } -uuid = { version = "1.4.1", features = ["serde", "v4"] } +uuid = { version = "1.5.0", features = ["serde", "v4"] } yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] } zeroize = { version = "1.6.0", features = ["std", "zeroize_derive"] } zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } @@ -110,11 +112,10 @@ bit-set = { version = "0.5.3" } bit-vec = { version = "0.6.3" } bitflags-dff4ba8e3ae991db = { package = "bitflags", version = "1.3.2" } bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["serde"] } -bitvec = { version = "1.0.1" } bstr-6f8ce4dd05d13bba = { package = "bstr", version = "0.2.17" } bstr-dff4ba8e3ae991db = { package = "bstr", version = "1.6.0" } +byteorder = { version = "1.5.0" } bytes = { version = "1.5.0", features = ["serde"] } -cc = { version = "1.0.83", default-features = false, features = ["parallel"] } chrono = { version = "0.4.31", features = ["alloc", "serde"] } cipher = { version = "0.4.4", default-features = false, features = ["block-padding", "zeroize"] } clap = { version = "4.4.3", features = ["derive", "env", "wrap_help"] } @@ -123,6 +124,7 @@ console = { version = "0.15.7" } const-oid = { version = "0.9.5", default-features = false, features = ["db", "std"] } crossbeam-epoch = { version = "0.9.15" } crossbeam-utils = { version = "0.8.16" } +crossterm = { version = "0.27.0", features = ["event-stream", "serde"] } crypto-common = { version = "0.1.6", default-features = false, features = ["getrandom", "std"] } diesel = { version = "2.1.3", features = ["chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes", "network-address", "postgres", "r2d2", "serde_json", "uuid"] } digest = { version = "0.10.7", features = ["mac", "oid", "std"] } @@ -138,17 +140,16 @@ futures-util = { version = "0.3.29", features = ["channel", "io", "sink"] } gateway-messages = { git = "https://github.com/oxidecomputer/management-gateway-service", rev = "2739c18e80697aa6bc235c935176d14b4d757ee9", features = ["std"] } generic-array = { version = "0.14.7", default-features = false, features = ["more_lengths", "zeroize"] } getrandom = { version = "0.2.10", default-features = false, features = ["js", "rdrand", "std"] } -hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14.0", features = ["raw"] } -hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13.2" } +hashbrown = { version = "0.13.2" } hex = { version = "0.4.3", features = ["serde"] } hyper = { version = "0.14.27", features = ["full"] } -indexmap = { version = "2.0.0", features = ["serde"] } +indexmap = { version = "2.1.0", features = ["serde"] } inout = { version = "0.1.3", default-features = false, features = ["std"] } ipnetwork = { version = "0.20.0", features = ["schemars"] } itertools = { version = "0.10.5" } lalrpop-util = { version = "0.19.12" } lazy_static = { version = "1.4.0", default-features = false, features = ["spin_no_std"] } -libc = { version = "0.2.149", features = ["extra_traits"] } +libc = { version = "0.2.150", features = ["extra_traits"] } log = { version = "0.4.20", default-features = false, features = ["std"] } managed = { version = "0.8.0", default-features = false, features = ["alloc", "map"] } memchr = { version = "2.6.3" } @@ -156,21 +157,22 @@ num-bigint = { version = "0.4.4", features = ["rand"] } num-integer = { version = "0.1.45", features = ["i128"] } num-iter = { version = "0.1.43", default-features = false, features = ["i128"] } num-traits = { version = "0.2.16", features = ["i128", "libm"] } -openapiv3 = { version = "1.0.3", default-features = false, features = ["skip_serializing_defaults"] } +openapiv3 = { version = "2.0.0-rc.1", default-features = false, features = ["skip_serializing_defaults"] } petgraph = { version = "0.6.4", features = ["serde-1"] } postgres-types = { version = "0.2.6", default-features = false, features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } ppv-lite86 = { version = "0.2.17", default-features = false, features = ["simd", "std"] } predicates = { version = "3.0.4" } -rand = { version = "0.8.5", features = ["min_const_gen", "small_rng"] } -rand_chacha = { version = "0.3.1" } +proc-macro2 = { version = "1.0.69" } +rand = { version = "0.8.5" } +rand_chacha = { version = "0.3.1", default-features = false, features = ["std"] } regex = { version = "1.10.2" } regex-automata = { version = "0.4.3", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8.2" } -reqwest = { version = "0.11.20", features = ["blocking", "json", "rustls-tls", "stream"] } +reqwest = { version = "0.11.22", features = ["blocking", "json", "rustls-tls", "stream"] } ring = { version = "0.16.20", features = ["std"] } schemars = { version = "0.8.13", features = ["bytes", "chrono", "uuid1"] } semver = { version = "1.0.20", features = ["serde"] } -serde = { version = "1.0.188", features = ["alloc", "derive", "rc"] } +serde = { version = "1.0.192", features = ["alloc", "derive", "rc"] } serde_json = { version = "1.0.108", features = ["raw_value"] } sha2 = { version = "0.10.8", features = ["oid"] } signature = { version = "2.1.0", default-features = false, features = ["digest", "rand_core", "std"] } @@ -187,13 +189,13 @@ tokio = { version = "1.33.0", features = ["full", "test-util"] } tokio-postgres = { version = "0.7.10", features = ["with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } tokio-stream = { version = "0.1.14", features = ["net"] } toml = { version = "0.7.8" } +toml_edit-647d43efb71741da = { package = "toml_edit", version = "0.21.0", features = ["serde"] } tracing = { version = "0.1.37", features = ["log"] } trust-dns-proto = { version = "0.22.0" } unicode-bidi = { version = "0.3.13" } unicode-normalization = { version = "0.1.22" } -unicode-xid = { version = "0.2.4" } usdt = { version = "0.3.5" } -uuid = { version = "1.4.1", features = ["serde", "v4"] } +uuid = { version = "1.5.0", features = ["serde", "v4"] } yasna = { version = "0.5.2", features = ["bit-vec", "num-bigint", "std", "time"] } zeroize = { version = "1.6.0", features = ["std", "zeroize_derive"] } zip = { version = "0.6.6", default-features = false, features = ["bzip2", "deflate"] } @@ -246,8 +248,8 @@ hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } -toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } -toml_edit = { version = "0.19.15", features = ["serde"] } +toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] } +toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } [target.x86_64-unknown-illumos.build-dependencies] bitflags-f595c2ba2a3f28df = { package = "bitflags", version = "2.4.0", default-features = false, features = ["std"] } @@ -255,7 +257,7 @@ hyper-rustls = { version = "0.24.2" } mio = { version = "0.8.8", features = ["net", "os-ext"] } once_cell = { version = "1.18.0", features = ["unstable"] } rustix = { version = "0.38.9", features = ["fs", "termios"] } -toml_datetime = { version = "0.6.3", default-features = false, features = ["serde"] } -toml_edit = { version = "0.19.15", features = ["serde"] } +toml_datetime = { version = "0.6.5", default-features = false, features = ["serde"] } +toml_edit-cdcf2f9584511fe6 = { package = "toml_edit", version = "0.19.15", features = ["serde"] } ### END HAKARI SECTION